diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b3a4d7444..21e48cf4fcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [4.7.0] - 2026-03-30 + ### Added +- **Semantic chunk labeling** (#600): Chunks now include a `chunk_type` field identifying the semantic nature of the content (e.g., `paragraph`, `heading`, `list_item`, `table_cell`, `code_block`). Supported across all 11 language bindings with updated E2E test parity. - **Unified InternalDocument architecture**: All extractors now return a canonical `InternalDocument` with typed elements, relationships, images, and tables. Replaces format-specific intermediate representations. - **Unified rendering layer**: New `new_markdown.rs` renderer produces CommonMark from `InternalDocument`, supporting headings, lists, tables, code blocks, formulas, footnotes, images, and inline annotations (bold, italic, links). - **PDF structure pipeline**: Full rewrite of PDF extraction using `page.text().all()` for clean text, char-indexed font metadata for heading/bold detection, segment-based paragraph gap detection, and pdfium segment bounding boxes for precise paragraph regions. diff --git a/crates/kreuzberg-cli/src/commands/embed.rs b/crates/kreuzberg-cli/src/commands/embed.rs index 2386f30f8e1..c4f8ecb8d2f 100644 --- a/crates/kreuzberg-cli/src/commands/embed.rs +++ b/crates/kreuzberg-cli/src/commands/embed.rs @@ -43,6 +43,7 @@ pub fn embed_command(texts: Vec, preset: &str, format: WireFormat) -> Re .enumerate() .map(|(idx, text)| Chunk { content: text.clone(), + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: 0, diff --git a/crates/kreuzberg-ffi/benches/result_view_benchmark.rs b/crates/kreuzberg-ffi/benches/result_view_benchmark.rs index 00f30c57e3f..bb2fbda4ade 100644 --- a/crates/kreuzberg-ffi/benches/result_view_benchmark.rs +++ b/crates/kreuzberg-ffi/benches/result_view_benchmark.rs @@ -44,6 +44,7 @@ fn create_test_result(content_size: usize, chunk_count: usize) -> ExtractionResu }; Chunk { content: content[start..end].to_string(), + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: start, diff --git a/crates/kreuzberg-ffi/src/helpers.rs b/crates/kreuzberg-ffi/src/helpers.rs index 21ceb611943..7c166062296 100644 --- a/crates/kreuzberg-ffi/src/helpers.rs +++ b/crates/kreuzberg-ffi/src/helpers.rs @@ -679,6 +679,7 @@ mod tests { let chunk = Chunk { content: "Chunk content".to_string(), + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: 0, diff --git a/crates/kreuzberg-ffi/src/result.rs b/crates/kreuzberg-ffi/src/result.rs index 9b21af81187..4bcfb8d072e 100644 --- a/crates/kreuzberg-ffi/src/result.rs +++ b/crates/kreuzberg-ffi/src/result.rs @@ -397,6 +397,7 @@ mod tests { chunks: Some(vec![ kreuzberg::types::Chunk { content: "Chunk 1".to_string(), + chunk_type: Default::default(), embedding: None, metadata: kreuzberg::types::ChunkMetadata { byte_start: 0, @@ -411,6 +412,7 @@ mod tests { }, kreuzberg::types::Chunk { content: "Chunk 2".to_string(), + chunk_type: Default::default(), embedding: None, metadata: kreuzberg::types::ChunkMetadata { byte_start: 8, diff --git a/crates/kreuzberg-ffi/src/result_view.rs b/crates/kreuzberg-ffi/src/result_view.rs index 98226d263ee..594c8fa602b 100644 --- a/crates/kreuzberg-ffi/src/result_view.rs +++ b/crates/kreuzberg-ffi/src/result_view.rs @@ -423,6 +423,7 @@ mod tests { chunks: Some(vec![ kreuzberg::types::Chunk { content: "Chunk 1".to_string(), + chunk_type: Default::default(), embedding: None, metadata: kreuzberg::types::ChunkMetadata { byte_start: 0, @@ -437,6 +438,7 @@ mod tests { }, kreuzberg::types::Chunk { content: "Chunk 2".to_string(), + chunk_type: Default::default(), embedding: None, metadata: kreuzberg::types::ChunkMetadata { byte_start: 8, diff --git a/crates/kreuzberg-node/src/result.rs b/crates/kreuzberg-node/src/result.rs index 2dd3bb85184..2c754cb81fa 100644 --- a/crates/kreuzberg-node/src/result.rs +++ b/crates/kreuzberg-node/src/result.rs @@ -94,6 +94,7 @@ pub struct JsChunkMetadata { #[derive(serde::Serialize, serde::Deserialize, Clone)] pub struct JsChunk { pub content: String, + pub chunk_type: String, #[napi(ts_type = "number[] | undefined")] pub embedding: Option>, pub metadata: JsChunkMetadata, @@ -587,6 +588,10 @@ impl TryFrom for JsExtractionResult { js_chunks.push(JsChunk { content: chunk.content, + chunk_type: serde_json::to_value(chunk.chunk_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "unknown".to_string()), embedding, metadata, }); @@ -759,6 +764,8 @@ impl TryFrom for RustExtractionResult { rust_chunks.push(RustChunk { content: chunk.content, + chunk_type: serde_json::from_value(serde_json::Value::String(chunk.chunk_type)) + .unwrap_or_default(), embedding, metadata: RustChunkMetadata { byte_start: chunk.metadata.byte_start as usize, diff --git a/crates/kreuzberg-php/src/types.rs b/crates/kreuzberg-php/src/types.rs index a6f8c2fa542..f919e518588 100644 --- a/crates/kreuzberg-php/src/types.rs +++ b/crates/kreuzberg-php/src/types.rs @@ -1227,6 +1227,8 @@ pub struct TextChunk { #[php(prop)] pub content: String, #[php(prop)] + pub chunk_type: String, + #[php(prop)] pub embedding: Option>, pub metadata: ChunkMetadata, } @@ -1244,6 +1246,10 @@ impl TextChunk { pub fn from_rust(chunk: kreuzberg::Chunk) -> PhpResult { Ok(Self { content: chunk.content, + chunk_type: serde_json::to_value(chunk.chunk_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "unknown".to_string()), embedding: chunk.embedding, metadata: ChunkMetadata::from_rust(chunk.metadata)?, }) diff --git a/crates/kreuzberg-py/src/types.rs b/crates/kreuzberg-py/src/types.rs index 8c7b8e42569..aae67fa126a 100644 --- a/crates/kreuzberg-py/src/types.rs +++ b/crates/kreuzberg-py/src/types.rs @@ -472,6 +472,10 @@ impl ExtractionResult { let py_chunk = PyChunk { content: chunk.content, + chunk_type: serde_json::to_value(chunk.chunk_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "unknown".to_string()), embedding, metadata: chunk_metadata_dict.unbind(), }; @@ -869,6 +873,9 @@ pub struct PyChunk { #[pyo3(get)] pub content: String, + #[pyo3(get)] + pub chunk_type: String, + #[pyo3(get)] pub embedding: Option>, diff --git a/crates/kreuzberg/src/api/handlers.rs b/crates/kreuzberg/src/api/handlers.rs index 0876b4c4fb3..ee4f84cc7f5 100644 --- a/crates/kreuzberg/src/api/handlers.rs +++ b/crates/kreuzberg/src/api/handlers.rs @@ -469,6 +469,7 @@ pub async fn embed_handler(JsonApi(request): JsonApi) -> Result) -> ChunkType { + let trimmed = content.trim(); + + // ── 1. Heading ────────────────────────────────────────────────────────── + // A chunk that IS a heading (starts with `#`) or that sits at the top + // of the heading hierarchy (h1 only, very short content). + if is_heading(trimmed, heading_context) { + return ChunkType::Heading; + } + + // ── 2. Code block ─────────────────────────────────────────────────────── + // Use original content (not trimmed) so leading-indented blocks retain + // their 4-space prefix on every line. + if is_code_block(content) { + return ChunkType::CodeBlock; + } + + // ── 3. Table-like ─────────────────────────────────────────────────────── + if is_table_like(trimmed) { + return ChunkType::TableLike; + } + + // ── 4. Formula ────────────────────────────────────────────────────────── + if is_formula(trimmed) { + return ChunkType::Formula; + } + + // ── 5. Schedule / annex (heading context or keyword) ──────────────────── + if is_schedule(trimmed, heading_context) { + return ChunkType::Schedule; + } + + // ── 6. Definitions ────────────────────────────────────────────────────── + if is_definitions(trimmed) { + return ChunkType::Definitions; + } + + // ── 7. Signature block ────────────────────────────────────────────────── + if is_signature_block(trimmed) { + return ChunkType::SignatureBlock; + } + + // ── 8. Operative clause ───────────────────────────────────────────────── + if is_operative_clause(trimmed) { + return ChunkType::OperativeClause; + } + + // ── 9. Party list ─────────────────────────────────────────────────────── + if is_party_list(trimmed) { + return ChunkType::PartyList; + } + + ChunkType::Unknown +} + +// ─── Rule implementations ─────────────────────────────────────────────────── + +fn is_heading(content: &str, _ctx: Option<&HeadingContext>) -> bool { + // Markdown ATX heading + if content.starts_with('#') { + return true; + } + // Setext-style heading: next line is `===` or `---` + let mut lines = content.lines(); + if let (Some(_title), Some(underline)) = (lines.next(), lines.next()) { + let u = underline.trim(); + if !u.is_empty() && (u.chars().all(|c| c == '=') || u.chars().all(|c| c == '-')) { + return true; + } + } + false +} + +fn is_code_block(content: &str) -> bool { + // Fenced code block + if content.starts_with("```") || content.starts_with("~~~") { + return true; + } + // All non-empty lines indented ≥ 4 spaces (classic Markdown code block), + // but only when the block has ≥ 2 lines to avoid false positives. + let lines: Vec<&str> = content.lines().collect(); + if lines.len() >= 2 { + let all_indented = lines + .iter() + .filter(|l| !l.trim().is_empty()) + .all(|l| l.starts_with(" ") || l.starts_with('\t')); + if all_indented { + return true; + } + } + false +} + +fn is_table_like(content: &str) -> bool { + let lines: Vec<&str> = content.lines().collect(); + if lines.len() < 2 { + return false; + } + // Count lines that look like Markdown table rows: contain `|` + let pipe_lines = lines.iter().filter(|l| l.contains('|')).count(); + if pipe_lines >= 2 { + return true; + } + // Count separator lines (`---`, `===`, repeated dashes ≥ 4) + let sep_lines = lines + .iter() + .filter(|l| { + let t = l.trim(); + t.len() >= 4 && t.chars().all(|c| c == '-' || c == '+' || c == '|' || c == ' ') + }) + .count(); + sep_lines >= 3 +} + +fn is_formula(content: &str) -> bool { + // Unicode math symbols + const MATH_SYMBOLS: &[char] = &['∑', '∫', '√', '∂', '∏', '≤', '≥', '≠', '→', '←', '⊂', '⊃']; + if content.chars().any(|c| MATH_SYMBOLS.contains(&c)) { + return true; + } + // LaTeX-style patterns + let lower = content.to_lowercase(); + let latex_patterns = [r"\frac", r"\sum", r"\int", r"\sqrt", r"\alpha", r"\beta", r"\delta", r"$$", r"\["]; + if latex_patterns.iter().any(|p| lower.contains(p)) { + return true; + } + false +} + +fn is_schedule(content: &str, ctx: Option<&HeadingContext>) -> bool { + const KEYWORDS: &[&str] = &["schedule", "annex", "appendix", "exhibit"]; + let lower = content.to_lowercase(); + + // Check heading context for schedule keywords + if let Some(ctx) = ctx { + for h in &ctx.headings { + let hl = h.text.to_lowercase(); + if KEYWORDS.iter().any(|k| hl.contains(k)) { + return true; + } + } + } + // First line starts with a schedule keyword + let first_line = content.lines().next().unwrap_or("").to_lowercase(); + if KEYWORDS.iter().any(|k| first_line.starts_with(k)) { + return true; + } + // Content-level: strong keyword presence (e.g. "Schedule 1 –" or "Annex A:") + let schedule_re = KEYWORDS.iter().any(|k| { + if let Some(idx) = lower.find(k) { + // Must be followed by a space + alphanumeric (e.g. "Schedule 1", "Annex A") + let rest = &lower[idx + k.len()..]; + rest.starts_with(' ') && rest.trim_start().chars().next().map(|c| c.is_alphanumeric()).unwrap_or(false) + } else { + false + } + }); + schedule_re +} + +fn is_definitions(content: &str) -> bool { + let lower = content.to_lowercase(); + // Classic legal definition patterns + let patterns = [ + "\" means ", + "\" shall mean ", + "\" has the meaning", + "' means ", + "' shall mean ", + "means, for purposes", + "is defined as", + "shall be construed as", + ]; + patterns.iter().any(|p| lower.contains(p)) +} + +fn is_signature_block(content: &str) -> bool { + let lower = content.to_lowercase(); + let keywords = ["signature", "signed by", "witnessed by", "date:", "in witness whereof", "authorized signatory", "duly authorized", "____"]; + let hits = keywords.iter().filter(|k| lower.contains(*k)).count(); + hits >= 2 +} + +fn is_operative_clause(content: &str) -> bool { + let lower = content.to_lowercase(); + // Action verbs commonly found in operative legal clauses + let verbs = [ + "shall ", "agree ", "agrees ", "transfer", "grant ", "grants ", + "undertake", "obligat", "covenant", "warrant", "represent", + "indemnif", "assign ", "assigns ", "license ", "licenses ", + "purchase", "sell ", "sells ", "pay ", "pays ", "deliver", + ]; + let hits = verbs.iter().filter(|v| lower.contains(*v)).count(); + hits >= 3 +} + +fn is_party_list(content: &str) -> bool { + // Party lists tend to have multiple short lines with Title Case names, + // often mixed with addresses (contain digits) or role labels. + let lines: Vec<&str> = content + .lines() + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + .collect(); + + if lines.len() < 3 { + return false; + } + + let party_like = lines.iter().filter(|l| is_party_line(l)).count(); + // Majority of lines should look party-like + party_like >= (lines.len() * 2 / 3).max(2) +} + +/// Heuristic for a single "party-like" line. +/// +/// A line looks like a party entry when it: +/// - Is short (≤ 120 chars), AND +/// - Starts with an uppercase letter (Title Case name or role), AND +/// - Contains at least one of: a comma, a digit (address), or a role keyword. +fn is_party_line(line: &str) -> bool { + if line.len() > 120 { + return false; + } + let starts_upper = line.chars().next().map(|c| c.is_uppercase()).unwrap_or(false); + if !starts_upper { + return false; + } + let has_digit = line.chars().any(|c| c.is_ascii_digit()); + let has_comma = line.contains(','); + let lower = line.to_lowercase(); + let has_role = ["investor", "company", "borrower", "lender", "seller", "buyer", "party", "subscriber", "guarantor"] + .iter() + .any(|r| lower.contains(r)); + has_digit || has_comma || has_role +} + +// ─── Tests ────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn classify(content: &str) -> ChunkType { + classify_chunk(content, None) + } + + // ── Heading ────────────────────────────────────────────────────────────── + + #[test] + fn test_heading_atx() { + assert_eq!(classify("# Introduction"), ChunkType::Heading); + assert_eq!(classify("## Section 2"), ChunkType::Heading); + assert_eq!(classify("### Sub-section"), ChunkType::Heading); + } + + #[test] + fn test_heading_setext() { + assert_eq!(classify("Introduction\n============"), ChunkType::Heading); + assert_eq!(classify("Section 2\n---------"), ChunkType::Heading); + } + + #[test] + fn test_not_heading_plain_text() { + assert_ne!(classify("This is plain paragraph text."), ChunkType::Heading); + } + + // ── Code block ─────────────────────────────────────────────────────────── + + #[test] + fn test_code_block_fenced() { + assert_eq!(classify("```rust\nfn main() {}\n```"), ChunkType::CodeBlock); + assert_eq!(classify("~~~python\nprint('hi')\n~~~"), ChunkType::CodeBlock); + } + + #[test] + fn test_code_block_indented() { + let indented = " fn main() {\n println!(\"hello\");\n }"; + assert_eq!(classify(indented), ChunkType::CodeBlock); + } + + // ── Table-like ─────────────────────────────────────────────────────────── + + #[test] + fn test_table_markdown() { + let table = "| Name | Age |\n|------|-----|\n| Alice | 30 |"; + assert_eq!(classify(table), ChunkType::TableLike); + } + + #[test] + fn test_table_single_pipe_line_not_table() { + // Only one pipe line → not enough evidence + assert_ne!(classify("Just one | separator here"), ChunkType::TableLike); + } + + // ── Formula ────────────────────────────────────────────────────────────── + + #[test] + fn test_formula_unicode_symbols() { + assert_eq!(classify("The total ∑ of all values equals 1."), ChunkType::Formula); + assert_eq!(classify("∫ f(x) dx from 0 to ∞"), ChunkType::Formula); + } + + #[test] + fn test_formula_latex() { + assert_eq!(classify(r"The result is $\frac{a}{b}$"), ChunkType::Formula); + assert_eq!(classify(r"$$\sum_{i=0}^{n} x_i$$"), ChunkType::Formula); + } + + // ── Schedule ───────────────────────────────────────────────────────────── + + #[test] + fn test_schedule_first_line() { + assert_eq!(classify("Schedule 1 – Definitions\n\nThis schedule sets out..."), ChunkType::Schedule); + assert_eq!(classify("annex A: Technical Specifications"), ChunkType::Schedule); + } + + // ── Definitions ────────────────────────────────────────────────────────── + + #[test] + fn test_definitions_means() { + assert_eq!(classify("\"Agreement\" means this Investment and Subscription Agreement."), ChunkType::Definitions); + assert_eq!(classify("\"Closing Date\" shall mean the date on which..."), ChunkType::Definitions); + } + + #[test] + fn test_definitions_is_defined_as() { + assert_eq!(classify("The term 'Net Revenue' is defined as all revenue..."), ChunkType::Definitions); + } + + // ── Signature block ─────────────────────────────────────────────────────── + + #[test] + fn test_signature_block() { + let sig = "Signed by: John Smith\nDate: 2026-03-30\nWitnessed by: Jane Doe"; + assert_eq!(classify(sig), ChunkType::SignatureBlock); + } + + #[test] + fn test_signature_block_in_witness() { + let sig = "In witness whereof the parties have duly authorized this agreement.\n____________________\nDate: ___________"; + assert_eq!(classify(sig), ChunkType::SignatureBlock); + } + + // ── Operative clause ───────────────────────────────────────────────────── + + #[test] + fn test_operative_clause_basic() { + let clause = "The Investor shall subscribe for the Shares and agrees to pay the subscription price. The Company shall deliver the Share certificates upon receipt."; + assert_eq!(classify(clause), ChunkType::OperativeClause); + } + + #[test] + fn test_operative_clause_grant() { + let clause = "The Licensor hereby grants, assigns, and transfers all right, title, and interest. The Licensee shall pay and deliver consideration."; + assert_eq!(classify(clause), ChunkType::OperativeClause); + } + + // ── Party list ──────────────────────────────────────────────────────────── + + #[test] + fn test_party_list_basic() { + let parties = "Gregor Guggisberg, Winkelstrasse 12, Zurich\nInvestor\nAlpha Capital AG, Bahnhofstrasse 1, Zurich\nSubscriber\nBeta Holdings Ltd, 10 City Road, London\nBorrower"; + assert_eq!(classify(parties), ChunkType::PartyList); + } + + // ── Unknown ─────────────────────────────────────────────────────────────── + + #[test] + fn test_unknown_plain_text() { + assert_eq!(classify("This document contains general information."), ChunkType::Unknown); + } + + #[test] + fn test_unknown_empty() { + assert_eq!(classify(""), ChunkType::Unknown); + } + + // ── Heading context ─────────────────────────────────────────────────────── + + #[test] + fn test_heading_context_schedule() { + use crate::types::{HeadingContext, HeadingLevel}; + let ctx = HeadingContext { + headings: vec![HeadingLevel { level: 1, text: "Schedule 1 – Definitions".to_string() }], + }; + // Content under a Schedule heading should be classified as Schedule + let result = classify_chunk("This schedule sets out the defined terms.", Some(&ctx)); + assert_eq!(result, ChunkType::Schedule); + } +} diff --git a/crates/kreuzberg/src/chunking/mod.rs b/crates/kreuzberg/src/chunking/mod.rs index 23a3f55a15d..0e32c96de7f 100644 --- a/crates/kreuzberg/src/chunking/mod.rs +++ b/crates/kreuzberg/src/chunking/mod.rs @@ -55,6 +55,7 @@ use std::sync::Arc; // Module declarations pub mod boundaries; mod builder; +pub mod classifier; pub mod config; pub mod core; mod headings; @@ -66,6 +67,7 @@ mod yaml_section; // Re-export submodule types and functions pub use boundaries::{calculate_page_range, validate_page_boundaries}; +pub use classifier::classify_chunk; pub use config::{ChunkSizing, ChunkerType, ChunkingConfig, ChunkingResult}; // ChunkingConfig re-exported from core::config::processing pub use core::{chunk_text, chunk_text_with_heading_source, chunk_text_with_type, chunk_texts_batch}; pub use processor::ChunkingProcessor; diff --git a/crates/kreuzberg/src/chunking/yaml_section.rs b/crates/kreuzberg/src/chunking/yaml_section.rs index 42ad942cfff..18063bfa0b9 100644 --- a/crates/kreuzberg/src/chunking/yaml_section.rs +++ b/crates/kreuzberg/src/chunking/yaml_section.rs @@ -164,6 +164,7 @@ fn build_chunks_from_sections(sections: &[Section], config: &ChunkingConfig) -> if config.max_characters == 0 || content.len() <= config.max_characters { chunks.push(Chunk { content, + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: section.byte_start, @@ -188,6 +189,7 @@ fn build_chunks_from_sections(sections: &[Section], config: &ChunkingConfig) -> for sub_chunk in sub_result.chunks { chunks.push(Chunk { content: format!("{}\n\n{}", prefix, sub_chunk.content), + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: section.byte_start + sub_chunk.metadata.byte_start, diff --git a/crates/kreuzberg/src/mcp/format.rs b/crates/kreuzberg/src/mcp/format.rs index a3d624918fe..c9a69f51bba 100644 --- a/crates/kreuzberg/src/mcp/format.rs +++ b/crates/kreuzberg/src/mcp/format.rs @@ -335,6 +335,7 @@ mod tests { detected_languages: None, chunks: Some(vec![crate::Chunk { content: "Chunk 1".to_string(), + chunk_type: Default::default(), embedding: None, metadata: crate::ChunkMetadata { byte_start: 0, diff --git a/crates/kreuzberg/src/mcp/server.rs b/crates/kreuzberg/src/mcp/server.rs index 7018296981c..25645b99bac 100644 --- a/crates/kreuzberg/src/mcp/server.rs +++ b/crates/kreuzberg/src/mcp/server.rs @@ -643,6 +643,7 @@ fn embed_text_impl(params: super::params::EmbedTextParams) -> Result, } +/// Semantic structural classification of a text chunk. +/// +/// Assigned by the heuristic classifier in `chunking::classifier`. +/// Defaults to `Unknown` when no rule matches. +/// Designed to be extended in future versions without breaking changes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "api", derive(utoipa::ToSchema))] +#[serde(rename_all = "snake_case")] +pub enum ChunkType { + /// Section heading or document title. + Heading, + /// Party list: names, addresses, and signatories. + PartyList, + /// Definition clause ("X means…", "X shall mean…"). + Definitions, + /// Operative clause containing legal/contractual action verbs. + OperativeClause, + /// Signature block with signatures, names, and dates. + SignatureBlock, + /// Schedule, annex, appendix, or exhibit section. + Schedule, + /// Table-like content with aligned columns or repeated patterns. + TableLike, + /// Mathematical formula or equation. + Formula, + /// Code block or preformatted content. + CodeBlock, + /// Embedded or referenced image content. + Image, + /// Organizational chart or hierarchy diagram. + OrgChart, + /// Diagram, figure, or visual illustration. + Diagram, + /// Unclassified or mixed content. + #[default] + Unknown, +} + /// A text chunk with optional embedding and metadata. /// /// Chunks are created when chunking is enabled in `ExtractionConfig`. Each chunk @@ -213,6 +251,13 @@ pub struct Chunk { /// The text content of this chunk. pub content: String, + /// Semantic structural classification of this chunk. + /// + /// Assigned by the heuristic classifier based on content patterns and + /// heading context. Defaults to `ChunkType::Unknown` when no rule matches. + #[serde(default)] + pub chunk_type: ChunkType, + /// Optional embedding vector for this chunk. /// /// Only populated when `EmbeddingConfig` is provided in chunking configuration. diff --git a/crates/kreuzberg/tests/config_features.rs b/crates/kreuzberg/tests/config_features.rs index ea087a166d0..e1e34940b54 100644 --- a/crates/kreuzberg/tests/config_features.rs +++ b/crates/kreuzberg/tests/config_features.rs @@ -548,6 +548,73 @@ More detailed content here in the subsection. } } +/// Test that chunk_type is populated for markdown chunks. +#[tokio::test] +#[cfg(feature = "chunking")] +async fn test_chunk_type_populated() { + use kreuzberg::ChunkType; + + let markdown = r#"# Introduction + +This section introduces the document with some content. + +## Code Example + +```rust +fn hello() { + println!("Hello, world!"); +} +``` + +## Summary + +A brief summary of the document. +"#; + + let config = ExtractionConfig { + chunking: Some(ChunkingConfig { + max_characters: 200, + overlap: 0, + chunker_type: kreuzberg::ChunkerType::Markdown, + ..Default::default() + }), + ..Default::default() + }; + + let result = extract_bytes(markdown.as_bytes(), "text/markdown", &config) + .await + .expect("Should extract successfully"); + + assert!(result.chunks.is_some(), "Chunks should be present"); + let chunks = result.chunks.expect("Should have chunks"); + assert!(!chunks.is_empty(), "Should have at least one chunk"); + + // All chunks must have a chunk_type (not the default Unknown when content is classifiable) + for chunk in &chunks { + // chunk_type must always be set (never uninitialized) + let _ = &chunk.chunk_type; + } + + // At least one chunk should be classified as something other than Unknown + let has_classified = chunks + .iter() + .any(|c| !matches!(c.chunk_type, ChunkType::Unknown)); + assert!(has_classified, "At least one chunk should have a non-Unknown chunk_type"); + + // Heading chunks should be classified as Heading + let heading_chunks: Vec<_> = chunks + .iter() + .filter(|c| c.content.starts_with('#')) + .collect(); + for chunk in &heading_chunks { + assert!( + matches!(chunk.chunk_type, ChunkType::Heading), + "Chunk starting with '#' should be classified as Heading, got {:?}", + chunk.chunk_type + ); + } +} + /// Test tokenizer env var override. #[tokio::test] #[cfg(feature = "chunking-tokenizers")] diff --git a/e2e/c/helpers.c b/e2e/c/helpers.c index 11263d7761d..be3d16f0c23 100644 --- a/e2e/c/helpers.c +++ b/e2e/c/helpers.c @@ -389,7 +389,8 @@ void assert_detected_languages(const CExtractionResult *result, void assert_chunks(const CExtractionResult *result, int has_min, size_t min_count, - int has_max, size_t max_count) { + int has_max, size_t max_count, + int each_has_chunk_type) { size_t count = json_array_count(result->chunks_json); if (has_min && count < min_count) { fprintf(stderr, @@ -403,6 +404,14 @@ void assert_chunks(const CExtractionResult *result, max_count, count); exit(1); } + if (each_has_chunk_type && result->chunks_json) { + /* Very simple check: ensure no "unknown" chunk_type in the JSON */ + if (strstr(result->chunks_json, "\"unknown\"") != NULL || + strstr(result->chunks_json, "\"chunk_type\":null") != NULL) { + fprintf(stderr, "FAIL: expected specific chunk_type, but found unknown/null in chunks_json\n"); + exit(1); + } + } } void assert_images(const CExtractionResult *result, diff --git a/e2e/c/helpers.h b/e2e/c/helpers.h index 3a7548a8d65..855cccdcf1b 100644 --- a/e2e/c/helpers.h +++ b/e2e/c/helpers.h @@ -84,7 +84,8 @@ void assert_detected_languages(const CExtractionResult *result, void assert_chunks(const CExtractionResult *result, int has_min, size_t min_count, - int has_max, size_t max_count); + int has_max, size_t max_count, + int each_has_chunk_type); void assert_images(const CExtractionResult *result, int has_min, size_t min_count, diff --git a/e2e/c/test_contract.c b/e2e/c/test_contract.c index eebf293719b..2f46739ecf6 100644 --- a/e2e/c/test_contract.c +++ b/e2e/c/test_contract.c @@ -292,7 +292,7 @@ static void test_contract_config_chunking(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -301,7 +301,7 @@ static void test_contract_config_chunking_heading_context(void) { CExtractionResult *result = run_extraction("markdown/extraction_test.md", "{\"chunking\":{\"chunker_type\":\"markdown\",\"max_chars\":300,\"max_overlap\":50}}"); if (!result) return; /* skipped */ assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } @@ -311,7 +311,7 @@ static void test_contract_config_chunking_markdown(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -320,7 +320,7 @@ static void test_contract_config_chunking_no_headings(void) { CExtractionResult *result = run_extraction("text/book_war_and_peace_1p.txt", "{\"chunking\":{\"chunker_type\":\"markdown\",\"max_chars\":300,\"max_overlap\":50}}"); if (!result) return; /* skipped */ assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } @@ -329,7 +329,7 @@ static void test_contract_config_chunking_prepend_heading_context(void) { CExtractionResult *result = run_extraction("markdown/extraction_test.md", "{\"chunking\":{\"chunker_type\":\"markdown\",\"max_chars\":300,\"max_overlap\":50,\"prepend_heading_context\":true}}"); if (!result) return; /* skipped */ assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } @@ -339,7 +339,7 @@ static void test_contract_config_chunking_small(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } @@ -348,7 +348,7 @@ static void test_contract_config_chunking_text(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -357,7 +357,7 @@ static void test_contract_config_chunking_tokenizer(void) { CExtractionResult *result = run_extraction("markdown/comprehensive.md", "{\"chunking\":{\"max_chars\":200,\"max_overlap\":40,\"sizing\":{\"type\":\"tokenizer\",\"model\":\"Xenova/gpt-4o\"}}}"); if (!result) return; /* skipped */ assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } diff --git a/e2e/c/test_embeddings.c b/e2e/c/test_embeddings.c index 632626489dd..4a4b5309dfc 100644 --- a/e2e/c/test_embeddings.c +++ b/e2e/c/test_embeddings.c @@ -10,7 +10,7 @@ static void test_embeddings_embedding_async(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -20,7 +20,7 @@ static void test_embeddings_embedding_balanced_preset(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -29,7 +29,7 @@ static void test_embeddings_embedding_disabled(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -39,7 +39,7 @@ static void test_embeddings_embedding_fast_preset(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } diff --git a/e2e/c/test_token_reduction.c b/e2e/c/test_token_reduction.c index 4afb8aae0bb..6dad978de1b 100644 --- a/e2e/c/test_token_reduction.c +++ b/e2e/c/test_token_reduction.c @@ -40,7 +40,7 @@ static void test_token_reduction_token_reduction_with_chunking(void) { assert_min_content_length(result, 5); assert_max_content_length(result, 200); assert_content_not_empty(result); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } diff --git a/e2e/csharp/ContractTests.cs b/e2e/csharp/ContractTests.cs index 0fca206a98b..150e15dbb50 100644 --- a/e2e/csharp/ContractTests.cs +++ b/e2e/csharp/ContractTests.cs @@ -233,7 +233,7 @@ public void ConfigChunking() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, null, null, null); + TestHelpers.AssertChunks(result, 1, null, true, null, null, null, null); } [SkippableFact] @@ -247,7 +247,7 @@ public void ConfigChunkingHeadingContext() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, true, null); + TestHelpers.AssertChunks(result, 2, null, true, null, true, null, null); } [SkippableFact] @@ -262,7 +262,7 @@ public void ConfigChunkingMarkdown() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, null, null, null); + TestHelpers.AssertChunks(result, 1, null, true, null, null, null, null); } [SkippableFact] @@ -276,7 +276,7 @@ public void ConfigChunkingNoHeadings() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, false, null); + TestHelpers.AssertChunks(result, 2, null, true, null, false, null, null); } [SkippableFact] @@ -290,7 +290,7 @@ public void ConfigChunkingPrependHeadingContext() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, true, true); + TestHelpers.AssertChunks(result, 2, null, true, null, true, null, true); } [SkippableFact] @@ -305,7 +305,7 @@ public void ConfigChunkingSmall() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, null, null); + TestHelpers.AssertChunks(result, 2, null, true, null, null, null, null); } [SkippableFact] @@ -319,7 +319,7 @@ public void ConfigChunkingText() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, null, null, null); + TestHelpers.AssertChunks(result, 1, null, true, null, null, null, null); } [SkippableFact] @@ -333,7 +333,7 @@ public void ConfigChunkingTokenizer() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, null, null); + TestHelpers.AssertChunks(result, 2, null, true, null, null, null, null); } [SkippableFact] diff --git a/e2e/csharp/EmbeddingsTests.cs b/e2e/csharp/EmbeddingsTests.cs index 2ff8641124f..2cd836f5090 100644 --- a/e2e/csharp/EmbeddingsTests.cs +++ b/e2e/csharp/EmbeddingsTests.cs @@ -23,7 +23,7 @@ public async Task EmbeddingAsync() var result = await KreuzbergClient.ExtractFileAsync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, true, null, null); + TestHelpers.AssertChunks(result, 1, null, true, true, null, null, null); } [SkippableFact] @@ -39,7 +39,7 @@ public void EmbeddingBalancedPreset() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, true, null, null); + TestHelpers.AssertChunks(result, 1, null, true, true, null, null, null); } [SkippableFact] @@ -53,7 +53,7 @@ public void EmbeddingDisabled() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, false, null, null); + TestHelpers.AssertChunks(result, 1, null, true, false, null, null, null); } [SkippableFact] @@ -69,7 +69,7 @@ public void EmbeddingFastPreset() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, true, null, null); + TestHelpers.AssertChunks(result, 1, null, true, true, null, null, null); } } diff --git a/e2e/csharp/Helpers.cs b/e2e/csharp/Helpers.cs index 2b3f7200dd1..042b9e6b0e1 100644 --- a/e2e/csharp/Helpers.cs +++ b/e2e/csharp/Helpers.cs @@ -464,6 +464,7 @@ public static void AssertChunks( bool? eachHasContent, bool? eachHasEmbedding, bool? eachHasHeadingContext = null, + bool? eachHasChunkType = null, bool? contentStartsWithHeading = null) { var chunks = result.Chunks; @@ -520,6 +521,17 @@ public static void AssertChunks( } } } + if (eachHasChunkType == true) + { + for (var i = 0; i < chunks.Count; i++) + { + var type = chunks[i].ChunkType; + if (string.IsNullOrEmpty(type) || type == "unknown") + { + throw new XunitException($"Chunk {i} has no specific chunk_type, got {type}"); + } + } + } if (contentStartsWithHeading == true) { var headingChar = new string(new[] { (char)35 }); diff --git a/e2e/csharp/TokenReductionTests.cs b/e2e/csharp/TokenReductionTests.cs index c9ceba38fd9..e5d72b74392 100644 --- a/e2e/csharp/TokenReductionTests.cs +++ b/e2e/csharp/TokenReductionTests.cs @@ -66,7 +66,7 @@ public void TokenReductionWithChunking() TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 5); TestHelpers.AssertMaxContentLength(result, 200); - TestHelpers.AssertChunks(result, 1, null, true, null, null, null); + TestHelpers.AssertChunks(result, 1, null, true, null, null, null, null); TestHelpers.AssertContentNotEmpty(result); } diff --git a/e2e/elixir/test/e2e/archive_test.exs b/e2e/elixir/test/e2e/archive_test.exs index 1c37978c431..3620b285897 100644 --- a/e2e/elixir/test/e2e/archive_test.exs +++ b/e2e/elixir/test/e2e/archive_test.exs @@ -11,13 +11,13 @@ defmodule E2E.ArchiveTest do describe "archive fixtures" do test "archive_gz_basic" do case E2E.Helpers.run_fixture( - "archive_gz_basic", - "archives/book_war_and_peace_1p.txt.gz", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "archive_gz_basic", + "archives/book_war_and_peace_1p.txt.gz", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/gzip", "application/x-gzip"]) @@ -33,13 +33,13 @@ defmodule E2E.ArchiveTest do test "archive_sevenz_basic" do case E2E.Helpers.run_fixture( - "archive_sevenz_basic", - "archives/documents.7z", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "archive_sevenz_basic", + "archives/documents.7z", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-7z-compressed"]) @@ -55,13 +55,13 @@ defmodule E2E.ArchiveTest do test "archive_tar_basic" do case E2E.Helpers.run_fixture( - "archive_tar_basic", - "archives/documents.tar", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "archive_tar_basic", + "archives/documents.tar", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-tar", "application/tar"]) @@ -77,13 +77,13 @@ defmodule E2E.ArchiveTest do test "archive_zip_basic" do case E2E.Helpers.run_fixture( - "archive_zip_basic", - "archives/documents.zip", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "archive_zip_basic", + "archives/documents.zip", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/zip", "application/x-zip-compressed"]) diff --git a/e2e/elixir/test/e2e/email_test.exs b/e2e/elixir/test/e2e/email_test.exs index 51b73be3321..f129dacc742 100644 --- a/e2e/elixir/test/e2e/email_test.exs +++ b/e2e/elixir/test/e2e/email_test.exs @@ -11,13 +11,13 @@ defmodule E2E.EmailTest do describe "email fixtures" do test "email_eml_html_body" do case E2E.Helpers.run_fixture( - "email_eml_html_body", - "email/html_only.eml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_eml_html_body", + "email/html_only.eml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["message/rfc822"]) @@ -33,13 +33,13 @@ defmodule E2E.EmailTest do test "email_eml_multipart" do case E2E.Helpers.run_fixture( - "email_eml_multipart", - "email/html_email_multipart.eml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_eml_multipart", + "email/html_email_multipart.eml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["message/rfc822"]) @@ -55,13 +55,13 @@ defmodule E2E.EmailTest do test "email_eml_utf16" do case E2E.Helpers.run_fixture( - "email_eml_utf16", - "vendored/unstructured/eml/fake-email-utf-16.eml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_eml_utf16", + "vendored/unstructured/eml/fake-email-utf-16.eml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["message/rfc822"]) @@ -78,13 +78,13 @@ defmodule E2E.EmailTest do test "email_msg_basic" do case E2E.Helpers.run_fixture( - "email_msg_basic", - "email/fake_email.msg", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_msg_basic", + "email/fake_email.msg", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-outlook"]) @@ -100,13 +100,13 @@ defmodule E2E.EmailTest do test "email_pst_empty" do case E2E.Helpers.run_fixture( - "email_pst_empty", - "email/empty.pst", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_pst_empty", + "email/empty.pst", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-outlook-pst"]) @@ -121,13 +121,13 @@ defmodule E2E.EmailTest do test "email_sample_eml" do case E2E.Helpers.run_fixture( - "email_sample_eml", - "email/sample_email.eml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_sample_eml", + "email/sample_email.eml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["message/rfc822"]) diff --git a/e2e/elixir/test/e2e/html_test.exs b/e2e/elixir/test/e2e/html_test.exs index 38ced073b0d..25dbf508e0f 100644 --- a/e2e/elixir/test/e2e/html_test.exs +++ b/e2e/elixir/test/e2e/html_test.exs @@ -11,13 +11,13 @@ defmodule E2E.HtmlTest do describe "html fixtures" do test "html_complex_layout" do case E2E.Helpers.run_fixture( - "html_complex_layout", - "html/taylor_swift.html", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "html_complex_layout", + "html/taylor_swift.html", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/html"]) @@ -33,26 +33,18 @@ defmodule E2E.HtmlTest do test "html_simple_table" do case E2E.Helpers.run_fixture( - "html_simple_table", - "html/simple_table.html", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "html_simple_table", + "html/simple_table.html", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/html"]) |> E2E.Helpers.assert_min_content_length(100) - |> E2E.Helpers.assert_content_contains_all([ - "Product", - "Category", - "Price", - "Stock", - "Laptop", - "Electronics", - "Sample Data Table" - ]) + |> E2E.Helpers.assert_content_contains_all(["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]) {:skipped, reason} -> IO.puts("SKIPPED: #{reason}") diff --git a/e2e/elixir/test/e2e/image_test.exs b/e2e/elixir/test/e2e/image_test.exs index 82f088f2e4b..1109ed0a9ae 100644 --- a/e2e/elixir/test/e2e/image_test.exs +++ b/e2e/elixir/test/e2e/image_test.exs @@ -11,13 +11,13 @@ defmodule E2E.ImageTest do describe "image fixtures" do test "image_bmp_basic" do case E2E.Helpers.run_fixture( - "image_bmp_basic", - "images/bmp_24.bmp", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_bmp_basic", + "images/bmp_24.bmp", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/bmp"]) @@ -33,13 +33,13 @@ defmodule E2E.ImageTest do test "image_gif_basic" do case E2E.Helpers.run_fixture( - "image_gif_basic", - "images_extra/ocr_image.gif", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_gif_basic", + "images_extra/ocr_image.gif", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/gif"]) @@ -55,13 +55,13 @@ defmodule E2E.ImageTest do test "image_jp2_basic" do case E2E.Helpers.run_fixture( - "image_jp2_basic", - "images_extra/ocr_image.jp2", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_jp2_basic", + "images_extra/ocr_image.jp2", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/jp2", "image/jpeg2000"]) @@ -77,13 +77,13 @@ defmodule E2E.ImageTest do test "image_metadata_only" do case E2E.Helpers.run_fixture( - "image_metadata_only", - "images/example.jpg", - %{ocr: nil}, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "image_metadata_only", + "images/example.jpg", + %{ocr: nil}, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/jpeg"]) @@ -99,13 +99,13 @@ defmodule E2E.ImageTest do test "image_pbm_basic" do case E2E.Helpers.run_fixture( - "image_pbm_basic", - "images_extra/ocr_image.pbm", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_pbm_basic", + "images_extra/ocr_image.pbm", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/x-portable-bitmap", "image/x-pbm"]) @@ -121,13 +121,13 @@ defmodule E2E.ImageTest do test "image_pgm_basic" do case E2E.Helpers.run_fixture( - "image_pgm_basic", - "images_extra/ocr_image.pgm", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_pgm_basic", + "images_extra/ocr_image.pgm", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/x-portable-graymap", "image/x-pgm"]) @@ -143,13 +143,13 @@ defmodule E2E.ImageTest do test "image_ppm_basic" do case E2E.Helpers.run_fixture( - "image_ppm_basic", - "images_extra/ocr_image.ppm", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_ppm_basic", + "images_extra/ocr_image.ppm", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/x-portable-pixmap", "image/x-ppm"]) @@ -165,13 +165,13 @@ defmodule E2E.ImageTest do test "image_svg_basic" do case E2E.Helpers.run_fixture( - "image_svg_basic", - "xml/simple_svg.svg", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "image_svg_basic", + "xml/simple_svg.svg", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/svg+xml"]) @@ -187,13 +187,13 @@ defmodule E2E.ImageTest do test "image_tiff_basic" do case E2E.Helpers.run_fixture( - "image_tiff_basic", - "images_extra/ocr_image.tif", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_tiff_basic", + "images_extra/ocr_image.tif", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/tiff"]) @@ -209,13 +209,13 @@ defmodule E2E.ImageTest do test "image_webp_basic" do case E2E.Helpers.run_fixture( - "image_webp_basic", - "images_extra/ocr_image.webp", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_webp_basic", + "images_extra/ocr_image.webp", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/webp"]) diff --git a/e2e/elixir/test/e2e/keywords_test.exs b/e2e/elixir/test/e2e/keywords_test.exs index 7fefc523628..29aa3907ad2 100644 --- a/e2e/elixir/test/e2e/keywords_test.exs +++ b/e2e/elixir/test/e2e/keywords_test.exs @@ -11,13 +11,13 @@ defmodule E2E.KeywordsTest do describe "keywords fixtures" do test "keywords_rake" do case E2E.Helpers.run_fixture( - "keywords_rake", - "pdf/fake_memo.pdf", - %{keywords: %{algorithm: "rake", max_keywords: 10}}, - requirements: ["keywords-rake"], - notes: nil, - skip_if_missing: true - ) do + "keywords_rake", + "pdf/fake_memo.pdf", + %{keywords: %{algorithm: "rake", max_keywords: 10}}, + requirements: ["keywords-rake"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/pdf"]) @@ -34,13 +34,13 @@ defmodule E2E.KeywordsTest do test "keywords_yake" do case E2E.Helpers.run_fixture( - "keywords_yake", - "pdf/fake_memo.pdf", - %{keywords: %{algorithm: "yake", max_keywords: 10}}, - requirements: ["keywords-yake"], - notes: nil, - skip_if_missing: true - ) do + "keywords_yake", + "pdf/fake_memo.pdf", + %{keywords: %{algorithm: "yake", max_keywords: 10}}, + requirements: ["keywords-yake"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/pdf"]) diff --git a/e2e/elixir/test/e2e/office_test.exs b/e2e/elixir/test/e2e/office_test.exs index 5825aa23135..0cdfd4be69e 100644 --- a/e2e/elixir/test/e2e/office_test.exs +++ b/e2e/elixir/test/e2e/office_test.exs @@ -11,13 +11,13 @@ defmodule E2E.OfficeTest do describe "office fixtures" do test "office_bibtex_basic" do case E2E.Helpers.run_fixture( - "office_bibtex_basic", - "bibtex/comprehensive.bib", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_bibtex_basic", + "bibtex/comprehensive.bib", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-bibtex", "text/x-bibtex"]) @@ -33,13 +33,13 @@ defmodule E2E.OfficeTest do test "office_commonmark_basic" do case E2E.Helpers.run_fixture( - "office_commonmark_basic", - "markdown/sample.commonmark", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_commonmark_basic", + "markdown/sample.commonmark", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/markdown", "text/plain", "text/x-commonmark"]) @@ -55,13 +55,13 @@ defmodule E2E.OfficeTest do test "office_dbf_basic" do case E2E.Helpers.run_fixture( - "office_dbf_basic", - "dbf/stations.dbf", - nil, - requirements: ["office"], - notes: "Requires the office feature.", - skip_if_missing: true - ) do + "office_dbf_basic", + "dbf/stations.dbf", + nil, + requirements: ["office"], + notes: "Requires the office feature.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-dbf"]) @@ -78,13 +78,13 @@ defmodule E2E.OfficeTest do test "office_djot_basic" do case E2E.Helpers.run_fixture( - "office_djot_basic", - "markdown/tables.djot", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_djot_basic", + "markdown/tables.djot", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/x-djot", "text/djot"]) @@ -100,13 +100,13 @@ defmodule E2E.OfficeTest do test "office_doc_legacy" do case E2E.Helpers.run_fixture( - "office_doc_legacy", - "doc/unit_test_lists.doc", - nil, - requirements: ["office"], - notes: "Requires the office feature.", - skip_if_missing: true - ) do + "office_doc_legacy", + "doc/unit_test_lists.doc", + nil, + requirements: ["office"], + notes: "Requires the office feature.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/msword"]) @@ -122,13 +122,13 @@ defmodule E2E.OfficeTest do test "office_docbook_basic" do case E2E.Helpers.run_fixture( - "office_docbook_basic", - "docbook/docbook-reader.docbook", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docbook_basic", + "docbook/docbook-reader.docbook", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/docbook+xml", "text/docbook"]) @@ -144,18 +144,16 @@ defmodule E2E.OfficeTest do test "office_docx_basic" do case E2E.Helpers.run_fixture( - "office_docx_basic", - "docx/sample_document.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_basic", + "docx/sample_document.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(10) {:skipped, reason} -> @@ -168,18 +166,16 @@ defmodule E2E.OfficeTest do test "office_docx_equations" do case E2E.Helpers.run_fixture( - "office_docx_equations", - "docx/equations.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_equations", + "docx/equations.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -192,18 +188,16 @@ defmodule E2E.OfficeTest do test "office_docx_fake" do case E2E.Helpers.run_fixture( - "office_docx_fake", - "docx/fake.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_fake", + "docx/fake.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -216,18 +210,16 @@ defmodule E2E.OfficeTest do test "office_docx_formatting" do case E2E.Helpers.run_fixture( - "office_docx_formatting", - "docx/unit_test_formatting.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_formatting", + "docx/unit_test_formatting.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -240,18 +232,16 @@ defmodule E2E.OfficeTest do test "office_docx_headers" do case E2E.Helpers.run_fixture( - "office_docx_headers", - "docx/unit_test_headers.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_headers", + "docx/unit_test_headers.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -264,18 +254,16 @@ defmodule E2E.OfficeTest do test "office_docx_lists" do case E2E.Helpers.run_fixture( - "office_docx_lists", - "docx/unit_test_lists.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_lists", + "docx/unit_test_lists.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -288,25 +276,18 @@ defmodule E2E.OfficeTest do test "office_docx_tables" do case E2E.Helpers.run_fixture( - "office_docx_tables", - "docx/docx_tables.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_tables", + "docx/docx_tables.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(50) - |> E2E.Helpers.assert_content_contains_all([ - "Simple uniform table", - "Nested Table", - "merged cells", - "Header Col" - ]) + |> E2E.Helpers.assert_content_contains_all(["Simple uniform table", "Nested Table", "merged cells", "Header Col"]) |> E2E.Helpers.assert_table_count(1, nil) {:skipped, reason} -> @@ -319,13 +300,13 @@ defmodule E2E.OfficeTest do test "office_epub_basic" do case E2E.Helpers.run_fixture( - "office_epub_basic", - "epub/features.epub", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_epub_basic", + "epub/features.epub", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/epub+zip"]) @@ -341,13 +322,13 @@ defmodule E2E.OfficeTest do test "office_fb2_basic" do case E2E.Helpers.run_fixture( - "office_fb2_basic", - "fictionbook/basic.fb2", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_fb2_basic", + "fictionbook/basic.fb2", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-fictionbook+xml"]) @@ -363,13 +344,13 @@ defmodule E2E.OfficeTest do test "office_fictionbook_basic" do case E2E.Helpers.run_fixture( - "office_fictionbook_basic", - "fictionbook/basic.fb2", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_fictionbook_basic", + "fictionbook/basic.fb2", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-fictionbook+xml", "application/x-fictionbook"]) @@ -385,13 +366,13 @@ defmodule E2E.OfficeTest do test "office_hwp_basic" do case E2E.Helpers.run_fixture( - "office_hwp_basic", - "hwp/converted_output.hwp", - nil, - requirements: ["office"], - notes: "Requires the office feature.", - skip_if_missing: true - ) do + "office_hwp_basic", + "hwp/converted_output.hwp", + nil, + requirements: ["office"], + notes: "Requires the office feature.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-hwp"]) @@ -407,14 +388,13 @@ defmodule E2E.OfficeTest do test "office_hwp_styled" do case E2E.Helpers.run_fixture( - "office_hwp_styled", - "hwp/styled_document.hwp", - nil, - requirements: ["hwp"], - notes: - "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", - skip_if_missing: true - ) do + "office_hwp_styled", + "hwp/styled_document.hwp", + nil, + requirements: ["hwp"], + notes: "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-hwp"]) @@ -429,13 +409,13 @@ defmodule E2E.OfficeTest do test "office_jats_basic" do case E2E.Helpers.run_fixture( - "office_jats_basic", - "jats/sample_article.jats", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_jats_basic", + "jats/sample_article.jats", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-jats+xml", "text/jats"]) @@ -451,13 +431,13 @@ defmodule E2E.OfficeTest do test "office_jupyter_basic" do case E2E.Helpers.run_fixture( - "office_jupyter_basic", - "jupyter/rank.ipynb", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_jupyter_basic", + "jupyter/rank.ipynb", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-ipynb+json"]) @@ -473,13 +453,13 @@ defmodule E2E.OfficeTest do test "office_keynote_basic" do case E2E.Helpers.run_fixture( - "office_keynote_basic", - "iwork/test.key", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_keynote_basic", + "iwork/test.key", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-iwork-keynote-sffkey"]) @@ -495,13 +475,13 @@ defmodule E2E.OfficeTest do test "office_latex_basic" do case E2E.Helpers.run_fixture( - "office_latex_basic", - "latex/basic_sections.tex", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_latex_basic", + "latex/basic_sections.tex", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-latex", "text/x-latex"]) @@ -517,13 +497,13 @@ defmodule E2E.OfficeTest do test "office_markdown_basic" do case E2E.Helpers.run_fixture( - "office_markdown_basic", - "markdown/comprehensive.md", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_markdown_basic", + "markdown/comprehensive.md", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/markdown"]) @@ -539,13 +519,13 @@ defmodule E2E.OfficeTest do test "office_mdx_basic" do case E2E.Helpers.run_fixture( - "office_mdx_basic", - "markdown/sample.mdx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_mdx_basic", + "markdown/sample.mdx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/mdx", "text/x-mdx"]) @@ -561,13 +541,13 @@ defmodule E2E.OfficeTest do test "office_mdx_getting_started" do case E2E.Helpers.run_fixture( - "office_mdx_getting_started", - "markdown/mdx_getting_started.mdx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_mdx_getting_started", + "markdown/mdx_getting_started.mdx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/mdx", "text/x-mdx"]) @@ -583,13 +563,13 @@ defmodule E2E.OfficeTest do test "office_mdx_troubleshooting" do case E2E.Helpers.run_fixture( - "office_mdx_troubleshooting", - "markdown/mdx_troubleshooting.mdx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_mdx_troubleshooting", + "markdown/mdx_troubleshooting.mdx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/mdx", "text/x-mdx"]) @@ -605,13 +585,13 @@ defmodule E2E.OfficeTest do test "office_mdx_using_mdx" do case E2E.Helpers.run_fixture( - "office_mdx_using_mdx", - "markdown/mdx_using_mdx.mdx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_mdx_using_mdx", + "markdown/mdx_using_mdx.mdx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/mdx", "text/x-mdx"]) @@ -627,13 +607,13 @@ defmodule E2E.OfficeTest do test "office_numbers_basic" do case E2E.Helpers.run_fixture( - "office_numbers_basic", - "iwork/test.numbers", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_numbers_basic", + "iwork/test.numbers", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-iwork-numbers-sffnumbers"]) @@ -649,13 +629,13 @@ defmodule E2E.OfficeTest do test "office_ods_basic" do case E2E.Helpers.run_fixture( - "office_ods_basic", - "data_formats/test_01.ods", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_ods_basic", + "data_formats/test_01.ods", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.spreadsheet"]) @@ -671,13 +651,13 @@ defmodule E2E.OfficeTest do test "office_odt_bold" do case E2E.Helpers.run_fixture( - "office_odt_bold", - "odt/bold.odt", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_odt_bold", + "odt/bold.odt", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.text"]) @@ -693,13 +673,13 @@ defmodule E2E.OfficeTest do test "office_odt_list" do case E2E.Helpers.run_fixture( - "office_odt_list", - "odt/unorderedList.odt", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_odt_list", + "odt/unorderedList.odt", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.text"]) @@ -716,13 +696,13 @@ defmodule E2E.OfficeTest do test "office_odt_simple" do case E2E.Helpers.run_fixture( - "office_odt_simple", - "odt/simple.odt", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_odt_simple", + "odt/simple.odt", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.text"]) @@ -739,13 +719,13 @@ defmodule E2E.OfficeTest do test "office_odt_table" do case E2E.Helpers.run_fixture( - "office_odt_table", - "odt/table.odt", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_odt_table", + "odt/table.odt", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.text"]) @@ -762,13 +742,13 @@ defmodule E2E.OfficeTest do test "office_opml_basic" do case E2E.Helpers.run_fixture( - "office_opml_basic", - "opml/outline.opml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_opml_basic", + "opml/outline.opml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/xml+opml", "text/x-opml", "application/x-opml+xml"]) @@ -784,13 +764,13 @@ defmodule E2E.OfficeTest do test "office_org_basic" do case E2E.Helpers.run_fixture( - "office_org_basic", - "org/comprehensive.org", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_org_basic", + "org/comprehensive.org", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/x-org", "text/org"]) @@ -806,13 +786,13 @@ defmodule E2E.OfficeTest do test "office_pages_basic" do case E2E.Helpers.run_fixture( - "office_pages_basic", - "iwork/test.pages", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pages_basic", + "iwork/test.pages", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-iwork-pages-sffpages"]) @@ -828,18 +808,16 @@ defmodule E2E.OfficeTest do test "office_ppsx_slideshow" do case E2E.Helpers.run_fixture( - "office_ppsx_slideshow", - "pptx/sample.ppsx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_ppsx_slideshow", + "pptx/sample.ppsx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.presentationml.slideshow" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]) |> E2E.Helpers.assert_min_content_length(10) {:skipped, reason} -> @@ -852,13 +830,13 @@ defmodule E2E.OfficeTest do test "office_ppt_legacy" do case E2E.Helpers.run_fixture( - "office_ppt_legacy", - "ppt/simple.ppt", - nil, - requirements: ["office"], - notes: "Requires the office feature.", - skip_if_missing: true - ) do + "office_ppt_legacy", + "ppt/simple.ppt", + nil, + requirements: ["office"], + notes: "Requires the office feature.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-powerpoint"]) @@ -874,19 +852,16 @@ defmodule E2E.OfficeTest do test "office_pptm_basic" do case E2E.Helpers.run_fixture( - "office_pptm_basic", - "pptx/powerpoint_with_image.pptm", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pptm_basic", + "pptx/powerpoint_with_image.pptm", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.ms-powerpoint.presentation.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation"]) |> E2E.Helpers.assert_content_not_empty() {:skipped, reason} -> @@ -899,18 +874,16 @@ defmodule E2E.OfficeTest do test "office_pptx_basic" do case E2E.Helpers.run_fixture( - "office_pptx_basic", - "pptx/simple.pptx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pptx_basic", + "pptx/simple.pptx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) |> E2E.Helpers.assert_min_content_length(50) {:skipped, reason} -> @@ -923,18 +896,16 @@ defmodule E2E.OfficeTest do test "office_pptx_images" do case E2E.Helpers.run_fixture( - "office_pptx_images", - "pptx/powerpoint_with_image.pptx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pptx_images", + "pptx/powerpoint_with_image.pptx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) |> E2E.Helpers.assert_min_content_length(15) {:skipped, reason} -> @@ -947,18 +918,16 @@ defmodule E2E.OfficeTest do test "office_pptx_pitch_deck" do case E2E.Helpers.run_fixture( - "office_pptx_pitch_deck", - "pptx/pitch_deck_presentation.pptx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pptx_pitch_deck", + "pptx/pitch_deck_presentation.pptx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) |> E2E.Helpers.assert_min_content_length(100) {:skipped, reason} -> @@ -971,13 +940,13 @@ defmodule E2E.OfficeTest do test "office_rst_basic" do case E2E.Helpers.run_fixture( - "office_rst_basic", - "rst/restructured_text.rst", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_rst_basic", + "rst/restructured_text.rst", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/x-rst", "text/prs.fallenstein.rst"]) @@ -993,13 +962,13 @@ defmodule E2E.OfficeTest do test "office_rtf_basic" do case E2E.Helpers.run_fixture( - "office_rtf_basic", - "rtf/extraction_test.rtf", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_rtf_basic", + "rtf/extraction_test.rtf", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/rtf", "text/rtf"]) @@ -1015,13 +984,13 @@ defmodule E2E.OfficeTest do test "office_typst_basic" do case E2E.Helpers.run_fixture( - "office_typst_basic", - "typst/headings.typ", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_typst_basic", + "typst/headings.typ", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-typst", "text/x-typst"]) @@ -1037,13 +1006,13 @@ defmodule E2E.OfficeTest do test "office_xls_legacy" do case E2E.Helpers.run_fixture( - "office_xls_legacy", - "xls/test_excel.xls", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xls_legacy", + "xls/test_excel.xls", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-excel"]) @@ -1059,19 +1028,16 @@ defmodule E2E.OfficeTest do test "office_xlsb_basic" do case E2E.Helpers.run_fixture( - "office_xlsb_basic", - "xlsx/test_xlsb.xlsb", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsb_basic", + "xlsx/test_xlsb.xlsb", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.ms-excel.sheet.binary.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) |> E2E.Helpers.assert_content_not_empty() {:skipped, reason} -> @@ -1084,19 +1050,16 @@ defmodule E2E.OfficeTest do test "office_xlsm_basic" do case E2E.Helpers.run_fixture( - "office_xlsm_basic", - "xlsx/test_01.xlsm", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsm_basic", + "xlsx/test_01.xlsm", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.ms-excel.sheet.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) |> E2E.Helpers.assert_content_not_empty() {:skipped, reason} -> @@ -1109,13 +1072,13 @@ defmodule E2E.OfficeTest do test "office_xlsx_basic" do case E2E.Helpers.run_fixture( - "office_xlsx_basic", - "xlsx/stanley_cups.xlsx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsx_basic", + "xlsx/stanley_cups.xlsx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) @@ -1135,13 +1098,13 @@ defmodule E2E.OfficeTest do test "office_xlsx_multi_sheet" do case E2E.Helpers.run_fixture( - "office_xlsx_multi_sheet", - "xlsx/excel_multi_sheet.xlsx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsx_multi_sheet", + "xlsx/excel_multi_sheet.xlsx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) @@ -1158,13 +1121,13 @@ defmodule E2E.OfficeTest do test "office_xlsx_office_example" do case E2E.Helpers.run_fixture( - "office_xlsx_office_example", - "xlsx/test_01.xlsx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsx_office_example", + "xlsx/test_01.xlsx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) diff --git a/e2e/elixir/test/e2e/plugin_apis_test.exs b/e2e/elixir/test/e2e/plugin_apis_test.exs index 277392f15a1..774a4e5ca1a 100644 --- a/e2e/elixir/test/e2e/plugin_apis_test.exs +++ b/e2e/elixir/test/e2e/plugin_apis_test.exs @@ -13,22 +13,19 @@ defmodule E2E.ConfigurationTest do File.mkdir_p!(tmpdir) on_exit(fn -> File.rm_rf!(tmpdir) end) config_path = Path.join(tmpdir, "kreuzberg.toml") - config_content = """ [chunking] max_chars = 50 """ - File.write!(config_path, config_content) subdir = Path.join(tmpdir, "subdir") File.mkdir_p!(subdir) prev_cwd = File.cwd!() - try do File.cd!(subdir) - {:ok, config} = Kreuzberg.ExtractionConfig.discover() + {:ok, config} = Kreuzberg.ExtractionConfig.discover assert config.chunking != nil assert config.chunking["max_chars"] == 50 @@ -42,7 +39,6 @@ defmodule E2E.ConfigurationTest do File.mkdir_p!(tmpdir) on_exit(fn -> File.rm_rf!(tmpdir) end) config_path = Path.join(tmpdir, "test_config.toml") - config_content = """ [chunking] max_chars = 100 @@ -51,7 +47,6 @@ defmodule E2E.ConfigurationTest do [language_detection] enabled = false """ - File.write!(config_path, config_content) {:ok, config} = Kreuzberg.ExtractionConfig.from_file(config_path) @@ -62,6 +57,7 @@ defmodule E2E.ConfigurationTest do assert config.language_detection != nil assert config.language_detection["enabled"] == false end + end end @@ -85,6 +81,7 @@ defmodule E2E.DocumentExtractorManagementTest do Kreuzberg.Plugin.unregister_document_extractor(:"nonexistent-extractor-xyz") # Should not raise an error end + end end @@ -114,6 +111,7 @@ defmodule E2E.MimeUtilitiesTest do assert is_list(result) assert Enum.member?(result, "pdf") end + end end @@ -137,6 +135,7 @@ defmodule E2E.OcrBackendManagementTest do Kreuzberg.Plugin.unregister_ocr_backend(:"nonexistent-backend-xyz") # Should not raise an error end + end end @@ -153,6 +152,7 @@ defmodule E2E.PostProcessorManagementTest do assert is_list(result) assert Enum.all?(result, &is_binary/1) end + end end @@ -171,5 +171,6 @@ defmodule E2E.ValidatorManagementTest do assert is_list(result) assert Enum.all?(result, &is_binary/1) end + end end diff --git a/e2e/elixir/test/e2e/render_test.exs b/e2e/elixir/test/e2e/render_test.exs index 77087559069..66639ab2396 100644 --- a/e2e/elixir/test/e2e/render_test.exs +++ b/e2e/elixir/test/e2e/render_test.exs @@ -28,13 +28,11 @@ defmodule E2E.RenderTest do else stream = Kreuzberg.render_pdf_pages_stream(document_path, dpi: 150) pages = Enum.to_list(stream) - Enum.each(pages, fn {_page_index, png_data} -> assert_is_png(png_data) end) - assert length(pages) >= 1, - "Expected at least 1 pages, got #{length(pages)}" + "Expected at least 1 pages, got #{length(pages)}" end end @@ -50,4 +48,5 @@ defmodule E2E.RenderTest do assert_min_byte_length(png_data, 100) end end + end diff --git a/e2e/elixir/test/e2e/structured_test.exs b/e2e/elixir/test/e2e/structured_test.exs index 1ac21bf0e4e..81ee80f1992 100644 --- a/e2e/elixir/test/e2e/structured_test.exs +++ b/e2e/elixir/test/e2e/structured_test.exs @@ -11,13 +11,13 @@ defmodule E2E.StructuredTest do describe "structured fixtures" do test "structured_csv_basic" do case E2E.Helpers.run_fixture( - "structured_csv_basic", - "csv/stanley_cups.csv", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_csv_basic", + "csv/stanley_cups.csv", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/csv"]) @@ -33,20 +33,16 @@ defmodule E2E.StructuredTest do test "structured_enw_basic" do case E2E.Helpers.run_fixture( - "structured_enw_basic", - "data_formats/sample.enw", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_enw_basic", + "data_formats/sample.enw", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/x-endnote-refer", - "application/x-endnote+xml", - "text/plain" - ]) + |> E2E.Helpers.assert_expected_mime(["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]) {:skipped, reason} -> IO.puts("SKIPPED: #{reason}") @@ -58,13 +54,13 @@ defmodule E2E.StructuredTest do test "structured_json_basic" do case E2E.Helpers.run_fixture( - "structured_json_basic", - "json/sample_document.json", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_json_basic", + "json/sample_document.json", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/json"]) @@ -81,13 +77,13 @@ defmodule E2E.StructuredTest do test "structured_json_simple" do case E2E.Helpers.run_fixture( - "structured_json_simple", - "json/simple.json", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_json_simple", + "json/simple.json", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/json"]) @@ -104,13 +100,13 @@ defmodule E2E.StructuredTest do test "structured_nbib_basic" do case E2E.Helpers.run_fixture( - "structured_nbib_basic", - "data_formats/sample.nbib", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_nbib_basic", + "data_formats/sample.nbib", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/nbib", "application/x-pubmed", "text/plain"]) @@ -126,13 +122,13 @@ defmodule E2E.StructuredTest do test "structured_ris_basic" do case E2E.Helpers.run_fixture( - "structured_ris_basic", - "data_formats/sample.ris", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_ris_basic", + "data_formats/sample.ris", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-research-info-systems", "text/plain"]) @@ -148,13 +144,13 @@ defmodule E2E.StructuredTest do test "structured_toml_basic" do case E2E.Helpers.run_fixture( - "structured_toml_basic", - "data_formats/cargo.toml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_toml_basic", + "data_formats/cargo.toml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/toml", "text/toml"]) @@ -170,13 +166,13 @@ defmodule E2E.StructuredTest do test "structured_tsv_basic" do case E2E.Helpers.run_fixture( - "structured_tsv_basic", - "data_formats/employees.tsv", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_tsv_basic", + "data_formats/employees.tsv", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/tab-separated-values", "text/plain"]) @@ -192,13 +188,13 @@ defmodule E2E.StructuredTest do test "structured_yaml_basic" do case E2E.Helpers.run_fixture( - "structured_yaml_basic", - "yaml/simple.yaml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_yaml_basic", + "yaml/simple.yaml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]) @@ -214,13 +210,13 @@ defmodule E2E.StructuredTest do test "structured_yaml_simple" do case E2E.Helpers.run_fixture( - "structured_yaml_simple", - "yaml/simple.yaml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_yaml_simple", + "yaml/simple.yaml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-yaml"]) diff --git a/e2e/elixir/test/e2e/xml_test.exs b/e2e/elixir/test/e2e/xml_test.exs index c77ab6555ac..9448447aeca 100644 --- a/e2e/elixir/test/e2e/xml_test.exs +++ b/e2e/elixir/test/e2e/xml_test.exs @@ -11,13 +11,13 @@ defmodule E2E.XmlTest do describe "xml fixtures" do test "xml_plant_catalog" do case E2E.Helpers.run_fixture( - "xml_plant_catalog", - "xml/plant_catalog.xml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "xml_plant_catalog", + "xml/plant_catalog.xml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/xml"]) diff --git a/e2e/elixir/test/support/e2e_helpers.ex b/e2e/elixir/test/support/e2e_helpers.ex index 215b2d36e20..4d178a6b9ac 100644 --- a/e2e/elixir/test/support/e2e_helpers.ex +++ b/e2e/elixir/test/support/e2e_helpers.ex @@ -17,7 +17,6 @@ defmodule E2E.Helpers do def build_config(nil), do: nil def build_config(raw) when is_map(raw) and map_size(raw) == 0, do: nil - def build_config(raw) when is_map(raw) do atomize_keys(raw) end @@ -38,12 +37,9 @@ defmodule E2E.Helpers do def skip_reason_for(error, fixture_id, requirements, notes \\ nil) do message = Exception.message(error) downcased = String.downcase(message) - - requirement_hit = - Enum.any?(requirements, fn req -> - String.contains?(downcased, String.downcase(req)) - end) - + requirement_hit = Enum.any?(requirements, fn req -> + String.contains?(downcased, String.downcase(req)) + end) missing_dependency = String.contains?(downcased, "missing dependency") unsupported_format = String.contains?(downcased, "unsupported format") @@ -78,11 +74,8 @@ defmodule E2E.Helpers do requirements = Keyword.get(opts, :requirements, []) notes = Keyword.get(opts, :notes, nil) skip_if_missing = Keyword.get(opts, :skip_if_missing, true) - run_fixture_with_method(fixture_id, relative_path, config_hash, :sync, :file, - requirements: requirements, - notes: notes, - skip_if_missing: skip_if_missing + requirements: requirements, notes: notes, skip_if_missing: skip_if_missing ) end @@ -195,7 +188,6 @@ defmodule E2E.Helpers do result else mime = result.mime_type || "" - if Enum.any?(expected, fn token -> String.contains?(mime, token) end) do result else @@ -206,7 +198,6 @@ defmodule E2E.Helpers do def assert_min_content_length(result, minimum) do content_len = String.length(result.content || "") - if content_len >= minimum do result else @@ -216,7 +207,6 @@ defmodule E2E.Helpers do def assert_max_content_length(result, maximum) do content_len = String.length(result.content || "") - if content_len <= maximum do result else @@ -229,7 +219,6 @@ defmodule E2E.Helpers do result else lowered = String.downcase(result.content || "") - if Enum.any?(snippets, fn snippet -> String.contains?(lowered, String.downcase(snippet)) end) do result else @@ -243,7 +232,6 @@ defmodule E2E.Helpers do result else lowered = String.downcase(result.content || "") - if Enum.all?(snippets, fn snippet -> String.contains?(lowered, String.downcase(snippet)) end) do result else @@ -302,7 +290,6 @@ defmodule E2E.Helpers do expectation when is_map(expectation) -> if Map.has_key?(expectation, :eq) do expected_val = Map.get(expectation, :eq) - if !values_equal?(value, expected_val) do flunk("Metadata path '#{path}' value #{inspect(value)} != #{inspect(expected_val)}") end @@ -310,7 +297,6 @@ defmodule E2E.Helpers do if Map.has_key?(expectation, :gte) do expected_val = Map.get(expectation, :gte) - if convert_numeric(value) < convert_numeric(expected_val) do flunk("Metadata path '#{path}' value #{inspect(value)} < #{inspect(expected_val)}") end @@ -318,7 +304,6 @@ defmodule E2E.Helpers do if Map.has_key?(expectation, :lte) do expected_val = Map.get(expectation, :lte) - if convert_numeric(value) > convert_numeric(expected_val) do flunk("Metadata path '#{path}' value #{inspect(value)} > #{inspect(expected_val)}") end @@ -393,6 +378,12 @@ defmodule E2E.Helpers do end end + if opts[:each_has_chunk_type] == true do + if !Enum.all?(chunks, fn chunk -> chunk.chunk_type && chunk.chunk_type != "unknown" end) do + flunk("Not all chunks have a specific chunk_type") + end + end + if opts[:content_starts_with_heading] == true do chunks_with_heading = Enum.filter(chunks, fn chunk -> @@ -486,18 +477,13 @@ defmodule E2E.Helpers do nodes |> Enum.map(fn node -> content = Map.get(node, :content) || Map.get(node, "content") - - if content, - do: Map.get(content, :node_type) || Map.get(content, "node_type"), - else: Map.get(node, :node_type) || Map.get(node, "node_type") + if content, do: Map.get(content, :node_type) || Map.get(content, "node_type"), else: Map.get(node, :node_type) || Map.get(node, "node_type") end) |> Enum.reject(&is_nil/1) |> Enum.uniq() if !Enum.all?(opts[:node_types_include], fn t -> Enum.member?(found_types, t) end) do - flunk( - "Document node types #{inspect(found_types)} do not include all of #{inspect(opts[:node_types_include])}" - ) + flunk("Document node types #{inspect(found_types)} do not include all of #{inspect(opts[:node_types_include])}") end end @@ -508,7 +494,7 @@ defmodule E2E.Helpers do node_type = if content do - if(is_struct(content), do: Map.get(content, :node_type), else: nil) || + (if is_struct(content), do: Map.get(content, :node_type), else: nil) || Map.get(content, :node_type) || Map.get(content, "node_type") else Map.get(node, :node_type) || Map.get(node, "node_type") @@ -567,7 +553,6 @@ defmodule E2E.Helpers do def assert_content_not_empty(result) do content_len = String.length(result.content || "") - if content_len > 0 do result else @@ -768,22 +753,18 @@ defmodule E2E.Helpers do defp lookup_metadata_path(_, _), do: nil defp values_equal?(lhs, rhs) when is_binary(lhs) and is_binary(rhs), do: lhs == rhs - defp values_equal?(lhs, rhs) when is_number(lhs) and is_number(rhs) do convert_numeric(lhs) == convert_numeric(rhs) end - defp values_equal?(lhs, rhs), do: lhs == rhs defp convert_numeric(value) when is_number(value), do: value - defp convert_numeric(value) when is_binary(value) do case Float.parse(value) do {num, ""} -> num _ -> 0.0 end end - defp convert_numeric(_), do: 0.0 def assert_is_png(data) do @@ -794,6 +775,6 @@ defmodule E2E.Helpers do def assert_min_byte_length(data, min_length) do assert byte_size(data) >= min_length, - "Expected at least #{min_length} bytes, got #{byte_size(data)}" + "Expected at least #{min_length} bytes, got #{byte_size(data)}" end end diff --git a/e2e/go/contract_test.go b/e2e/go/contract_test.go index c4f10804181..13fb1bfbf65 100644 --- a/e2e/go/contract_test.go +++ b/e2e/go/contract_test.go @@ -150,7 +150,7 @@ func TestContractConfigChunking(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigChunkingHeadingContext(t *testing.T) { @@ -163,7 +163,7 @@ func TestContractConfigChunkingHeadingContext(t *testing.T) { } }`)) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(true), nil) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(true), nil, nil) } func TestContractConfigChunkingMarkdown(t *testing.T) { @@ -177,7 +177,7 @@ func TestContractConfigChunkingMarkdown(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigChunkingNoHeadings(t *testing.T) { @@ -190,7 +190,7 @@ func TestContractConfigChunkingNoHeadings(t *testing.T) { } }`)) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(false), nil) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(false), nil, nil) } func TestContractConfigChunkingPrependHeadingContext(t *testing.T) { @@ -204,7 +204,7 @@ func TestContractConfigChunkingPrependHeadingContext(t *testing.T) { } }`)) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(true), boolPtr(true)) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(true), nil, boolPtr(true)) } func TestContractConfigChunkingSmall(t *testing.T) { @@ -217,7 +217,7 @@ func TestContractConfigChunkingSmall(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigChunkingText(t *testing.T) { @@ -230,7 +230,7 @@ func TestContractConfigChunkingText(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigChunkingTokenizer(t *testing.T) { @@ -246,7 +246,7 @@ func TestContractConfigChunkingTokenizer(t *testing.T) { } }`)) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigDisableOcr(t *testing.T) { diff --git a/e2e/go/embeddings_test.go b/e2e/go/embeddings_test.go index e2198309523..573a53fcd94 100644 --- a/e2e/go/embeddings_test.go +++ b/e2e/go/embeddings_test.go @@ -22,7 +22,7 @@ func TestEmbeddingsEmbeddingAsync(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil, nil) } func TestEmbeddingsEmbeddingBalancedPreset(t *testing.T) { @@ -42,7 +42,7 @@ func TestEmbeddingsEmbeddingBalancedPreset(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil, nil) } func TestEmbeddingsEmbeddingDisabled(t *testing.T) { @@ -54,7 +54,7 @@ func TestEmbeddingsEmbeddingDisabled(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(false), nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(false), nil, nil, nil) } func TestEmbeddingsEmbeddingFastPreset(t *testing.T) { @@ -74,5 +74,5 @@ func TestEmbeddingsEmbeddingFastPreset(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil, nil) } diff --git a/e2e/go/helpers_test.go b/e2e/go/helpers_test.go index d3bc4661544..eb52da6be5c 100644 --- a/e2e/go/helpers_test.go +++ b/e2e/go/helpers_test.go @@ -10,7 +10,7 @@ import ( "testing" "unicode" - kreuzberg "github.com/kreuzberg-dev/kreuzberg/packages/go/v4" + "github.com/kreuzberg-dev/kreuzberg/packages/go/v4" ) var ( @@ -241,7 +241,7 @@ func boolPtr(value bool) *bool { return &value } -func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, contentStartsWithHeading *bool) { +func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, eachHasChunkType, contentStartsWithHeading *bool) { t.Helper() count := len(result.Chunks) if minCount != nil && count < *minCount { @@ -278,6 +278,13 @@ func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, ma } } } + if eachHasChunkType != nil && *eachHasChunkType { + for i, chunk := range result.Chunks { + if chunk.ChunkType == "" || chunk.ChunkType == "unknown" { + t.Fatalf("chunk %d has no specific chunk_type, got %q", i, chunk.ChunkType) + } + } + } if contentStartsWithHeading != nil && *contentStartsWithHeading { for i, chunk := range result.Chunks { if chunk.Metadata == nil || chunk.Metadata.HeadingContext == nil { diff --git a/e2e/go/render_test.go b/e2e/go/render_test.go index 92a6690da28..dee7c2303ac 100644 --- a/e2e/go/render_test.go +++ b/e2e/go/render_test.go @@ -7,7 +7,7 @@ import ( "os" "testing" - kreuzberg "github.com/kreuzberg-dev/kreuzberg/packages/go/v4" + "github.com/kreuzberg-dev/kreuzberg/packages/go/v4" ) func TestRenderRenderCustomDpi(t *testing.T) { diff --git a/e2e/go/token_reduction_test.go b/e2e/go/token_reduction_test.go index 031461a4e43..afad76d71b0 100644 --- a/e2e/go/token_reduction_test.go +++ b/e2e/go/token_reduction_test.go @@ -53,6 +53,6 @@ func TestTokenReductionTokenReductionWithChunking(t *testing.T) { assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 5) assertMaxContentLength(t, result, 200) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil, nil) assertContentNotEmpty(t, result) } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/ArchiveTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/ArchiveTest.java index c610400fc62..b6745ee3b07 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/ArchiveTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/ArchiveTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,73 +26,74 @@ /** Tests for archive fixtures. */ public class ArchiveTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void archiveGzBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "archive_gz_basic", + "archives/book_war_and_peace_1p.txt.gz", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/gzip", "application/x-gzip")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void archiveGzBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "archive_gz_basic", - "archives/book_war_and_peace_1p.txt.gz", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/gzip", "application/x-gzip")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void archiveSevenzBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "archive_sevenz_basic", + "archives/documents.7z", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-7z-compressed")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void archiveSevenzBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "archive_sevenz_basic", - "archives/documents.7z", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/x-7z-compressed")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void archiveTarBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "archive_tar_basic", + "archives/documents.tar", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-tar", "application/tar")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void archiveTarBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "archive_tar_basic", - "archives/documents.tar", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/x-tar", "application/tar")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void archiveZipBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "archive_zip_basic", + "archives/documents.zip", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/zip", "application/x-zip-compressed")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void archiveZipBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "archive_zip_basic", - "archives/documents.zip", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/zip", "application/x-zip-compressed")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/ContractTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/ContractTest.java index 2584fcae2b9..82a409e26df 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/ContractTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/ContractTest.java @@ -553,7 +553,7 @@ public void configChunking() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null, null); }); } @@ -572,7 +572,7 @@ public void configChunkingHeadingContext() throws Exception { true, result -> { E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, true, null); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, true, null, null); }); } @@ -592,7 +592,7 @@ public void configChunkingMarkdown() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null, null); }); } @@ -611,7 +611,7 @@ public void configChunkingNoHeadings() throws Exception { true, result -> { E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, false, null); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, false, null, null); }); } @@ -630,7 +630,7 @@ public void configChunkingPrependHeadingContext() throws Exception { true, result -> { E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, true, true); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, true, null, true); }); } @@ -648,7 +648,7 @@ public void configChunkingSmall() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, null, null, null); }); } @@ -667,7 +667,7 @@ public void configChunkingText() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null, null); }); } @@ -686,7 +686,7 @@ public void configChunkingTokenizer() throws Exception { true, result -> { E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, null, null, null); }); } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java b/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java index 2751d9651d0..400bcd75d52 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java @@ -343,6 +343,7 @@ public static void assertChunks( Boolean eachHasContent, Boolean eachHasEmbedding, Boolean eachHasHeadingContext, + Boolean eachHasChunkType, Boolean contentStartsWithHeading) { var chunks = result.getChunks(); int count = chunks != null ? chunks.size() : 0; @@ -381,6 +382,13 @@ public static void assertChunks( "Expected each chunk to have no heading_context"); } } + if (chunks != null && eachHasChunkType != null && eachHasChunkType) { + for (var chunk : chunks) { + String type = chunk.getChunkType(); + assertTrue(type != null && !"unknown".equals(type), + "Expected each chunk to have a specific chunk_type"); + } + } if (chunks != null && contentStartsWithHeading != null && contentStartsWithHeading) { String headingPrefix = String.valueOf((char) 35); for (var chunk : chunks) { diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/EmailTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/EmailTest.java index 901544393b1..767c6b9f448 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/EmailTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/EmailTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,104 +26,108 @@ /** Tests for email fixtures. */ public class EmailTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void emailEmlHtmlBody() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_eml_html_body", + "email/html_only.eml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void emailEmlHtmlBody() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_eml_html_body", - "email/html_only.eml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void emailEmlMultipart() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_eml_multipart", + "email/html_email_multipart.eml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void emailEmlMultipart() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_eml_multipart", - "email/html_email_multipart.eml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void emailEmlUtf16() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_eml_utf16", + "vendored/unstructured/eml/fake-email-utf-16.eml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); + E2EHelpers.Assertions.assertMinContentLength(result, 50); + E2EHelpers.Assertions.assertContentContainsAny(result, Arrays.asList("Test Email", "Roses are red")); + } + ); + } - @Test - public void emailEmlUtf16() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_eml_utf16", - "vendored/unstructured/eml/fake-email-utf-16.eml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); - E2EHelpers.Assertions.assertMinContentLength(result, 50); - E2EHelpers.Assertions.assertContentContainsAny( - result, Arrays.asList("Test Email", "Roses are red")); - }); - } + @Test + public void emailMsgBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_msg_basic", + "email/fake_email.msg", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/vnd.ms-outlook")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void emailMsgBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_msg_basic", - "email/fake_email.msg", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/vnd.ms-outlook")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void emailPstEmpty() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_pst_empty", + "email/empty.pst", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/vnd.ms-outlook-pst")); + } + ); + } - @Test - public void emailPstEmpty() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_pst_empty", - "email/empty.pst", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/vnd.ms-outlook-pst")); - }); - } + @Test + public void emailSampleEml() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_sample_eml", + "email/sample_email.eml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); + E2EHelpers.Assertions.assertMinContentLength(result, 20); + } + ); + } - @Test - public void emailSampleEml() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_sample_eml", - "email/sample_email.eml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); - E2EHelpers.Assertions.assertMinContentLength(result, 20); - }); - } } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/EmbeddingsTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/EmbeddingsTest.java index 60efee5b620..4f3d535ee0e 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/EmbeddingsTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/EmbeddingsTest.java @@ -69,7 +69,7 @@ public void embeddingAsync() throws Exception { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null, null); } @Test @@ -94,7 +94,7 @@ public void embeddingBalancedPreset() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null, null); }); } @@ -111,7 +111,7 @@ public void embeddingDisabled() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, false, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, false, null, null, null); }); } @@ -137,7 +137,7 @@ public void embeddingFastPreset() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null, null); }); } } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/HtmlTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/HtmlTest.java index 3092fa89dc5..46a08ad3305 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/HtmlTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/HtmlTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,47 +26,41 @@ /** Tests for html fixtures. */ public class HtmlTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); - - @Test - public void htmlComplexLayout() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "html_complex_layout", - "html/taylor_swift.html", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/html")); - E2EHelpers.Assertions.assertMinContentLength(result, 1000); - }); - } - - @Test - public void htmlSimpleTable() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "html_simple_table", - "html/simple_table.html", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/html")); - E2EHelpers.Assertions.assertMinContentLength(result, 100); - E2EHelpers.Assertions.assertContentContainsAll( - result, - Arrays.asList( - "Product", - "Category", - "Price", - "Stock", - "Laptop", - "Electronics", - "Sample Data Table")); - }); - } + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void htmlComplexLayout() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "html_complex_layout", + "html/taylor_swift.html", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/html")); + E2EHelpers.Assertions.assertMinContentLength(result, 1000); + } + ); + } + + @Test + public void htmlSimpleTable() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "html_simple_table", + "html/simple_table.html", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/html")); + E2EHelpers.Assertions.assertMinContentLength(result, 100); + E2EHelpers.Assertions.assertContentContainsAll(result, Arrays.asList("Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table")); + } + ); + } + } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/PluginAPIsTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/PluginAPIsTest.java index 26700b42a76..d8c4f173336 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/PluginAPIsTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/PluginAPIsTest.java @@ -3,9 +3,9 @@ import static org.junit.jupiter.api.Assertions.*; +import dev.kreuzberg.config.ExtractionConfig; import dev.kreuzberg.Kreuzberg; import dev.kreuzberg.KreuzbergException; -import dev.kreuzberg.config.ExtractionConfig; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -17,25 +17,23 @@ /** * E2E tests for plugin/config/utility APIs. * - *

Generated from plugin API fixtures. To regenerate: cargo run -p kreuzberg-e2e-generator -- - * generate --lang java + *

Generated from plugin API fixtures. + * To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang java * * @since 4.0.0 */ @DisplayName("Plugin API Tests") class PluginAPIsTest { - // Configuration Tests + // Configuration Tests - // SKIPPED: config_discover - System.setProperty("user.dir") doesn't affect FFI working directory + // SKIPPED: config_discover - System.setProperty("user.dir") doesn't affect FFI working directory - @Test - @DisplayName("Load configuration from a TOML file") - void configFromFile(@TempDir Path tempDir) throws IOException, KreuzbergException { - Path configPath = tempDir.resolve("test_config.toml"); - Files.writeString( - configPath, -""" + @Test + @DisplayName("Load configuration from a TOML file") + void configFromFile(@TempDir Path tempDir) throws IOException, KreuzbergException { + Path configPath = tempDir.resolve("test_config.toml"); + Files.writeString(configPath, """ [chunking] max_chars = 100 max_overlap = 20 @@ -44,121 +42,122 @@ void configFromFile(@TempDir Path tempDir) throws IOException, KreuzbergExceptio enabled = false """); - ExtractionConfig config = ExtractionConfig.fromFile(configPath.toString()); - assertNotNull(config.getChunking()); - assertEquals(100, config.getChunking().getMaxChars()); - assertEquals(20, config.getChunking().getMaxOverlap()); - assertNotNull(config.getLanguageDetection()); - assertFalse(config.getLanguageDetection().isEnabled()); - } - - // Document Extractor Management Tests - - @Test - @DisplayName("Clear all document extractors and verify list is empty") - void extractorsClear() throws KreuzbergException { - Kreuzberg.clearDocumentExtractors(); - List result = Kreuzberg.listDocumentExtractors(); - assertEquals(0, result.size()); - } - - @Test - @DisplayName("List all registered document extractors") - void extractorsList() throws KreuzbergException { - List result = Kreuzberg.listDocumentExtractors(); - assertNotNull(result); - assertTrue(result.stream().allMatch(item -> item instanceof String)); - } - - @Test - @DisplayName("Unregister nonexistent document extractor gracefully") - void extractorsUnregister() throws KreuzbergException { - assertDoesNotThrow(() -> Kreuzberg.unregisterDocumentExtractor("nonexistent-extractor-xyz")); - } - - // Mime Utilities Tests - - @Test - @DisplayName("Detect MIME type from file bytes") - void mimeDetectBytes() throws KreuzbergException { - byte[] testBytes = "%PDF-1.4\n".getBytes(); - String result = Kreuzberg.detectMimeType(testBytes); - assertTrue(result.toLowerCase().contains("pdf")); - } - - @Test - @DisplayName("Detect MIME type from file path") - void mimeDetectPath(@TempDir Path tempDir) throws IOException, KreuzbergException { - Path testFile = tempDir.resolve("test.txt"); - Files.writeString(testFile, "Hello, world!"); - - String result = Kreuzberg.detectMimeTypeFromPath(testFile.toString()); - assertTrue(result.toLowerCase().contains("text")); - } - - @Test - @DisplayName("Get file extensions for a MIME type") - void mimeGetExtensions() throws KreuzbergException { - List result = Kreuzberg.getExtensionsForMime("application/pdf"); - assertNotNull(result); - assertTrue(result.contains("pdf")); - } - - // Ocr Backend Management Tests - - @Test - @DisplayName("Clear all OCR backends and verify list is empty") - void ocrBackendsClear() throws KreuzbergException { - Kreuzberg.clearOCRBackends(); - List result = Kreuzberg.listOCRBackends(); - assertEquals(0, result.size()); - } - - @Test - @DisplayName("List all registered OCR backends") - void ocrBackendsList() throws KreuzbergException { - List result = Kreuzberg.listOCRBackends(); - assertNotNull(result); - assertTrue(result.stream().allMatch(item -> item instanceof String)); - } - - @Test - @DisplayName("Unregister nonexistent OCR backend gracefully") - void ocrBackendsUnregister() throws KreuzbergException { - assertDoesNotThrow(() -> Kreuzberg.unregisterOCRBackend("nonexistent-backend-xyz")); - } - - // Post Processor Management Tests - - @Test - @DisplayName("Clear all post-processors and verify list is empty") - void postProcessorsClear() throws KreuzbergException { - Kreuzberg.clearPostProcessors(); - } - - @Test - @DisplayName("List all registered post-processors") - void postProcessorsList() throws KreuzbergException { - List result = Kreuzberg.listPostProcessors(); - assertNotNull(result); - assertTrue(result.stream().allMatch(item -> item instanceof String)); - } - - // Validator Management Tests - - @Test - @DisplayName("Clear all validators and verify list is empty") - void validatorsClear() throws KreuzbergException { - Kreuzberg.clearValidators(); - List result = Kreuzberg.listValidators(); - assertEquals(0, result.size()); - } - - @Test - @DisplayName("List all registered validators") - void validatorsList() throws KreuzbergException { - List result = Kreuzberg.listValidators(); - assertNotNull(result); - assertTrue(result.stream().allMatch(item -> item instanceof String)); - } + ExtractionConfig config = ExtractionConfig.fromFile(configPath.toString()); + assertNotNull(config.getChunking()); + assertEquals(100, config.getChunking().getMaxChars()); + assertEquals(20, config.getChunking().getMaxOverlap()); + assertNotNull(config.getLanguageDetection()); + assertFalse(config.getLanguageDetection().isEnabled()); + } + + // Document Extractor Management Tests + + @Test + @DisplayName("Clear all document extractors and verify list is empty") + void extractorsClear() throws KreuzbergException { + Kreuzberg.clearDocumentExtractors(); + List result = Kreuzberg.listDocumentExtractors(); + assertEquals(0, result.size()); + } + + @Test + @DisplayName("List all registered document extractors") + void extractorsList() throws KreuzbergException { + List result = Kreuzberg.listDocumentExtractors(); + assertNotNull(result); + assertTrue(result.stream().allMatch(item -> item instanceof String)); + } + + @Test + @DisplayName("Unregister nonexistent document extractor gracefully") + void extractorsUnregister() throws KreuzbergException { + assertDoesNotThrow(() -> Kreuzberg.unregisterDocumentExtractor("nonexistent-extractor-xyz")); + } + + // Mime Utilities Tests + + @Test + @DisplayName("Detect MIME type from file bytes") + void mimeDetectBytes() throws KreuzbergException { + byte[] testBytes = "%PDF-1.4\n".getBytes(); + String result = Kreuzberg.detectMimeType(testBytes); + assertTrue(result.toLowerCase().contains("pdf")); + } + + @Test + @DisplayName("Detect MIME type from file path") + void mimeDetectPath(@TempDir Path tempDir) throws IOException, KreuzbergException { + Path testFile = tempDir.resolve("test.txt"); + Files.writeString(testFile, "Hello, world!"); + + String result = Kreuzberg.detectMimeTypeFromPath(testFile.toString()); + assertTrue(result.toLowerCase().contains("text")); + } + + @Test + @DisplayName("Get file extensions for a MIME type") + void mimeGetExtensions() throws KreuzbergException { + List result = Kreuzberg.getExtensionsForMime("application/pdf"); + assertNotNull(result); + assertTrue(result.contains("pdf")); + } + + // Ocr Backend Management Tests + + @Test + @DisplayName("Clear all OCR backends and verify list is empty") + void ocrBackendsClear() throws KreuzbergException { + Kreuzberg.clearOCRBackends(); + List result = Kreuzberg.listOCRBackends(); + assertEquals(0, result.size()); + } + + @Test + @DisplayName("List all registered OCR backends") + void ocrBackendsList() throws KreuzbergException { + List result = Kreuzberg.listOCRBackends(); + assertNotNull(result); + assertTrue(result.stream().allMatch(item -> item instanceof String)); + } + + @Test + @DisplayName("Unregister nonexistent OCR backend gracefully") + void ocrBackendsUnregister() throws KreuzbergException { + assertDoesNotThrow(() -> Kreuzberg.unregisterOCRBackend("nonexistent-backend-xyz")); + } + + // Post Processor Management Tests + + @Test + @DisplayName("Clear all post-processors and verify list is empty") + void postProcessorsClear() throws KreuzbergException { + Kreuzberg.clearPostProcessors(); + } + + @Test + @DisplayName("List all registered post-processors") + void postProcessorsList() throws KreuzbergException { + List result = Kreuzberg.listPostProcessors(); + assertNotNull(result); + assertTrue(result.stream().allMatch(item -> item instanceof String)); + } + + // Validator Management Tests + + @Test + @DisplayName("Clear all validators and verify list is empty") + void validatorsClear() throws KreuzbergException { + Kreuzberg.clearValidators(); + List result = Kreuzberg.listValidators(); + assertEquals(0, result.size()); + } + + @Test + @DisplayName("List all registered validators") + void validatorsList() throws KreuzbergException { + List result = Kreuzberg.listValidators(); + assertNotNull(result); + assertTrue(result.stream().allMatch(item -> item instanceof String)); + } + } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/RenderTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/RenderTest.java index a78e19fe3ce..d264f27c20a 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/RenderTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/RenderTest.java @@ -3,49 +3,55 @@ // Code generated by kreuzberg-e2e-generator. DO NOT EDIT. // To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang java -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - import dev.kreuzberg.Kreuzberg; -import dev.kreuzberg.Kreuzberg.PageResult; import dev.kreuzberg.Kreuzberg.PdfPageIterator; +import dev.kreuzberg.Kreuzberg.PageResult; +import org.junit.jupiter.api.Test; + import java.nio.file.Files; import java.nio.file.Path; -import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** Tests for render fixtures. */ public class RenderTest { - @Test - public void renderCustomDpi() throws Exception { - Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); - assumeTrue(Files.exists(documentPath), "Skipping render_custom_dpi: missing document"); - byte[] pngData = Kreuzberg.renderPdfPage(documentPath, 0, 72); - E2EHelpers.assertIsPng(pngData); - E2EHelpers.assertMinByteLength(pngData, 50); - } - - @Test - public void renderIterator() throws Exception { - Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); - assumeTrue(Files.exists(documentPath), "Skipping render_iterator: missing document"); - int pageCount = 0; - try (PdfPageIterator iter = PdfPageIterator.open(documentPath, 150)) { - while (iter.hasNext()) { - PageResult page = iter.next(); - E2EHelpers.assertIsPng(page.data()); - pageCount++; - } + @Test + public void renderCustomDpi() throws Exception { + Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); + assumeTrue(Files.exists(documentPath), + "Skipping render_custom_dpi: missing document"); + byte[] pngData = Kreuzberg.renderPdfPage(documentPath, 0, 72); + E2EHelpers.assertIsPng(pngData); + E2EHelpers.assertMinByteLength(pngData, 50); + } + + @Test + public void renderIterator() throws Exception { + Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); + assumeTrue(Files.exists(documentPath), + "Skipping render_iterator: missing document"); + int pageCount = 0; + try (PdfPageIterator iter = PdfPageIterator.open(documentPath, 150)) { + while (iter.hasNext()) { + PageResult page = iter.next(); + E2EHelpers.assertIsPng(page.data()); + pageCount++; + } + } + assertTrue(pageCount >= 1, + String.format("Expected at least 1 pages, got %d", pageCount)); + } + + @Test + public void renderSinglePage() throws Exception { + Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); + assumeTrue(Files.exists(documentPath), + "Skipping render_single_page: missing document"); + byte[] pngData = Kreuzberg.renderPdfPage(documentPath, 0, 150); + E2EHelpers.assertIsPng(pngData); + E2EHelpers.assertMinByteLength(pngData, 100); } - assertTrue(pageCount >= 1, String.format("Expected at least 1 pages, got %d", pageCount)); - } - - @Test - public void renderSinglePage() throws Exception { - Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); - assumeTrue(Files.exists(documentPath), "Skipping render_single_page: missing document"); - byte[] pngData = Kreuzberg.renderPdfPage(documentPath, 0, 150); - E2EHelpers.assertIsPng(pngData); - E2EHelpers.assertMinByteLength(pngData, 100); - } + } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/StructuredTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/StructuredTest.java index e33c142d505..4dfdf3123d2 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/StructuredTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/StructuredTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,176 +26,177 @@ /** Tests for structured fixtures. */ public class StructuredTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); - - @Test - public void structuredCsvBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_csv_basic", - "csv/stanley_cups.csv", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/csv")); - E2EHelpers.Assertions.assertMinContentLength(result, 20); - }); - } - - @Test - public void structuredEnwBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_enw_basic", - "data_formats/sample.enw", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, - Arrays.asList( - "application/x-endnote-refer", "application/x-endnote+xml", "text/plain")); - }); - } - - @Test - public void structuredJsonBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_json_basic", - "json/sample_document.json", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/json")); - E2EHelpers.Assertions.assertMinContentLength(result, 20); - E2EHelpers.Assertions.assertContentContainsAny( - result, Arrays.asList("Sample Document", "Test Author")); - }); - } - - @Test - public void structuredJsonSimple() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_json_simple", - "json/simple.json", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/json")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertContentContainsAny(result, Arrays.asList("{", "name")); - }); - } - - @Test - public void structuredNbibBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_nbib_basic", - "data_formats/sample.nbib", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/nbib", "application/x-pubmed", "text/plain")); - E2EHelpers.Assertions.assertContentNotEmpty(result); - }); - } - - @Test - public void structuredRisBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_ris_basic", - "data_formats/sample.ris", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/x-research-info-systems", "text/plain")); - E2EHelpers.Assertions.assertContentNotEmpty(result); - }); - } - - @Test - public void structuredTomlBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_toml_basic", - "data_formats/cargo.toml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/toml", "text/toml")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } - - @Test - public void structuredTsvBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_tsv_basic", - "data_formats/employees.tsv", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("text/tab-separated-values", "text/plain")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } - - @Test - public void structuredYamlBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_yaml_basic", - "yaml/simple.yaml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, - Arrays.asList("application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } - - @Test - public void structuredYamlSimple() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_yaml_simple", - "yaml/simple.yaml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-yaml")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void structuredCsvBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_csv_basic", + "csv/stanley_cups.csv", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/csv")); + E2EHelpers.Assertions.assertMinContentLength(result, 20); + } + ); + } + + @Test + public void structuredEnwBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_enw_basic", + "data_formats/sample.enw", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-endnote-refer", "application/x-endnote+xml", "text/plain")); + } + ); + } + + @Test + public void structuredJsonBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_json_basic", + "json/sample_document.json", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/json")); + E2EHelpers.Assertions.assertMinContentLength(result, 20); + E2EHelpers.Assertions.assertContentContainsAny(result, Arrays.asList("Sample Document", "Test Author")); + } + ); + } + + @Test + public void structuredJsonSimple() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_json_simple", + "json/simple.json", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/json")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + E2EHelpers.Assertions.assertContentContainsAny(result, Arrays.asList("{", "name")); + } + ); + } + + @Test + public void structuredNbibBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_nbib_basic", + "data_formats/sample.nbib", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/nbib", "application/x-pubmed", "text/plain")); + E2EHelpers.Assertions.assertContentNotEmpty(result); + } + ); + } + + @Test + public void structuredRisBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_ris_basic", + "data_formats/sample.ris", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-research-info-systems", "text/plain")); + E2EHelpers.Assertions.assertContentNotEmpty(result); + } + ); + } + + @Test + public void structuredTomlBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_toml_basic", + "data_formats/cargo.toml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/toml", "text/toml")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } + + @Test + public void structuredTsvBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_tsv_basic", + "data_formats/employees.tsv", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/tab-separated-values", "text/plain")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } + + @Test + public void structuredYamlBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_yaml_basic", + "yaml/simple.yaml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } + + @Test + public void structuredYamlSimple() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_yaml_simple", + "yaml/simple.yaml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-yaml")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } + } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/TokenReductionTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/TokenReductionTest.java index 6b347b41493..b00176ad3d9 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/TokenReductionTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/TokenReductionTest.java @@ -87,7 +87,7 @@ public void tokenReductionWithChunking() throws Exception { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 5); E2EHelpers.Assertions.assertMaxContentLength(result, 200); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null, null); E2EHelpers.Assertions.assertContentNotEmpty(result); }); } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/XmlTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/XmlTest.java index 2e2b38d908c..923e7997ae0 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/XmlTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/XmlTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,21 +26,23 @@ /** Tests for xml fixtures. */ public class XmlTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void xmlPlantCatalog() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "xml_plant_catalog", + "xml/plant_catalog.xml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/xml")); + E2EHelpers.Assertions.assertMinContentLength(result, 100); + } + ); + } - @Test - public void xmlPlantCatalog() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "xml_plant_catalog", - "xml/plant_catalog.xml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/xml")); - E2EHelpers.Assertions.assertMinContentLength(result, 100); - }); - } } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/package-info.java b/e2e/java/src/test/java/com/kreuzberg/e2e/package-info.java index af8c3c4a235..0d7096f1b66 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/package-info.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/package-info.java @@ -1,8 +1,8 @@ /** * E2E test utilities and generated test classes for Kreuzberg. * - *

This package contains auto-generated test classes organized by fixture category. Tests use - * JUnit 5 and validate document extraction across multiple formats. + *

This package contains auto-generated test classes organized by fixture category. + * Tests use JUnit 5 and validate document extraction across multiple formats. * * @since 4.0.0 */ diff --git a/e2e/php/tests/ContractTest.php b/e2e/php/tests/ContractTest.php index 52ed3e84568..2d311d5321a 100644 --- a/e2e/php/tests/ContractTest.php +++ b/e2e/php/tests/ContractTest.php @@ -334,7 +334,7 @@ public function test_config_chunking(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertChunks($result, 1, null, true, null, null, null, null); } /** @@ -355,7 +355,7 @@ public function test_config_chunking_heading_context(): void $result = $kreuzberg->extractFile($documentPath); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, true, null); + Helpers::assertChunks($result, 2, null, true, null, true, null, null); } /** @@ -377,7 +377,7 @@ public function test_config_chunking_markdown(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertChunks($result, 1, null, true, null, null, null, null); } /** @@ -398,7 +398,7 @@ public function test_config_chunking_no_headings(): void $result = $kreuzberg->extractFile($documentPath); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, false, null); + Helpers::assertChunks($result, 2, null, true, null, false, null, null); } /** @@ -419,7 +419,7 @@ public function test_config_chunking_prepend_heading_context(): void $result = $kreuzberg->extractFile($documentPath); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, true, true); + Helpers::assertChunks($result, 2, null, true, null, true, null, true); } /** @@ -441,7 +441,7 @@ public function test_config_chunking_small(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, null, null); + Helpers::assertChunks($result, 2, null, true, null, null, null, null); } /** @@ -461,7 +461,7 @@ public function test_config_chunking_text(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertChunks($result, 1, null, true, null, null, null, null); } /** @@ -482,7 +482,7 @@ public function test_config_chunking_tokenizer(): void $result = $kreuzberg->extractFile($documentPath); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, null, null); + Helpers::assertChunks($result, 2, null, true, null, null, null, null); } /** diff --git a/e2e/php/tests/EmbeddingsTest.php b/e2e/php/tests/EmbeddingsTest.php index f906aa9abbe..1baf60227fb 100644 --- a/e2e/php/tests/EmbeddingsTest.php +++ b/e2e/php/tests/EmbeddingsTest.php @@ -35,7 +35,7 @@ public function test_embedding_async(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); + Helpers::assertChunks($result, 1, null, true, true, null, null, null); } /** @@ -57,7 +57,7 @@ public function test_embedding_balanced_preset(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); + Helpers::assertChunks($result, 1, null, true, true, null, null, null); } /** @@ -77,7 +77,7 @@ public function test_embedding_disabled(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, false, null, null); + Helpers::assertChunks($result, 1, null, true, false, null, null, null); } /** @@ -99,7 +99,7 @@ public function test_embedding_fast_preset(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); + Helpers::assertChunks($result, 1, null, true, true, null, null, null); } } diff --git a/e2e/php/tests/Helpers.php b/e2e/php/tests/Helpers.php index 88caaf899d3..17bf9d44f92 100644 --- a/e2e/php/tests/Helpers.php +++ b/e2e/php/tests/Helpers.php @@ -209,6 +209,7 @@ public static function assertChunks( ?bool $eachHasContent, ?bool $eachHasEmbedding, ?bool $eachHasHeadingContext = null, + ?bool $eachHasChunkType = null, ?bool $contentStartsWithHeading = null ): void { $chunks = $result->chunks ?? []; @@ -264,6 +265,13 @@ public static function assertChunks( ); } } + if ($eachHasChunkType === true) { + foreach ($chunks as $i => $chunk) { + $type = $chunk->chunk_type ?? null; + Assert::assertNotNull($type, sprintf("Chunk %d should have chunk_type", $i)); + Assert::assertNotSame('unknown', $type, sprintf("Chunk %d should have specific chunk_type, got 'unknown'", $i)); + } + } if ($contentStartsWithHeading === true) { foreach ($chunks as $i => $chunk) { if (empty($chunk->metadata->heading_context ?? null)) { diff --git a/e2e/php/tests/Token_reductionTest.php b/e2e/php/tests/Token_reductionTest.php index e09b33f23bb..c89cce2d86c 100644 --- a/e2e/php/tests/Token_reductionTest.php +++ b/e2e/php/tests/Token_reductionTest.php @@ -95,7 +95,7 @@ public function test_token_reduction_with_chunking(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 5); Helpers::assertMaxContentLength($result, 200); - Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertChunks($result, 1, null, true, null, null, null, null); Helpers::assertContentNotEmpty($result); } diff --git a/e2e/python/tests/helpers.py b/e2e/python/tests/helpers.py index 81696dc4589..4efa020cc14 100644 --- a/e2e/python/tests/helpers.py +++ b/e2e/python/tests/helpers.py @@ -336,6 +336,13 @@ def _assert_chunks_heading_context(chunks: Any, expected: bool) -> None: pytest.fail(f"Chunk {i} should have no heading_context") +def _assert_chunks_chunk_type(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + chunk_type = getattr(chunk, "chunk_type", None) + if chunk_type is None or str(chunk_type) == "unknown": + pytest.fail(f"Chunk {i} has no specific chunk_type, got {chunk_type}") + + def _assert_chunks_heading_prefix(chunks: Any) -> None: for i, chunk in enumerate(chunks): meta = getattr(chunk, "metadata", None) @@ -354,6 +361,7 @@ def assert_chunks( each_has_content: bool | None = None, each_has_embedding: bool | None = None, each_has_heading_context: bool | None = None, + each_has_chunk_type: bool | None = None, content_starts_with_heading: bool | None = None, ) -> None: chunks = getattr(result, "chunks", None) @@ -370,6 +378,8 @@ def assert_chunks( _assert_chunks_embedding(chunks) if each_has_heading_context is not None: _assert_chunks_heading_context(chunks, each_has_heading_context) + if each_has_chunk_type: + _assert_chunks_chunk_type(chunks) if content_starts_with_heading: _assert_chunks_heading_prefix(chunks) diff --git a/e2e/r/tests/testthat/helper-kreuzberg.R b/e2e/r/tests/testthat/helper-kreuzberg.R index d1072a4cf0e..c0fe48bc82e 100644 --- a/e2e/r/tests/testthat/helper-kreuzberg.R +++ b/e2e/r/tests/testthat/helper-kreuzberg.R @@ -10,21 +10,15 @@ find_workspace_root <- function() { if (!is.null(ofile)) { # From tests/testthat/helper-kreuzberg.R, go up 4 levels to repo root candidate <- normalizePath(file.path(dirname(ofile), "..", "..", "..", ".."), mustWork = FALSE) - if (dir.exists(file.path(candidate, "test_documents"))) { - return(candidate) - } + if (dir.exists(file.path(candidate, "test_documents"))) return(candidate) } # Fallback: testthat sets working dir to package root (e2e/r/) candidate <- normalizePath(file.path(getwd(), "..", ".."), mustWork = FALSE) - if (dir.exists(file.path(candidate, "test_documents"))) { - return(candidate) - } + if (dir.exists(file.path(candidate, "test_documents"))) return(candidate) # Last resort: walk up from current dir d <- getwd() for (i in seq_len(10)) { - if (dir.exists(file.path(d, "test_documents"))) { - return(d) - } + if (dir.exists(file.path(d, "test_documents"))) return(d) d <- dirname(d) } stop("Could not find workspace root (test_documents directory)") @@ -37,16 +31,12 @@ resolve_document <- function(relative) { } build_config <- function(raw) { - if (is.null(raw) || length(raw) == 0) { - return(NULL) - } + if (is.null(raw) || length(raw) == 0) return(NULL) raw } build_extraction_config <- function(raw) { - if (is.null(raw) || length(raw) == 0) { - return(NULL) - } + if (is.null(raw) || length(raw) == 0) return(NULL) # Convert the raw config map into extraction_config() call do.call(extraction_config, raw) } @@ -58,9 +48,7 @@ skip_reason_for <- function(error, fixture_id, requirements, notes = NULL) { missing_dependency <- grepl("missing dependency", downcased, fixed = TRUE) unsupported_format <- grepl("unsupported format", downcased, fixed = TRUE) - if (!missing_dependency && !unsupported_format && !requirement_hit) { - return(NULL) - } + if (!missing_dependency && !unsupported_format && !requirement_hit) return(NULL) reason <- if (missing_dependency) { "missing dependency" @@ -88,12 +76,11 @@ skip_if_feature_unavailable <- function(feature) { run_fixture <- function(fixture_id, relative_path, config_hash, requirements, notes, skip_if_missing = TRUE) { run_fixture_with_method(fixture_id, relative_path, config_hash, "sync", "file", - requirements = requirements, notes = notes, skip_if_missing = skip_if_missing - ) + requirements = requirements, notes = notes, skip_if_missing = skip_if_missing) } run_fixture_with_method <- function(fixture_id, relative_path, config_hash, method, input_type, - requirements, notes, skip_if_missing = TRUE) { + requirements, notes, skip_if_missing = TRUE) { document_path <- resolve_document(relative_path) if (skip_if_missing && !file.exists(document_path)) { @@ -149,9 +136,7 @@ perform_extraction <- function(document_path, config, method, input_type) { # Assertion helpers assert_expected_mime <- function(result, expected) { - if (length(expected) == 0) { - return(invisible(NULL)) - } + if (length(expected) == 0) return(invisible(NULL)) testthat::expect_true(any(vapply(expected, function(token) grepl(token, result$mime_type, fixed = TRUE), logical(1)))) } @@ -164,17 +149,13 @@ assert_max_content_length <- function(result, maximum) { } assert_content_contains_any <- function(result, snippets) { - if (length(snippets) == 0) { - return(invisible(NULL)) - } + if (length(snippets) == 0) return(invisible(NULL)) lowered <- tolower(result$content) testthat::expect_true(any(vapply(snippets, function(s) grepl(tolower(s), lowered, fixed = TRUE), logical(1)))) } assert_content_contains_all <- function(result, snippets) { - if (length(snippets) == 0) { - return(invisible(NULL)) - } + if (length(snippets) == 0) return(invisible(NULL)) lowered <- tolower(result$content) testthat::expect_true(all(vapply(snippets, function(s) grepl(tolower(s), lowered, fixed = TRUE), logical(1)))) } @@ -186,9 +167,7 @@ assert_table_count <- function(result, minimum = NULL, maximum = NULL) { } assert_detected_languages <- function(result, expected, min_confidence = NULL) { - if (length(expected) == 0) { - return(invisible(NULL)) - } + if (length(expected) == 0) return(invisible(NULL)) languages <- result$detected_languages testthat::expect_false(is.null(languages)) testthat::expect_true(all(expected %in% languages)) @@ -237,6 +216,7 @@ assert_metadata_expectation <- function(result, path, expectation) { assert_chunks <- function(result, min_count = NULL, max_count = NULL, each_has_content = NULL, each_has_embedding = NULL, each_has_heading_context = NULL, + each_has_chunk_type = NULL, content_starts_with_heading = NULL) { chunks <- if (is.null(result$chunks)) list() else result$chunks if (!is.null(min_count)) testthat::expect_gte(length(chunks), min_count) @@ -253,6 +233,14 @@ assert_chunks <- function(result, min_count = NULL, max_count = NULL, if (isFALSE(each_has_heading_context)) { for (chunk in chunks) testthat::expect_true(is.null(chunk$metadata$heading_context)) } + if (isTRUE(each_has_chunk_type)) { + for (chunk in chunks) { + chunk_type <- chunk[["chunk_type"]] + if (is.null(chunk_type) || chunk_type == "unknown") { + testthat::fail(paste0("Expected specific chunk_type, got: ", chunk_type)) + } + } + } if (isTRUE(content_starts_with_heading)) { heading_char <- rawToChar(as.raw(35)) for (chunk in chunks) { @@ -292,15 +280,13 @@ assert_elements <- function(result, min_count = NULL, types_include = NULL) { } assert_ocr_elements <- function(result, has_elements = NULL, elements_have_geometry = NULL, - elements_have_confidence = NULL, min_count = NULL) { + elements_have_confidence = NULL, min_count = NULL) { ocr_elements <- result$ocr_elements if (isTRUE(has_elements)) { testthat::expect_false(is.null(ocr_elements)) testthat::expect_gt(length(ocr_elements), 0) } - if (!is.list(ocr_elements)) { - return(invisible(NULL)) - } + if (!is.list(ocr_elements)) return(invisible(NULL)) if (!is.null(min_count)) testthat::expect_gte(length(ocr_elements), min_count) if (isTRUE(elements_have_geometry)) { for (el in ocr_elements) { @@ -317,7 +303,7 @@ assert_ocr_elements <- function(result, has_elements = NULL, elements_have_geome } assert_document <- function(result, has_document = FALSE, min_node_count = NULL, - node_types_include = NULL, has_groups = NULL) { + node_types_include = NULL, has_groups = NULL) { document <- result$document if (has_document) { testthat::expect_false(is.null(document)) @@ -378,9 +364,7 @@ assert_table_bounding_boxes <- function(result) { } assert_table_content_contains_any <- function(result, snippets) { - if (length(snippets) == 0) { - return(invisible(NULL)) - } + if (length(snippets) == 0) return(invisible(NULL)) tables <- if (is.null(result$tables)) list() else result$tables all_content <- tolower(paste(vapply(tables, function(t) t$content %||% "", character(1)), collapse = " ")) testthat::expect_true(any(vapply(snippets, function(s) grepl(tolower(s), all_content, fixed = TRUE), logical(1)))) @@ -435,13 +419,9 @@ assert_annotations <- function(result, has_annotations = FALSE, min_count = NULL # Internal helpers fetch_metadata_value <- function(metadata, path) { value <- lookup_metadata_path(metadata, path) - if (!is.null(value)) { - return(value) - } + if (!is.null(value)) return(value) format_data <- metadata$format - if (is.list(format_data)) { - return(lookup_metadata_path(format_data, path)) - } + if (is.list(format_data)) return(lookup_metadata_path(format_data, path)) NULL } @@ -449,24 +429,16 @@ lookup_metadata_path <- function(metadata, path) { current <- metadata segments <- strsplit(path, ".", fixed = TRUE)[[1]] for (segment in segments) { - if (!is.list(current)) { - return(NULL) - } + if (!is.list(current)) return(NULL) current <- current[[segment]] - if (is.null(current)) { - return(NULL) - } + if (is.null(current)) return(NULL) } current } values_equal <- function(lhs, rhs) { - if (is.character(lhs) && is.character(rhs)) { - return(identical(lhs, rhs)) - } - if (is.numeric(lhs) && is.numeric(rhs)) { - return(as.numeric(lhs) == as.numeric(rhs)) - } + if (is.character(lhs) && is.character(rhs)) return(identical(lhs, rhs)) + if (is.numeric(lhs) && is.numeric(rhs)) return(as.numeric(lhs) == as.numeric(rhs)) identical(lhs, rhs) } diff --git a/e2e/r/tests/testthat/test-archive.R b/e2e/r/tests/testthat/test-archive.R index d4938c3bff3..c40afe3ce02 100644 --- a/e2e/r/tests/testthat/test-archive.R +++ b/e2e/r/tests/testthat/test-archive.R @@ -13,8 +13,8 @@ test_that("archive_gz_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/gzip", "application/x-gzip")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/gzip", "application/x-gzip")) + assert_min_content_length(result, 10L) }) test_that("archive_sevenz_basic", { @@ -26,8 +26,8 @@ test_that("archive_sevenz_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-7z-compressed")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-7z-compressed")) + assert_min_content_length(result, 10L) }) test_that("archive_tar_basic", { @@ -39,8 +39,8 @@ test_that("archive_tar_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-tar", "application/tar")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-tar", "application/tar")) + assert_min_content_length(result, 10L) }) test_that("archive_zip_basic", { @@ -52,7 +52,7 @@ test_that("archive_zip_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/zip", "application/x-zip-compressed")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/zip", "application/x-zip-compressed")) + assert_min_content_length(result, 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-contract.R b/e2e/r/tests/testthat/test-contract.R index a1f7c629212..20a69e5b001 100644 --- a/e2e/r/tests/testthat/test-contract.R +++ b/e2e/r/tests/testthat/test-contract.R @@ -15,9 +15,9 @@ test_that("api_batch_bytes_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_batch_bytes_sync", { @@ -31,9 +31,9 @@ test_that("api_batch_bytes_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_batch_bytes_with_configs_async", { @@ -47,8 +47,8 @@ test_that("api_batch_bytes_with_configs_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_batch_bytes_with_configs_sync", { @@ -62,8 +62,8 @@ test_that("api_batch_bytes_with_configs_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_batch_file_async", { @@ -77,9 +77,9 @@ test_that("api_batch_file_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_batch_file_sync", { @@ -93,9 +93,9 @@ test_that("api_batch_file_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_batch_file_with_configs_async", { @@ -109,8 +109,8 @@ test_that("api_batch_file_with_configs_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_batch_file_with_configs_sync", { @@ -124,8 +124,8 @@ test_that("api_batch_file_with_configs_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_batch_file_with_timeout_sync", { @@ -139,8 +139,8 @@ test_that("api_batch_file_with_timeout_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_extract_bytes_async", { @@ -154,9 +154,9 @@ test_that("api_extract_bytes_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_extract_bytes_sync", { @@ -170,9 +170,9 @@ test_that("api_extract_bytes_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_extract_file_async", { @@ -186,9 +186,9 @@ test_that("api_extract_file_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_extract_file_sync", { @@ -200,9 +200,9 @@ test_that("api_extract_file_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("config_acceleration_cpu_provider", { @@ -214,9 +214,9 @@ test_that("config_acceleration_cpu_provider", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern")) }) test_that("config_chunking", { @@ -228,9 +228,9 @@ test_that("config_chunking", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE) }) test_that("config_chunking_heading_context", { @@ -243,8 +243,8 @@ test_that("config_chunking_heading_context", { notes = NULL, skip_if_missing = TRUE ) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = TRUE) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = TRUE) }) test_that("config_chunking_markdown", { @@ -257,9 +257,9 @@ test_that("config_chunking_markdown", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE) }) test_that("config_chunking_no_headings", { @@ -272,8 +272,8 @@ test_that("config_chunking_no_headings", { notes = NULL, skip_if_missing = TRUE ) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = FALSE) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = FALSE) }) test_that("config_chunking_prepend_heading_context", { @@ -286,8 +286,8 @@ test_that("config_chunking_prepend_heading_context", { notes = NULL, skip_if_missing = TRUE ) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = TRUE, content_starts_with_heading = TRUE) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = TRUE, content_starts_with_heading = TRUE) }) test_that("config_chunking_small", { @@ -300,9 +300,9 @@ test_that("config_chunking_small", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE) }) test_that("config_chunking_text", { @@ -314,9 +314,9 @@ test_that("config_chunking_text", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE) }) test_that("config_chunking_tokenizer", { @@ -329,8 +329,8 @@ test_that("config_chunking_tokenizer", { notes = "Requires network access for HuggingFace Hub tokenizer download", skip_if_missing = TRUE ) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE) }) test_that("config_disable_ocr", { @@ -356,8 +356,8 @@ test_that("config_djot_content", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("config_document_structure", { @@ -369,8 +369,8 @@ test_that("config_document_structure", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_document(result, has_document = TRUE, min_node_count = 1L, node_types_include = c("paragraph")) + assert_expected_mime(result, c("application/pdf")) + assert_document(result, has_document = TRUE, min_node_count = 1L, node_types_include = c("paragraph")) }) test_that("config_document_structure_disabled", { @@ -382,8 +382,8 @@ test_that("config_document_structure_disabled", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_document(result, has_document = FALSE) + assert_expected_mime(result, c("application/pdf")) + assert_document(result, has_document = FALSE) }) test_that("config_document_structure_groups", { @@ -396,8 +396,8 @@ test_that("config_document_structure_groups", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_document(result, has_document = TRUE, has_groups = TRUE) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_document(result, has_document = TRUE, has_groups = TRUE) }) test_that("config_document_structure_headings", { @@ -410,8 +410,8 @@ test_that("config_document_structure_headings", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_document(result, has_document = TRUE, min_node_count = 1L, node_types_include = c("heading", "paragraph")) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_document(result, has_document = TRUE, min_node_count = 1L, node_types_include = c("heading", "paragraph")) }) test_that("config_document_structure_with_headings", { @@ -423,8 +423,8 @@ test_that("config_document_structure_with_headings", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_document(result, has_document = TRUE, min_node_count = 1L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_document(result, has_document = TRUE, min_node_count = 1L) }) test_that("config_element_types", { @@ -437,8 +437,8 @@ test_that("config_element_types", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_elements(result, min_count = 1L, types_include = c("narrative_text")) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_elements(result, min_count = 1L, types_include = c("narrative_text")) }) test_that("config_email_msg_fallback_codepage", { @@ -450,8 +450,8 @@ test_that("config_email_msg_fallback_codepage", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-outlook")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.ms-outlook")) + assert_min_content_length(result, 10L) }) test_that("config_extraction_timeout", { @@ -463,8 +463,8 @@ test_that("config_extraction_timeout", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("config_force_ocr", { @@ -477,8 +477,8 @@ test_that("config_force_ocr", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) }) test_that("config_force_ocr_pages", { @@ -491,8 +491,8 @@ test_that("config_force_ocr_pages", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 1L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 1L) }) test_that("config_html_options", { @@ -504,9 +504,9 @@ test_that("config_html_options", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/html")) - assert_min_content_length(result, 10L) - assert_content_not_empty(result) + assert_expected_mime(result, c("text/html")) + assert_min_content_length(result, 10L) + assert_content_not_empty(result) }) test_that("config_images", { @@ -518,8 +518,8 @@ test_that("config_images", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_images(result, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_images(result, min_count = 1L) }) test_that("config_images_with_formats", { @@ -531,8 +531,8 @@ test_that("config_images_with_formats", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_images(result, min_count = 1L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_images(result, min_count = 1L) }) test_that("config_keywords", { @@ -545,9 +545,9 @@ test_that("config_keywords", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_keywords(result, has_keywords = TRUE, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_keywords(result, has_keywords = TRUE, min_count = 1L) }) test_that("config_language_detection", { @@ -559,9 +559,9 @@ test_that("config_language_detection", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_detected_languages(result, c("eng"), 0.5) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_detected_languages(result, c("eng"), 0.5) }) test_that("config_language_detection_multi", { @@ -573,9 +573,9 @@ test_that("config_language_detection_multi", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_detected_languages(result, c("eng"), NULL) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_detected_languages(result, c("eng"), NULL) }) test_that("config_language_multi", { @@ -588,9 +588,9 @@ test_that("config_language_multi", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_detected_languages(result, c("eng"), NULL) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_detected_languages(result, c("eng"), NULL) }) test_that("config_pages", { @@ -603,9 +603,9 @@ test_that("config_pages", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("PAGE")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("PAGE")) }) test_that("config_pages_exact_count", { @@ -618,9 +618,9 @@ test_that("config_pages_exact_count", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_pages(result, exact_count = 5L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_pages(result, exact_count = 5L) }) test_that("config_pages_extract", { @@ -633,9 +633,9 @@ test_that("config_pages_extract", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_pages(result, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_pages(result, min_count = 1L) }) test_that("config_pages_markers", { @@ -648,9 +648,9 @@ test_that("config_pages_markers", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("PAGE")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("PAGE")) }) test_that("config_pdf_annotations_count", { @@ -663,8 +663,8 @@ test_that("config_pdf_annotations_count", { notes = "PDFium ARM Linux binary does not support annotation extraction", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_annotations(result, has_annotations = TRUE, min_count = 3L) + assert_expected_mime(result, c("application/pdf")) + assert_annotations(result, has_annotations = TRUE, min_count = 3L) }) test_that("config_pdf_hierarchy", { @@ -677,8 +677,8 @@ test_that("config_pdf_hierarchy", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) }) test_that("config_pdf_margins", { @@ -691,8 +691,8 @@ test_that("config_pdf_margins", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) }) test_that("config_postprocessor", { @@ -704,9 +704,9 @@ test_that("config_postprocessor", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_not_empty(result) }) test_that("config_processing_warnings_empty", { @@ -718,9 +718,9 @@ test_that("config_processing_warnings_empty", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_processing_warnings(result, is_empty = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_processing_warnings(result, is_empty = TRUE) }) test_that("config_quality_disabled", { @@ -732,9 +732,9 @@ test_that("config_quality_disabled", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_not_empty(result) }) test_that("config_quality_enabled", { @@ -747,9 +747,9 @@ test_that("config_quality_enabled", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_quality_score(result, has_score = TRUE, min_score = 0, max_score = 1) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_quality_score(result, has_score = TRUE, min_score = 0, max_score = 1) }) test_that("config_quality_score_range", { @@ -762,8 +762,8 @@ test_that("config_quality_score_range", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_quality_score(result, has_score = TRUE, min_score = 0.1) + assert_expected_mime(result, c("application/pdf")) + assert_quality_score(result, has_score = TRUE, min_score = 0.1) }) test_that("config_security_limits", { @@ -775,8 +775,8 @@ test_that("config_security_limits", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/zip", "application/x-zip-compressed")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/zip", "application/x-zip-compressed")) + assert_min_content_length(result, 10L) }) test_that("config_structured_output", { @@ -789,8 +789,8 @@ test_that("config_structured_output", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("config_tables_content", { @@ -802,8 +802,8 @@ test_that("config_tables_content", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("config_tree_sitter", { @@ -843,8 +843,8 @@ test_that("config_use_cache_false", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_bytes_markdown", { @@ -858,8 +858,8 @@ test_that("output_format_bytes_markdown", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_djot", { @@ -871,8 +871,8 @@ test_that("output_format_djot", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_html", { @@ -884,8 +884,8 @@ test_that("output_format_html", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_markdown", { @@ -897,8 +897,8 @@ test_that("output_format_markdown", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_plain", { @@ -910,8 +910,8 @@ test_that("output_format_plain", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("result_format_element_based", { @@ -923,8 +923,8 @@ test_that("result_format_element_based", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_elements(result, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_elements(result, min_count = 1L) }) test_that("result_format_unified", { @@ -936,7 +936,7 @@ test_that("result_format_unified", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-email.R b/e2e/r/tests/testthat/test-email.R index ddc219c5349..78e0da909f6 100644 --- a/e2e/r/tests/testthat/test-email.R +++ b/e2e/r/tests/testthat/test-email.R @@ -13,8 +13,8 @@ test_that("email_eml_html_body", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("message/rfc822")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("message/rfc822")) + assert_min_content_length(result, 10L) }) test_that("email_eml_multipart", { @@ -26,8 +26,8 @@ test_that("email_eml_multipart", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("message/rfc822")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("message/rfc822")) + assert_min_content_length(result, 10L) }) test_that("email_eml_utf16", { @@ -39,9 +39,9 @@ test_that("email_eml_utf16", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("message/rfc822")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("Test Email", "Roses are red")) + assert_expected_mime(result, c("message/rfc822")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("Test Email", "Roses are red")) }) test_that("email_msg_basic", { @@ -53,8 +53,8 @@ test_that("email_msg_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-outlook")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.ms-outlook")) + assert_min_content_length(result, 10L) }) test_that("email_pst_empty", { @@ -66,7 +66,7 @@ test_that("email_pst_empty", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-outlook-pst")) + assert_expected_mime(result, c("application/vnd.ms-outlook-pst")) }) test_that("email_sample_eml", { @@ -78,7 +78,7 @@ test_that("email_sample_eml", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("message/rfc822")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("message/rfc822")) + assert_min_content_length(result, 20L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-embeddings.R b/e2e/r/tests/testthat/test-embeddings.R index ccf6faaff51..03b89266d2c 100644 --- a/e2e/r/tests/testthat/test-embeddings.R +++ b/e2e/r/tests/testthat/test-embeddings.R @@ -16,9 +16,9 @@ test_that("embedding_async", { notes = "Embeddings not supported on Windows", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) }) test_that("embedding_balanced_preset", { @@ -31,9 +31,9 @@ test_that("embedding_balanced_preset", { notes = "Embeddings not supported on Windows", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) }) test_that("embedding_disabled", { @@ -45,9 +45,9 @@ test_that("embedding_disabled", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = FALSE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = FALSE) }) test_that("embedding_fast_preset", { @@ -60,8 +60,8 @@ test_that("embedding_fast_preset", { notes = "Embeddings not supported on Windows", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) }) # nolint end diff --git a/e2e/r/tests/testthat/test-html.R b/e2e/r/tests/testthat/test-html.R index 2e2d9899f41..eae8c08984d 100644 --- a/e2e/r/tests/testthat/test-html.R +++ b/e2e/r/tests/testthat/test-html.R @@ -13,8 +13,8 @@ test_that("html_complex_layout", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/html")) - assert_min_content_length(result, 1000L) + assert_expected_mime(result, c("text/html")) + assert_min_content_length(result, 1000L) }) test_that("html_simple_table", { @@ -26,8 +26,8 @@ test_that("html_simple_table", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/html")) - assert_min_content_length(result, 100L) - assert_content_contains_all(result, c("Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table")) + assert_expected_mime(result, c("text/html")) + assert_min_content_length(result, 100L) + assert_content_contains_all(result, c("Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table")) }) # nolint end diff --git a/e2e/r/tests/testthat/test-image.R b/e2e/r/tests/testthat/test-image.R index 157031c245b..fe26fd7631b 100644 --- a/e2e/r/tests/testthat/test-image.R +++ b/e2e/r/tests/testthat/test-image.R @@ -14,8 +14,8 @@ test_that("image_bmp_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/bmp")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/bmp")) + assert_content_not_empty(result) }) test_that("image_gif_basic", { @@ -28,8 +28,8 @@ test_that("image_gif_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/gif")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/gif")) + assert_content_not_empty(result) }) test_that("image_jp2_basic", { @@ -42,8 +42,8 @@ test_that("image_jp2_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jp2", "image/jpeg2000")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/jp2", "image/jpeg2000")) + assert_content_not_empty(result) }) test_that("image_metadata_only", { @@ -55,8 +55,8 @@ test_that("image_metadata_only", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jpeg")) - assert_max_content_length(result, 200L) + assert_expected_mime(result, c("image/jpeg")) + assert_max_content_length(result, 200L) }) test_that("image_pbm_basic", { @@ -69,8 +69,8 @@ test_that("image_pbm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/x-portable-bitmap", "image/x-pbm")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/x-portable-bitmap", "image/x-pbm")) + assert_content_not_empty(result) }) test_that("image_pgm_basic", { @@ -83,8 +83,8 @@ test_that("image_pgm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/x-portable-graymap", "image/x-pgm")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/x-portable-graymap", "image/x-pgm")) + assert_content_not_empty(result) }) test_that("image_ppm_basic", { @@ -97,8 +97,8 @@ test_that("image_ppm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/x-portable-pixmap", "image/x-ppm")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/x-portable-pixmap", "image/x-ppm")) + assert_content_not_empty(result) }) test_that("image_svg_basic", { @@ -110,8 +110,8 @@ test_that("image_svg_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/svg+xml")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("image/svg+xml")) + assert_min_content_length(result, 5L) }) test_that("image_tiff_basic", { @@ -124,8 +124,8 @@ test_that("image_tiff_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/tiff")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/tiff")) + assert_content_not_empty(result) }) test_that("image_webp_basic", { @@ -138,7 +138,7 @@ test_that("image_webp_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/webp")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/webp")) + assert_content_not_empty(result) }) # nolint end diff --git a/e2e/r/tests/testthat/test-keywords.R b/e2e/r/tests/testthat/test-keywords.R index 4a80972d76c..08a0fe1a40e 100644 --- a/e2e/r/tests/testthat/test-keywords.R +++ b/e2e/r/tests/testthat/test-keywords.R @@ -14,9 +14,9 @@ test_that("keywords_rake", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_keywords(result, has_keywords = TRUE, min_count = 1L, max_count = 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_keywords(result, has_keywords = TRUE, min_count = 1L, max_count = 10L) }) test_that("keywords_yake", { @@ -29,8 +29,8 @@ test_that("keywords_yake", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_keywords(result, has_keywords = TRUE, min_count = 1L, max_count = 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_keywords(result, has_keywords = TRUE, min_count = 1L, max_count = 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-ocr.R b/e2e/r/tests/testthat/test-ocr.R index 2e0dc0f1fce..3c731000ec1 100644 --- a/e2e/r/tests/testthat/test-ocr.R +++ b/e2e/r/tests/testthat/test-ocr.R @@ -14,9 +14,9 @@ test_that("ocr_image_hello_world", { notes = "Requires Tesseract OCR for image text extraction.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_content_contains_any(result, c("hello", "world")) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_content_contains_any(result, c("hello", "world")) }) test_that("ocr_image_no_text", { @@ -29,8 +29,8 @@ test_that("ocr_image_no_text", { notes = "Skip when Tesseract is unavailable.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jpeg")) - assert_max_content_length(result, 300L) + assert_expected_mime(result, c("image/jpeg")) + assert_max_content_length(result, 300L) }) test_that("ocr_paddle_confidence_filter", { @@ -44,8 +44,8 @@ test_that("ocr_paddle_confidence_filter", { notes = "Tests confidence threshold filtering with PaddleOCR", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jpeg")) - assert_min_content_length(result, 1L) + assert_expected_mime(result, c("image/jpeg")) + assert_min_content_length(result, 1L) }) test_that("ocr_paddle_element_hierarchy", { @@ -59,9 +59,9 @@ test_that("ocr_paddle_element_hierarchy", { notes = "Requires PaddleOCR with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) }) test_that("ocr_paddle_element_levels", { @@ -75,9 +75,9 @@ test_that("ocr_paddle_element_levels", { notes = "Requires PaddleOCR with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, min_count = 1L) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, min_count = 1L) }) test_that("ocr_paddle_image_chinese", { @@ -91,8 +91,8 @@ test_that("ocr_paddle_image_chinese", { notes = "Requires PaddleOCR with Chinese models", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jpeg")) - assert_min_content_length(result, 1L) + assert_expected_mime(result, c("image/jpeg")) + assert_min_content_length(result, 1L) }) test_that("ocr_paddle_image_english", { @@ -106,9 +106,9 @@ test_that("ocr_paddle_image_english", { notes = "Requires PaddleOCR with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_content_contains_any(result, c("hello", "Hello", "world", "World")) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_content_contains_any(result, c("hello", "Hello", "world", "World")) }) test_that("ocr_paddle_markdown", { @@ -122,9 +122,9 @@ test_that("ocr_paddle_markdown", { notes = "Tests markdown output format parity with Tesseract", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_content_contains_any(result, c("hello", "Hello", "world", "World")) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_content_contains_any(result, c("hello", "Hello", "world", "World")) }) test_that("ocr_paddle_pdf_scanned", { @@ -138,9 +138,9 @@ test_that("ocr_paddle_pdf_scanned", { notes = "Requires PaddleOCR with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 20L) - assert_content_contains_any(result, c("Docling", "Markdown", "JSON")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 20L) + assert_content_contains_any(result, c("Docling", "Markdown", "JSON")) }) test_that("ocr_paddle_structured", { @@ -154,9 +154,9 @@ test_that("ocr_paddle_structured", { notes = "Tests structured output with bbox/confidence preservation", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) }) test_that("ocr_paddle_table_detection", { @@ -170,9 +170,9 @@ test_that("ocr_paddle_table_detection", { notes = "Tests table detection capability with PaddleOCR. ONNX Runtime model loading unstable on ARM Linux.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 10L) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 10L) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("ocr_pdf_image_only_german", { @@ -185,9 +185,9 @@ test_that("ocr_pdf_image_only_german", { notes = "Requires Tesseract OCR with German language data.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 20L) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 20L) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("ocr_pdf_rotated_90", { @@ -200,8 +200,8 @@ test_that("ocr_pdf_rotated_90", { notes = "Skip automatically when OCR backend is missing.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("ocr_pdf_tesseract", { @@ -214,9 +214,9 @@ test_that("ocr_pdf_tesseract", { notes = "Skip automatically if OCR backend is unavailable.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 20L) - assert_content_contains_any(result, c("Docling", "Markdown", "JSON")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 20L) + assert_content_contains_any(result, c("Docling", "Markdown", "JSON")) }) test_that("ocr_tesseract_elements", { @@ -229,9 +229,9 @@ test_that("ocr_tesseract_elements", { notes = "Requires Tesseract OCR backend", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) }) test_that("ocr_tesseract_elements_min_count", { @@ -244,9 +244,9 @@ test_that("ocr_tesseract_elements_min_count", { notes = "Requires Tesseract OCR backend", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, min_count = 1L) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, min_count = 1L) }) test_that("ocr_tesseract_language_german", { @@ -259,7 +259,7 @@ test_that("ocr_tesseract_language_german", { notes = "Requires Tesseract OCR with German language data (deu)", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 20L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-office.R b/e2e/r/tests/testthat/test-office.R index 50ef8f5dfac..df5a485eecd 100644 --- a/e2e/r/tests/testthat/test-office.R +++ b/e2e/r/tests/testthat/test-office.R @@ -13,8 +13,8 @@ test_that("office_bibtex_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-bibtex", "text/x-bibtex")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-bibtex", "text/x-bibtex")) + assert_min_content_length(result, 10L) }) test_that("office_commonmark_basic", { @@ -26,8 +26,8 @@ test_that("office_commonmark_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/markdown", "text/plain", "text/x-commonmark")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("text/markdown", "text/plain", "text/x-commonmark")) + assert_min_content_length(result, 5L) }) test_that("office_dbf_basic", { @@ -40,9 +40,9 @@ test_that("office_dbf_basic", { notes = "Requires the office feature.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-dbf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("|")) + assert_expected_mime(result, c("application/x-dbf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("|")) }) test_that("office_djot_basic", { @@ -54,8 +54,8 @@ test_that("office_djot_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/x-djot", "text/djot")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("text/x-djot", "text/djot")) + assert_min_content_length(result, 10L) }) test_that("office_doc_legacy", { @@ -68,8 +68,8 @@ test_that("office_doc_legacy", { notes = "Requires the office feature.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/msword")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/msword")) + assert_min_content_length(result, 20L) }) test_that("office_docbook_basic", { @@ -81,8 +81,8 @@ test_that("office_docbook_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/docbook+xml", "text/docbook")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/docbook+xml", "text/docbook")) + assert_min_content_length(result, 10L) }) test_that("office_docx_basic", { @@ -94,8 +94,8 @@ test_that("office_docx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 10L) }) test_that("office_docx_equations", { @@ -107,8 +107,8 @@ test_that("office_docx_equations", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_fake", { @@ -120,8 +120,8 @@ test_that("office_docx_fake", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_formatting", { @@ -133,8 +133,8 @@ test_that("office_docx_formatting", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_headers", { @@ -146,8 +146,8 @@ test_that("office_docx_headers", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_lists", { @@ -159,8 +159,8 @@ test_that("office_docx_lists", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_tables", { @@ -172,10 +172,10 @@ test_that("office_docx_tables", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 50L) - assert_content_contains_all(result, c("Simple uniform table", "Nested Table", "merged cells", "Header Col")) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 50L) + assert_content_contains_all(result, c("Simple uniform table", "Nested Table", "merged cells", "Header Col")) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("office_epub_basic", { @@ -187,8 +187,8 @@ test_that("office_epub_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/epub+zip")) - assert_min_content_length(result, 50L) + assert_expected_mime(result, c("application/epub+zip")) + assert_min_content_length(result, 50L) }) test_that("office_fb2_basic", { @@ -200,8 +200,8 @@ test_that("office_fb2_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-fictionbook+xml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-fictionbook+xml")) + assert_min_content_length(result, 10L) }) test_that("office_fictionbook_basic", { @@ -213,8 +213,8 @@ test_that("office_fictionbook_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-fictionbook+xml", "application/x-fictionbook")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-fictionbook+xml", "application/x-fictionbook")) + assert_min_content_length(result, 10L) }) test_that("office_hwp_basic", { @@ -227,8 +227,8 @@ test_that("office_hwp_basic", { notes = "Requires the office feature.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-hwp")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-hwp")) + assert_min_content_length(result, 10L) }) test_that("office_hwp_styled", { @@ -241,7 +241,7 @@ test_that("office_hwp_styled", { notes = "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-hwp")) + assert_expected_mime(result, c("application/x-hwp")) }) test_that("office_jats_basic", { @@ -253,8 +253,8 @@ test_that("office_jats_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-jats+xml", "text/jats")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-jats+xml", "text/jats")) + assert_min_content_length(result, 10L) }) test_that("office_jupyter_basic", { @@ -266,8 +266,8 @@ test_that("office_jupyter_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-ipynb+json")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-ipynb+json")) + assert_min_content_length(result, 10L) }) test_that("office_keynote_basic", { @@ -279,8 +279,8 @@ test_that("office_keynote_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-iwork-keynote-sffkey")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/x-iwork-keynote-sffkey")) + assert_min_content_length(result, 5L) }) test_that("office_latex_basic", { @@ -292,8 +292,8 @@ test_that("office_latex_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-latex", "text/x-latex")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/x-latex", "text/x-latex")) + assert_min_content_length(result, 20L) }) test_that("office_markdown_basic", { @@ -305,8 +305,8 @@ test_that("office_markdown_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/markdown")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("text/markdown")) + assert_min_content_length(result, 10L) }) test_that("office_mdx_basic", { @@ -318,8 +318,8 @@ test_that("office_mdx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/mdx", "text/x-mdx")) - assert_min_content_length(result, 50L) + assert_expected_mime(result, c("text/mdx", "text/x-mdx")) + assert_min_content_length(result, 50L) }) test_that("office_mdx_getting_started", { @@ -331,8 +331,8 @@ test_that("office_mdx_getting_started", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/mdx", "text/x-mdx")) - assert_min_content_length(result, 2000L) + assert_expected_mime(result, c("text/mdx", "text/x-mdx")) + assert_min_content_length(result, 2000L) }) test_that("office_mdx_troubleshooting", { @@ -344,8 +344,8 @@ test_that("office_mdx_troubleshooting", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/mdx", "text/x-mdx")) - assert_min_content_length(result, 2000L) + assert_expected_mime(result, c("text/mdx", "text/x-mdx")) + assert_min_content_length(result, 2000L) }) test_that("office_mdx_using_mdx", { @@ -357,8 +357,8 @@ test_that("office_mdx_using_mdx", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/mdx", "text/x-mdx")) - assert_min_content_length(result, 2000L) + assert_expected_mime(result, c("text/mdx", "text/x-mdx")) + assert_min_content_length(result, 2000L) }) test_that("office_numbers_basic", { @@ -370,8 +370,8 @@ test_that("office_numbers_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-iwork-numbers-sffnumbers")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-iwork-numbers-sffnumbers")) + assert_min_content_length(result, 10L) }) test_that("office_ods_basic", { @@ -383,8 +383,8 @@ test_that("office_ods_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.spreadsheet")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.spreadsheet")) + assert_min_content_length(result, 10L) }) test_that("office_odt_bold", { @@ -396,8 +396,8 @@ test_that("office_odt_bold", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) + assert_min_content_length(result, 10L) }) test_that("office_odt_list", { @@ -409,9 +409,9 @@ test_that("office_odt_list", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) - assert_min_content_length(result, 30L) - assert_content_contains_any(result, c("list item", "New level", "Pushed us")) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) + assert_min_content_length(result, 30L) + assert_content_contains_any(result, c("list item", "New level", "Pushed us")) }) test_that("office_odt_simple", { @@ -423,9 +423,9 @@ test_that("office_odt_simple", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("favorite things", "Parrots", "Analysis")) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("favorite things", "Parrots", "Analysis")) }) test_that("office_odt_table", { @@ -437,9 +437,9 @@ test_that("office_odt_table", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) - assert_min_content_length(result, 10L) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) + assert_min_content_length(result, 10L) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("office_opml_basic", { @@ -451,8 +451,8 @@ test_that("office_opml_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/xml+opml", "text/x-opml", "application/x-opml+xml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/xml+opml", "text/x-opml", "application/x-opml+xml")) + assert_min_content_length(result, 10L) }) test_that("office_org_basic", { @@ -464,8 +464,8 @@ test_that("office_org_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/x-org", "text/org")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("text/x-org", "text/org")) + assert_min_content_length(result, 20L) }) test_that("office_pages_basic", { @@ -477,8 +477,8 @@ test_that("office_pages_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-iwork-pages-sffpages")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/x-iwork-pages-sffpages")) + assert_min_content_length(result, 5L) }) test_that("office_ppsx_slideshow", { @@ -490,8 +490,8 @@ test_that("office_ppsx_slideshow", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.slideshow")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.slideshow")) + assert_min_content_length(result, 10L) }) test_that("office_ppt_legacy", { @@ -504,8 +504,8 @@ test_that("office_ppt_legacy", { notes = "Requires the office feature.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-powerpoint")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.ms-powerpoint")) + assert_min_content_length(result, 10L) }) test_that("office_pptm_basic", { @@ -517,8 +517,8 @@ test_that("office_pptm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_content_not_empty(result) }) test_that("office_pptx_basic", { @@ -530,8 +530,8 @@ test_that("office_pptx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_min_content_length(result, 50L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_min_content_length(result, 50L) }) test_that("office_pptx_images", { @@ -543,8 +543,8 @@ test_that("office_pptx_images", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_min_content_length(result, 15L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_min_content_length(result, 15L) }) test_that("office_pptx_pitch_deck", { @@ -556,8 +556,8 @@ test_that("office_pptx_pitch_deck", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_min_content_length(result, 100L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_min_content_length(result, 100L) }) test_that("office_rst_basic", { @@ -569,8 +569,8 @@ test_that("office_rst_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/x-rst", "text/prs.fallenstein.rst")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("text/x-rst", "text/prs.fallenstein.rst")) + assert_min_content_length(result, 20L) }) test_that("office_rtf_basic", { @@ -582,8 +582,8 @@ test_that("office_rtf_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/rtf", "text/rtf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/rtf", "text/rtf")) + assert_min_content_length(result, 10L) }) test_that("office_typst_basic", { @@ -595,8 +595,8 @@ test_that("office_typst_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-typst", "text/x-typst")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-typst", "text/x-typst")) + assert_min_content_length(result, 10L) }) test_that("office_xls_legacy", { @@ -608,8 +608,8 @@ test_that("office_xls_legacy", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-excel")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.ms-excel")) + assert_min_content_length(result, 10L) }) test_that("office_xlsb_basic", { @@ -621,8 +621,8 @@ test_that("office_xlsb_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_content_not_empty(result) }) test_that("office_xlsm_basic", { @@ -634,8 +634,8 @@ test_that("office_xlsm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_content_not_empty(result) }) test_that("office_xlsx_basic", { @@ -647,12 +647,12 @@ test_that("office_xlsx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_min_content_length(result, 100L) - assert_content_contains_all(result, c("Team", "Location", "Stanley Cups")) - assert_table_count(result, minimum = 1L, maximum = NULL) - assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) - assert_metadata_expectation(result, "sheet_names", list(contains = c("Stanley Cups"))) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_min_content_length(result, 100L) + assert_content_contains_all(result, c("Team", "Location", "Stanley Cups")) + assert_table_count(result, minimum = 1L, maximum = NULL) + assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) + assert_metadata_expectation(result, "sheet_names", list(contains = c("Stanley Cups"))) }) test_that("office_xlsx_multi_sheet", { @@ -664,9 +664,9 @@ test_that("office_xlsx_multi_sheet", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_min_content_length(result, 20L) - assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_min_content_length(result, 20L) + assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) }) test_that("office_xlsx_office_example", { @@ -678,7 +678,7 @@ test_that("office_xlsx_office_example", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_min_content_length(result, 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-pdf.R b/e2e/r/tests/testthat/test-pdf.R index 4cbbd3ba819..aab8d272886 100644 --- a/e2e/r/tests/testthat/test-pdf.R +++ b/e2e/r/tests/testthat/test-pdf.R @@ -13,8 +13,8 @@ test_that("pdf_annotations", { notes = "PDFium ARM Linux binary does not support annotation extraction", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_annotations(result, has_annotations = TRUE, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_annotations(result, has_annotations = TRUE, min_count = 1L) }) test_that("pdf_assembly_technical", { @@ -26,10 +26,10 @@ test_that("pdf_assembly_technical", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5000L) - assert_content_contains_any(result, c("assembly", "register", "instruction")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5000L) + assert_content_contains_any(result, c("assembly", "register", "instruction")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_bayesian_data_analysis", { @@ -41,10 +41,10 @@ test_that("pdf_bayesian_data_analysis", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10000L) - assert_content_contains_any(result, c("Bayesian", "probability", "distribution")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10000L) + assert_content_contains_any(result, c("Bayesian", "probability", "distribution")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_bounding_boxes", { @@ -57,10 +57,10 @@ test_that("pdf_bounding_boxes", { notes = "ONNX Runtime model loading unstable on ARM Linux; table detection returns 0 tables", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_table_count(result, minimum = 1L, maximum = NULL) - assert_table_bounding_boxes(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_table_count(result, minimum = 1L, maximum = NULL) + assert_table_bounding_boxes(result) }) test_that("pdf_code_and_formula", { @@ -72,8 +72,8 @@ test_that("pdf_code_and_formula", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 100L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 100L) }) test_that("pdf_deep_learning", { @@ -85,10 +85,10 @@ test_that("pdf_deep_learning", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 1000L) - assert_content_contains_any(result, c("neural", "network", "deep learning")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 1000L) + assert_content_contains_any(result, c("neural", "network", "deep learning")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_embedded_images", { @@ -100,9 +100,9 @@ test_that("pdf_embedded_images", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_table_count(result, minimum = 0L, maximum = NULL) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_table_count(result, minimum = 0L, maximum = NULL) }) test_that("pdf_google_doc", { @@ -114,9 +114,9 @@ test_that("pdf_google_doc", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_large_ciml", { @@ -128,10 +128,10 @@ test_that("pdf_large_ciml", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10000L) - assert_content_contains_any(result, c("machine learning", "algorithm", "training")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10000L) + assert_content_contains_any(result, c("machine learning", "algorithm", "training")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_layout_detection", { @@ -144,9 +144,9 @@ test_that("pdf_layout_detection", { notes = "Requires layout-detection feature with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 100L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 100L) + assert_content_not_empty(result) }) test_that("pdf_non_english_german", { @@ -158,10 +158,10 @@ test_that("pdf_non_english_german", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 100L) - assert_content_contains_any(result, c("Intel", "paging")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 100L) + assert_content_contains_any(result, c("Intel", "paging")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_password_protected", { @@ -173,9 +173,9 @@ test_that("pdf_password_protected", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("LayoutParser", "document image analysis", "deep learning")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("LayoutParser", "document image analysis", "deep learning")) }) test_that("pdf_right_to_left", { @@ -187,9 +187,9 @@ test_that("pdf_right_to_left", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_simple_text", { @@ -201,9 +201,9 @@ test_that("pdf_simple_text", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern", "Mallori")) }) test_that("pdf_tables_large", { @@ -215,8 +215,8 @@ test_that("pdf_tables_large", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 500L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 500L) }) test_that("pdf_tables_medium", { @@ -228,8 +228,8 @@ test_that("pdf_tables_medium", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 100L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 100L) }) test_that("pdf_tables_small", { @@ -242,10 +242,10 @@ test_that("pdf_tables_small", { notes = "PDF table extraction requires OCR feature. ONNX Runtime model loading unstable on ARM Linux.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_all(result, c("Table 1", "Selected Numbers", "Celsius", "Fahrenheit", "Water Freezing Point", "Water Boiling Point")) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_all(result, c("Table 1", "Selected Numbers", "Celsius", "Fahrenheit", "Water Freezing Point", "Water Boiling Point")) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("pdf_technical_stat_learning", { @@ -257,9 +257,9 @@ test_that("pdf_technical_stat_learning", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10000L) - assert_content_contains_any(result, c("statistical", "regression", "learning")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10000L) + assert_content_contains_any(result, c("statistical", "regression", "learning")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) # nolint end diff --git a/e2e/r/tests/testthat/test-plugin-apis.R b/e2e/r/tests/testthat/test-plugin-apis.R index fd35b85d1c3..e0e2caffc53 100644 --- a/e2e/r/tests/testthat/test-plugin-apis.R +++ b/e2e/r/tests/testthat/test-plugin-apis.R @@ -7,11 +7,10 @@ library(kreuzberg) test_that("Configuration API tests", { # Discover configuration from current or parent directories - withr::with_tempdir({ - tmpdir <- getwd() + withr::with_tempdir({ tmpdir <- getwd() config_path <- file.path(tmpdir, "kreuzberg.toml") writeLines(c( - "[chunking]", "max_chars = 50" + "[chunking]", "max_chars = 50" ), config_path) subdir <- file.path(tmpdir, "subdir") @@ -25,11 +24,10 @@ test_that("Configuration API tests", { }) # Load configuration from a TOML file - withr::with_tempdir({ - tmpdir <- getwd() + withr::with_tempdir({ tmpdir <- getwd() config_path <- file.path(tmpdir, "test_config.toml") writeLines(c( - "[chunking]", "max_chars = 100", "max_overlap = 20", "", "[language_detection]", "enabled = false" + "[chunking]", "max_chars = 100", "max_overlap = 20", "", "[language_detection]", "enabled = false" ), config_path) config <- from_file(config_path) @@ -39,6 +37,7 @@ test_that("Configuration API tests", { expect_false(is.null(config$language_detection)) expect_equal(config$language_detection$enabled, FALSE) }) + }) test_that("Document Extractor Management API tests", { @@ -53,6 +52,7 @@ test_that("Document Extractor Management API tests", { # Unregister nonexistent document extractor gracefully expect_no_error(unregister_document_extractor("nonexistent-extractor-xyz")) + }) test_that("Mime Utilities API tests", { @@ -62,8 +62,7 @@ test_that("Mime Utilities API tests", { expect_true(grepl("pdf", tolower(result), fixed = TRUE)) # Detect MIME type from file path - withr::with_tempdir({ - tmpdir <- getwd() + withr::with_tempdir({ tmpdir <- getwd() test_file <- file.path(tmpdir, "test.txt") writeLines("Hello, world!", test_file) @@ -75,6 +74,7 @@ test_that("Mime Utilities API tests", { result <- get_extensions_for_mime("application/pdf") expect_true(is.character(result)) expect_true("pdf" %in% result) + }) test_that("Ocr Backend Management API tests", { @@ -89,6 +89,7 @@ test_that("Ocr Backend Management API tests", { # Unregister nonexistent OCR backend gracefully expect_no_error(unregister_ocr_backend("nonexistent-backend-xyz")) + }) test_that("Post Processor Management API tests", { @@ -98,6 +99,7 @@ test_that("Post Processor Management API tests", { # List all registered post-processors result <- list_post_processors() expect_true(is.character(result) || is.list(result)) + }) test_that("Validator Management API tests", { @@ -109,6 +111,7 @@ test_that("Validator Management API tests", { # List all registered validators result <- list_validators() expect_true(is.character(result) || is.list(result)) + }) # nolint end diff --git a/e2e/r/tests/testthat/test-smoke.R b/e2e/r/tests/testthat/test-smoke.R index 14f85b1f308..ad812a20add 100644 --- a/e2e/r/tests/testthat/test-smoke.R +++ b/e2e/r/tests/testthat/test-smoke.R @@ -13,8 +13,8 @@ test_that("smoke_cache_namespace", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/plain")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("text/plain")) + assert_min_content_length(result, 5L) }) test_that("smoke_docx_basic", { @@ -26,9 +26,9 @@ test_that("smoke_docx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) - assert_content_contains_any(result, c("Lorem", "ipsum", "document", "text")) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) + assert_content_contains_any(result, c("Lorem", "ipsum", "document", "text")) }) test_that("smoke_html_basic", { @@ -40,9 +40,9 @@ test_that("smoke_html_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/html")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("Sample Data Table", "Laptop", "Electronics", "Product")) + assert_expected_mime(result, c("text/html")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("Sample Data Table", "Laptop", "Electronics", "Product")) }) test_that("smoke_image_png", { @@ -54,8 +54,8 @@ test_that("smoke_image_png", { notes = "Image extraction requires image processing dependencies", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_metadata_expectation(result, "format", "PNG") + assert_expected_mime(result, c("image/png")) + assert_metadata_expectation(result, "format", "PNG") }) test_that("smoke_json_basic", { @@ -67,8 +67,8 @@ test_that("smoke_json_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/json")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/json")) + assert_min_content_length(result, 5L) }) test_that("smoke_pdf_basic", { @@ -80,9 +80,9 @@ test_that("smoke_pdf_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern")) }) test_that("smoke_txt_basic", { @@ -94,8 +94,8 @@ test_that("smoke_txt_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/plain")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("text/plain")) + assert_min_content_length(result, 5L) }) test_that("smoke_xlsx_basic", { @@ -107,11 +107,11 @@ test_that("smoke_xlsx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_min_content_length(result, 100L) - assert_content_contains_all(result, c("Team", "Location", "Stanley Cups", "Blues", "Flyers", "Maple Leafs", "STL", "PHI", "TOR")) - assert_table_count(result, minimum = 1L, maximum = NULL) - assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) - assert_metadata_expectation(result, "sheet_names", list(contains = c("Stanley Cups"))) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_min_content_length(result, 100L) + assert_content_contains_all(result, c("Team", "Location", "Stanley Cups", "Blues", "Flyers", "Maple Leafs", "STL", "PHI", "TOR")) + assert_table_count(result, minimum = 1L, maximum = NULL) + assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) + assert_metadata_expectation(result, "sheet_names", list(contains = c("Stanley Cups"))) }) # nolint end diff --git a/e2e/r/tests/testthat/test-structured.R b/e2e/r/tests/testthat/test-structured.R index 98e4c6d1166..ffad8ff30bb 100644 --- a/e2e/r/tests/testthat/test-structured.R +++ b/e2e/r/tests/testthat/test-structured.R @@ -13,8 +13,8 @@ test_that("structured_csv_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/csv")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("text/csv")) + assert_min_content_length(result, 20L) }) test_that("structured_enw_basic", { @@ -26,7 +26,7 @@ test_that("structured_enw_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-endnote-refer", "application/x-endnote+xml", "text/plain")) + assert_expected_mime(result, c("application/x-endnote-refer", "application/x-endnote+xml", "text/plain")) }) test_that("structured_json_basic", { @@ -38,9 +38,9 @@ test_that("structured_json_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/json")) - assert_min_content_length(result, 20L) - assert_content_contains_any(result, c("Sample Document", "Test Author")) + assert_expected_mime(result, c("application/json")) + assert_min_content_length(result, 20L) + assert_content_contains_any(result, c("Sample Document", "Test Author")) }) test_that("structured_json_simple", { @@ -52,9 +52,9 @@ test_that("structured_json_simple", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/json")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("{", "name")) + assert_expected_mime(result, c("application/json")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("{", "name")) }) test_that("structured_nbib_basic", { @@ -66,8 +66,8 @@ test_that("structured_nbib_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/nbib", "application/x-pubmed", "text/plain")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/nbib", "application/x-pubmed", "text/plain")) + assert_content_not_empty(result) }) test_that("structured_ris_basic", { @@ -79,8 +79,8 @@ test_that("structured_ris_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-research-info-systems", "text/plain")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/x-research-info-systems", "text/plain")) + assert_content_not_empty(result) }) test_that("structured_toml_basic", { @@ -92,8 +92,8 @@ test_that("structured_toml_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/toml", "text/toml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/toml", "text/toml")) + assert_min_content_length(result, 10L) }) test_that("structured_tsv_basic", { @@ -105,8 +105,8 @@ test_that("structured_tsv_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/tab-separated-values", "text/plain")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("text/tab-separated-values", "text/plain")) + assert_min_content_length(result, 10L) }) test_that("structured_yaml_basic", { @@ -118,8 +118,8 @@ test_that("structured_yaml_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml")) + assert_min_content_length(result, 10L) }) test_that("structured_yaml_simple", { @@ -131,7 +131,7 @@ test_that("structured_yaml_simple", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-yaml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-yaml")) + assert_min_content_length(result, 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-token_reduction.R b/e2e/r/tests/testthat/test-token_reduction.R index 282a9220055..22e91918b62 100644 --- a/e2e/r/tests/testthat/test-token_reduction.R +++ b/e2e/r/tests/testthat/test-token_reduction.R @@ -13,10 +13,10 @@ test_that("token_reduction_aggressive", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) - assert_max_content_length(result, 150L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) + assert_max_content_length(result, 150L) + assert_content_not_empty(result) }) test_that("token_reduction_basic", { @@ -28,10 +28,10 @@ test_that("token_reduction_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) - assert_max_content_length(result, 200L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) + assert_max_content_length(result, 200L) + assert_content_not_empty(result) }) test_that("token_reduction_light", { @@ -43,9 +43,9 @@ test_that("token_reduction_light", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_not_empty(result) }) test_that("token_reduction_with_chunking", { @@ -57,10 +57,10 @@ test_that("token_reduction_with_chunking", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) - assert_max_content_length(result, 200L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) + assert_max_content_length(result, 200L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE) + assert_content_not_empty(result) }) # nolint end diff --git a/e2e/r/tests/testthat/test-xml.R b/e2e/r/tests/testthat/test-xml.R index c830ad2e83e..1ed8a2ab184 100644 --- a/e2e/r/tests/testthat/test-xml.R +++ b/e2e/r/tests/testthat/test-xml.R @@ -13,7 +13,7 @@ test_that("xml_plant_catalog", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/xml")) - assert_min_content_length(result, 100L) + assert_expected_mime(result, c("application/xml")) + assert_min_content_length(result, 100L) }) # nolint end diff --git a/e2e/ruby/spec/archive_spec.rb b/e2e/ruby/spec/archive_spec.rb index 0c6489dc4bf..11853a9e82c 100644 --- a/e2e/ruby/spec/archive_spec.rb +++ b/e2e/ruby/spec/archive_spec.rb @@ -5,6 +5,7 @@ # # Tests for archive fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'archive fixtures' do @@ -76,3 +77,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/email_spec.rb b/e2e/ruby/spec/email_spec.rb index 297032f5175..1d77894c648 100644 --- a/e2e/ruby/spec/email_spec.rb +++ b/e2e/ruby/spec/email_spec.rb @@ -5,6 +5,7 @@ # # Tests for email fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'email fixtures' do @@ -110,3 +111,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/helpers.rb b/e2e/ruby/spec/helpers.rb index c72edf57123..29c6baf629b 100644 --- a/e2e/ruby/spec/helpers.rb +++ b/e2e/ruby/spec/helpers.rb @@ -240,7 +240,7 @@ def self.assert_metadata_expectation(result, path, expectation) end end - def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, content_starts_with_heading: nil) + def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, each_has_chunk_type: nil, content_starts_with_heading: nil) chunks = Array(result.chunks) expect(chunks.length).to be >= min_count if min_count expect(chunks.length).to be <= max_count if max_count @@ -251,6 +251,12 @@ def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: elsif each_has_heading_context == false chunks.each { |chunk| expect(chunk.metadata&.heading_context).to be_nil } end + if each_has_chunk_type == true + chunks.each { |chunk| + expect(chunk.chunk_type).not_to be_nil + expect(chunk.chunk_type).not_to eq("unknown") + } + end return unless content_starts_with_heading == true chunks.each do |chunk| diff --git a/e2e/ruby/spec/image_spec.rb b/e2e/ruby/spec/image_spec.rb index c9b628cb2b7..c53a0c49670 100644 --- a/e2e/ruby/spec/image_spec.rb +++ b/e2e/ruby/spec/image_spec.rb @@ -5,6 +5,7 @@ # # Tests for image fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'image fixtures' do @@ -186,3 +187,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/keywords_spec.rb b/e2e/ruby/spec/keywords_spec.rb index 195e45085fb..6386e08ce6d 100644 --- a/e2e/ruby/spec/keywords_spec.rb +++ b/e2e/ruby/spec/keywords_spec.rb @@ -5,6 +5,7 @@ # # Tests for keywords fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'keywords fixtures' do @@ -46,3 +47,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/plugin_apis_spec.rb b/e2e/ruby/spec/plugin_apis_spec.rb index e2a10a87960..92759aeb3ef 100644 --- a/e2e/ruby/spec/plugin_apis_spec.rb +++ b/e2e/ruby/spec/plugin_apis_spec.rb @@ -7,6 +7,8 @@ # Generated from plugin API fixtures. # To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang ruby +# rubocop:disable Metrics/BlockLength + require 'spec_helper' require 'tmpdir' require 'fileutils' @@ -14,131 +16,140 @@ RSpec.describe 'Plugin APIs' do describe 'Configuration' do it 'Discover configuration from current or parent directories' do - Dir.mktmpdir do |tmpdir| - config_path = File.join(tmpdir, 'kreuzberg.toml') - File.write(config_path, <<~TOML) - [chunking] - max_chars = 50 - TOML - - subdir = File.join(tmpdir, 'subdir') - FileUtils.mkdir_p(subdir) - - FileUtils.cd(subdir) do - config = Kreuzberg::Config::Extraction.discover - - expect(config.chunking).not_to be_nil - expect(config.chunking.max_chars).to eq(50) - end - end - end - - it 'Load configuration from a TOML file' do - Dir.mktmpdir do |tmpdir| - config_path = File.join(tmpdir, 'test_config.toml') - File.write(config_path, <<~TOML) - [chunking] - max_chars = 100 - max_overlap = 20 + Dir.mktmpdir do |tmpdir| + config_path = File.join(tmpdir, 'kreuzberg.toml') + File.write(config_path, <<~TOML) + [chunking] + max_chars = 50 + TOML - [language_detection] - enabled = false - TOML + subdir = File.join(tmpdir, 'subdir') + FileUtils.mkdir_p(subdir) - config = Kreuzberg::Config::Extraction.from_file(config_path) + FileUtils.cd(subdir) do + config = Kreuzberg::Config::Extraction.discover expect(config.chunking).not_to be_nil - expect(config.chunking.max_chars).to eq(100) - expect(config.chunking.max_overlap).to eq(20) - expect(config.language_detection).not_to be_nil - expect(config.language_detection.enabled).to be(false) + expect(config.chunking.max_chars).to eq(50) end end + end + + it 'Load configuration from a TOML file' do + Dir.mktmpdir do |tmpdir| + config_path = File.join(tmpdir, 'test_config.toml') + File.write(config_path, <<~TOML) + [chunking] + max_chars = 100 + max_overlap = 20 + + [language_detection] + enabled = false + TOML + + config = Kreuzberg::Config::Extraction.from_file(config_path) + + expect(config.chunking).not_to be_nil + expect(config.chunking.max_chars).to eq(100) + expect(config.chunking.max_overlap).to eq(20) + expect(config.language_detection).not_to be_nil + expect(config.language_detection.enabled).to eq(false) + end + end + end describe 'Document Extractor Management' do it 'Clear all document extractors and verify list is empty' do - Kreuzberg.clear_document_extractors - result = Kreuzberg.list_document_extractors - expect(result).to be_empty + Kreuzberg.clear_document_extractors + result = Kreuzberg.list_document_extractors + expect(result).to be_empty end it 'List all registered document extractors' do - result = Kreuzberg.list_document_extractors - expect(result).to be_an(Array) - expect(result).to all(be_a(String)) + result = Kreuzberg.list_document_extractors + expect(result).to be_an(Array) + expect(result).to all(be_a(String)) end it 'Unregister nonexistent document extractor gracefully' do - expect { Kreuzberg.unregister_document_extractor('nonexistent-extractor-xyz') }.not_to raise_error + expect { Kreuzberg.unregister_document_extractor('nonexistent-extractor-xyz') }.not_to raise_error end + end describe 'Mime Utilities' do it 'Detect MIME type from file bytes' do - test_bytes = (+'%PDF-1.4\\n').force_encoding('ASCII-8BIT') - result = Kreuzberg.detect_mime_type(test_bytes) - expect(result.downcase).to include('pdf') + test_bytes = '%PDF-1.4\\n'.dup.force_encoding('ASCII-8BIT') + result = Kreuzberg.detect_mime_type(test_bytes) + expect(result.downcase).to include('pdf') end it 'Detect MIME type from file path' do - Dir.mktmpdir do |tmpdir| - test_file = File.join(tmpdir, 'test.txt') - File.write(test_file, 'Hello, world!') + Dir.mktmpdir do |tmpdir| + test_file = File.join(tmpdir, 'test.txt') + File.write(test_file, 'Hello, world!') - result = Kreuzberg.detect_mime_type_from_path(test_file) - expect(result.downcase).to include('text') - end + result = Kreuzberg.detect_mime_type_from_path(test_file) + expect(result.downcase).to include('text') + end end it 'Get file extensions for a MIME type' do - result = Kreuzberg.get_extensions_for_mime('application/pdf') - expect(result).to be_an(Array) - expect(result).to include('pdf') + result = Kreuzberg.get_extensions_for_mime('application/pdf') + expect(result).to be_an(Array) + expect(result).to include('pdf') end + end describe 'Ocr Backend Management' do it 'Clear all OCR backends and verify list is empty' do - Kreuzberg.clear_ocr_backends - result = Kreuzberg.list_ocr_backends - expect(result).to be_empty + Kreuzberg.clear_ocr_backends + result = Kreuzberg.list_ocr_backends + expect(result).to be_empty end it 'List all registered OCR backends' do - result = Kreuzberg.list_ocr_backends - expect(result).to be_an(Array) - expect(result).to all(be_a(String)) + result = Kreuzberg.list_ocr_backends + expect(result).to be_an(Array) + expect(result).to all(be_a(String)) end it 'Unregister nonexistent OCR backend gracefully' do - expect { Kreuzberg.unregister_ocr_backend('nonexistent-backend-xyz') }.not_to raise_error + expect { Kreuzberg.unregister_ocr_backend('nonexistent-backend-xyz') }.not_to raise_error end + end describe 'Post Processor Management' do it 'Clear all post-processors and verify list is empty' do - Kreuzberg.clear_post_processors + Kreuzberg.clear_post_processors end it 'List all registered post-processors' do - result = Kreuzberg.list_post_processors - expect(result).to be_an(Array) - expect(result).to all(be_a(String)) + result = Kreuzberg.list_post_processors + expect(result).to be_an(Array) + expect(result).to all(be_a(String)) end + end describe 'Validator Management' do it 'Clear all validators and verify list is empty' do - Kreuzberg.clear_validators - result = Kreuzberg.list_validators - expect(result).to be_empty + Kreuzberg.clear_validators + result = Kreuzberg.list_validators + expect(result).to be_empty end it 'List all registered validators' do - result = Kreuzberg.list_validators - expect(result).to be_an(Array) - expect(result).to all(be_a(String)) + result = Kreuzberg.list_validators + expect(result).to be_an(Array) + expect(result).to all(be_a(String)) end + end + end + +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/render_spec.rb b/e2e/ruby/spec/render_spec.rb index 34b637a9d7a..ce570d0731d 100644 --- a/e2e/ruby/spec/render_spec.rb +++ b/e2e/ruby/spec/render_spec.rb @@ -3,6 +3,8 @@ # Code generated by kreuzberg-e2e-generator. DO NOT EDIT. # To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang ruby +# rubocop:disable Layout/LineLength + require 'spec_helper' require 'kreuzberg' @@ -34,3 +36,5 @@ E2ERuby.assert_min_byte_length(png_data, 100) end end + +# rubocop:enable Layout/LineLength diff --git a/e2e/ruby/spec/structured_spec.rb b/e2e/ruby/spec/structured_spec.rb index 90e2bd4421e..7b3e91c5200 100644 --- a/e2e/ruby/spec/structured_spec.rb +++ b/e2e/ruby/spec/structured_spec.rb @@ -5,6 +5,7 @@ # # Tests for structured fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'structured fixtures' do @@ -179,3 +180,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/token_reduction_spec.rb b/e2e/ruby/spec/token_reduction_spec.rb index 20c8b71054c..72cc2e85e00 100644 --- a/e2e/ruby/spec/token_reduction_spec.rb +++ b/e2e/ruby/spec/token_reduction_spec.rb @@ -5,6 +5,7 @@ # # Tests for token_reduction fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'token_reduction fixtures' do @@ -84,3 +85,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/xml_spec.rb b/e2e/ruby/spec/xml_spec.rb index 357bb622a5c..b877836cd40 100644 --- a/e2e/ruby/spec/xml_spec.rb +++ b/e2e/ruby/spec/xml_spec.rb @@ -5,6 +5,7 @@ # # Tests for xml fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'xml fixtures' do @@ -25,3 +26,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/rust/src/lib.rs b/e2e/rust/src/lib.rs index 3c0767fb5ea..cbb3e4a243e 100644 --- a/e2e/rust/src/lib.rs +++ b/e2e/rust/src/lib.rs @@ -233,6 +233,7 @@ pub mod assertions { each_has_content: Option, each_has_embedding: Option, each_has_heading_context: Option, + each_has_chunk_type: Option, content_starts_with_heading: Option, ) { let chunks = result.chunks.as_ref().expect("Expected chunks in result"); @@ -279,6 +280,15 @@ pub mod assertions { } } + if each_has_chunk_type == Some(true) { + for (i, chunk) in chunks.iter().enumerate() { + assert!( + chunk.chunk_type != kreuzberg::types::ChunkType::Unknown, + "Expected chunk {i} to have a specific chunk_type, got Unknown" + ); + } + } + if content_starts_with_heading == Some(true) { for (i, chunk) in chunks.iter().enumerate() { if chunk.metadata.heading_context.is_none() { diff --git a/e2e/rust/tests/e2e.rs b/e2e/rust/tests/e2e.rs new file mode 100644 index 00000000000..f3e51325b79 --- /dev/null +++ b/e2e/rust/tests/e2e.rs @@ -0,0 +1,3 @@ +// Code generated by kreuzberg-e2e-generator. DO NOT EDIT. + +mod fixtures; diff --git a/e2e/rust/tests/archive_test.rs b/e2e/rust/tests/fixtures/archive_test.rs similarity index 100% rename from e2e/rust/tests/archive_test.rs rename to e2e/rust/tests/fixtures/archive_test.rs diff --git a/e2e/rust/tests/contract_test.rs b/e2e/rust/tests/fixtures/contract_test.rs similarity index 99% rename from e2e/rust/tests/contract_test.rs rename to e2e/rust/tests/fixtures/contract_test.rs index 7a2492a0b4d..5c1e64f31e2 100644 --- a/e2e/rust/tests/contract_test.rs +++ b/e2e/rust/tests/fixtures/contract_test.rs @@ -402,7 +402,7 @@ fn test_config_chunking() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None, None); } #[test] @@ -452,7 +452,7 @@ fn test_config_chunking_heading_context() { }; assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(true), None); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(true), None, None); } #[test] @@ -500,7 +500,7 @@ fn test_config_chunking_markdown() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None, None); } #[test] @@ -550,7 +550,7 @@ fn test_config_chunking_no_headings() { }; assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(false), None); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(false), None, None); } #[test] @@ -601,7 +601,7 @@ fn test_config_chunking_prepend_heading_context() { }; assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(true), Some(true)); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(true), None, Some(true)); } #[test] @@ -648,7 +648,7 @@ fn test_config_chunking_small() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, None, None, None); } #[test] @@ -681,7 +681,7 @@ fn test_config_chunking_text() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None, None); } #[test] @@ -734,7 +734,7 @@ fn test_config_chunking_tokenizer() { }; assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, None, None, None); } #[test] diff --git a/e2e/rust/tests/email_test.rs b/e2e/rust/tests/fixtures/email_test.rs similarity index 100% rename from e2e/rust/tests/email_test.rs rename to e2e/rust/tests/fixtures/email_test.rs diff --git a/e2e/rust/tests/embeddings_test.rs b/e2e/rust/tests/fixtures/embeddings_test.rs similarity index 97% rename from e2e/rust/tests/embeddings_test.rs rename to e2e/rust/tests/fixtures/embeddings_test.rs index 8eab943b887..6178418b428 100644 --- a/e2e/rust/tests/embeddings_test.rs +++ b/e2e/rust/tests/fixtures/embeddings_test.rs @@ -58,7 +58,7 @@ async fn test_embedding_async() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None, None); } #[test] @@ -115,7 +115,7 @@ fn test_embedding_balanced_preset() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None, None); } #[test] @@ -147,7 +147,7 @@ fn test_embedding_disabled() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), Some(false), None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), Some(false), None, None, None); } #[test] @@ -201,5 +201,5 @@ fn test_embedding_fast_preset() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None, None); } diff --git a/e2e/rust/tests/html_test.rs b/e2e/rust/tests/fixtures/html_test.rs similarity index 100% rename from e2e/rust/tests/html_test.rs rename to e2e/rust/tests/fixtures/html_test.rs diff --git a/e2e/rust/tests/image_test.rs b/e2e/rust/tests/fixtures/image_test.rs similarity index 100% rename from e2e/rust/tests/image_test.rs rename to e2e/rust/tests/fixtures/image_test.rs diff --git a/e2e/rust/tests/keywords_test.rs b/e2e/rust/tests/fixtures/keywords_test.rs similarity index 100% rename from e2e/rust/tests/keywords_test.rs rename to e2e/rust/tests/fixtures/keywords_test.rs diff --git a/e2e/rust/tests/fixtures/mod.rs b/e2e/rust/tests/fixtures/mod.rs new file mode 100644 index 00000000000..7341521d833 --- /dev/null +++ b/e2e/rust/tests/fixtures/mod.rs @@ -0,0 +1,18 @@ +// Code generated by kreuzberg-e2e-generator. DO NOT EDIT. + +pub mod archive_test; +pub mod contract_test; +pub mod email_test; +pub mod embeddings_test; +pub mod html_test; +pub mod image_test; +pub mod keywords_test; +pub mod ocr_test; +pub mod office_test; +pub mod pdf_test; +pub mod plugin_apis_test; +pub mod render_test; +pub mod smoke_test; +pub mod structured_test; +pub mod token_reduction_test; +pub mod xml_test; diff --git a/e2e/rust/tests/ocr_test.rs b/e2e/rust/tests/fixtures/ocr_test.rs similarity index 100% rename from e2e/rust/tests/ocr_test.rs rename to e2e/rust/tests/fixtures/ocr_test.rs diff --git a/e2e/rust/tests/office_test.rs b/e2e/rust/tests/fixtures/office_test.rs similarity index 100% rename from e2e/rust/tests/office_test.rs rename to e2e/rust/tests/fixtures/office_test.rs diff --git a/e2e/rust/tests/pdf_test.rs b/e2e/rust/tests/fixtures/pdf_test.rs similarity index 100% rename from e2e/rust/tests/pdf_test.rs rename to e2e/rust/tests/fixtures/pdf_test.rs diff --git a/e2e/rust/tests/plugin_apis_test.rs b/e2e/rust/tests/fixtures/plugin_apis_test.rs similarity index 100% rename from e2e/rust/tests/plugin_apis_test.rs rename to e2e/rust/tests/fixtures/plugin_apis_test.rs diff --git a/e2e/rust/tests/render_test.rs b/e2e/rust/tests/fixtures/render_test.rs similarity index 100% rename from e2e/rust/tests/render_test.rs rename to e2e/rust/tests/fixtures/render_test.rs diff --git a/e2e/rust/tests/smoke_test.rs b/e2e/rust/tests/fixtures/smoke_test.rs similarity index 100% rename from e2e/rust/tests/smoke_test.rs rename to e2e/rust/tests/fixtures/smoke_test.rs diff --git a/e2e/rust/tests/structured_test.rs b/e2e/rust/tests/fixtures/structured_test.rs similarity index 100% rename from e2e/rust/tests/structured_test.rs rename to e2e/rust/tests/fixtures/structured_test.rs diff --git a/e2e/rust/tests/token_reduction_test.rs b/e2e/rust/tests/fixtures/token_reduction_test.rs similarity index 99% rename from e2e/rust/tests/token_reduction_test.rs rename to e2e/rust/tests/fixtures/token_reduction_test.rs index 19b3ddff7d3..8284c84ede2 100644 --- a/e2e/rust/tests/token_reduction_test.rs +++ b/e2e/rust/tests/fixtures/token_reduction_test.rs @@ -134,6 +134,6 @@ fn test_token_reduction_with_chunking() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 5); assertions::assert_max_content_length(&result, 200); - assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None, None); assertions::assert_content_not_empty(&result); } diff --git a/e2e/rust/tests/xml_test.rs b/e2e/rust/tests/fixtures/xml_test.rs similarity index 100% rename from e2e/rust/tests/xml_test.rs rename to e2e/rust/tests/fixtures/xml_test.rs diff --git a/e2e/typescript/tests/archive.spec.ts b/e2e/typescript/tests/archive.spec.ts index 722d3d2c04a..e8aee483ea5 100644 --- a/e2e/typescript/tests/archive.spec.ts +++ b/e2e/typescript/tests/archive.spec.ts @@ -4,10 +4,10 @@ // Tests for archive fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; diff --git a/e2e/typescript/tests/contract.spec.ts b/e2e/typescript/tests/contract.spec.ts index d710be46a31..dfe7166d856 100644 --- a/e2e/typescript/tests/contract.spec.ts +++ b/e2e/typescript/tests/contract.spec.ts @@ -453,7 +453,7 @@ describe("contract fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, null, null, null); + chunkAssertions.assertChunks(result, 1, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -480,7 +480,7 @@ describe("contract fixtures", () => { return; } assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, true, null); + chunkAssertions.assertChunks(result, 2, null, true, null, true, null, null); }, TEST_TIMEOUT_MS, ); @@ -508,7 +508,7 @@ describe("contract fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, null, null, null); + chunkAssertions.assertChunks(result, 1, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -535,7 +535,7 @@ describe("contract fixtures", () => { return; } assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, false, null); + chunkAssertions.assertChunks(result, 2, null, true, null, false, null, null); }, TEST_TIMEOUT_MS, ); @@ -564,7 +564,7 @@ describe("contract fixtures", () => { return; } assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, true, true); + chunkAssertions.assertChunks(result, 2, null, true, null, true, null, true); }, TEST_TIMEOUT_MS, ); @@ -592,7 +592,7 @@ describe("contract fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, null, null); + chunkAssertions.assertChunks(result, 2, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -620,7 +620,7 @@ describe("contract fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, null, null, null); + chunkAssertions.assertChunks(result, 1, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -657,7 +657,7 @@ describe("contract fixtures", () => { return; } assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, null, null); + chunkAssertions.assertChunks(result, 2, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); diff --git a/e2e/typescript/tests/email.spec.ts b/e2e/typescript/tests/email.spec.ts index df1bbe0d3a6..0858980f436 100644 --- a/e2e/typescript/tests/email.spec.ts +++ b/e2e/typescript/tests/email.spec.ts @@ -4,10 +4,10 @@ // Tests for email fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; diff --git a/e2e/typescript/tests/embeddings.spec.ts b/e2e/typescript/tests/embeddings.spec.ts index e87cc6e4bab..0b2177cfb8d 100644 --- a/e2e/typescript/tests/embeddings.spec.ts +++ b/e2e/typescript/tests/embeddings.spec.ts @@ -46,7 +46,7 @@ describe("embeddings fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, true, null, null); + chunkAssertions.assertChunks(result, 1, null, true, true, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -87,7 +87,7 @@ describe("embeddings fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, true, null, null); + chunkAssertions.assertChunks(result, 1, null, true, true, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -115,7 +115,7 @@ describe("embeddings fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, false, null, null); + chunkAssertions.assertChunks(result, 1, null, true, false, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -154,7 +154,7 @@ describe("embeddings fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, true, null, null); + chunkAssertions.assertChunks(result, 1, null, true, true, null, null, null); }, TEST_TIMEOUT_MS, ); diff --git a/e2e/typescript/tests/helpers.ts b/e2e/typescript/tests/helpers.ts index 0e86b0c191e..05bb3585196 100644 --- a/e2e/typescript/tests/helpers.ts +++ b/e2e/typescript/tests/helpers.ts @@ -415,9 +415,8 @@ export function buildConfig(raw: unknown): ExtractionConfig { if (isPlainRecord(source.email)) { const email = source.email as PlainRecord; - target.email = { - ...(typeof email.msg_fallback_codepage === "number" ? { msgFallbackCodepage: email.msg_fallback_codepage } : {}), - }; + target.email = + typeof email.msg_fallback_codepage === "number" ? { msgFallbackCodepage: email.msg_fallback_codepage } : {}; } if (isPlainRecord(source.tree_sitter)) { @@ -783,6 +782,7 @@ export const chunkAssertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -816,6 +816,13 @@ export const chunkAssertions = { expect(((chunk as PlainRecord).metadata as PlainRecord)?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const meta = (chunk as PlainRecord).metadata as PlainRecord | undefined; diff --git a/e2e/typescript/tests/html.spec.ts b/e2e/typescript/tests/html.spec.ts index a1c2eadb7ad..7f7b177f00f 100644 --- a/e2e/typescript/tests/html.spec.ts +++ b/e2e/typescript/tests/html.spec.ts @@ -4,10 +4,10 @@ // Tests for html fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; diff --git a/e2e/typescript/tests/image.spec.ts b/e2e/typescript/tests/image.spec.ts index 2a12439ae3c..d7ea275a8b5 100644 --- a/e2e/typescript/tests/image.spec.ts +++ b/e2e/typescript/tests/image.spec.ts @@ -4,10 +4,10 @@ // Tests for image fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; diff --git a/e2e/typescript/tests/keywords.spec.ts b/e2e/typescript/tests/keywords.spec.ts index 4a1d37cd8b3..6797169a9fc 100644 --- a/e2e/typescript/tests/keywords.spec.ts +++ b/e2e/typescript/tests/keywords.spec.ts @@ -4,10 +4,10 @@ // Tests for keywords fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; diff --git a/e2e/typescript/tests/office.spec.ts b/e2e/typescript/tests/office.spec.ts index 9d69785faa0..a15c3d307ac 100644 --- a/e2e/typescript/tests/office.spec.ts +++ b/e2e/typescript/tests/office.spec.ts @@ -4,10 +4,10 @@ // Tests for office fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; diff --git a/e2e/typescript/tests/plugin-apis.spec.ts b/e2e/typescript/tests/plugin-apis.spec.ts index efd53e0f245..86552eb9c7f 100644 --- a/e2e/typescript/tests/plugin-apis.spec.ts +++ b/e2e/typescript/tests/plugin-apis.spec.ts @@ -9,8 +9,8 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import * as kreuzberg from "@kreuzberg/node"; import { describe, expect, it } from "vitest"; +import * as kreuzberg from "@kreuzberg/node"; describe("Configuration", () => { it("Discover configuration from current or parent directories", () => { diff --git a/e2e/typescript/tests/render.spec.ts b/e2e/typescript/tests/render.spec.ts index 0bf23800dc2..4d8f42a3731 100644 --- a/e2e/typescript/tests/render.spec.ts +++ b/e2e/typescript/tests/render.spec.ts @@ -3,9 +3,9 @@ // Tests for render fixtures. +import { describe, it, expect } from "vitest"; import { existsSync } from "node:fs"; -import { PdfPageIterator, renderPdfPageSync } from "@kreuzberg/node"; -import { describe, expect, it } from "vitest"; +import { renderPdfPageSync, PdfPageIterator } from "@kreuzberg/node"; import { assertions, resolveDocument } from "./helpers.js"; describe("render", () => { diff --git a/e2e/typescript/tests/structured.spec.ts b/e2e/typescript/tests/structured.spec.ts index 1f7ff5d2156..e954274567f 100644 --- a/e2e/typescript/tests/structured.spec.ts +++ b/e2e/typescript/tests/structured.spec.ts @@ -4,10 +4,10 @@ // Tests for structured fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; diff --git a/e2e/typescript/tests/token-reduction.spec.ts b/e2e/typescript/tests/token-reduction.spec.ts index 826eb54f251..510d6caddd5 100644 --- a/e2e/typescript/tests/token-reduction.spec.ts +++ b/e2e/typescript/tests/token-reduction.spec.ts @@ -125,7 +125,7 @@ describe("token_reduction fixtures", () => { assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 5); assertions.assertMaxContentLength(result, 200); - chunkAssertions.assertChunks(result, 1, null, true, null, null, null); + chunkAssertions.assertChunks(result, 1, null, true, null, null, null, null); assertions.assertContentNotEmpty(result); }, TEST_TIMEOUT_MS, diff --git a/e2e/typescript/tests/xml.spec.ts b/e2e/typescript/tests/xml.spec.ts index 7b1fdb7debb..1a3191978a7 100644 --- a/e2e/typescript/tests/xml.spec.ts +++ b/e2e/typescript/tests/xml.spec.ts @@ -4,10 +4,10 @@ // Tests for xml fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; diff --git a/e2e/wasm-deno/archive.test.ts b/e2e/wasm-deno/archive.test.ts index e96e9bcdd75..eee27332263 100644 --- a/e2e/wasm-deno/archive.test.ts +++ b/e2e/wasm-deno/archive.test.ts @@ -3,7 +3,6 @@ // Tests for archive fixtures. Run with: deno test --allow-read -import type { ExtractionResult } from "./helpers.ts"; import { assertions, buildConfig, @@ -13,6 +12,7 @@ import { resolveDocument, shouldSkipFixture, } from "./helpers.ts"; +import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); diff --git a/e2e/wasm-deno/contract.test.ts b/e2e/wasm-deno/contract.test.ts index 53e93adfaf5..b69b1d29ec8 100644 --- a/e2e/wasm-deno/contract.test.ts +++ b/e2e/wasm-deno/contract.test.ts @@ -334,7 +334,7 @@ Deno.test("config_chunking", { permissions: { read: true, net: true } }, async ( } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); Deno.test("config_chunking_heading_context", { permissions: { read: true, net: true } }, async () => { @@ -354,7 +354,7 @@ Deno.test("config_chunking_heading_context", { permissions: { read: true, net: t return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, true, null); + assertions.assertChunks(result, 2, null, true, null, true, null, null); }); Deno.test("config_chunking_markdown", { permissions: { read: true, net: true } }, async () => { @@ -375,7 +375,7 @@ Deno.test("config_chunking_markdown", { permissions: { read: true, net: true } } } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); Deno.test("config_chunking_no_headings", { permissions: { read: true, net: true } }, async () => { @@ -395,7 +395,7 @@ Deno.test("config_chunking_no_headings", { permissions: { read: true, net: true return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, false, null); + assertions.assertChunks(result, 2, null, true, null, false, null, null); }); Deno.test("config_chunking_prepend_heading_context", { permissions: { read: true, net: true } }, async () => { @@ -417,7 +417,7 @@ Deno.test("config_chunking_prepend_heading_context", { permissions: { read: true return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, true, true); + assertions.assertChunks(result, 2, null, true, null, true, null, true); }); Deno.test("config_chunking_small", { permissions: { read: true, net: true } }, async () => { @@ -438,7 +438,7 @@ Deno.test("config_chunking_small", { permissions: { read: true, net: true } }, a } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, null, null); + assertions.assertChunks(result, 2, null, true, null, null, null, null); }); Deno.test("config_chunking_text", { permissions: { read: true, net: true } }, async () => { @@ -459,7 +459,7 @@ Deno.test("config_chunking_text", { permissions: { read: true, net: true } }, as } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); Deno.test("config_disable_ocr", { permissions: { read: true, net: true } }, async () => { diff --git a/e2e/wasm-deno/email.test.ts b/e2e/wasm-deno/email.test.ts index 1624161a568..b1c903543a9 100644 --- a/e2e/wasm-deno/email.test.ts +++ b/e2e/wasm-deno/email.test.ts @@ -3,7 +3,6 @@ // Tests for email fixtures. Run with: deno test --allow-read -import type { ExtractionResult } from "./helpers.ts"; import { assertions, buildConfig, @@ -13,6 +12,7 @@ import { resolveDocument, shouldSkipFixture, } from "./helpers.ts"; +import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); diff --git a/e2e/wasm-deno/embeddings.test.ts b/e2e/wasm-deno/embeddings.test.ts index 8ccc4b19e2e..abf97a996d4 100644 --- a/e2e/wasm-deno/embeddings.test.ts +++ b/e2e/wasm-deno/embeddings.test.ts @@ -3,7 +3,6 @@ // Tests for embeddings fixtures. Run with: deno test --allow-read -import type { ExtractionResult } from "./helpers.ts"; import { assertions, buildConfig, @@ -13,6 +12,7 @@ import { resolveDocument, shouldSkipFixture, } from "./helpers.ts"; +import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); @@ -36,5 +36,5 @@ Deno.test("embedding_disabled", { permissions: { read: true, net: true } }, asyn } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, false, null, null); + assertions.assertChunks(result, 1, null, true, false, null, null, null); }); diff --git a/e2e/wasm-deno/helpers.ts b/e2e/wasm-deno/helpers.ts index 16120b565a6..4525260becd 100644 --- a/e2e/wasm-deno/helpers.ts +++ b/e2e/wasm-deno/helpers.ts @@ -445,6 +445,7 @@ export const assertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -482,6 +483,13 @@ export const assertions = { ); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + assertExists(chunkType, "Chunk missing chunk_type"); + assertEquals(chunkType !== "unknown", true, "Chunk chunk_type should not be 'unknown'"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const meta = (chunk as PlainRecord).metadata as PlainRecord | undefined; diff --git a/e2e/wasm-deno/html.test.ts b/e2e/wasm-deno/html.test.ts index 6f4e35f761a..d633dc5b6ae 100644 --- a/e2e/wasm-deno/html.test.ts +++ b/e2e/wasm-deno/html.test.ts @@ -3,7 +3,6 @@ // Tests for html fixtures. Run with: deno test --allow-read -import type { ExtractionResult } from "./helpers.ts"; import { assertions, buildConfig, @@ -13,6 +12,7 @@ import { resolveDocument, shouldSkipFixture, } from "./helpers.ts"; +import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); diff --git a/e2e/wasm-deno/image.test.ts b/e2e/wasm-deno/image.test.ts index 2bc3e0ee61a..c7f94d2bbba 100644 --- a/e2e/wasm-deno/image.test.ts +++ b/e2e/wasm-deno/image.test.ts @@ -3,7 +3,6 @@ // Tests for image fixtures. Run with: deno test --allow-read -import type { ExtractionResult } from "./helpers.ts"; import { assertions, buildConfig, @@ -13,6 +12,7 @@ import { resolveDocument, shouldSkipFixture, } from "./helpers.ts"; +import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); diff --git a/e2e/wasm-deno/office.test.ts b/e2e/wasm-deno/office.test.ts index cd53e6eba10..c73c36f91d9 100644 --- a/e2e/wasm-deno/office.test.ts +++ b/e2e/wasm-deno/office.test.ts @@ -3,7 +3,6 @@ // Tests for office fixtures. Run with: deno test --allow-read -import type { ExtractionResult } from "./helpers.ts"; import { assertions, buildConfig, @@ -13,6 +12,7 @@ import { resolveDocument, shouldSkipFixture, } from "./helpers.ts"; +import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); diff --git a/e2e/wasm-deno/plugin-apis.test.ts b/e2e/wasm-deno/plugin-apis.test.ts index 7244dbfd890..6252a9a31fb 100644 --- a/e2e/wasm-deno/plugin-apis.test.ts +++ b/e2e/wasm-deno/plugin-apis.test.ts @@ -6,6 +6,7 @@ * To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang wasm-deno */ +import { assertEquals } from "@std/assert"; // @deno-types="../../crates/kreuzberg-wasm/dist/index.d.ts" import { clearOcrBackends, @@ -19,7 +20,6 @@ import { listValidators, unregisterOcrBackend, } from "npm:@kreuzberg/wasm@^4.0.0"; -import { assertEquals } from "@std/assert"; await initWasm(); diff --git a/e2e/wasm-deno/structured.test.ts b/e2e/wasm-deno/structured.test.ts index befae7fd0d3..d8d45b451fb 100644 --- a/e2e/wasm-deno/structured.test.ts +++ b/e2e/wasm-deno/structured.test.ts @@ -3,7 +3,6 @@ // Tests for structured fixtures. Run with: deno test --allow-read -import type { ExtractionResult } from "./helpers.ts"; import { assertions, buildConfig, @@ -13,6 +12,7 @@ import { resolveDocument, shouldSkipFixture, } from "./helpers.ts"; +import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); diff --git a/e2e/wasm-deno/token_reduction.test.ts b/e2e/wasm-deno/token_reduction.test.ts index 55901f8e37c..3de61fd4f13 100644 --- a/e2e/wasm-deno/token_reduction.test.ts +++ b/e2e/wasm-deno/token_reduction.test.ts @@ -102,6 +102,6 @@ Deno.test("token_reduction_with_chunking", { permissions: { read: true, net: tru assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 5); assertions.assertMaxContentLength(result, 200); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); assertions.assertContentNotEmpty(result); }); diff --git a/e2e/wasm-deno/xml.test.ts b/e2e/wasm-deno/xml.test.ts index 917a57ebc57..e38eb793059 100644 --- a/e2e/wasm-deno/xml.test.ts +++ b/e2e/wasm-deno/xml.test.ts @@ -3,7 +3,6 @@ // Tests for xml fixtures. Run with: deno test --allow-read -import type { ExtractionResult } from "./helpers.ts"; import { assertions, buildConfig, @@ -13,6 +12,7 @@ import { resolveDocument, shouldSkipFixture, } from "./helpers.ts"; +import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); diff --git a/e2e/wasm-workers/tests/archive.spec.ts b/e2e/wasm-workers/tests/archive.spec.ts index 92e7a40f454..9b96d9a8122 100644 --- a/e2e/wasm-workers/tests/archive.spec.ts +++ b/e2e/wasm-workers/tests/archive.spec.ts @@ -3,10 +3,10 @@ // Tests for archive fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("archive", () => { it("archive_gz_basic", async () => { diff --git a/e2e/wasm-workers/tests/contract.spec.ts b/e2e/wasm-workers/tests/contract.spec.ts index bab0f5c6481..dd7dd60a9fa 100644 --- a/e2e/wasm-workers/tests/contract.spec.ts +++ b/e2e/wasm-workers/tests/contract.spec.ts @@ -400,7 +400,7 @@ describe("contract", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); it("config_chunking_heading_context", async () => { @@ -424,7 +424,7 @@ describe("contract", () => { return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, true, null); + assertions.assertChunks(result, 2, null, true, null, true, null, null); }); it("config_chunking_markdown", async () => { @@ -449,7 +449,7 @@ describe("contract", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); it("config_chunking_no_headings", async () => { @@ -473,7 +473,7 @@ describe("contract", () => { return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, false, null); + assertions.assertChunks(result, 2, null, true, null, false, null, null); }); it("config_chunking_prepend_heading_context", async () => { @@ -499,7 +499,7 @@ describe("contract", () => { return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, true, true); + assertions.assertChunks(result, 2, null, true, null, true, null, true); }); it("config_chunking_small", async () => { @@ -524,7 +524,7 @@ describe("contract", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, null, null); + assertions.assertChunks(result, 2, null, true, null, null, null, null); }); it("config_chunking_text", async () => { @@ -549,7 +549,7 @@ describe("contract", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); it("config_disable_ocr", async () => { diff --git a/e2e/wasm-workers/tests/email.spec.ts b/e2e/wasm-workers/tests/email.spec.ts index 4d5d0e35ef0..91523f05e91 100644 --- a/e2e/wasm-workers/tests/email.spec.ts +++ b/e2e/wasm-workers/tests/email.spec.ts @@ -3,10 +3,10 @@ // Tests for email fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("email", () => { it("email_eml_html_body", async () => { diff --git a/e2e/wasm-workers/tests/embeddings.spec.ts b/e2e/wasm-workers/tests/embeddings.spec.ts index 885699a333b..776d11ba60f 100644 --- a/e2e/wasm-workers/tests/embeddings.spec.ts +++ b/e2e/wasm-workers/tests/embeddings.spec.ts @@ -3,10 +3,10 @@ // Tests for embeddings fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("embeddings", () => { it("embedding_disabled", async () => { @@ -31,6 +31,6 @@ describe("embeddings", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, false, null, null); + assertions.assertChunks(result, 1, null, true, false, null, null, null); }); }); diff --git a/e2e/wasm-workers/tests/helpers.ts b/e2e/wasm-workers/tests/helpers.ts index 6af8502e30e..2a07c055ccc 100644 --- a/e2e/wasm-workers/tests/helpers.ts +++ b/e2e/wasm-workers/tests/helpers.ts @@ -396,6 +396,7 @@ export const assertions = { eachHasContent: boolean | null, eachHasEmbedding: boolean | null, eachHasHeadingContext: boolean | null, + eachHasChunkType: boolean | null, contentStartsWithHeading: boolean | null, ): void { const chunks = Array.isArray(result.chunks) ? result.chunks : []; @@ -425,6 +426,13 @@ export const assertions = { expect(chunk.metadata?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as unknown as PlainRecord).chunkType ?? (chunk as unknown as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { if (chunk.metadata?.headingContext == null) continue; diff --git a/e2e/wasm-workers/tests/html.spec.ts b/e2e/wasm-workers/tests/html.spec.ts index 55463ea7374..d67201ded95 100644 --- a/e2e/wasm-workers/tests/html.spec.ts +++ b/e2e/wasm-workers/tests/html.spec.ts @@ -3,10 +3,10 @@ // Tests for html fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("html", () => { it("html_complex_layout", async () => { diff --git a/e2e/wasm-workers/tests/image.spec.ts b/e2e/wasm-workers/tests/image.spec.ts index ca2d46a73cc..31e14599f1a 100644 --- a/e2e/wasm-workers/tests/image.spec.ts +++ b/e2e/wasm-workers/tests/image.spec.ts @@ -3,10 +3,10 @@ // Tests for image fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("image", () => { it("image_bmp_basic", async () => { diff --git a/e2e/wasm-workers/tests/plugin-apis.spec.ts b/e2e/wasm-workers/tests/plugin-apis.spec.ts index 7236aa7fa44..97162bad852 100644 --- a/e2e/wasm-workers/tests/plugin-apis.spec.ts +++ b/e2e/wasm-workers/tests/plugin-apis.spec.ts @@ -6,6 +6,7 @@ * To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang wasm-workers */ +import { describe, it, expect } from "vitest"; import { clearOcrBackends, clearPostProcessors, @@ -15,7 +16,6 @@ import { listValidators, unregisterOcrBackend, } from "@kreuzberg/wasm"; -import { describe, expect, it } from "vitest"; describe("Configuration", () => { it.skip("Discover configuration from current or parent directories (not available in WASM)", () => {}); diff --git a/e2e/wasm-workers/tests/structured.spec.ts b/e2e/wasm-workers/tests/structured.spec.ts index 53935b15c95..2e529cbb3a4 100644 --- a/e2e/wasm-workers/tests/structured.spec.ts +++ b/e2e/wasm-workers/tests/structured.spec.ts @@ -3,10 +3,10 @@ // Tests for structured fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("structured", () => { it("structured_csv_basic", async () => { diff --git a/e2e/wasm-workers/tests/token_reduction.spec.ts b/e2e/wasm-workers/tests/token_reduction.spec.ts index f2d271fea9f..d8ae8136689 100644 --- a/e2e/wasm-workers/tests/token_reduction.spec.ts +++ b/e2e/wasm-workers/tests/token_reduction.spec.ts @@ -112,7 +112,7 @@ describe("token_reduction", () => { assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 5); assertions.assertMaxContentLength(result, 200); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); assertions.assertContentNotEmpty(result); }); }); diff --git a/e2e/wasm-workers/tests/xml.spec.ts b/e2e/wasm-workers/tests/xml.spec.ts index fd864cbb062..cdd7cb3de01 100644 --- a/e2e/wasm-workers/tests/xml.spec.ts +++ b/e2e/wasm-workers/tests/xml.spec.ts @@ -3,10 +3,10 @@ // Tests for xml fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("xml", () => { it("xml_plant_catalog", async () => { diff --git a/e2e/wasm-workers/vitest.config.ts b/e2e/wasm-workers/vitest.config.ts index b085d5ed386..ef4fdf8df08 100644 --- a/e2e/wasm-workers/vitest.config.ts +++ b/e2e/wasm-workers/vitest.config.ts @@ -1,5 +1,5 @@ -import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; import { defineConfig } from "vitest/config"; +import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; export default defineConfig({ plugins: [ diff --git a/fixtures/schema.json b/fixtures/schema.json index 5e23d2168f5..aeaabc3c196 100644 --- a/fixtures/schema.json +++ b/fixtures/schema.json @@ -181,6 +181,14 @@ "each_has_heading_context": { "type": "boolean", "description": "Assert each chunk has heading_context populated (markdown chunker only)." + }, + "each_has_chunk_type": { + "type": "boolean", + "description": "Assert each chunk has a specific (non-Unknown) chunk_type classification." + }, + "content_starts_with_heading": { + "type": "boolean", + "description": "Assert each chunk's content starts with a Markdown heading (#)." } }, "additionalProperties": false diff --git a/packages/csharp/Kreuzberg/Models.cs b/packages/csharp/Kreuzberg/Models.cs index 268b1215c80..a74a3d49322 100644 --- a/packages/csharp/Kreuzberg/Models.cs +++ b/packages/csharp/Kreuzberg/Models.cs @@ -794,6 +794,12 @@ public sealed class Chunk /// [JsonPropertyName("metadata")] public ChunkMetadata Metadata { get; set; } = new(); + + ///

+ /// Semantic type classification of this chunk. + /// + [JsonPropertyName("chunk_type")] + public string ChunkType { get; set; } = "unknown"; } /// diff --git a/packages/elixir/lib/kreuzberg/chunk.ex b/packages/elixir/lib/kreuzberg/chunk.ex index 994072c73dc..6fe33bcfd87 100644 --- a/packages/elixir/lib/kreuzberg/chunk.ex +++ b/packages/elixir/lib/kreuzberg/chunk.ex @@ -9,15 +9,20 @@ defmodule Kreuzberg.Chunk do * `:content` - The text content of this chunk * `:embedding` - Vector embedding (list of floats) for semantic search * `:metadata` - ChunkMetadata struct with position and token info + * `:chunk_type` - Semantic type classification of this chunk (e.g. "heading", "unknown") """ @type t :: %__MODULE__{ content: String.t(), embedding: list(float()) | nil, - metadata: Kreuzberg.ChunkMetadata.t() + metadata: Kreuzberg.ChunkMetadata.t(), + chunk_type: String.t() } - defstruct content: "", embedding: nil, metadata: %Kreuzberg.ChunkMetadata{} + defstruct content: "", + embedding: nil, + metadata: %Kreuzberg.ChunkMetadata{}, + chunk_type: "unknown" @doc """ Creates a new Chunk struct. @@ -25,14 +30,15 @@ defmodule Kreuzberg.Chunk do ## Parameters * `content` - The text content of the chunk - * `opts` - Optional keyword list with `:embedding` and `:metadata` + * `opts` - Optional keyword list with `:embedding`, `:metadata`, and `:chunk_type` """ @spec new(String.t(), keyword()) :: t() def new(content, opts \\ []) when is_binary(content) do %__MODULE__{ content: content, embedding: Keyword.get(opts, :embedding), - metadata: Keyword.get(opts, :metadata, %Kreuzberg.ChunkMetadata{}) + metadata: Keyword.get(opts, :metadata, %Kreuzberg.ChunkMetadata{}), + chunk_type: Keyword.get(opts, :chunk_type, "unknown") } end @@ -42,7 +48,7 @@ defmodule Kreuzberg.Chunk do ## Examples iex> Kreuzberg.Chunk.from_map(%{"content" => "chunk text", "embedding" => [0.1, 0.2]}) - %Kreuzberg.Chunk{content: "chunk text", embedding: [0.1, 0.2]} + %Kreuzberg.Chunk{content: "chunk text", embedding: [0.1, 0.2], metadata: %Kreuzberg.ChunkMetadata{}, chunk_type: "unknown"} """ @spec from_map(map()) :: t() def from_map(data) when is_map(data) do @@ -56,7 +62,8 @@ defmodule Kreuzberg.Chunk do %__MODULE__{ content: data["content"] || "", embedding: data["embedding"], - metadata: metadata + metadata: metadata, + chunk_type: data["chunk_type"] || "unknown" } end @@ -68,7 +75,8 @@ defmodule Kreuzberg.Chunk do %{ "content" => chunk.content, "embedding" => chunk.embedding, - "metadata" => Kreuzberg.ChunkMetadata.to_map(chunk.metadata) + "metadata" => Kreuzberg.ChunkMetadata.to_map(chunk.metadata), + "chunk_type" => chunk.chunk_type } end end diff --git a/packages/go/v4/types.go b/packages/go/v4/types.go index 18393f098d4..0841f90edb6 100644 --- a/packages/go/v4/types.go +++ b/packages/go/v4/types.go @@ -190,6 +190,7 @@ type Chunk struct { Content string `json:"content"` Embedding []float32 `json:"embedding,omitempty"` Metadata ChunkMetadata `json:"metadata"` + ChunkType string `json:"chunk_type,omitempty"` } // ChunkMetadata provides positional information for a chunk. diff --git a/packages/java/src/main/java/dev/kreuzberg/Chunk.java b/packages/java/src/main/java/dev/kreuzberg/Chunk.java index 13321b88279..f2340644d5f 100644 --- a/packages/java/src/main/java/dev/kreuzberg/Chunk.java +++ b/packages/java/src/main/java/dev/kreuzberg/Chunk.java @@ -16,10 +16,11 @@ public final class Chunk { private final String content; private final List embedding; private final ChunkMetadata metadata; + private final String chunkType; @JsonCreator public Chunk(@JsonProperty("content") String content, @JsonProperty("embedding") List embedding, - @JsonProperty("metadata") ChunkMetadata metadata) { + @JsonProperty("metadata") ChunkMetadata metadata, @JsonProperty("chunk_type") String chunkType) { this.content = Objects.requireNonNull(content, "content must not be null"); this.metadata = Objects.requireNonNull(metadata, "metadata must not be null"); if (embedding != null) { @@ -27,6 +28,7 @@ public Chunk(@JsonProperty("content") String content, @JsonProperty("embedding") } else { this.embedding = null; } + this.chunkType = chunkType != null ? chunkType : "unknown"; } public String getContent() { @@ -41,6 +43,10 @@ public ChunkMetadata getMetadata() { return metadata; } + public String getChunkType() { + return chunkType; + } + @Override public boolean equals(Object obj) { if (this == obj) { @@ -51,17 +57,17 @@ public boolean equals(Object obj) { } Chunk other = (Chunk) obj; return Objects.equals(content, other.content) && Objects.equals(embedding, other.embedding) - && Objects.equals(metadata, other.metadata); + && Objects.equals(metadata, other.metadata) && Objects.equals(chunkType, other.chunkType); } @Override public int hashCode() { - return Objects.hash(content, embedding, metadata); + return Objects.hash(content, embedding, metadata, chunkType); } @Override public String toString() { return "Chunk{" + "contentLength=" + content.length() + ", embeddingDimensions=" - + (embedding == null ? 0 : embedding.size()) + ", metadata=" + metadata + '}'; + + (embedding == null ? 0 : embedding.size()) + ", metadata=" + metadata + ", chunkType=" + chunkType + '}'; } } diff --git a/packages/ruby/lib/kreuzberg/result.rb b/packages/ruby/lib/kreuzberg/result.rb index 48747d3258a..e1581718e28 100644 --- a/packages/ruby/lib/kreuzberg/result.rb +++ b/packages/ruby/lib/kreuzberg/result.rb @@ -52,6 +52,7 @@ def to_h :total_chunks, :first_page, :last_page, + :chunk_type, :embedding ) do def to_h @@ -64,6 +65,7 @@ def to_h total_chunks: total_chunks, first_page: first_page, last_page: last_page, + chunk_type: chunk_type, embedding: embedding } end @@ -525,6 +527,7 @@ def parse_chunks(chunks_data) total_chunks: chunk_hash['total_chunks'], first_page: chunk_hash['first_page'], last_page: chunk_hash['last_page'], + chunk_type: chunk_hash['chunk_type'], embedding: chunk_hash['embedding'] ) end diff --git a/packages/typescript/core/src/types/results.ts b/packages/typescript/core/src/types/results.ts index c696d8325ed..cd7e0072b35 100644 --- a/packages/typescript/core/src/types/results.ts +++ b/packages/typescript/core/src/types/results.ts @@ -48,10 +48,26 @@ export interface ChunkMetadata { headingContext?: HeadingContext | null; } +export type ChunkType = + | "heading" + | "party_list" + | "definitions" + | "operative_clause" + | "signature_block" + | "schedule" + | "table_like" + | "formula" + | "code_block" + | "image" + | "org_chart" + | "diagram" + | "unknown"; + export interface Chunk { content: string; embedding?: number[] | null; metadata: ChunkMetadata; + chunkType: ChunkType; } export interface ExtractedImage { diff --git a/tools/benchmark-harness/src/embed_benchmark.rs b/tools/benchmark-harness/src/embed_benchmark.rs index 21c4426fc74..8b0d751702a 100644 --- a/tools/benchmark-harness/src/embed_benchmark.rs +++ b/tools/benchmark-harness/src/embed_benchmark.rs @@ -158,6 +158,7 @@ fn generate_test_chunks(count: usize, words_per_chunk: usize) -> Vec { Chunk { content: text, embedding: None, + chunk_type: Default::default(), metadata: ChunkMetadata { byte_start: 0, byte_end, diff --git a/tools/e2e-generator/e2e/go/helpers_test.go b/tools/e2e-generator/e2e/go/helpers_test.go index e00eafcc94c..dcbde55239a 100644 --- a/tools/e2e-generator/e2e/go/helpers_test.go +++ b/tools/e2e-generator/e2e/go/helpers_test.go @@ -241,7 +241,7 @@ func boolPtr(value bool) *bool { return &value } -func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext *bool) { +func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, eachHasChunkType, contentStartsWithHeading *bool) { t.Helper() count := len(result.Chunks) if minCount != nil && count < *minCount { @@ -278,6 +278,20 @@ func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, ma } } } + if eachHasChunkType != nil && *eachHasChunkType { + for i, chunk := range result.Chunks { + if chunk.ChunkType == "" || chunk.ChunkType == "unknown" { + t.Fatalf("chunk %d has no specific chunk_type, got %q", i, chunk.ChunkType) + } + } + } + if contentStartsWithHeading != nil && *contentStartsWithHeading { + for i, chunk := range result.Chunks { + if !strings.HasPrefix(chunk.Content, "#") { + t.Fatalf("chunk %d content does not start with '#'", i) + } + } + } } func assertImages(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, formatsInclude []string) { diff --git a/tools/e2e-generator/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java b/tools/e2e-generator/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java index b3bbc49e243..bb584aa5386 100644 --- a/tools/e2e-generator/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java +++ b/tools/e2e-generator/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java @@ -337,7 +337,9 @@ public static void assertChunks( Integer maxCount, Boolean eachHasContent, Boolean eachHasEmbedding, - Boolean eachHasHeadingContext) { + Boolean eachHasHeadingContext, + Boolean eachHasChunkType, + Boolean contentStartsWithHeading) { var chunks = result.getChunks(); int count = chunks != null ? chunks.size() : 0; if (minCount != null) { @@ -375,6 +377,22 @@ public static void assertChunks( "Expected each chunk to have no heading_context"); } } + if (chunks != null && eachHasChunkType != null && eachHasChunkType) { + for (var chunk : chunks) { + String type = chunk.getChunkType(); + assertTrue(type != null && !"unknown".equals(type), + "Expected each chunk to have a specific chunk_type"); + } + } + if (chunks != null && contentStartsWithHeading != null && contentStartsWithHeading) { + String headingPrefix = String.valueOf((char) 35); + for (var chunk : chunks) { + String content = chunk.getContent(); + assertTrue( + content != null && content.startsWith(headingPrefix), + "Expected each chunk content to start with a heading"); + } + } } public static void assertImages( diff --git a/tools/e2e-generator/e2e/php/tests/Helpers.php b/tools/e2e-generator/e2e/php/tests/Helpers.php index 245c73886b0..49b433e68ac 100644 --- a/tools/e2e-generator/e2e/php/tests/Helpers.php +++ b/tools/e2e-generator/e2e/php/tests/Helpers.php @@ -208,7 +208,9 @@ public static function assertChunks( ?int $maxCount, ?bool $eachHasContent, ?bool $eachHasEmbedding, - ?bool $eachHasHeadingContext = null + ?bool $eachHasHeadingContext = null, + ?bool $eachHasChunkType = null, + ?bool $contentStartsWithHeading = null ): void { $chunks = $result->chunks ?? []; $count = count($chunks); @@ -263,6 +265,22 @@ public static function assertChunks( ); } } + if ($eachHasChunkType === true) { + foreach ($chunks as $i => $chunk) { + $type = $chunk->chunk_type ?? null; + Assert::assertNotNull($type, sprintf("Chunk %d should have chunk_type", $i)); + Assert::assertNotSame('unknown', $type, sprintf("Chunk %d should have specific chunk_type, got 'unknown'", $i)); + } + } + if ($contentStartsWithHeading === true) { + foreach ($chunks as $i => $chunk) { + Assert::assertStringStartsWith( + '#', + $chunk->content ?? '', + sprintf("Chunk %d content should start with a heading (#)", $i) + ); + } + } } public static function assertImages( diff --git a/tools/e2e-generator/e2e/python/tests/helpers.py b/tools/e2e-generator/e2e/python/tests/helpers.py index bdc46ca3ecb..24d26f1c66a 100644 --- a/tools/e2e-generator/e2e/python/tests/helpers.py +++ b/tools/e2e-generator/e2e/python/tests/helpers.py @@ -30,6 +30,8 @@ PostProcessorConfig, ResultFormat, TokenReductionConfig, + TreeSitterConfig, + TreeSitterProcessConfig, ) _WORKSPACE_ROOT = Path(__file__).resolve().parent.parent.parent.parent @@ -104,6 +106,12 @@ def _build_config_objects(config: dict[str, Any], kwargs: dict[str, Any]) -> Non kwargs["acceleration"] = AccelerationConfig(**accel_data) if (email_data := config.get("email")) is not None and isinstance(email_data, dict): kwargs["email"] = EmailConfig(**email_data) + if (tree_sitter_data := config.get("tree_sitter")) is not None: + ts = dict(tree_sitter_data) + process_data = ts.pop("process", None) + if process_data is not None: + ts["process"] = TreeSitterProcessConfig(**process_data) + kwargs["tree_sitter"] = TreeSitterConfig(**ts) def build_config(config: dict[str, Any] | None) -> ExtractionConfig: @@ -114,10 +122,20 @@ def build_config(config: dict[str, Any] | None) -> ExtractionConfig: kwargs: dict[str, Any] = {} - for key in ("use_cache", "enable_quality_processing", "force_ocr", "include_document_structure"): + for key in ( + "use_cache", + "enable_quality_processing", + "force_ocr", + "disable_ocr", + "force_ocr_pages", + "include_document_structure", + ): if key in config: kwargs[key] = config[key] + if (extraction_timeout_secs := config.get("extraction_timeout_secs")) is not None: + kwargs["extraction_timeout_secs"] = int(extraction_timeout_secs) + _build_config_objects(config, kwargs) if (output_format := config.get("output_format")) is not None: @@ -137,10 +155,19 @@ def build_file_config(config: dict[str, Any] | None) -> FileExtractionConfig: kwargs: dict[str, Any] = {} - for key in ("enable_quality_processing", "force_ocr", "include_document_structure"): + for key in ( + "enable_quality_processing", + "force_ocr", + "disable_ocr", + "force_ocr_pages", + "include_document_structure", + ): if key in config: kwargs[key] = config[key] + if (timeout_secs := config.get("timeout_secs")) is not None: + kwargs["timeout_secs"] = int(timeout_secs) + _build_config_objects(config, kwargs) if (output_format := config.get("output_format")) is not None: @@ -288,6 +315,41 @@ def _get_heading_context(chunk: Any) -> Any: return metadata.get("heading_context") if isinstance(metadata, dict) else getattr(metadata, "heading_context", None) +def _assert_chunks_content(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + if not getattr(chunk, "content", None): + pytest.fail(f"Chunk {i} has no content") + + +def _assert_chunks_embedding(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + if not getattr(chunk, "embedding", None): + pytest.fail(f"Chunk {i} has no embedding") + + +def _assert_chunks_heading_context(chunks: Any, expected: bool) -> None: + for i, chunk in enumerate(chunks): + hc = _get_heading_context(chunk) + if expected and hc is None: + pytest.fail(f"Chunk {i} has no heading_context") + if not expected and hc is not None: + pytest.fail(f"Chunk {i} should have no heading_context") + + +def _assert_chunks_chunk_type(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + chunk_type = getattr(chunk, "chunk_type", None) + if chunk_type is None or str(chunk_type) == "unknown": + pytest.fail(f"Chunk {i} has no specific chunk_type, got {chunk_type}") + + +def _assert_chunks_heading_prefix(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + content = getattr(chunk, "content", None) + if not isinstance(content, str) or content[0:1] != chr(35): + pytest.fail(f"Chunk {i} content does not start with a heading") + + def assert_chunks( result: Any, min_count: int | None = None, @@ -295,6 +357,8 @@ def assert_chunks( each_has_content: bool | None = None, each_has_embedding: bool | None = None, each_has_heading_context: bool | None = None, + each_has_chunk_type: bool | None = None, + content_starts_with_heading: bool | None = None, ) -> None: chunks = getattr(result, "chunks", None) if chunks is None: @@ -305,20 +369,15 @@ def assert_chunks( if max_count is not None and count > max_count: pytest.fail(f"Expected at most {max_count} chunks, found {count}") if each_has_content: - for i, chunk in enumerate(chunks): - if not getattr(chunk, "content", None): - pytest.fail(f"Chunk {i} has no content") + _assert_chunks_content(chunks) if each_has_embedding: - for i, chunk in enumerate(chunks): - if not getattr(chunk, "embedding", None): - pytest.fail(f"Chunk {i} has no embedding") + _assert_chunks_embedding(chunks) if each_has_heading_context is not None: - for i, chunk in enumerate(chunks): - hc = _get_heading_context(chunk) - if each_has_heading_context and hc is None: - pytest.fail(f"Chunk {i} has no heading_context") - if not each_has_heading_context and hc is not None: - pytest.fail(f"Chunk {i} should have no heading_context") + _assert_chunks_heading_context(chunks, each_has_heading_context) + if each_has_chunk_type: + _assert_chunks_chunk_type(chunks) + if content_starts_with_heading: + _assert_chunks_heading_prefix(chunks) def assert_images( @@ -622,3 +681,14 @@ def assert_annotations(result: Any, has_annotations: bool = False, min_count: in and len(annotations) < min_count ): pytest.fail(f"Expected at least {min_count} annotations, found {len(annotations)}") + + +def assert_is_png(data: bytes) -> None: + """Assert data starts with PNG magic bytes.""" + assert len(data) >= 4, f"Data too short for PNG: {len(data)} bytes" + assert data[:4] == b"\x89PNG", f"Missing PNG magic bytes, got: {data[:4]!r}" + + +def assert_min_byte_length(data: bytes, min_length: int) -> None: + """Assert data is at least min_length bytes.""" + assert len(data) >= min_length, f"Expected at least {min_length} bytes, got {len(data)}" diff --git a/tools/e2e-generator/e2e/python/tests/test_archive.py b/tools/e2e-generator/e2e/python/tests/test_archive.py new file mode 100644 index 00000000000..bba1e42eea0 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_archive.py @@ -0,0 +1,73 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for archive fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_archive_gz_basic() -> None: + """Gzip compressed file extraction.""" + + document_path = helpers.resolve_document("archives/book_war_and_peace_1p.txt.gz") + if not document_path.exists(): + pytest.skip(f"Skipping archive_gz_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/gzip", "application/x-gzip"]) + helpers.assert_min_content_length(result, 10) + + +def test_archive_sevenz_basic() -> None: + """7-Zip archive extraction.""" + + document_path = helpers.resolve_document("archives/documents.7z") + if not document_path.exists(): + pytest.skip(f"Skipping archive_sevenz_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-7z-compressed"]) + helpers.assert_min_content_length(result, 10) + + +def test_archive_tar_basic() -> None: + """TAR archive extraction.""" + + document_path = helpers.resolve_document("archives/documents.tar") + if not document_path.exists(): + pytest.skip(f"Skipping archive_tar_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-tar", "application/tar"]) + helpers.assert_min_content_length(result, 10) + + +def test_archive_zip_basic() -> None: + """ZIP archive extraction.""" + + document_path = helpers.resolve_document("archives/documents.zip") + if not document_path.exists(): + pytest.skip(f"Skipping archive_zip_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/zip", "application/x-zip-compressed"]) + helpers.assert_min_content_length(result, 10) diff --git a/tools/e2e-generator/e2e/python/tests/test_code.py b/tools/e2e-generator/e2e/python/tests/test_code.py new file mode 100644 index 00000000000..63ed1e4f5cc --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_code.py @@ -0,0 +1,104 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for code fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + detect_mime_type_from_path, + extract_bytes_sync, + extract_file_sync, +) + +from . import helpers + + +def test_code_javascript_basic() -> None: + """Smoke test: JavaScript source code file extraction""" + + document_path = helpers.resolve_document("code/app.js") + if not document_path.exists(): + pytest.skip(f"Skipping code_javascript_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["class Application", "export function", "import"]) + + +def test_code_python_basic() -> None: + """Smoke test: Python source code file extraction""" + + document_path = helpers.resolve_document("code/hello.py") + if not document_path.exists(): + pytest.skip(f"Skipping code_python_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["def greet", "class Greeter", "import os"]) + + +def test_code_rust_basic() -> None: + """Smoke test: Rust source code file extraction""" + + document_path = helpers.resolve_document("code/main.rs") + if not document_path.exists(): + pytest.skip(f"Skipping code_rust_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["struct Store", "fn main", "HashMap"]) + + +def test_code_shebang_detection() -> None: + """Test language detection from shebang line via bytes input""" + + document_path = helpers.resolve_document("code/script.sh") + if not document_path.exists(): + pytest.skip(f"Skipping code_shebang_detection: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + result = extract_bytes_sync(file_bytes, mime_type, config=config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["build", "clean"]) + + +def test_code_tree_sitter_config() -> None: + """Test tree-sitter config with structure, comments, and docstrings enabled""" + + document_path = helpers.resolve_document("code/hello.py") + if not document_path.exists(): + pytest.skip(f"Skipping code_tree_sitter_config: missing document at {document_path}") + + config = helpers.build_config( + { + "tree_sitter": { + "process": {"structure": True, "imports": True, "exports": True, "comments": True, "docstrings": True} + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["def greet", "class Greeter"]) diff --git a/tools/e2e-generator/e2e/python/tests/test_contract.py b/tools/e2e-generator/e2e/python/tests/test_contract.py new file mode 100644 index 00000000000..76a8b16aebf --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_contract.py @@ -0,0 +1,1145 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for contract fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + batch_extract_bytes, + batch_extract_bytes_sync, + batch_extract_files, + batch_extract_files_sync, + detect_mime_type_from_path, + extract_bytes, + extract_bytes_sync, + extract_file, + extract_file_sync, +) + +from . import helpers + + +@pytest.mark.asyncio +async def test_api_batch_bytes_async() -> None: + """Tests async batch bytes extraction API (batch_extract_bytes)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_bytes_async: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + results = await batch_extract_bytes([file_bytes], [mime_type], config=config) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_api_batch_bytes_sync() -> None: + """Tests sync batch bytes extraction API (batch_extract_bytes_sync)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_bytes_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + results = batch_extract_bytes_sync([file_bytes], [mime_type], config=config) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +@pytest.mark.asyncio +async def test_api_batch_bytes_with_configs_async() -> None: + """Tests async batch bytes extraction with per-file configs (batch_extract_bytes with file_configs parameter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_bytes_with_configs_async: missing document at {document_path}") + + config = helpers.build_config(None) + + file_configs = [helpers.build_file_config({"output_format": "markdown"})] + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + results = await batch_extract_bytes([file_bytes], [mime_type], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_api_batch_bytes_with_configs_sync() -> None: + """Tests sync batch bytes extraction with per-file configs (batch_extract_bytes_sync with file_configs parameter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_bytes_with_configs_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + file_configs = [helpers.build_file_config({"output_format": "markdown"})] + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + results = batch_extract_bytes_sync([file_bytes], [mime_type], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +@pytest.mark.asyncio +async def test_api_batch_file_async() -> None: + """Tests async batch file extraction API (batch_extract_file)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_async: missing document at {document_path}") + + config = helpers.build_config(None) + + results = await batch_extract_files([document_path], config=config) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_api_batch_file_sync() -> None: + """Tests sync batch file extraction API (batch_extract_file_sync)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + results = batch_extract_files_sync([document_path], config=config) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +@pytest.mark.asyncio +async def test_api_batch_file_with_configs_async() -> None: + """Tests async batch file extraction with per-file configs (batch_extract_files with file_configs parameter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_with_configs_async: missing document at {document_path}") + + config = helpers.build_config(None) + + file_configs = [helpers.build_file_config({"output_format": "markdown"})] + + results = await batch_extract_files([document_path], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_api_batch_file_with_configs_sync() -> None: + """Tests sync batch file extraction with per-file configs (batch_extract_files_sync with file_configs parameter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_with_configs_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + file_configs = [helpers.build_file_config({"output_format": "markdown"})] + + results = batch_extract_files_sync([document_path], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_api_batch_file_with_timeout_sync() -> None: + """Tests sync batch file extraction with per-file timeout config override""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_with_timeout_sync: missing document at {document_path}") + + config = helpers.build_config({"extraction_timeout_secs": 300}) + + file_configs = [helpers.build_file_config({"timeout_secs": 600})] + + results = batch_extract_files_sync([document_path], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + + +@pytest.mark.asyncio +async def test_api_extract_bytes_async() -> None: + """Tests async bytes extraction API (extract_bytes)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_extract_bytes_async: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + result = await extract_bytes(file_bytes, mime_type, config=config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_api_extract_bytes_sync() -> None: + """Tests sync bytes extraction API (extract_bytes_sync)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_extract_bytes_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + result = extract_bytes_sync(file_bytes, mime_type, config=config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +@pytest.mark.asyncio +async def test_api_extract_file_async() -> None: + """Tests async file extraction API (extract_file)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_extract_file_async: missing document at {document_path}") + + config = helpers.build_config(None) + + result = await extract_file(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_api_extract_file_sync() -> None: + """Tests sync file extraction API (extract_file_sync)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_extract_file_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_config_acceleration_cpu_provider() -> None: + """Tests explicit CPU acceleration provider configuration""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_acceleration_cpu_provider: missing document at {document_path}") + + config = helpers.build_config({"acceleration": {"provider": "cpu", "device_id": 0}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["May 5, 2023", "To Whom it May Concern"]) + + +def test_config_chunking() -> None: + """Tests chunking configuration with chunk assertions""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"max_chars": 500, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True) + + +def test_config_chunking_heading_context() -> None: + """Tests markdown chunker populates heading context on chunks""" + + document_path = helpers.resolve_document("markdown/extraction_test.md") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_heading_context: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"chunker_type": "markdown", "max_chars": 300, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=2, each_has_content=True, each_has_heading_context=True) + + +def test_config_chunking_markdown() -> None: + """Tests markdown-aware chunker type""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_markdown: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"chunker_type": "markdown", "max_chars": 500, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True) + + +def test_config_chunking_no_headings() -> None: + """Tests markdown chunker on text with no headings produces null heading_context""" + + document_path = helpers.resolve_document("text/book_war_and_peace_1p.txt") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_no_headings: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"chunker_type": "markdown", "max_chars": 300, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=2, each_has_content=True, each_has_heading_context=False) + + +def test_config_chunking_prepend_heading_context() -> None: + """Tests markdown chunker prepends heading hierarchy to chunk content""" + + document_path = helpers.resolve_document("markdown/extraction_test.md") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_prepend_heading_context: missing document at {document_path}") + + config = helpers.build_config( + {"chunking": {"chunker_type": "markdown", "max_chars": 300, "max_overlap": 50, "prepend_heading_context": True}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks( + result, min_count=2, each_has_content=True, each_has_heading_context=True, content_starts_with_heading=True + ) + + +def test_config_chunking_small() -> None: + """Tests chunking with very small chunk size produces more chunks""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_small: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"max_chars": 100, "max_overlap": 20}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=2, each_has_content=True) + + +def test_config_chunking_text() -> None: + """Tests text chunker type (generic whitespace/punctuation splitter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_text: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"chunker_type": "text", "max_chars": 500, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True) + + +def test_config_chunking_tokenizer() -> None: + """Tests token-based chunk sizing with HuggingFace tokenizer""" + + document_path = helpers.resolve_document("markdown/comprehensive.md") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_tokenizer: missing document at {document_path}") + + config = helpers.build_config( + {"chunking": {"max_chars": 200, "max_overlap": 40, "sizing": {"type": "tokenizer", "model": "Xenova/gpt-4o"}}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=2, each_has_content=True) + + +def test_config_disable_ocr() -> None: + """Tests disable_ocr configuration option - OCR is skipped for images""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping config_disable_ocr: missing document at {document_path}") + + config = helpers.build_config({"disable_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_max_content_length(result, 5) + + +def test_config_djot_content() -> None: + """Tests djot output format converts content to djot markup""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_djot_content: missing document at {document_path}") + + config = helpers.build_config({"output_format": "djot"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "djot") + + +def test_config_document_structure() -> None: + """Tests include_document_structure config produces document tree""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure: missing document at {document_path}") + + config = helpers.build_config({"include_document_structure": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_document(result, has_document=True, min_node_count=1, node_types_include=["paragraph"]) + + +def test_config_document_structure_disabled() -> None: + """Tests document field is null when include_document_structure is false""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure_disabled: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_document(result, has_document=False) + + +def test_config_document_structure_groups() -> None: + """Tests document structure extraction with group node assertion on DOCX with headings""" + + document_path = helpers.resolve_document("docx/unit_test_headers.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure_groups: missing document at {document_path}") + + config = helpers.build_config({"include_document_structure": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_document(result, has_document=True, has_groups=True) + + +def test_config_document_structure_headings() -> None: + """Tests document structure extraction with heading nodes on a DOCX""" + + document_path = helpers.resolve_document("docx/unit_test_headers.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure_headings: missing document at {document_path}") + + config = helpers.build_config({"include_document_structure": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_document(result, has_document=True, min_node_count=1, node_types_include=["heading", "paragraph"]) + + +def test_config_document_structure_with_headings() -> None: + """Tests document structure with DOCX heading-driven nesting""" + + document_path = helpers.resolve_document("docx/fake.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure_with_headings: missing document at {document_path}") + + config = helpers.build_config({"include_document_structure": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_document(result, has_document=True, min_node_count=1) + + +def test_config_element_types() -> None: + """Tests element-based result format with element type assertions on DOCX""" + + document_path = helpers.resolve_document("docx/unit_test_headers.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_element_types: missing document at {document_path}") + + config = helpers.build_config({"result_format": "element_based"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_elements(result, min_count=1, types_include=["narrative_text"]) + helpers.assert_result_format(result, "element_based") + + +def test_config_email_msg_fallback_codepage() -> None: + """Tests MSG extraction with custom fallback codepage for Cyrillic""" + + document_path = helpers.resolve_document("email/fake_email.msg") + if not document_path.exists(): + pytest.skip(f"Skipping config_email_msg_fallback_codepage: missing document at {document_path}") + + config = helpers.build_config({"email": {"msg_fallback_codepage": 1251}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-outlook"]) + helpers.assert_min_content_length(result, 10) + + +def test_config_extraction_timeout() -> None: + """Tests that extraction_timeout_secs config field is accepted and does not affect fast extractions""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_extraction_timeout: missing document at {document_path}") + + config = helpers.build_config({"extraction_timeout_secs": 300}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + + +def test_config_force_ocr() -> None: + """Tests force_ocr configuration option""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_force_ocr: missing document at {document_path}") + + config = helpers.build_config({"force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + + +def test_config_force_ocr_pages() -> None: + """Tests that force_ocr_pages config field is accepted for selective page OCR""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_force_ocr_pages: missing document at {document_path}") + + config = helpers.build_config({"force_ocr_pages": [1], "ocr": {"backend": "tesseract", "language": "eng"}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 1) + + +def test_config_html_options() -> None: + """Tests extraction with HTML conversion options configured""" + + document_path = helpers.resolve_document("html/complex_table.html") + if not document_path.exists(): + pytest.skip(f"Skipping config_html_options: missing document at {document_path}") + + config = helpers.build_config({"html_options": {"extractMetadata": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/html"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_not_empty(result) + + +def test_config_images() -> None: + """Tests image extraction configuration with image assertions""" + + document_path = helpers.resolve_document("pdf/embedded_images_tables.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_images: missing document at {document_path}") + + config = helpers.build_config({"images": {"extract_images": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_images(result, min_count=1) + + +def test_config_images_with_formats() -> None: + """Tests image extraction on PPTX containing embedded images""" + + document_path = helpers.resolve_document("pptx/powerpoint_with_image.pptx") + if not document_path.exists(): + pytest.skip(f"Skipping config_images_with_formats: missing document at {document_path}") + + config = helpers.build_config({"images": {"extract_images": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) + helpers.assert_images(result, min_count=1) + + +def test_config_keywords() -> None: + """Tests keyword extraction via YAKE algorithm""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_keywords: missing document at {document_path}") + + config = helpers.build_config({"keywords": {"algorithm": "yake", "max_keywords": 10}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_keywords(result, has_keywords=True, min_count=1) + + +def test_config_language_detection() -> None: + """Tests language detection configuration""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_language_detection: missing document at {document_path}") + + config = helpers.build_config({"language_detection": {"enabled": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_detected_languages(result, ["eng"], 0.5) + + +def test_config_language_detection_multi() -> None: + """Tests language detection with detect_multiple and min_confidence options""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_language_detection_multi: missing document at {document_path}") + + config = helpers.build_config( + {"language_detection": {"enabled": True, "detect_multiple": True, "min_confidence": 0.3}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_detected_languages(result, ["eng"], None) + + +def test_config_language_multi() -> None: + """Tests multi-language detection config""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_language_multi: missing document at {document_path}") + + config = helpers.build_config({"language_detection": {"enabled": True, "detect_multiple": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_detected_languages(result, ["eng"], None) + + +def test_config_pages() -> None: + """Tests page extraction and page marker configuration""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pages: missing document at {document_path}") + + config = helpers.build_config({"pages": {"extract_pages": True, "insert_page_markers": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["PAGE"]) + + +def test_config_pages_exact_count() -> None: + """Tests page extraction with exact page count assertion on multi-page PDF""" + + document_path = helpers.resolve_document("pdf/multi_page.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pages_exact_count: missing document at {document_path}") + + config = helpers.build_config({"pages": {"extract_pages": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_pages(result, exact_count=5) + + +def test_config_pages_extract() -> None: + """Tests page extraction config producing per-page content array""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pages_extract: missing document at {document_path}") + + config = helpers.build_config({"pages": {"extract_pages": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_pages(result, min_count=1) + + +def test_config_pages_markers() -> None: + """Tests page marker insertion in extracted content""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pages_markers: missing document at {document_path}") + + config = helpers.build_config({"pages": {"insert_page_markers": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["PAGE"]) + + +def test_config_pdf_annotations_count() -> None: + """Tests PDF annotation extraction with min_count assertion""" + + document_path = helpers.resolve_document("vendored/pdfplumber/pdf/annotations.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pdf_annotations_count: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "aarch64" and _platform.system() == "Linux": + pytest.skip("Skipping config_pdf_annotations_count: not supported on this platform") + + config = helpers.build_config({"pdf_options": {"extract_annotations": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_annotations(result, has_annotations=True, min_count=3) + + +def test_config_pdf_hierarchy() -> None: + """Tests PDF hierarchy extraction config with block-level structure""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pdf_hierarchy: missing document at {document_path}") + + config = helpers.build_config( + {"pdf_options": {"hierarchy": {"enabled": True, "include_bbox": True}}, "pages": {"extract_pages": True}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + + +def test_config_pdf_margins() -> None: + """Tests PDF margin exclusion configuration""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pdf_margins: missing document at {document_path}") + + config = helpers.build_config({"pdf_options": {"top_margin_fraction": 0.1, "bottom_margin_fraction": 0.1}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + + +def test_config_postprocessor() -> None: + """Tests postprocessor config is accepted and extraction succeeds""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_postprocessor: missing document at {document_path}") + + config = helpers.build_config({"postprocessor": {"enabled": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_not_empty(result) + + +def test_config_processing_warnings_empty() -> None: + """Tests that a clean PDF extraction produces no processing warnings""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_processing_warnings_empty: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_processing_warnings(result, is_empty=True) + + +def test_config_quality_disabled() -> None: + """Tests extraction with quality processing explicitly disabled""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_quality_disabled: missing document at {document_path}") + + config = helpers.build_config({"enable_quality_processing": False}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_not_empty(result) + + +def test_config_quality_enabled() -> None: + """Tests quality scoring produces a score value in [0.0, 1.0]""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_quality_enabled: missing document at {document_path}") + + config = helpers.build_config({"enable_quality_processing": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_quality_score(result, has_score=True, min_score=0, max_score=1) + + +def test_config_quality_score_range() -> None: + """Tests quality scoring produces a score with minimum bound assertion""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_quality_score_range: missing document at {document_path}") + + config = helpers.build_config({"enable_quality_processing": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_quality_score(result, has_score=True, min_score=0.1) + + +def test_config_security_limits() -> None: + """Tests archive extraction with custom security limits""" + + document_path = helpers.resolve_document("archives/documents.zip") + if not document_path.exists(): + pytest.skip(f"Skipping config_security_limits: missing document at {document_path}") + + config = helpers.build_config( + {"security_limits": {"max_archive_size": 104857600, "max_compression_ratio": 50, "max_files_in_archive": 100}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/zip", "application/x-zip-compressed"]) + helpers.assert_min_content_length(result, 10) + + +def test_config_structured_output() -> None: + """Tests structured (JSON) output format config""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_structured_output: missing document at {document_path}") + + config = helpers.build_config({"output_format": "structured"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "structured") + + +def test_config_tables_content() -> None: + """Tests table extraction with content_contains_any assertion on DOCX with tables""" + + document_path = helpers.resolve_document("docx/docx_tables.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_tables_content: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_table_count(result, 1, None) + + +def test_config_tree_sitter() -> None: + """Tests tree-sitter configuration round-trip""" + + document_path = helpers.resolve_document("code/hello.py") + if not document_path.exists(): + pytest.skip(f"Skipping config_tree_sitter: missing document at {document_path}") + + config = helpers.build_config( + { + "tree_sitter": { + "languages": ["python", "rust"], + "groups": ["web"], + "process": { + "structure": True, + "imports": True, + "exports": True, + "comments": False, + "docstrings": False, + "symbols": False, + "diagnostics": False, + }, + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 5) + + +def test_config_tree_sitter_process() -> None: + """Tests tree-sitter process config with all options enabled""" + + document_path = helpers.resolve_document("code/hello.py") + if not document_path.exists(): + pytest.skip(f"Skipping config_tree_sitter_process: missing document at {document_path}") + + config = helpers.build_config( + { + "tree_sitter": { + "process": { + "structure": True, + "imports": True, + "exports": True, + "comments": True, + "docstrings": True, + "symbols": True, + "diagnostics": True, + "chunk_max_size": 2000, + } + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 5) + + +def test_config_use_cache_false() -> None: + """Tests use_cache=false configuration option""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_use_cache_false: missing document at {document_path}") + + config = helpers.build_config({"use_cache": False}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + + +def test_output_format_bytes_markdown() -> None: + """Tests markdown output format via bytes extraction API""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_bytes_markdown: missing document at {document_path}") + + config = helpers.build_config({"output_format": "markdown"}) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + result = extract_bytes_sync(file_bytes, mime_type, config=config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_output_format_djot() -> None: + """Tests Djot output format""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_djot: missing document at {document_path}") + + config = helpers.build_config({"output_format": "djot"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "djot") + + +def test_output_format_html() -> None: + """Tests HTML output format""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_html: missing document at {document_path}") + + config = helpers.build_config({"output_format": "html"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "html") + + +def test_output_format_markdown() -> None: + """Tests Markdown output format""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_markdown: missing document at {document_path}") + + config = helpers.build_config({"output_format": "markdown"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_output_format_plain() -> None: + """Tests Plain output format""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_plain: missing document at {document_path}") + + config = helpers.build_config({"output_format": "plain"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "plain") + + +def test_result_format_element_based() -> None: + """Tests ElementBased result format with element assertions""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping result_format_element_based: missing document at {document_path}") + + config = helpers.build_config({"result_format": "element_based"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_elements(result, min_count=1) + helpers.assert_result_format(result, "element_based") + + +def test_result_format_unified() -> None: + """Tests Unified result format (default)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping result_format_unified: missing document at {document_path}") + + config = helpers.build_config({"result_format": "unified"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_result_format(result, "unified") diff --git a/tools/e2e-generator/e2e/python/tests/test_email.py b/tools/e2e-generator/e2e/python/tests/test_email.py new file mode 100644 index 00000000000..53dc11347f9 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_email.py @@ -0,0 +1,103 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for email fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_email_eml_html_body() -> None: + """EML with HTML body content.""" + + document_path = helpers.resolve_document("email/html_only.eml") + if not document_path.exists(): + pytest.skip(f"Skipping email_eml_html_body: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["message/rfc822"]) + helpers.assert_min_content_length(result, 10) + + +def test_email_eml_multipart() -> None: + """EML with multipart MIME content.""" + + document_path = helpers.resolve_document("email/html_email_multipart.eml") + if not document_path.exists(): + pytest.skip(f"Skipping email_eml_multipart: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["message/rfc822"]) + helpers.assert_min_content_length(result, 10) + + +def test_email_eml_utf16() -> None: + """UTF-16 encoded EML file with BOM.""" + + document_path = helpers.resolve_document("vendored/unstructured/eml/fake-email-utf-16.eml") + if not document_path.exists(): + pytest.skip(f"Skipping email_eml_utf16: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["message/rfc822"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["Test Email", "Roses are red"]) + + +def test_email_msg_basic() -> None: + """Outlook MSG file extraction.""" + + document_path = helpers.resolve_document("email/fake_email.msg") + if not document_path.exists(): + pytest.skip(f"Skipping email_msg_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-outlook"]) + helpers.assert_min_content_length(result, 10) + + +def test_email_pst_empty() -> None: + """Empty Outlook PST archive with no messages.""" + + document_path = helpers.resolve_document("email/empty.pst") + if not document_path.exists(): + pytest.skip(f"Skipping email_pst_empty: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-outlook-pst"]) + + +def test_email_sample_eml() -> None: + """Sample EML email file to verify email parsing.""" + + document_path = helpers.resolve_document("email/sample_email.eml") + if not document_path.exists(): + pytest.skip(f"Skipping email_sample_eml: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["message/rfc822"]) + helpers.assert_min_content_length(result, 20) diff --git a/tools/e2e-generator/e2e/python/tests/test_embeddings.py b/tools/e2e-generator/e2e/python/tests/test_embeddings.py new file mode 100644 index 00000000000..6e3ad344a9b --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_embeddings.py @@ -0,0 +1,118 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for embeddings fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file, + extract_file_sync, +) + +from . import helpers + + +@pytest.mark.asyncio +async def test_embedding_async() -> None: + """Tests embedding generation via async extraction path""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping embedding_async: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "AMD64" and _platform.system() == "Windows": + pytest.skip("Skipping embedding_async: not supported on this platform") + + config = helpers.build_config( + { + "chunking": { + "max_chars": 500, + "max_overlap": 50, + "embedding": {"model": {"type": "preset", "name": "balanced"}, "normalize": True}, + } + } + ) + + result = await extract_file(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True, each_has_embedding=True) + + +def test_embedding_balanced_preset() -> None: + """Tests embedding generation with balanced preset model via chunking""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping embedding_balanced_preset: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "AMD64" and _platform.system() == "Windows": + pytest.skip("Skipping embedding_balanced_preset: not supported on this platform") + + config = helpers.build_config( + { + "chunking": { + "max_chars": 500, + "max_overlap": 50, + "embedding": {"model": {"type": "preset", "name": "balanced"}, "normalize": True}, + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True, each_has_embedding=True) + + +def test_embedding_disabled() -> None: + """Tests chunking without embeddings - chunks should not have embedding vectors""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping embedding_disabled: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"max_chars": 500, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True, each_has_embedding=False) + + +def test_embedding_fast_preset() -> None: + """Tests embedding generation with fast preset model""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping embedding_fast_preset: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "AMD64" and _platform.system() == "Windows": + pytest.skip("Skipping embedding_fast_preset: not supported on this platform") + + config = helpers.build_config( + { + "chunking": { + "max_chars": 500, + "max_overlap": 50, + "embedding": {"model": {"type": "preset", "name": "fast"}, "normalize": True}, + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True, each_has_embedding=True) diff --git a/tools/e2e-generator/e2e/python/tests/test_html.py b/tools/e2e-generator/e2e/python/tests/test_html.py new file mode 100644 index 00000000000..264832f14c6 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_html.py @@ -0,0 +1,46 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for html fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_html_complex_layout() -> None: + """Large Wikipedia HTML page to validate complex conversion.""" + + document_path = helpers.resolve_document("html/taylor_swift.html") + if not document_path.exists(): + pytest.skip(f"Skipping html_complex_layout: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/html"]) + helpers.assert_min_content_length(result, 1000) + + +def test_html_simple_table() -> None: + """HTML table converted to markdown should retain structure.""" + + document_path = helpers.resolve_document("html/simple_table.html") + if not document_path.exists(): + pytest.skip(f"Skipping html_simple_table: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/html"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_contains_all( + result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"] + ) diff --git a/tools/e2e-generator/e2e/python/tests/test_image.py b/tools/e2e-generator/e2e/python/tests/test_image.py new file mode 100644 index 00000000000..32e4b367d3d --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_image.py @@ -0,0 +1,163 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for image fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_image_bmp_basic() -> None: + """BMP image extraction via OCR.""" + + document_path = helpers.resolve_document("images/bmp_24.bmp") + if not document_path.exists(): + pytest.skip(f"Skipping image_bmp_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/bmp"]) + helpers.assert_content_not_empty(result) + + +def test_image_gif_basic() -> None: + """GIF image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.gif") + if not document_path.exists(): + pytest.skip(f"Skipping image_gif_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/gif"]) + helpers.assert_content_not_empty(result) + + +def test_image_jp2_basic() -> None: + """JPEG 2000 image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.jp2") + if not document_path.exists(): + pytest.skip(f"Skipping image_jp2_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jp2", "image/jpeg2000"]) + helpers.assert_content_not_empty(result) + + +def test_image_metadata_only() -> None: + """JPEG image to validate metadata extraction without OCR.""" + + document_path = helpers.resolve_document("images/example.jpg") + if not document_path.exists(): + pytest.skip(f"Skipping image_metadata_only: missing document at {document_path}") + + config = helpers.build_config({"ocr": None}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jpeg"]) + helpers.assert_max_content_length(result, 200) + + +def test_image_pbm_basic() -> None: + """PBM (portable bitmap) image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.pbm") + if not document_path.exists(): + pytest.skip(f"Skipping image_pbm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/x-portable-bitmap", "image/x-pbm"]) + helpers.assert_content_not_empty(result) + + +def test_image_pgm_basic() -> None: + """PGM (portable graymap) image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.pgm") + if not document_path.exists(): + pytest.skip(f"Skipping image_pgm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/x-portable-graymap", "image/x-pgm"]) + helpers.assert_content_not_empty(result) + + +def test_image_ppm_basic() -> None: + """PPM (portable pixmap) image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.ppm") + if not document_path.exists(): + pytest.skip(f"Skipping image_ppm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/x-portable-pixmap", "image/x-ppm"]) + helpers.assert_content_not_empty(result) + + +def test_image_svg_basic() -> None: + """SVG image extraction.""" + + document_path = helpers.resolve_document("xml/simple_svg.svg") + if not document_path.exists(): + pytest.skip(f"Skipping image_svg_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/svg+xml"]) + helpers.assert_min_content_length(result, 5) + + +def test_image_tiff_basic() -> None: + """TIFF image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.tif") + if not document_path.exists(): + pytest.skip(f"Skipping image_tiff_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/tiff"]) + helpers.assert_content_not_empty(result) + + +def test_image_webp_basic() -> None: + """WebP image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.webp") + if not document_path.exists(): + pytest.skip(f"Skipping image_webp_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/webp"]) + helpers.assert_content_not_empty(result) diff --git a/tools/e2e-generator/e2e/python/tests/test_keywords.py b/tools/e2e-generator/e2e/python/tests/test_keywords.py new file mode 100644 index 00000000000..5a3949a87d9 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_keywords.py @@ -0,0 +1,45 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for keywords fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_keywords_rake() -> None: + """Tests keyword extraction using RAKE algorithm""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping keywords_rake: missing document at {document_path}") + + config = helpers.build_config({"keywords": {"algorithm": "rake", "max_keywords": 10}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_keywords(result, has_keywords=True, min_count=1, max_count=10) + + +def test_keywords_yake() -> None: + """Tests keyword extraction using YAKE algorithm""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping keywords_yake: missing document at {document_path}") + + config = helpers.build_config({"keywords": {"algorithm": "yake", "max_keywords": 10}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_keywords(result, has_keywords=True, min_count=1, max_count=10) diff --git a/tools/e2e-generator/e2e/python/tests/test_ocr.py b/tools/e2e-generator/e2e/python/tests/test_ocr.py new file mode 100644 index 00000000000..50551498698 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_ocr.py @@ -0,0 +1,332 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for ocr fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_ocr_image_hello_world() -> None: + """PNG image with visible English text for OCR validation.""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_image_hello_world: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "eng"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_content_contains_any(result, ["hello", "world"]) + + +def test_ocr_image_no_text() -> None: + """Image with no text to ensure OCR handles empty results gracefully.""" + + document_path = helpers.resolve_document("images/flower_no_text.jpg") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_image_no_text: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "eng"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jpeg"]) + helpers.assert_max_content_length(result, 300) + + +def test_ocr_paddle_confidence_filter() -> None: + """PaddleOCR with minimum confidence threshold filtering.""" + + document_path = helpers.resolve_document("images/ocr_image.jpg") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_confidence_filter: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "paddle-ocr", "language": "en", "paddle_ocr_config": {"min_confidence": 80.0}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jpeg"]) + helpers.assert_min_content_length(result, 1) + + +def test_ocr_paddle_element_hierarchy() -> None: + """Tests PaddleOCR with element hierarchy building enabled""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_element_hierarchy: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": { + "backend": "paddle-ocr", + "language": "en", + "element_config": {"include_elements": True, "build_hierarchy": True}, + }, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, elements_have_geometry=True, elements_have_confidence=True) + + +def test_ocr_paddle_element_levels() -> None: + """Tests PaddleOCR with word-level element extraction""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_element_levels: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": { + "backend": "paddle-ocr", + "language": "en", + "element_config": {"include_elements": True, "min_level": "word"}, + }, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, elements_have_geometry=True, min_count=1) + + +def test_ocr_paddle_image_chinese() -> None: + """Chinese OCR with PaddleOCR - its core strength.""" + + document_path = helpers.resolve_document("images/chi_sim_image.jpeg") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_image_chinese: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "paddle-ocr", "language": "ch"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jpeg"]) + helpers.assert_min_content_length(result, 1) + + +def test_ocr_paddle_image_english() -> None: + """Simple English image OCR with PaddleOCR backend.""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_image_english: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "paddle-ocr", "language": "en"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_content_contains_any(result, ["hello", "Hello", "world", "World"]) + + +def test_ocr_paddle_markdown() -> None: + """PaddleOCR with markdown output format.""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_markdown: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "paddle-ocr", "language": "en", "paddle_ocr_config": {"output_format": "markdown"}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_content_contains_any(result, ["hello", "Hello", "world", "World"]) + + +def test_ocr_paddle_pdf_scanned() -> None: + """Scanned PDF requires PaddleOCR to extract text.""" + + document_path = helpers.resolve_document("pdf/ocr_test.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_pdf_scanned: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "paddle-ocr", "language": "en"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_content_contains_any(result, ["Docling", "Markdown", "JSON"]) + + +def test_ocr_paddle_structured() -> None: + """PaddleOCR with structured output preserving all metadata.""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_structured: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "paddle-ocr", "language": "en", "element_config": {"include_elements": True}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, elements_have_geometry=True, elements_have_confidence=True) + + +def test_ocr_paddle_table_detection() -> None: + """Table detection and extraction with PaddleOCR.""" + + document_path = helpers.resolve_document("images/simple_table.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_table_detection: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "paddle-ocr", "language": "en", "paddle_ocr_config": {"enable_table_detection": True}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_table_count(result, 1, None) + + +def test_ocr_pdf_image_only_german() -> None: + """Image-only German PDF requiring OCR to extract text.""" + + document_path = helpers.resolve_document("pdf/image_only_german_pdf.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_pdf_image_only_german: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "deu"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_ocr_pdf_rotated_90() -> None: + """Rotated page PDF requiring OCR to verify orientation handling.""" + + document_path = helpers.resolve_document("pdf/ocr_test_rotated_90.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_pdf_rotated_90: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "eng"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + + +def test_ocr_pdf_tesseract() -> None: + """Scanned PDF requires OCR to extract text.""" + + document_path = helpers.resolve_document("pdf/ocr_test.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_pdf_tesseract: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "eng"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_content_contains_any(result, ["Docling", "Markdown", "JSON"]) + + +def test_ocr_tesseract_elements() -> None: + """Tests Tesseract OCR with element-level structured output including geometry""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_tesseract_elements: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "tesseract", "language": "eng", "element_config": {"include_elements": True}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, elements_have_geometry=True, elements_have_confidence=True) + + +def test_ocr_tesseract_elements_min_count() -> None: + """Tests Tesseract OCR element output with min_count assertion""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_tesseract_elements_min_count: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": { + "backend": "tesseract", + "language": "eng", + "element_config": {"include_elements": True, "min_level": "line"}, + }, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, min_count=1) + + +def test_ocr_tesseract_language_german() -> None: + """Tests Tesseract OCR with German language configuration""" + + document_path = helpers.resolve_document("pdf/image_only_german_pdf.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_tesseract_language_german: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "deu"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 20) diff --git a/tools/e2e-generator/e2e/python/tests/test_office.py b/tools/e2e-generator/e2e/python/tests/test_office.py new file mode 100644 index 00000000000..72239fb4306 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_office.py @@ -0,0 +1,811 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for office fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_office_bibtex_basic() -> None: + """BibTeX bibliography extraction.""" + + document_path = helpers.resolve_document("bibtex/comprehensive.bib") + if not document_path.exists(): + pytest.skip(f"Skipping office_bibtex_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-bibtex", "text/x-bibtex"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_commonmark_basic() -> None: + """CommonMark (.commonmark) text extraction.""" + + document_path = helpers.resolve_document("markdown/sample.commonmark") + if not document_path.exists(): + pytest.skip(f"Skipping office_commonmark_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/markdown", "text/plain", "text/x-commonmark"]) + helpers.assert_min_content_length(result, 5) + + +def test_office_dbf_basic() -> None: + """dBASE (.dbf) table extraction as markdown.""" + + document_path = helpers.resolve_document("dbf/stations.dbf") + if not document_path.exists(): + pytest.skip(f"Skipping office_dbf_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-dbf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["|"]) + + +def test_office_djot_basic() -> None: + """Djot markup text extraction.""" + + document_path = helpers.resolve_document("markdown/tables.djot") + if not document_path.exists(): + pytest.skip(f"Skipping office_djot_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-djot", "text/djot"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_doc_legacy() -> None: + """Legacy .doc document extraction via native OLE/CFB parsing.""" + + document_path = helpers.resolve_document("doc/unit_test_lists.doc") + if not document_path.exists(): + pytest.skip(f"Skipping office_doc_legacy: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/msword"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docbook_basic() -> None: + """DocBook XML document extraction.""" + + document_path = helpers.resolve_document("docbook/docbook-reader.docbook") + if not document_path.exists(): + pytest.skip(f"Skipping office_docbook_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/docbook+xml", "text/docbook"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_docx_basic() -> None: + """DOCX document extraction baseline.""" + + document_path = helpers.resolve_document("docx/sample_document.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_docx_equations() -> None: + """DOCX file containing equations to validate math extraction.""" + + document_path = helpers.resolve_document("docx/equations.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_equations: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_fake() -> None: + """Simple DOCX document to verify baseline extraction.""" + + document_path = helpers.resolve_document("docx/fake.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_fake: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_formatting() -> None: + """DOCX document heavy on formatting for style preservation.""" + + document_path = helpers.resolve_document("docx/unit_test_formatting.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_formatting: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_headers() -> None: + """DOCX document with complex headers.""" + + document_path = helpers.resolve_document("docx/unit_test_headers.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_headers: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_lists() -> None: + """DOCX document emphasizing list formatting.""" + + document_path = helpers.resolve_document("docx/unit_test_lists.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_lists: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_tables() -> None: + """DOCX document containing tables for table-aware extraction.""" + + document_path = helpers.resolve_document("docx/docx_tables.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_tables: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_all(result, ["Simple uniform table", "Nested Table", "merged cells", "Header Col"]) + helpers.assert_table_count(result, 1, None) + + +def test_office_epub_basic() -> None: + """EPUB book extraction with text content.""" + + document_path = helpers.resolve_document("epub/features.epub") + if not document_path.exists(): + pytest.skip(f"Skipping office_epub_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/epub+zip"]) + helpers.assert_min_content_length(result, 50) + + +def test_office_fb2_basic() -> None: + """FictionBook (FB2) document extraction baseline.""" + + document_path = helpers.resolve_document("fictionbook/basic.fb2") + if not document_path.exists(): + pytest.skip(f"Skipping office_fb2_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-fictionbook+xml"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_fictionbook_basic() -> None: + """FictionBook (.fb2) text extraction.""" + + document_path = helpers.resolve_document("fictionbook/basic.fb2") + if not document_path.exists(): + pytest.skip(f"Skipping office_fictionbook_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_hwp_basic() -> None: + """Hangul Word Processor (.hwp) text extraction.""" + + document_path = helpers.resolve_document("hwp/converted_output.hwp") + if not document_path.exists(): + pytest.skip(f"Skipping office_hwp_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-hwp"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_hwp_styled() -> None: + """Hangul Word Processor (.hwp) styled document extraction.""" + + document_path = helpers.resolve_document("hwp/styled_document.hwp") + if not document_path.exists(): + pytest.skip(f"Skipping office_hwp_styled: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "aarch64" and _platform.system() == "Linux": + pytest.skip("Skipping office_hwp_styled: not supported on this platform") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-hwp"]) + + +def test_office_jats_basic() -> None: + """JATS scientific article extraction.""" + + document_path = helpers.resolve_document("jats/sample_article.jats") + if not document_path.exists(): + pytest.skip(f"Skipping office_jats_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-jats+xml", "text/jats"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_jupyter_basic() -> None: + """Jupyter notebook extraction.""" + + document_path = helpers.resolve_document("jupyter/rank.ipynb") + if not document_path.exists(): + pytest.skip(f"Skipping office_jupyter_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-ipynb+json"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_keynote_basic() -> None: + """Keynote document extraction baseline.""" + + document_path = helpers.resolve_document("iwork/test.key") + if not document_path.exists(): + pytest.skip(f"Skipping office_keynote_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-iwork-keynote-sffkey"]) + helpers.assert_min_content_length(result, 5) + + +def test_office_latex_basic() -> None: + """LaTeX document text extraction.""" + + document_path = helpers.resolve_document("latex/basic_sections.tex") + if not document_path.exists(): + pytest.skip(f"Skipping office_latex_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-latex", "text/x-latex"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_markdown_basic() -> None: + """Markdown document extraction baseline.""" + + document_path = helpers.resolve_document("markdown/comprehensive.md") + if not document_path.exists(): + pytest.skip(f"Skipping office_markdown_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/markdown"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_mdx_basic() -> None: + """MDX document extraction with JSX stripping and frontmatter.""" + + document_path = helpers.resolve_document("markdown/sample.mdx") + if not document_path.exists(): + pytest.skip(f"Skipping office_mdx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/mdx", "text/x-mdx"]) + helpers.assert_min_content_length(result, 50) + + +def test_office_mdx_getting_started() -> None: + """Real-world MDX extraction from mdx-js/mdx getting-started.mdx with imports, exports, JSX components, code blocks, and reference links.""" + + document_path = helpers.resolve_document("markdown/mdx_getting_started.mdx") + if not document_path.exists(): + pytest.skip(f"Skipping office_mdx_getting_started: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/mdx", "text/x-mdx"]) + helpers.assert_min_content_length(result, 2000) + + +def test_office_mdx_troubleshooting() -> None: + """Real-world MDX extraction from mdx-js/mdx troubleshooting-mdx.mdx with error documentation, code blocks, and JSX comments.""" + + document_path = helpers.resolve_document("markdown/mdx_troubleshooting.mdx") + if not document_path.exists(): + pytest.skip(f"Skipping office_mdx_troubleshooting: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/mdx", "text/x-mdx"]) + helpers.assert_min_content_length(result, 2000) + + +def test_office_mdx_using_mdx() -> None: + """Real-world MDX extraction from mdx-js/mdx using-mdx.mdx with complex JSX examples, component passing, and MDX provider patterns.""" + + document_path = helpers.resolve_document("markdown/mdx_using_mdx.mdx") + if not document_path.exists(): + pytest.skip(f"Skipping office_mdx_using_mdx: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/mdx", "text/x-mdx"]) + helpers.assert_min_content_length(result, 2000) + + +def test_office_numbers_basic() -> None: + """Numbers document extraction baseline.""" + + document_path = helpers.resolve_document("iwork/test.numbers") + if not document_path.exists(): + pytest.skip(f"Skipping office_numbers_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-iwork-numbers-sffnumbers"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_ods_basic() -> None: + """Basic ODS spreadsheet extraction.""" + + document_path = helpers.resolve_document("data_formats/test_01.ods") + if not document_path.exists(): + pytest.skip(f"Skipping office_ods_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.spreadsheet"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_odt_bold() -> None: + """ODT document with bold formatting.""" + + document_path = helpers.resolve_document("odt/bold.odt") + if not document_path.exists(): + pytest.skip(f"Skipping office_odt_bold: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.text"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_odt_list() -> None: + """ODT document containing unordered lists with nesting.""" + + document_path = helpers.resolve_document("odt/unorderedList.odt") + if not document_path.exists(): + pytest.skip(f"Skipping office_odt_list: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.text"]) + helpers.assert_min_content_length(result, 30) + helpers.assert_content_contains_any(result, ["list item", "New level", "Pushed us"]) + + +def test_office_odt_simple() -> None: + """Basic ODT document with paragraphs and headings.""" + + document_path = helpers.resolve_document("odt/simple.odt") + if not document_path.exists(): + pytest.skip(f"Skipping office_odt_simple: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.text"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["favorite things", "Parrots", "Analysis"]) + + +def test_office_odt_table() -> None: + """ODT document with a table structure.""" + + document_path = helpers.resolve_document("odt/table.odt") + if not document_path.exists(): + pytest.skip(f"Skipping office_odt_table: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.text"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_table_count(result, 1, None) + + +def test_office_opml_basic() -> None: + """OPML outline document extraction.""" + + document_path = helpers.resolve_document("opml/outline.opml") + if not document_path.exists(): + pytest.skip(f"Skipping office_opml_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_org_basic() -> None: + """Org-mode document text extraction.""" + + document_path = helpers.resolve_document("org/comprehensive.org") + if not document_path.exists(): + pytest.skip(f"Skipping office_org_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-org", "text/org"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_pages_basic() -> None: + """Pages document extraction baseline.""" + + document_path = helpers.resolve_document("iwork/test.pages") + if not document_path.exists(): + pytest.skip(f"Skipping office_pages_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-iwork-pages-sffpages"]) + helpers.assert_min_content_length(result, 5) + + +def test_office_ppsx_slideshow() -> None: + """PPSX (PowerPoint Show) files should extract slides content identical to PPTX. GitHub Issue #321 Bug 2.""" + + document_path = helpers.resolve_document("pptx/sample.ppsx") + if not document_path.exists(): + pytest.skip(f"Skipping office_ppsx_slideshow: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_ppt_legacy() -> None: + """Legacy PowerPoint .ppt extraction via native OLE/CFB parsing.""" + + document_path = helpers.resolve_document("ppt/simple.ppt") + if not document_path.exists(): + pytest.skip(f"Skipping office_ppt_legacy: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-powerpoint"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_pptm_basic() -> None: + """PowerPoint macro-enabled presentation (.pptm) extraction.""" + + document_path = helpers.resolve_document("pptx/powerpoint_with_image.pptm") + if not document_path.exists(): + pytest.skip(f"Skipping office_pptm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime( + result, + [ + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ], + ) + helpers.assert_content_not_empty(result) + + +def test_office_pptx_basic() -> None: + """PPTX deck should extract slides content.""" + + document_path = helpers.resolve_document("pptx/simple.pptx") + if not document_path.exists(): + pytest.skip(f"Skipping office_pptx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) + helpers.assert_min_content_length(result, 50) + + +def test_office_pptx_images() -> None: + """PPTX presentation containing images to ensure metadata extraction.""" + + document_path = helpers.resolve_document("pptx/powerpoint_with_image.pptx") + if not document_path.exists(): + pytest.skip(f"Skipping office_pptx_images: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) + helpers.assert_min_content_length(result, 15) + + +def test_office_pptx_pitch_deck() -> None: + """Pitch deck PPTX used to validate large slide extraction.""" + + document_path = helpers.resolve_document("pptx/pitch_deck_presentation.pptx") + if not document_path.exists(): + pytest.skip(f"Skipping office_pptx_pitch_deck: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) + helpers.assert_min_content_length(result, 100) + + +def test_office_rst_basic() -> None: + """reStructuredText document extraction.""" + + document_path = helpers.resolve_document("rst/restructured_text.rst") + if not document_path.exists(): + pytest.skip(f"Skipping office_rst_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-rst", "text/prs.fallenstein.rst"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_rtf_basic() -> None: + """RTF document text extraction.""" + + document_path = helpers.resolve_document("rtf/extraction_test.rtf") + if not document_path.exists(): + pytest.skip(f"Skipping office_rtf_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/rtf", "text/rtf"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_typst_basic() -> None: + """Typst document text extraction.""" + + document_path = helpers.resolve_document("typst/headings.typ") + if not document_path.exists(): + pytest.skip(f"Skipping office_typst_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-typst", "text/x-typst"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_xls_legacy() -> None: + """Legacy XLS spreadsheet to ensure backward compatibility.""" + + document_path = helpers.resolve_document("xls/test_excel.xls") + if not document_path.exists(): + pytest.skip(f"Skipping office_xls_legacy: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-excel"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_xlsb_basic() -> None: + """Excel binary workbook (.xlsb) extraction.""" + + document_path = helpers.resolve_document("xlsx/test_xlsb.xlsb") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsb_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime( + result, + [ + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ], + ) + helpers.assert_content_not_empty(result) + + +def test_office_xlsm_basic() -> None: + """Excel macro-enabled workbook (.xlsm) extraction.""" + + document_path = helpers.resolve_document("xlsx/test_01.xlsm") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime( + result, + [ + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ], + ) + helpers.assert_content_not_empty(result) + + +def test_office_xlsx_basic() -> None: + """XLSX spreadsheet should produce metadata and table content.""" + + document_path = helpers.resolve_document("xlsx/stanley_cups.xlsx") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_contains_all(result, ["Team", "Location", "Stanley Cups"]) + helpers.assert_table_count(result, 1, None) + helpers.assert_metadata_expectation(result, "sheet_count", {"gte": 2}) + helpers.assert_metadata_expectation(result, "sheet_names", {"contains": ["Stanley Cups"]}) + + +def test_office_xlsx_multi_sheet() -> None: + """XLSX workbook with multiple sheets.""" + + document_path = helpers.resolve_document("xlsx/excel_multi_sheet.xlsx") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsx_multi_sheet: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_metadata_expectation(result, "sheet_count", {"gte": 2}) + + +def test_office_xlsx_office_example() -> None: + """Simple XLSX spreadsheet shipped alongside office integration tests.""" + + document_path = helpers.resolve_document("xlsx/test_01.xlsx") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsx_office_example: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) + helpers.assert_min_content_length(result, 10) diff --git a/tools/e2e-generator/e2e/python/tests/test_parity.py b/tools/e2e-generator/e2e/python/tests/test_parity.py new file mode 100644 index 00000000000..1699413b55f --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_parity.py @@ -0,0 +1,158 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python + +from __future__ import annotations + +from kreuzberg import ExtractionConfig, extract_file_sync + +from . import helpers + + +def test_extraction_result_field_parity(): + """Verify ExtractionResult has all expected fields.""" + result = extract_file_sync(helpers.resolve_document("pdf/fake_memo.pdf")) + + required_fields = [ + "content", + "metadata", + "mime_type", + "tables", + ] + for field in required_fields: + assert hasattr(result, field), f"ExtractionResult missing required field: {field}" + + all_fields = [ + "annotations", + "children", + "chunks", + "content", + "detected_languages", + "djot_content", + "document", + "elements", + "extracted_keywords", + "images", + "metadata", + "mime_type", + "ocr_elements", + "pages", + "processing_warnings", + "quality_score", + "tables", + "uris", + ] + for field in all_fields: + assert hasattr(result, field), f"ExtractionResult missing field: {field}" + + +def test_extraction_config_field_parity(): + """Verify ExtractionConfig accepts all expected fields.""" + config = ExtractionConfig() + + all_fields = [ + "acceleration", + "cache_namespace", + "cache_ttl_secs", + "chunking", + "concurrency", + "disable_ocr", + "email", + "enable_quality_processing", + "extraction_timeout_secs", + "force_ocr", + "force_ocr_pages", + "html_options", + "images", + "include_document_structure", + "keywords", + "language_detection", + "layout", + "max_archive_depth", + "max_concurrent_extractions", + "ocr", + "output_format", + "pages", + "pdf_options", + "postprocessor", + "result_format", + "security_limits", + "token_reduction", + "tree_sitter", + "use_cache", + ] + for field in all_fields: + assert hasattr(config, field), f"ExtractionConfig missing field: {field}" + + +def test_archive_entry_field_parity(): + """Verify ArchiveEntry has all expected fields in the manifest.""" + expected_fields = [ + "mime_type", + "path", + "result", + ] + assert len(expected_fields) > 0, "ArchiveEntry should have at least one field" + + +def test_bounding_box_field_parity(): + """Verify BoundingBox has all expected fields in the manifest.""" + expected_fields = [ + "x0", + "x1", + "y0", + "y1", + ] + assert len(expected_fields) > 0, "BoundingBox should have at least one field" + + +def test_keyword_field_parity(): + """Verify Keyword has all expected fields in the manifest.""" + expected_fields = [ + "algorithm", + "positions", + "score", + "text", + ] + assert len(expected_fields) > 0, "Keyword should have at least one field" + + +def test_pdf_annotation_field_parity(): + """Verify PdfAnnotation has all expected fields in the manifest.""" + expected_fields = [ + "annotation_type", + "bounding_box", + "content", + "page_number", + ] + assert len(expected_fields) > 0, "PdfAnnotation should have at least one field" + + +def test_processing_warning_field_parity(): + """Verify ProcessingWarning has all expected fields in the manifest.""" + expected_fields = [ + "message", + "source", + ] + assert len(expected_fields) > 0, "ProcessingWarning should have at least one field" + + +def test_table_field_parity(): + """Verify Table has all expected fields in the manifest.""" + expected_fields = [ + "bounding_box", + "cells", + "markdown", + "page_number", + ] + assert len(expected_fields) > 0, "Table should have at least one field" + + +def test_uri_field_parity(): + """Verify Uri has all expected fields in the manifest.""" + expected_fields = [ + "kind", + "label", + "page", + "url", + ] + assert len(expected_fields) > 0, "Uri should have at least one field" diff --git a/tools/e2e-generator/e2e/python/tests/test_pdf.py b/tools/e2e-generator/e2e/python/tests/test_pdf.py new file mode 100644 index 00000000000..bb88f8ca96e --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_pdf.py @@ -0,0 +1,316 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for pdf fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_pdf_annotations() -> None: + """PDF with annotations should extract annotation data when enabled.""" + + document_path = helpers.resolve_document("pdf/test_article.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_annotations: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "aarch64" and _platform.system() == "Linux": + pytest.skip("Skipping pdf_annotations: not supported on this platform") + + config = helpers.build_config({"pdf_options": {"extract_annotations": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_annotations(result, has_annotations=True, min_count=1) + + +def test_pdf_assembly_technical() -> None: + """Assembly language technical manual with large body of text.""" + + document_path = helpers.resolve_document("pdf/assembly_language_for_beginners_al4_b_en.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_assembly_technical: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5000) + helpers.assert_content_contains_any(result, ["assembly", "register", "instruction"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_bayesian_data_analysis() -> None: + """Bayesian data analysis textbook PDF with large content volume.""" + + document_path = helpers.resolve_document("pdf/bayesian_data_analysis_third_edition_13th_feb_2020.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_bayesian_data_analysis: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10000) + helpers.assert_content_contains_any(result, ["Bayesian", "probability", "distribution"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_bounding_boxes() -> None: + """Tests bounding box extraction on PDF tables and images""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_bounding_boxes: missing document at {document_path}") + + config = helpers.build_config({"images": {"extract_images": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_table_count(result, 1, None) + helpers.assert_table_bounding_boxes(result, True) + + +def test_pdf_code_and_formula() -> None: + """PDF containing code snippets and formulas should retain substantial content.""" + + document_path = helpers.resolve_document("pdf/code_and_formula.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_code_and_formula: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 100) + + +def test_pdf_deep_learning() -> None: + """Deep learning textbook PDF to ensure long-form extraction quality.""" + + document_path = helpers.resolve_document("pdf/fundamentals_of_deep_learning_2014.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_deep_learning: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 1000) + helpers.assert_content_contains_any(result, ["neural", "network", "deep learning"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_embedded_images() -> None: + """PDF with embedded images should extract text and tables when present.""" + + document_path = helpers.resolve_document("pdf/embedded_images_tables.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_embedded_images: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_table_count(result, 0, None) + + +def test_pdf_google_doc() -> None: + """Google Docs exported PDF to verify conversion fidelity.""" + + document_path = helpers.resolve_document("pdf/google_doc_document.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_google_doc: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_large_ciml() -> None: + """Large machine learning textbook PDF to stress extraction length.""" + + document_path = helpers.resolve_document("pdf/a_course_in_machine_learning_ciml_v0_9_all.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_large_ciml: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10000) + helpers.assert_content_contains_any(result, ["machine learning", "algorithm", "training"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_layout_detection() -> None: + """PDF extraction with layout detection enabled should produce content from a document with mixed structure.""" + + document_path = helpers.resolve_document("pdf/docling.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_layout_detection: missing document at {document_path}") + + config = helpers.build_config( + {"layout": {"preset": "accurate", "table_model": "tatr"}, "output_format": "markdown"} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_not_empty(result) + + +def test_pdf_non_english_german() -> None: + """German technical PDF to ensure non-ASCII content extraction.""" + + document_path = helpers.resolve_document("pdf/5_level_paging_and_5_level_ept_intel_revision_1_1_may_2017.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_non_english_german: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_contains_any(result, ["Intel", "paging"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_password_protected() -> None: + """Copy-protected PDF should extract content (pdfium handles copy-protection transparently).""" + + document_path = helpers.resolve_document("pdf/copy_protected.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_password_protected: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["LayoutParser", "document image analysis", "deep learning"]) + + +def test_pdf_right_to_left() -> None: + """Right-to-left language PDF to verify RTL extraction.""" + + document_path = helpers.resolve_document("pdf/right_to_left_01.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_right_to_left: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_simple_text() -> None: + """Simple text-heavy PDF should extract content without OCR or tables.""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_simple_text: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["May 5, 2023", "To Whom it May Concern", "Mallori"]) + + +def test_pdf_tables_large() -> None: + """Large PDF with extensive tables to stress table extraction.""" + + document_path = helpers.resolve_document("pdf/large.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_tables_large: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 500) + + +def test_pdf_tables_medium() -> None: + """Medium-sized PDF with multiple tables.""" + + document_path = helpers.resolve_document("pdf/medium.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_tables_medium: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 100) + + +def test_pdf_tables_small() -> None: + """Small PDF containing tables to validate table extraction.""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_tables_small: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_all( + result, ["Table 1", "Selected Numbers", "Celsius", "Fahrenheit", "Water Freezing Point", "Water Boiling Point"] + ) + helpers.assert_table_count(result, 1, None) + + +def test_pdf_technical_stat_learning() -> None: + """Technical statistical learning PDF requiring substantial extraction.""" + + document_path = helpers.resolve_document( + "pdf/an_introduction_to_statistical_learning_with_applications_in_r_islr_sixth_printing.pdf" + ) + if not document_path.exists(): + pytest.skip(f"Skipping pdf_technical_stat_learning: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10000) + helpers.assert_content_contains_any(result, ["statistical", "regression", "learning"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) diff --git a/tools/e2e-generator/e2e/python/tests/test_plugin_apis.py b/tools/e2e-generator/e2e/python/tests/test_plugin_apis.py new file mode 100644 index 00000000000..54a4011bbcb --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_plugin_apis.py @@ -0,0 +1,162 @@ +# Auto-generated from fixtures/plugin_api/ - DO NOT EDIT +""" +E2E tests for plugin/config/utility APIs. + +Generated from plugin API fixtures. +To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import kreuzberg +from kreuzberg import ExtractionConfig + +if TYPE_CHECKING: + from pathlib import Path + + +# Configuration Tests + + +def test_config_discover(tmp_path: Path, monkeypatch) -> None: + """Discover configuration from current or parent directories""" + config_path = tmp_path / "kreuzberg.toml" + config_path.write_text("""[chunking] +max_chars = 50 +""") + + subdir = tmp_path / "subdir" + subdir.mkdir() + monkeypatch.chdir(subdir) + + config = ExtractionConfig.discover() + assert config is not None + + assert config.chunking is not None + assert config.chunking.max_chars == 50 + + +def test_config_from_file(tmp_path: Path) -> None: + """Load configuration from a TOML file""" + config_path = tmp_path / "test_config.toml" + config_path.write_text("""[chunking] +max_chars = 100 +max_overlap = 20 + +[language_detection] +enabled = false +""") + + config = ExtractionConfig.from_file(str(config_path)) + + assert config.chunking is not None + assert config.chunking.max_chars == 100 + assert config.chunking.max_overlap == 20 + assert config.language_detection is not None + assert config.language_detection.enabled is False + + +# Document Extractor Management Tests + + +def test_extractors_clear() -> None: + """Clear all document extractors and verify list is empty""" + kreuzberg.clear_document_extractors() + result = kreuzberg.list_document_extractors() + assert len(result) == 0 + + +def test_extractors_list() -> None: + """List all registered document extractors""" + result = kreuzberg.list_document_extractors() + assert isinstance(result, list) + assert all(isinstance(item, str) for item in result) + + +def test_extractors_unregister() -> None: + """Unregister nonexistent document extractor gracefully""" + kreuzberg.unregister_document_extractor("nonexistent-extractor-xyz") + + +# Mime Utilities Tests + + +def test_mime_detect_bytes() -> None: + """Detect MIME type from file bytes""" + test_bytes = b"%PDF-1.4\n" + result = kreuzberg.detect_mime_type(test_bytes) + + assert "pdf" in result.lower() + + +def test_mime_detect_path(tmp_path: Path) -> None: + """Detect MIME type from file path""" + test_file = tmp_path / "test.txt" + test_file.write_text("Hello, world!") + + result = kreuzberg.detect_mime_type_from_path(str(test_file)) + + assert "text" in result.lower() + + +def test_mime_get_extensions() -> None: + """Get file extensions for a MIME type""" + result = kreuzberg.get_extensions_for_mime("application/pdf") + assert isinstance(result, list) + assert "pdf" in result + + +# Ocr Backend Management Tests + + +def test_ocr_backends_clear() -> None: + """Clear all OCR backends and verify list is empty""" + kreuzberg.clear_ocr_backends() + result = kreuzberg.list_ocr_backends() + assert len(result) == 0 + + +def test_ocr_backends_list() -> None: + """List all registered OCR backends""" + result = kreuzberg.list_ocr_backends() + assert isinstance(result, list) + assert all(isinstance(item, str) for item in result) + + +def test_ocr_backends_unregister() -> None: + """Unregister nonexistent OCR backend gracefully""" + kreuzberg.unregister_ocr_backend("nonexistent-backend-xyz") + + +# Post Processor Management Tests + + +def test_post_processors_clear() -> None: + """Clear all post-processors and verify list is empty""" + kreuzberg.clear_post_processors() + + +def test_post_processors_list() -> None: + """List all registered post-processors""" + result = kreuzberg.list_post_processors() + assert isinstance(result, list) + assert all(isinstance(item, str) for item in result) + + +# Validator Management Tests + + +def test_validators_clear() -> None: + """Clear all validators and verify list is empty""" + kreuzberg.clear_validators() + result = kreuzberg.list_validators() + assert len(result) == 0 + + +def test_validators_list() -> None: + """List all registered validators""" + result = kreuzberg.list_validators() + assert isinstance(result, list) + assert all(isinstance(item, str) for item in result) diff --git a/tools/e2e-generator/e2e/python/tests/test_render.py b/tools/e2e-generator/e2e/python/tests/test_render.py new file mode 100644 index 00000000000..517cff43756 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_render.py @@ -0,0 +1,53 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for render fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import PdfPageIterator, render_pdf_page + +from . import helpers + + +def test_render_custom_dpi() -> None: + """Render PDF page with custom DPI""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping render_custom_dpi: missing document at {document_path}") + + png_data = render_pdf_page(document_path, 0, dpi=72) + + helpers.assert_is_png(png_data) + helpers.assert_min_byte_length(png_data, 50) + + +def test_render_iterator() -> None: + """Iterate all PDF pages as PNG images""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping render_iterator: missing document at {document_path}") + + pages = [] + with PdfPageIterator(str(document_path), dpi=150) as page_iter: + for _page_index, png_data in page_iter: + helpers.assert_is_png(png_data) + pages.append(png_data) + + assert len(pages) >= 1, f"Expected at least 1 pages, got {len(pages)}" + + +def test_render_single_page() -> None: + """Render a single PDF page to PNG""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping render_single_page: missing document at {document_path}") + + png_data = render_pdf_page(document_path, 0, dpi=150) + + helpers.assert_is_png(png_data) + helpers.assert_min_byte_length(png_data, 100) diff --git a/tools/e2e-generator/e2e/python/tests/test_smoke.py b/tools/e2e-generator/e2e/python/tests/test_smoke.py new file mode 100644 index 00000000000..be976dd45d1 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_smoke.py @@ -0,0 +1,142 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for smoke fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_smoke_cache_namespace() -> None: + """Smoke test: Extraction with cache namespace and TTL configuration""" + + document_path = helpers.resolve_document("text/report.txt") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_cache_namespace: missing document at {document_path}") + + config = helpers.build_config({"cache_namespace": "test_tenant", "cache_ttl_secs": 3600, "use_cache": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/plain"]) + helpers.assert_min_content_length(result, 5) + + +def test_smoke_docx_basic() -> None: + """Smoke test: DOCX with formatted text""" + + document_path = helpers.resolve_document("docx/fake.docx") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_docx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_content_contains_any(result, ["Lorem", "ipsum", "document", "text"]) + + +def test_smoke_html_basic() -> None: + """Smoke test: HTML table extraction""" + + document_path = helpers.resolve_document("html/simple_table.html") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_html_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/html"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["Sample Data Table", "Laptop", "Electronics", "Product"]) + + +def test_smoke_image_png() -> None: + """Smoke test: PNG image (without OCR, metadata only)""" + + document_path = helpers.resolve_document("images/sample.png") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_image_png: missing document at {document_path}") + + config = helpers.build_config({"disable_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_metadata_expectation(result, "format", {"eq": "PNG"}) + + +def test_smoke_json_basic() -> None: + """Smoke test: JSON file extraction""" + + document_path = helpers.resolve_document("json/simple.json") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_json_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/json"]) + helpers.assert_min_content_length(result, 5) + + +def test_smoke_pdf_basic() -> None: + """Smoke test: PDF with simple text extraction""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_pdf_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["May 5, 2023", "To Whom it May Concern"]) + + +def test_smoke_txt_basic() -> None: + """Smoke test: Plain text file""" + + document_path = helpers.resolve_document("text/report.txt") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_txt_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/plain"]) + helpers.assert_min_content_length(result, 5) + + +def test_smoke_xlsx_basic() -> None: + """Smoke test: XLSX with basic spreadsheet data including tables""" + + document_path = helpers.resolve_document("xlsx/stanley_cups.xlsx") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_xlsx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_contains_all( + result, ["Team", "Location", "Stanley Cups", "Blues", "Flyers", "Maple Leafs", "STL", "PHI", "TOR"] + ) + helpers.assert_table_count(result, 1, None) + helpers.assert_metadata_expectation(result, "sheet_count", {"gte": 2}) + helpers.assert_metadata_expectation(result, "sheet_names", {"contains": ["Stanley Cups"]}) diff --git a/tools/e2e-generator/e2e/python/tests/test_structured.py b/tools/e2e-generator/e2e/python/tests/test_structured.py new file mode 100644 index 00000000000..4cbf2907372 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_structured.py @@ -0,0 +1,164 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for structured fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_structured_csv_basic() -> None: + """CSV data file extraction.""" + + document_path = helpers.resolve_document("csv/stanley_cups.csv") + if not document_path.exists(): + pytest.skip(f"Skipping structured_csv_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/csv"]) + helpers.assert_min_content_length(result, 20) + + +def test_structured_enw_basic() -> None: + """EndNote ENW citation format extraction.""" + + document_path = helpers.resolve_document("data_formats/sample.enw") + if not document_path.exists(): + pytest.skip(f"Skipping structured_enw_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]) + + +def test_structured_json_basic() -> None: + """Structured JSON extraction should stream and preserve content.""" + + document_path = helpers.resolve_document("json/sample_document.json") + if not document_path.exists(): + pytest.skip(f"Skipping structured_json_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/json"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_content_contains_any(result, ["Sample Document", "Test Author"]) + + +def test_structured_json_simple() -> None: + """Simple JSON document to verify structured extraction.""" + + document_path = helpers.resolve_document("json/simple.json") + if not document_path.exists(): + pytest.skip(f"Skipping structured_json_simple: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/json"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["{", "name"]) + + +def test_structured_nbib_basic() -> None: + """PubMed NBIB citation format extraction.""" + + document_path = helpers.resolve_document("data_formats/sample.nbib") + if not document_path.exists(): + pytest.skip(f"Skipping structured_nbib_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/nbib", "application/x-pubmed", "text/plain"]) + helpers.assert_content_not_empty(result) + + +def test_structured_ris_basic() -> None: + """RIS citation format extraction.""" + + document_path = helpers.resolve_document("data_formats/sample.ris") + if not document_path.exists(): + pytest.skip(f"Skipping structured_ris_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-research-info-systems", "text/plain"]) + helpers.assert_content_not_empty(result) + + +def test_structured_toml_basic() -> None: + """TOML configuration file extraction.""" + + document_path = helpers.resolve_document("data_formats/cargo.toml") + if not document_path.exists(): + pytest.skip(f"Skipping structured_toml_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/toml", "text/toml"]) + helpers.assert_min_content_length(result, 10) + + +def test_structured_tsv_basic() -> None: + """TSV (tab-separated values) data file extraction.""" + + document_path = helpers.resolve_document("data_formats/employees.tsv") + if not document_path.exists(): + pytest.skip(f"Skipping structured_tsv_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/tab-separated-values", "text/plain"]) + helpers.assert_min_content_length(result, 10) + + +def test_structured_yaml_basic() -> None: + """YAML file text extraction.""" + + document_path = helpers.resolve_document("yaml/simple.yaml") + if not document_path.exists(): + pytest.skip(f"Skipping structured_yaml_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]) + helpers.assert_min_content_length(result, 10) + + +def test_structured_yaml_simple() -> None: + """Simple YAML document to validate structured extraction.""" + + document_path = helpers.resolve_document("yaml/simple.yaml") + if not document_path.exists(): + pytest.skip(f"Skipping structured_yaml_simple: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-yaml"]) + helpers.assert_min_content_length(result, 10) diff --git a/tools/e2e-generator/e2e/python/tests/test_token_reduction.py b/tools/e2e-generator/e2e/python/tests/test_token_reduction.py new file mode 100644 index 00000000000..73e3ed5168c --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_token_reduction.py @@ -0,0 +1,83 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for token_reduction fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_token_reduction_aggressive() -> None: + """Tests aggressive token reduction mode significantly reduces content""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping token_reduction_aggressive: missing document at {document_path}") + + config = helpers.build_config({"token_reduction": {"mode": "aggressive"}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_max_content_length(result, 150) + helpers.assert_content_not_empty(result) + + +def test_token_reduction_basic() -> None: + """Tests basic token reduction on PDF document""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping token_reduction_basic: missing document at {document_path}") + + config = helpers.build_config({"token_reduction": {"mode": "moderate"}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_max_content_length(result, 200) + helpers.assert_content_not_empty(result) + + +def test_token_reduction_light() -> None: + """Tests light token reduction mode preserves most content""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping token_reduction_light: missing document at {document_path}") + + config = helpers.build_config({"token_reduction": {"mode": "light"}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_not_empty(result) + + +def test_token_reduction_with_chunking() -> None: + """Tests token reduction combined with chunking""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping token_reduction_with_chunking: missing document at {document_path}") + + config = helpers.build_config( + {"token_reduction": {"mode": "moderate"}, "chunking": {"max_chars": 500, "max_overlap": 50}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_max_content_length(result, 200) + helpers.assert_chunks(result, min_count=1, each_has_content=True) + helpers.assert_content_not_empty(result) diff --git a/tools/e2e-generator/e2e/python/tests/test_xml.py b/tools/e2e-generator/e2e/python/tests/test_xml.py new file mode 100644 index 00000000000..6b08fd44216 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_xml.py @@ -0,0 +1,28 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for xml fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_xml_plant_catalog() -> None: + """XML plant catalog to validate streaming XML extraction.""" + + document_path = helpers.resolve_document("xml/plant_catalog.xml") + if not document_path.exists(): + pytest.skip(f"Skipping xml_plant_catalog: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/xml"]) + helpers.assert_min_content_length(result, 100) diff --git a/tools/e2e-generator/e2e/r/tests/testthat/helper-kreuzberg.R b/tools/e2e-generator/e2e/r/tests/testthat/helper-kreuzberg.R index 8dd336080fc..2000f975e87 100644 --- a/tools/e2e-generator/e2e/r/tests/testthat/helper-kreuzberg.R +++ b/tools/e2e-generator/e2e/r/tests/testthat/helper-kreuzberg.R @@ -236,7 +236,9 @@ assert_metadata_expectation <- function(result, path, expectation) { assert_chunks <- function(result, min_count = NULL, max_count = NULL, each_has_content = NULL, each_has_embedding = NULL, - each_has_heading_context = NULL) { + each_has_heading_context = NULL, + each_has_chunk_type = NULL, + content_starts_with_heading = NULL) { chunks <- if (is.null(result$chunks)) list() else result$chunks if (!is.null(min_count)) testthat::expect_gte(length(chunks), min_count) if (!is.null(max_count)) testthat::expect_lte(length(chunks), max_count) @@ -252,6 +254,18 @@ assert_chunks <- function(result, min_count = NULL, max_count = NULL, if (isFALSE(each_has_heading_context)) { for (chunk in chunks) testthat::expect_true(is.null(chunk$metadata$heading_context)) } + if (isTRUE(each_has_chunk_type)) { + for (chunk in chunks) { + chunk_type <- chunk[["chunk_type"]] + if (is.null(chunk_type) || chunk_type == "unknown") { + testthat::fail(paste0("Expected specific chunk_type, got: ", chunk_type)) + } + } + } + if (isTRUE(content_starts_with_heading)) { + heading_char <- rawToChar(as.raw(35)) + for (chunk in chunks) testthat::expect_true(startsWith(chunk$content %||% "", heading_char)) + } } assert_images <- function(result, min_count = NULL, max_count = NULL, formats_include = NULL) { diff --git a/tools/e2e-generator/e2e/ruby/spec/helpers.rb b/tools/e2e-generator/e2e/ruby/spec/helpers.rb index d21bf81edef..266301ca9de 100644 --- a/tools/e2e-generator/e2e/ruby/spec/helpers.rb +++ b/tools/e2e-generator/e2e/ruby/spec/helpers.rb @@ -230,7 +230,7 @@ def self.assert_metadata_expectation(result, path, expectation) end end - def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil) + def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, each_has_chunk_type: nil, content_starts_with_heading: nil) chunks = Array(result.chunks) expect(chunks.length).to be >= min_count if min_count expect(chunks.length).to be <= max_count if max_count @@ -241,6 +241,15 @@ def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: elsif each_has_heading_context == false chunks.each { |chunk| expect(chunk.metadata&.heading_context).to be_nil } end + if each_has_chunk_type == true + chunks.each { |chunk| + expect(chunk.chunk_type).not_to be_nil + expect(chunk.chunk_type).not_to eq("unknown") + } + end + return unless content_starts_with_heading == true + + chunks.each { |chunk| expect(chunk.content).to start_with('#') } end def self.assert_images(result, min_count: nil, max_count: nil, formats_include: nil) diff --git a/tools/e2e-generator/e2e/typescript/tests/helpers.ts b/tools/e2e-generator/e2e/typescript/tests/helpers.ts index d5a29e83619..cb131978a12 100644 --- a/tools/e2e-generator/e2e/typescript/tests/helpers.ts +++ b/tools/e2e-generator/e2e/typescript/tests/helpers.ts @@ -726,6 +726,7 @@ export const chunkAssertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -759,6 +760,13 @@ export const chunkAssertions = { expect(((chunk as PlainRecord).metadata as PlainRecord)?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const headingContext = ((chunk as PlainRecord).metadata as PlainRecord | undefined)?.headingContext; diff --git a/tools/e2e-generator/e2e/wasm-deno/helpers.ts b/tools/e2e-generator/e2e/wasm-deno/helpers.ts index d78edc59fbc..3c56e93849d 100644 --- a/tools/e2e-generator/e2e/wasm-deno/helpers.ts +++ b/tools/e2e-generator/e2e/wasm-deno/helpers.ts @@ -402,6 +402,8 @@ export const assertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, + contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; assertExists(chunks, "Expected chunks to be defined"); @@ -438,6 +440,23 @@ export const assertions = { ); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + assertExists(chunkType, "Chunk missing chunk_type"); + assertEquals(chunkType !== "unknown", true, "Chunk chunk_type should not be 'unknown'"); + } + } + if (contentStartsWithHeading === true) { + for (const chunk of chunks) { + const content = (chunk as PlainRecord).content; + assertEquals( + typeof content === "string" && (content as string).charCodeAt(0) === 35, + true, + "Chunk content should start with heading", + ); + } + } }, assertImages( diff --git a/tools/e2e-generator/e2e/wasm-workers/tests/helpers.ts b/tools/e2e-generator/e2e/wasm-workers/tests/helpers.ts index f51c836a572..9685a2a042a 100644 --- a/tools/e2e-generator/e2e/wasm-workers/tests/helpers.ts +++ b/tools/e2e-generator/e2e/wasm-workers/tests/helpers.ts @@ -353,6 +353,8 @@ export const assertions = { eachHasContent: boolean | null, eachHasEmbedding: boolean | null, eachHasHeadingContext: boolean | null, + eachHasChunkType: boolean | null, + contentStartsWithHeading: boolean | null, ): void { const chunks = Array.isArray(result.chunks) ? result.chunks : []; if (typeof minCount === "number") { @@ -381,6 +383,18 @@ export const assertions = { expect(chunk.metadata?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } + if (contentStartsWithHeading === true) { + for (const chunk of chunks) { + expect(typeof chunk.content === "string" && chunk.content.charCodeAt(0) === 35).toBe(true); + } + } }, assertImages( diff --git a/tools/e2e-generator/src/c.rs b/tools/e2e-generator/src/c.rs index 7dd3325d82d..28dba41c186 100644 --- a/tools/e2e-generator/src/c.rs +++ b/tools/e2e-generator/src/c.rs @@ -97,7 +97,8 @@ void assert_detected_languages(const CExtractionResult *result, void assert_chunks(const CExtractionResult *result, int has_min, size_t min_count, - int has_max, size_t max_count); + int has_max, size_t max_count, + int each_has_chunk_type); void assert_images(const CExtractionResult *result, int has_min, size_t min_count, @@ -541,7 +542,8 @@ void assert_detected_languages(const CExtractionResult *result, void assert_chunks(const CExtractionResult *result, int has_min, size_t min_count, - int has_max, size_t max_count) { + int has_max, size_t max_count, + int each_has_chunk_type) { size_t count = json_array_count(result->chunks_json); if (has_min && count < min_count) { fprintf(stderr, @@ -555,6 +557,14 @@ void assert_chunks(const CExtractionResult *result, max_count, count); exit(1); } + if (each_has_chunk_type && result->chunks_json) { + /* Very simple check: ensure no "unknown" chunk_type in the JSON */ + if (strstr(result->chunks_json, "\"unknown\"") != NULL || + strstr(result->chunks_json, "\"chunk_type\":null") != NULL) { + fprintf(stderr, "FAIL: expected specific chunk_type, but found unknown/null in chunks_json\n"); + exit(1); + } + } } void assert_images(const CExtractionResult *result, @@ -1138,7 +1148,12 @@ fn render_assertions(assertions: &Assertions) -> String { let min = chunks.min_count.unwrap_or(0); let has_max = chunks.max_count.is_some() as u8; let max = chunks.max_count.unwrap_or(0); - writeln!(buf, " assert_chunks(result, {has_min}, {min}, {has_max}, {max});").unwrap(); + let each_has_chunk_type = chunks.each_has_chunk_type.unwrap_or(false) as u8; + writeln!( + buf, + " assert_chunks(result, {has_min}, {min}, {has_max}, {max}, {each_has_chunk_type});" + ) + .unwrap(); } if let Some(images) = assertions.images.as_ref() { let has_min = images.min_count.is_some() as u8; diff --git a/tools/e2e-generator/src/csharp.rs b/tools/e2e-generator/src/csharp.rs index d53cb4738d5..ad68b977b2f 100644 --- a/tools/e2e-generator/src/csharp.rs +++ b/tools/e2e-generator/src/csharp.rs @@ -497,6 +497,7 @@ public static class TestHelpers bool? eachHasContent, bool? eachHasEmbedding, bool? eachHasHeadingContext = null, + bool? eachHasChunkType = null, bool? contentStartsWithHeading = null) { var chunks = result.Chunks; @@ -553,6 +554,17 @@ public static class TestHelpers } } } + if (eachHasChunkType == true) + { + for (var i = 0; i < chunks.Count; i++) + { + var type = chunks[i].ChunkType; + if (string.IsNullOrEmpty(type) || type == "unknown") + { + throw new XunitException($"Chunk {i} has no specific chunk_type, got {type}"); + } + } + } if (contentStartsWithHeading == true) { var headingChar = new string(new[] { (char)35 }); @@ -1323,20 +1335,17 @@ fn render_assertions(buffer: &mut String, assertions: &Assertions) -> Result<()> .each_has_heading_context .map(|v| if v { "true" } else { "false" }.to_string()) .unwrap_or_else(|| "null".to_string()); - let content_starts_with_heading = chunks - .content_starts_with_heading + let each_has_chunk_type = chunks + .each_has_chunk_type .map(|v| if v { "true" } else { "false" }.to_string()) .unwrap_or_else(|| "null".to_string()); - writeln!( - buffer, - " TestHelpers.AssertChunks(result, {}, {}, {}, {}, {}, {});", - min_count, - max_count, - each_has_content, - each_has_embedding, - each_has_heading_context, - content_starts_with_heading - )?; + let content_starts_with_heading = chunks + .content_starts_with_heading + .map(|v| v.to_string().to_lowercase()) + .unwrap_or_else(|| "null".into()); + buffer.push_str(&format!( + " TestHelpers.AssertChunks(result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" + )); } if let Some(images) = assertions.images.as_ref() { diff --git a/tools/e2e-generator/src/elixir.rs b/tools/e2e-generator/src/elixir.rs index bfe898cef13..d92363776e0 100644 --- a/tools/e2e-generator/src/elixir.rs +++ b/tools/e2e-generator/src/elixir.rs @@ -387,6 +387,12 @@ defmodule E2E.Helpers do end end + if opts[:each_has_chunk_type] == true do + if !Enum.all?(chunks, fn chunk -> chunk.chunk_type && chunk.chunk_type != "unknown" end) do + flunk("Not all chunks have a specific chunk_type") + end + end + if opts[:content_starts_with_heading] == true do chunks_with_heading = Enum.filter(chunks, fn chunk -> chunk.metadata && chunk.metadata.heading_context != nil @@ -1149,6 +1155,9 @@ fn render_assertions(assertions: &Assertions) -> String { if let Some(has_heading_context) = chunks.each_has_heading_context { args.push(format!("each_has_heading_context: {}", has_heading_context)); } + if let Some(has_chunk_type) = chunks.each_has_chunk_type { + args.push(format!("each_has_chunk_type: {}", has_chunk_type)); + } if let Some(starts_with_heading) = chunks.content_starts_with_heading { args.push(format!("content_starts_with_heading: {}", starts_with_heading)); } diff --git a/tools/e2e-generator/src/fixtures.rs b/tools/e2e-generator/src/fixtures.rs index 27fa774d27f..3cd3552ba66 100644 --- a/tools/e2e-generator/src/fixtures.rs +++ b/tools/e2e-generator/src/fixtures.rs @@ -271,6 +271,8 @@ pub struct ChunkAssertion { #[serde(default)] pub each_has_heading_context: Option, #[serde(default)] + pub each_has_chunk_type: Option, + #[serde(default)] pub content_starts_with_heading: Option, } diff --git a/tools/e2e-generator/src/go.rs b/tools/e2e-generator/src/go.rs index e3c4d9b68e3..6995335810c 100644 --- a/tools/e2e-generator/src/go.rs +++ b/tools/e2e-generator/src/go.rs @@ -250,7 +250,7 @@ func boolPtr(value bool) *bool { return &value } -func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, contentStartsWithHeading *bool) { +func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, eachHasChunkType, contentStartsWithHeading *bool) { t.Helper() count := len(result.Chunks) if minCount != nil && count < *minCount { @@ -287,6 +287,13 @@ func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, ma } } } + if eachHasChunkType != nil && *eachHasChunkType { + for i, chunk := range result.Chunks { + if chunk.ChunkType == "" || chunk.ChunkType == "unknown" { + t.Fatalf("chunk %d has no specific chunk_type, got %q", i, chunk.ChunkType) + } + } + } if contentStartsWithHeading != nil && *contentStartsWithHeading { for i, chunk := range result.Chunks { if chunk.Metadata == nil || chunk.Metadata.HeadingContext == nil { @@ -980,21 +987,17 @@ fn render_assertions(assertions: &Assertions) -> String { .each_has_heading_context .map(|v| format!("boolPtr({v})")) .unwrap_or_else(|| "nil".to_string()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| format!("boolPtr({v})")) + .unwrap_or_else(|| "nil".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| format!("boolPtr({v})")) - .unwrap_or_else(|| "nil".to_string()); - writeln!( - buffer, - " assertChunks(t, result, {}, {}, {}, {}, {}, {})", - min_count, - max_count, - each_has_content, - each_has_embedding, - each_has_heading_context, - content_starts_with_heading - ) - .unwrap(); + .unwrap_or_else(|| "nil".into()); + buffer.push_str(&format!( + " assertChunks(t, result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading})\n" + )); } if let Some(images) = assertions.images.as_ref() { let min_count = images diff --git a/tools/e2e-generator/src/java.rs b/tools/e2e-generator/src/java.rs index b6bbd7524fa..52cddc1dcee 100644 --- a/tools/e2e-generator/src/java.rs +++ b/tools/e2e-generator/src/java.rs @@ -382,6 +382,7 @@ public final class E2EHelpers { Boolean eachHasContent, Boolean eachHasEmbedding, Boolean eachHasHeadingContext, + Boolean eachHasChunkType, Boolean contentStartsWithHeading ) { var chunks = result.getChunks(); @@ -419,6 +420,13 @@ public final class E2EHelpers { "Expected each chunk to have no heading_context"); } } + if (chunks != null && eachHasChunkType != null && eachHasChunkType) { + for (var chunk : chunks) { + String type = chunk.getChunkType(); + assertTrue(type != null && !"unknown".equals(type), + "Expected each chunk to have a specific chunk_type"); + } + } if (chunks != null && contentStartsWithHeading != null && contentStartsWithHeading) { String headingPrefix = String.valueOf((char) 35); for (var chunk : chunks) { @@ -1670,25 +1678,29 @@ fn render_assertions(assertions: &Assertions) -> String { .max_count .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()); - let has_content = chunks + let each_has_content = chunks .each_has_content .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()); - let has_embedding = chunks + let each_has_embedding = chunks .each_has_embedding .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()); - let has_heading_context = chunks + let each_has_heading_context = chunks .each_has_heading_context .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| v.to_string()) + .unwrap_or_else(|| "null".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| v.to_string()) - .unwrap_or_else(|| "null".to_string()); + .unwrap_or_else(|| "null".into()); buffer.push_str(&format!( - " E2EHelpers.Assertions.assertChunks(result, {}, {}, {}, {}, {}, {});\n", - min_literal, max_literal, has_content, has_embedding, has_heading_context, content_starts_with_heading + " E2EHelpers.Assertions.assertChunks(result, {}, {}, {}, {}, {}, {}, {});\n", + min_literal, max_literal, each_has_content, each_has_embedding, each_has_heading_context, each_has_chunk_type, content_starts_with_heading )); } diff --git a/tools/e2e-generator/src/manifest.rs b/tools/e2e-generator/src/manifest.rs index d4febe3f607..bedc2e03126 100644 --- a/tools/e2e-generator/src/manifest.rs +++ b/tools/e2e-generator/src/manifest.rs @@ -256,6 +256,7 @@ fn sample_extraction_result() -> Value { let chunk = kreuzberg::types::extraction::Chunk { content: "chunk text".to_string(), + chunk_type: Default::default(), embedding: Some(vec![0.1, 0.2]), metadata: kreuzberg::types::extraction::ChunkMetadata { byte_start: 0, diff --git a/tools/e2e-generator/src/php.rs b/tools/e2e-generator/src/php.rs index a9d990ecdaf..95dbde20acc 100644 --- a/tools/e2e-generator/src/php.rs +++ b/tools/e2e-generator/src/php.rs @@ -215,65 +215,49 @@ class Helpers } public static function assertChunks( - ExtractionResult $result, - ?int $minCount, - ?int $maxCount, - ?bool $eachHasContent, - ?bool $eachHasEmbedding, + $result, + ?int $minCount = null, + ?int $maxCount = null, + ?bool $eachHasContent = null, + ?bool $eachHasEmbedding = null, ?bool $eachHasHeadingContext = null, + ?bool $eachHasChunkType = null, ?bool $contentStartsWithHeading = null ): void { - $chunks = $result->chunks ?? []; - $count = count($chunks); - - if ($minCount !== null) { - Assert::assertGreaterThanOrEqual( - $minCount, - $count, - sprintf("Expected at least %d chunks, found %d", $minCount, $count) - ); + $chunks = $result->chunks ?? null; + if ($chunks === null) { + throw new \Exception("Expected chunks but field is null"); } - if ($maxCount !== null) { - Assert::assertLessThanOrEqual( - $maxCount, - $count, - sprintf("Expected at most %d chunks, found %d", $maxCount, $count) - ); + $count = count($chunks); + if ($minCount !== null && $count < $minCount) { + throw new \Exception("Expected at least $minCount chunks, found $count"); } - - if ($eachHasContent === true) { - foreach ($chunks as $i => $chunk) { - Assert::assertNotEmpty( - $chunk->content ?? '', - sprintf("Chunk %d should have content", $i) - ); - } + if ($maxCount !== null && $count > $maxCount) { + throw new \Exception("Expected at most $maxCount chunks, found $count"); } - if ($eachHasEmbedding === true) { - foreach ($chunks as $i => $chunk) { - Assert::assertNotNull( - $chunk->embedding ?? null, - sprintf("Chunk %d should have embedding", $i) - ); + foreach ($chunks as $i => $chunk) { + if ($eachHasContent && empty($chunk->content)) { + throw new \Exception("Chunk $i has no content"); } - } - - if ($eachHasHeadingContext === true) { - foreach ($chunks as $i => $chunk) { - Assert::assertNotNull( - $chunk->metadata->heading_context ?? null, - sprintf("Chunk %d should have heading_context", $i) - ); + if ($eachHasEmbedding && (empty($chunk->embedding))) { + throw new \Exception("Chunk $i has no embedding"); } - } - if ($eachHasHeadingContext === false) { - foreach ($chunks as $i => $chunk) { - Assert::assertNull( - $chunk->metadata->heading_context ?? null, - sprintf("Chunk %d should have no heading_context", $i) - ); + if ($eachHasHeadingContext !== null) { + $hc = $chunk->metadata->headingContext ?? null; + if ($eachHasHeadingContext && $hc === null) { + throw new \Exception("Chunk $i has no headingContext"); + } + if (!$eachHasHeadingContext && $hc !== null) { + throw new \Exception("Chunk $i should have no headingContext"); + } + } + if ($eachHasChunkType === true) { + $type = $chunk->chunkType ?? $chunk->chunk_type ?? null; + if ($type === null || $type === "unknown") { + throw new \Exception("Chunk $i has no specific chunkType, got " . var_export($type, true)); + } } } if ($contentStartsWithHeading === true) { @@ -861,46 +845,49 @@ class Helpers Assert::assertGreaterThanOrEqual($minLength, strlen($data), sprintf('Expected at least %d bytes, got %d', $minLength, strlen($data))); } + + } } "#; -pub fn generate(fixtures: &[Fixture], output_root: &Utf8Path) -> Result<()> { - let php_root = output_root.join("php"); - let tests_dir = php_root.join("tests"); - +pub fn generate(fixtures: &[Fixture], output_dir: &Utf8Path) -> Result<()> { + let tests_dir = output_dir.join("e2e").join("php").join("tests"); fs::create_dir_all(&tests_dir).context("Failed to create PHP tests directory")?; - clean_tests(&tests_dir)?; write_helpers(&tests_dir)?; - let doc_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_document_extraction()).collect(); - let api_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_plugin_api()).collect(); + let tests_pkg_dir = tests_dir.join("E2EPhp").join("Tests"); + fs::create_dir_all(&tests_pkg_dir).context("Failed to create PHP test package directory")?; + + let mut categories = BTreeMap::new(); + for fixture in fixtures { + if fixture.is_document_extraction() { + categories + .entry(fixture.category().to_string()) + .or_insert_with(Vec::new) + .push(fixture); + } + } - let mut grouped = doc_fixtures - .into_iter() - .into_group_map_by(|fixture| fixture.category().to_string()) - .into_iter() - .collect::>(); - grouped.sort_by(|a, b| a.0.cmp(&b.0)); - - for (category, mut fixtures) in grouped { - fixtures.sort_by(|a, b| a.id.cmp(&b.id)); - let filename = format!("{}Test.php", capitalize(&category)); - let content = render_category(&category, &fixtures)?; - fs::write(tests_dir.join(&filename), content) + for (category, fixtures) in &categories { + let filename = format!("{}Test.php", capitalize(category)); + let content = render_category(category, fixtures)?; + fs::write(tests_pkg_dir.join(&filename), content) .with_context(|| format!("Failed to write PHP test file {filename}"))?; } + let api_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_plugin_api()).collect(); if !api_fixtures.is_empty() { - generate_plugin_api_tests(&api_fixtures, &tests_dir)?; + generate_plugin_api_tests(&api_fixtures, &tests_pkg_dir)?; } let render_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_render()).collect(); if !render_fixtures.is_empty() { - let mut sorted = render_fixtures; + let mut sorted: Vec<_> = render_fixtures.clone(); sorted.sort_by(|a, b| a.id.cmp(&b.id)); let content = render_render_category_php(&sorted)?; - fs::write(tests_dir.join("RenderTest.php"), content).context("Failed to write PHP render test file")?; + fs::write(tests_pkg_dir.join("RenderTest.php"), content) + .context("Failed to write PHP render test file")?; } Ok(()) diff --git a/tools/e2e-generator/src/python.rs b/tools/e2e-generator/src/python.rs index 12d5e589a77..5fd53d44db8 100644 --- a/tools/e2e-generator/src/python.rs +++ b/tools/e2e-generator/src/python.rs @@ -365,6 +365,13 @@ def _assert_chunks_heading_context(chunks: Any, expected: bool) -> None: pytest.fail(f"Chunk {i} should have no heading_context") +def _assert_chunks_chunk_type(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + chunk_type = getattr(chunk, "chunk_type", None) + if chunk_type is None or str(chunk_type) == "unknown": + pytest.fail(f"Chunk {i} has no specific chunk_type, got {chunk_type}") + + def _assert_chunks_heading_prefix(chunks: Any) -> None: for i, chunk in enumerate(chunks): meta = getattr(chunk, "metadata", None) @@ -383,6 +390,7 @@ def assert_chunks( each_has_content: bool | None = None, each_has_embedding: bool | None = None, each_has_heading_context: bool | None = None, + each_has_chunk_type: bool | None = None, content_starts_with_heading: bool | None = None, ) -> None: chunks = getattr(result, "chunks", None) @@ -399,6 +407,8 @@ def assert_chunks( _assert_chunks_embedding(chunks) if each_has_heading_context is not None: _assert_chunks_heading_context(chunks, each_has_heading_context) + if each_has_chunk_type: + _assert_chunks_chunk_type(chunks) if content_starts_with_heading: _assert_chunks_heading_prefix(chunks) @@ -1067,6 +1077,12 @@ fn render_assertions(assertions: &Assertions) -> String { if has_heading_context { "True" } else { "False" } )); } + if let Some(has_chunk_type) = chunks.each_has_chunk_type { + args.push(format!( + "each_has_chunk_type={}", + if has_chunk_type { "True" } else { "False" } + )); + } if let Some(starts_with_heading) = chunks.content_starts_with_heading { args.push(format!( "content_starts_with_heading={}", diff --git a/tools/e2e-generator/src/r/assertions.rs b/tools/e2e-generator/src/r/assertions.rs index 90e22e17cd8..c36b8963ad4 100644 --- a/tools/e2e-generator/src/r/assertions.rs +++ b/tools/e2e-generator/src/r/assertions.rs @@ -114,6 +114,12 @@ pub fn render_assertions(assertions: &Assertions) -> String { if has_heading_context { "TRUE" } else { "FALSE" } )); } + if let Some(has_chunk_type) = chunks.each_has_chunk_type { + args.push(format!( + "each_has_chunk_type = {}", + if has_chunk_type { "TRUE" } else { "FALSE" } + )); + } if let Some(starts_with_heading) = chunks.content_starts_with_heading { args.push(format!( "content_starts_with_heading = {}", diff --git a/tools/e2e-generator/src/r/helpers.rs b/tools/e2e-generator/src/r/helpers.rs index 6b3d7f58e9e..4919472bf24 100644 --- a/tools/e2e-generator/src/r/helpers.rs +++ b/tools/e2e-generator/src/r/helpers.rs @@ -216,6 +216,7 @@ assert_metadata_expectation <- function(result, path, expectation) { assert_chunks <- function(result, min_count = NULL, max_count = NULL, each_has_content = NULL, each_has_embedding = NULL, each_has_heading_context = NULL, + each_has_chunk_type = NULL, content_starts_with_heading = NULL) { chunks <- if (is.null(result$chunks)) list() else result$chunks if (!is.null(min_count)) testthat::expect_gte(length(chunks), min_count) @@ -232,6 +233,14 @@ assert_chunks <- function(result, min_count = NULL, max_count = NULL, if (isFALSE(each_has_heading_context)) { for (chunk in chunks) testthat::expect_true(is.null(chunk$metadata$heading_context)) } + if (isTRUE(each_has_chunk_type)) { + for (chunk in chunks) { + chunk_type <- chunk[["chunk_type"]] + if (is.null(chunk_type) || chunk_type == "unknown") { + testthat::fail(paste0("Expected specific chunk_type, got: ", chunk_type)) + } + } + } if (isTRUE(content_starts_with_heading)) { heading_char <- rawToChar(as.raw(35)) for (chunk in chunks) { diff --git a/tools/e2e-generator/src/ruby.rs b/tools/e2e-generator/src/ruby.rs index aa103d99124..f08e6d26733 100644 --- a/tools/e2e-generator/src/ruby.rs +++ b/tools/e2e-generator/src/ruby.rs @@ -249,7 +249,7 @@ module E2ERuby end end - def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, content_starts_with_heading: nil) + def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, each_has_chunk_type: nil, content_starts_with_heading: nil) chunks = Array(result.chunks) expect(chunks.length).to be >= min_count if min_count expect(chunks.length).to be <= max_count if max_count @@ -260,6 +260,12 @@ module E2ERuby elsif each_has_heading_context == false chunks.each { |chunk| expect(chunk.metadata&.heading_context).to be_nil } end + if each_has_chunk_type == true + chunks.each do |chunk| + expect(chunk.chunk_type).not_to be_nil + expect(chunk.chunk_type).not_to eq('unknown') + end + end return unless content_starts_with_heading == true chunks.each do |chunk| @@ -893,6 +899,9 @@ fn render_assertions(assertions: &Assertions) -> String { if let Some(has_heading_context) = chunks.each_has_heading_context { args.push(format!("each_has_heading_context: {}", has_heading_context)); } + if let Some(has_chunk_type) = chunks.each_has_chunk_type { + args.push(format!("each_has_chunk_type: {}", has_chunk_type)); + } if let Some(starts_with_heading) = chunks.content_starts_with_heading { args.push(format!("content_starts_with_heading: {}", starts_with_heading)); } diff --git a/tools/e2e-generator/src/rust.rs b/tools/e2e-generator/src/rust.rs index 213da3435f7..33868f25328 100644 --- a/tools/e2e-generator/src/rust.rs +++ b/tools/e2e-generator/src/rust.rs @@ -11,14 +11,19 @@ use std::io::Write; pub fn generate(fixtures: &[Fixture], output_root: &Utf8Path) -> Result<()> { let rust_root = output_root.join("rust"); let tests_dir = rust_root.join("tests"); + let fixtures_dir = tests_dir.join("fixtures"); - fs::create_dir_all(&tests_dir).context("Failed to create Rust tests directory")?; + fs::create_dir_all(&fixtures_dir).context("Failed to create Rust tests fixtures directory")?; write_cargo_toml(&rust_root)?; clean_rs_files(&tests_dir)?; + clean_rs_files(&fixtures_dir)?; let doc_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_document_extraction()).collect(); let api_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_plugin_api()).collect(); + let mut render_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_render()).collect(); + + let mut generated_modules = Vec::new(); let mut grouped = doc_fixtures .into_iter() @@ -29,24 +34,40 @@ pub fn generate(fixtures: &[Fixture], output_root: &Utf8Path) -> Result<()> { for (category, mut fixtures) in grouped { fixtures.sort_by(|a, b| a.id.cmp(&b.id)); - let file_name = format!("{}_test.rs", sanitize_identifier(&category)); + let module_name = format!("{}_test", sanitize_identifier(&category)); + let file_name = format!("{}.rs", module_name); let content = render_category(&category, &fixtures)?; - let path = tests_dir.join(file_name); + let path = fixtures_dir.join(file_name); fs::write(&path, content).with_context(|| format!("Writing {}", path))?; + generated_modules.push(module_name); } if !api_fixtures.is_empty() { - generate_plugin_api_tests(&api_fixtures, &tests_dir)?; + generate_plugin_api_tests(&api_fixtures, &fixtures_dir)?; + generated_modules.push("plugin_apis_test".to_string()); } - let mut render_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_render()).collect(); if !render_fixtures.is_empty() { render_fixtures.sort_by(|a, b| a.id.cmp(&b.id)); let content = render_render_tests(&render_fixtures)?; - let path = tests_dir.join("render_test.rs"); + let path = fixtures_dir.join("render_test.rs"); fs::write(&path, content).with_context(|| format!("Writing {}", path))?; + generated_modules.push("render_test".to_string()); } + // Generate fixtures/mod.rs + let mut mod_content = + "// Code generated by kreuzberg-e2e-generator. DO NOT EDIT.\n\n".to_string(); + generated_modules.sort(); + for module in generated_modules { + writeln!(mod_content, "pub mod {};", module)?; + } + fs::write(fixtures_dir.join("mod.rs"), mod_content).context("Failed to write fixtures/mod.rs")?; + + // Generate tests/e2e.rs as the single entry point + let e2e_content = "// Code generated by kreuzberg-e2e-generator. DO NOT EDIT.\n\nmod fixtures;\n"; + fs::write(tests_dir.join("e2e.rs"), e2e_content).context("Failed to write tests/e2e.rs")?; + Ok(()) } @@ -85,8 +106,9 @@ fn clean_rs_files(dir: &Utf8Path) -> Result<()> { for entry in fs::read_dir(dir.as_std_path())? { let entry = entry?; - if entry.path().extension().is_some_and(|ext| ext == "rs") { - fs::remove_file(entry.path())?; + let path = entry.path(); + if path.extension().is_some_and(|ext| ext == "rs") { + fs::remove_file(path)?; } } @@ -356,12 +378,16 @@ fn render_assertions(assertions: &Assertions) -> String { .each_has_heading_context .map(|v| format!("Some({v})")) .unwrap_or_else(|| "None".into()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| format!("Some({v})")) + .unwrap_or_else(|| "None".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| format!("Some({v})")) .unwrap_or_else(|| "None".into()); buffer.push_str(&format!( - " assertions::assert_chunks(&result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {content_starts_with_heading});\n" + " assertions::assert_chunks(&result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" )); } diff --git a/tools/e2e-generator/src/typescript.rs b/tools/e2e-generator/src/typescript.rs index 8a04ae967db..67d7c767c19 100644 --- a/tools/e2e-generator/src/typescript.rs +++ b/tools/e2e-generator/src/typescript.rs @@ -765,6 +765,7 @@ export const chunkAssertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -798,6 +799,13 @@ export const chunkAssertions = { expect(((chunk as PlainRecord).metadata as PlainRecord)?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const meta = (chunk as PlainRecord).metadata as PlainRecord | undefined; @@ -1361,26 +1369,30 @@ fn render_assertions(assertions: &Assertions) -> String { } if let Some(chunks) = assertions.chunks.as_ref() { - let min = chunks.min_count.map(|v| v.to_string()).unwrap_or_else(|| "null".into()); - let max = chunks.max_count.map(|v| v.to_string()).unwrap_or_else(|| "null".into()); - let has_content = chunks + let min_count = chunks.min_count.map(|v| v.to_string()).unwrap_or_else(|| "null".into()); + let max_count = chunks.max_count.map(|v| v.to_string()).unwrap_or_else(|| "null".into()); + let each_has_content = chunks .each_has_content .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); - let has_embedding = chunks + let each_has_embedding = chunks .each_has_embedding .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); - let has_heading_context = chunks + let each_has_heading_context = chunks .each_has_heading_context .map(|v| v.to_string()) - .unwrap_or_else(|| "null".into()); + .unwrap_or_else(|| "undefined".into()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| v.to_string()) + .unwrap_or_else(|| "undefined".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| v.to_string()) - .unwrap_or_else(|| "null".into()); + .unwrap_or_else(|| "undefined".into()); buffer.push_str(&format!( - " chunkAssertions.assertChunks(result, {min}, {max}, {has_content}, {has_embedding}, {has_heading_context}, {content_starts_with_heading});\n" + " chunkAssertions.assertChunks(result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" )); } diff --git a/tools/e2e-generator/src/wasm_deno.rs b/tools/e2e-generator/src/wasm_deno.rs index 71965dbeb25..8d140294d85 100644 --- a/tools/e2e-generator/src/wasm_deno.rs +++ b/tools/e2e-generator/src/wasm_deno.rs @@ -442,6 +442,7 @@ export const assertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -475,6 +476,13 @@ export const assertions = { assertEquals(((chunk as PlainRecord).metadata as PlainRecord)?.headingContext ?? null, null, "Chunk should have no heading_context"); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + assertExists(chunkType, "Chunk missing chunk_type"); + assertEquals(chunkType !== "unknown", true, "Chunk chunk_type should not be 'unknown'"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const meta = (chunk as PlainRecord).metadata as PlainRecord | undefined; @@ -1158,12 +1166,16 @@ fn render_assertions(assertions: &Assertions, _requirements: &[String]) -> Strin .each_has_heading_context .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| v.to_string()) + .unwrap_or_else(|| "null".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); buffer.push_str(&format!( - " assertions.assertChunks(result, {min}, {max}, {has_content}, {has_embedding}, {has_heading_context}, {content_starts_with_heading});\n" + " assertions.assertChunks(result, {min}, {max}, {has_content}, {has_embedding}, {has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" )); } diff --git a/tools/e2e-generator/src/wasm_workers.rs b/tools/e2e-generator/src/wasm_workers.rs index 20e513ca863..e57ce0b1c5d 100644 --- a/tools/e2e-generator/src/wasm_workers.rs +++ b/tools/e2e-generator/src/wasm_workers.rs @@ -411,6 +411,7 @@ export const assertions = { eachHasContent: boolean | null, eachHasEmbedding: boolean | null, eachHasHeadingContext: boolean | null, + eachHasChunkType: boolean | null, contentStartsWithHeading: boolean | null, ): void { const chunks = Array.isArray(result.chunks) ? result.chunks : []; @@ -440,6 +441,13 @@ export const assertions = { expect(chunk.metadata?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { if (chunk.metadata?.headingContext == null) continue; @@ -1097,12 +1105,16 @@ fn render_assertions(assertions: &Assertions) -> String { .each_has_heading_context .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| v.to_string()) + .unwrap_or_else(|| "null".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); buffer.push_str(&format!( - " assertions.assertChunks(result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {content_starts_with_heading});\n" + " assertions.assertChunks(result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" )); }