From 68906ae738a57a19be538213995a75978cb3cce3 Mon Sep 17 00:00:00 2001 From: kh3rld Date: Tue, 31 Mar 2026 08:46:37 -0400 Subject: [PATCH 01/12] feat(types): add ChunkType enum and chunk_type field to Chunk (#600) Extend Chunk with a chunk_type field classifying each chunk's structural role. ChunkType has 13 variants: heading, party_list, definitions, operative_clause, signature_block, schedule, table_like, formula, code_block, image, org_chart, diagram, unknown. Field is serde(default) so existing JSON without it deserializes cleanly. --- crates/kreuzberg/src/types/extraction.rs | 45 ++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/crates/kreuzberg/src/types/extraction.rs b/crates/kreuzberg/src/types/extraction.rs index d9e4872f2cc..77d65299a59 100644 --- a/crates/kreuzberg/src/types/extraction.rs +++ b/crates/kreuzberg/src/types/extraction.rs @@ -202,6 +202,44 @@ pub struct ProcessingWarning { pub message: Cow<'static, str>, } +/// Semantic structural classification of a text chunk. +/// +/// Assigned by the heuristic classifier in `chunking::classifier`. +/// Defaults to `Unknown` when no rule matches. +/// Designed to be extended in future versions without breaking changes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "api", derive(utoipa::ToSchema))] +#[serde(rename_all = "snake_case")] +pub enum ChunkType { + /// Section heading or document title. + Heading, + /// Party list: names, addresses, and signatories. + PartyList, + /// Definition clause ("X means…", "X shall mean…"). + Definitions, + /// Operative clause containing legal/contractual action verbs. + OperativeClause, + /// Signature block with signatures, names, and dates. + SignatureBlock, + /// Schedule, annex, appendix, or exhibit section. + Schedule, + /// Table-like content with aligned columns or repeated patterns. + TableLike, + /// Mathematical formula or equation. + Formula, + /// Code block or preformatted content. + CodeBlock, + /// Embedded or referenced image content. + Image, + /// Organizational chart or hierarchy diagram. + OrgChart, + /// Diagram, figure, or visual illustration. + Diagram, + /// Unclassified or mixed content. + #[default] + Unknown, +} + /// A text chunk with optional embedding and metadata. /// /// Chunks are created when chunking is enabled in `ExtractionConfig`. Each chunk @@ -213,6 +251,13 @@ pub struct Chunk { /// The text content of this chunk. pub content: String, + /// Semantic structural classification of this chunk. + /// + /// Assigned by the heuristic classifier based on content patterns and + /// heading context. Defaults to `ChunkType::Unknown` when no rule matches. + #[serde(default)] + pub chunk_type: ChunkType, + /// Optional embedding vector for this chunk. /// /// Only populated when `EmbeddingConfig` is provided in chunking configuration. From 6b748fb3c8130166e7e62a9f374c7af33ae1f72e Mon Sep 17 00:00:00 2001 From: kh3rld Date: Tue, 31 Mar 2026 08:46:43 -0400 Subject: [PATCH 02/12] feat(chunking): add heuristic classifier and wire into chunk builder Add classifier.rs with 9 ordered rules (heading, code_block, table_like, formula, schedule, definitions, signature_block, operative_clause, party_list) covering all proposed ChunkType variants. First match wins; falls back to Unknown. Includes 20 unit tests. Wire classify_chunk() into build_chunks() so every produced chunk gets a semantic label automatically. --- crates/kreuzberg/src/chunking/builder.rs | 2 + crates/kreuzberg/src/chunking/classifier.rs | 436 ++++++++++++++++++ crates/kreuzberg/src/chunking/mod.rs | 2 + crates/kreuzberg/src/chunking/yaml_section.rs | 2 + 4 files changed, 442 insertions(+) create mode 100644 crates/kreuzberg/src/chunking/classifier.rs diff --git a/crates/kreuzberg/src/chunking/builder.rs b/crates/kreuzberg/src/chunking/builder.rs index df287bb4b96..a3bd05ae735 100644 --- a/crates/kreuzberg/src/chunking/builder.rs +++ b/crates/kreuzberg/src/chunking/builder.rs @@ -8,6 +8,7 @@ use crate::types::{Chunk, ChunkMetadata, PageBoundary}; use text_splitter::{Characters, ChunkCapacity, ChunkConfig}; use super::boundaries::calculate_page_range; +use super::classifier::classify_chunk; /// Build a ChunkConfig from chunking parameters. /// @@ -78,6 +79,7 @@ where chunks.push(Chunk { content: chunk_text.to_string(), + chunk_type: classify_chunk(chunk_text, None), embedding: None, metadata: ChunkMetadata { byte_start, diff --git a/crates/kreuzberg/src/chunking/classifier.rs b/crates/kreuzberg/src/chunking/classifier.rs new file mode 100644 index 00000000000..db17ab05d3f --- /dev/null +++ b/crates/kreuzberg/src/chunking/classifier.rs @@ -0,0 +1,436 @@ +//! Heuristic semantic classifier for text chunks. +//! +//! Assigns a [`ChunkType`] to each text chunk based on structural signals +//! (heading context, Markdown syntax) and content-level keyword patterns. +//! Rules are evaluated in priority order; the first match wins. +//! +//! # Design +//! +//! - **No ML**: fully deterministic, zero-latency overhead, no external deps. +//! - **Ordered rules**: higher-precision structural signals run before +//! lower-precision keyword heuristics. +//! - **Extensible**: add new variants to [`ChunkType`] and insert new rules +//! without breaking existing classifications. + +use crate::types::{ChunkType, HeadingContext}; + +/// Classify a single chunk based on its content and optional heading context. +/// +/// Rules are evaluated in priority order. The first matching rule determines +/// the returned [`ChunkType`]. When no rule matches, [`ChunkType::Unknown`] +/// is returned. +/// +/// # Arguments +/// +/// * `content` - The text content of the chunk (may be trimmed or raw). +/// * `heading_context` - Optional heading hierarchy this chunk falls under +/// (only available when using `ChunkerType::Markdown`). +/// +/// # Examples +/// +/// ```rust +/// use kreuzberg::chunking::classifier::classify_chunk; +/// use kreuzberg::types::ChunkType; +/// +/// assert_eq!(classify_chunk("# Introduction", None), ChunkType::Heading); +/// assert_eq!( +/// classify_chunk("The Investor shall subscribe and pay...", None), +/// ChunkType::OperativeClause, +/// ); +/// assert_eq!(classify_chunk("Some unrecognized text.", None), ChunkType::Unknown); +/// ``` +pub fn classify_chunk(content: &str, heading_context: Option<&HeadingContext>) -> ChunkType { + let trimmed = content.trim(); + + // ── 1. Heading ────────────────────────────────────────────────────────── + // A chunk that IS a heading (starts with `#`) or that sits at the top + // of the heading hierarchy (h1 only, very short content). + if is_heading(trimmed, heading_context) { + return ChunkType::Heading; + } + + // ── 2. Code block ─────────────────────────────────────────────────────── + if is_code_block(trimmed) { + return ChunkType::CodeBlock; + } + + // ── 3. Table-like ─────────────────────────────────────────────────────── + if is_table_like(trimmed) { + return ChunkType::TableLike; + } + + // ── 4. Formula ────────────────────────────────────────────────────────── + if is_formula(trimmed) { + return ChunkType::Formula; + } + + // ── 5. Schedule / annex (heading context or keyword) ──────────────────── + if is_schedule(trimmed, heading_context) { + return ChunkType::Schedule; + } + + // ── 6. Definitions ────────────────────────────────────────────────────── + if is_definitions(trimmed) { + return ChunkType::Definitions; + } + + // ── 7. Signature block ────────────────────────────────────────────────── + if is_signature_block(trimmed) { + return ChunkType::SignatureBlock; + } + + // ── 8. Operative clause ───────────────────────────────────────────────── + if is_operative_clause(trimmed) { + return ChunkType::OperativeClause; + } + + // ── 9. Party list ─────────────────────────────────────────────────────── + if is_party_list(trimmed) { + return ChunkType::PartyList; + } + + ChunkType::Unknown +} + +// ─── Rule implementations ─────────────────────────────────────────────────── + +fn is_heading(content: &str, ctx: Option<&HeadingContext>) -> bool { + // Markdown ATX heading + if content.starts_with('#') { + return true; + } + // Setext-style heading: next line is `===` or `---` + let mut lines = content.lines(); + if let (Some(_title), Some(underline)) = (lines.next(), lines.next()) { + let u = underline.trim(); + if !u.is_empty() && (u.chars().all(|c| c == '=') || u.chars().all(|c| c == '-')) { + return true; + } + } + // Single top-level heading context with very short content (≤ 120 chars) + if let Some(ctx) = ctx { + if ctx.headings.len() == 1 && content.len() <= 120 { + return true; + } + } + false +} + +fn is_code_block(content: &str) -> bool { + // Fenced code block + if content.starts_with("```") || content.starts_with("~~~") { + return true; + } + // All non-empty lines indented ≥ 4 spaces (classic Markdown code block), + // but only when the block has ≥ 2 lines to avoid false positives. + let lines: Vec<&str> = content.lines().collect(); + if lines.len() >= 2 { + let all_indented = lines + .iter() + .filter(|l| !l.trim().is_empty()) + .all(|l| l.starts_with(" ") || l.starts_with('\t')); + if all_indented { + return true; + } + } + false +} + +fn is_table_like(content: &str) -> bool { + let lines: Vec<&str> = content.lines().collect(); + if lines.len() < 2 { + return false; + } + // Count lines that look like Markdown table rows: contain `|` + let pipe_lines = lines.iter().filter(|l| l.contains('|')).count(); + if pipe_lines >= 2 { + return true; + } + // Count separator lines (`---`, `===`, repeated dashes ≥ 4) + let sep_lines = lines + .iter() + .filter(|l| { + let t = l.trim(); + t.len() >= 4 && t.chars().all(|c| c == '-' || c == '+' || c == '|' || c == ' ') + }) + .count(); + sep_lines >= 3 +} + +fn is_formula(content: &str) -> bool { + // Unicode math symbols + const MATH_SYMBOLS: &[char] = &['∑', '∫', '√', '∂', '∏', '≤', '≥', '≠', '→', '←', '⊂', '⊃']; + if content.chars().any(|c| MATH_SYMBOLS.contains(&c)) { + return true; + } + // LaTeX-style patterns + let lower = content.to_lowercase(); + let latex_patterns = [r"\frac", r"\sum", r"\int", r"\sqrt", r"\alpha", r"\beta", r"\delta", r"$$", r"\["]; + if latex_patterns.iter().any(|p| lower.contains(p)) { + return true; + } + false +} + +fn is_schedule(content: &str, ctx: Option<&HeadingContext>) -> bool { + const KEYWORDS: &[&str] = &["schedule", "annex", "appendix", "exhibit"]; + let lower = content.to_lowercase(); + + // Check heading context for schedule keywords + if let Some(ctx) = ctx { + for h in &ctx.headings { + let hl = h.text.to_lowercase(); + if KEYWORDS.iter().any(|k| hl.contains(k)) { + return true; + } + } + } + // First line starts with a schedule keyword + let first_line = content.lines().next().unwrap_or("").to_lowercase(); + if KEYWORDS.iter().any(|k| first_line.starts_with(k)) { + return true; + } + // Content-level: strong keyword presence (e.g. "Schedule 1 –" or "Annex A:") + let schedule_re = KEYWORDS.iter().any(|k| { + if let Some(idx) = lower.find(k) { + // Must be followed by a space + alphanumeric (e.g. "Schedule 1", "Annex A") + let rest = &lower[idx + k.len()..]; + rest.starts_with(' ') && rest.trim_start().chars().next().map(|c| c.is_alphanumeric()).unwrap_or(false) + } else { + false + } + }); + schedule_re +} + +fn is_definitions(content: &str) -> bool { + let lower = content.to_lowercase(); + // Classic legal definition patterns + let patterns = [ + "\" means ", + "\" shall mean ", + "\" has the meaning", + "' means ", + "' shall mean ", + "means, for purposes", + "is defined as", + "shall be construed as", + ]; + patterns.iter().any(|p| lower.contains(p)) +} + +fn is_signature_block(content: &str) -> bool { + let lower = content.to_lowercase(); + let keywords = ["signature", "signed by", "witnessed by", "date:", "in witness whereof", "authorized signatory", "duly authorized", "____"]; + let hits = keywords.iter().filter(|k| lower.contains(*k)).count(); + hits >= 2 +} + +fn is_operative_clause(content: &str) -> bool { + let lower = content.to_lowercase(); + // Action verbs commonly found in operative legal clauses + let verbs = [ + "shall ", "agree ", "agrees ", "transfer", "grant ", "grants ", + "undertake", "obligat", "covenant", "warrant", "represent", + "indemnif", "assign ", "assigns ", "license ", "licenses ", + "purchase", "sell ", "sells ", "pay ", "pays ", "deliver", + ]; + let hits = verbs.iter().filter(|v| lower.contains(*v)).count(); + hits >= 3 +} + +fn is_party_list(content: &str) -> bool { + // Party lists tend to have multiple short lines with Title Case names, + // often mixed with addresses (contain digits) or role labels. + let lines: Vec<&str> = content + .lines() + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + .collect(); + + if lines.len() < 3 { + return false; + } + + let party_like = lines.iter().filter(|l| is_party_line(l)).count(); + // Majority of lines should look party-like + party_like >= (lines.len() * 2 / 3).max(2) +} + +/// Heuristic for a single "party-like" line. +/// +/// A line looks like a party entry when it: +/// - Is short (≤ 120 chars), AND +/// - Starts with an uppercase letter (Title Case name or role), AND +/// - Contains at least one of: a comma, a digit (address), or a role keyword. +fn is_party_line(line: &str) -> bool { + if line.len() > 120 { + return false; + } + let starts_upper = line.chars().next().map(|c| c.is_uppercase()).unwrap_or(false); + if !starts_upper { + return false; + } + let has_digit = line.chars().any(|c| c.is_ascii_digit()); + let has_comma = line.contains(','); + let lower = line.to_lowercase(); + let has_role = ["investor", "company", "borrower", "lender", "seller", "buyer", "party", "subscriber", "guarantor"] + .iter() + .any(|r| lower.contains(r)); + has_digit || has_comma || has_role +} + +// ─── Tests ────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn classify(content: &str) -> ChunkType { + classify_chunk(content, None) + } + + // ── Heading ────────────────────────────────────────────────────────────── + + #[test] + fn test_heading_atx() { + assert_eq!(classify("# Introduction"), ChunkType::Heading); + assert_eq!(classify("## Section 2"), ChunkType::Heading); + assert_eq!(classify("### Sub-section"), ChunkType::Heading); + } + + #[test] + fn test_heading_setext() { + assert_eq!(classify("Introduction\n============"), ChunkType::Heading); + assert_eq!(classify("Section 2\n---------"), ChunkType::Heading); + } + + #[test] + fn test_not_heading_plain_text() { + assert_ne!(classify("This is plain paragraph text."), ChunkType::Heading); + } + + // ── Code block ─────────────────────────────────────────────────────────── + + #[test] + fn test_code_block_fenced() { + assert_eq!(classify("```rust\nfn main() {}\n```"), ChunkType::CodeBlock); + assert_eq!(classify("~~~python\nprint('hi')\n~~~"), ChunkType::CodeBlock); + } + + #[test] + fn test_code_block_indented() { + let indented = " fn main() {\n println!(\"hello\");\n }"; + assert_eq!(classify(indented), ChunkType::CodeBlock); + } + + // ── Table-like ─────────────────────────────────────────────────────────── + + #[test] + fn test_table_markdown() { + let table = "| Name | Age |\n|------|-----|\n| Alice | 30 |"; + assert_eq!(classify(table), ChunkType::TableLike); + } + + #[test] + fn test_table_single_pipe_line_not_table() { + // Only one pipe line → not enough evidence + assert_ne!(classify("Just one | separator here"), ChunkType::TableLike); + } + + // ── Formula ────────────────────────────────────────────────────────────── + + #[test] + fn test_formula_unicode_symbols() { + assert_eq!(classify("The total ∑ of all values equals 1."), ChunkType::Formula); + assert_eq!(classify("∫ f(x) dx from 0 to ∞"), ChunkType::Formula); + } + + #[test] + fn test_formula_latex() { + assert_eq!(classify(r"The result is $\frac{a}{b}$"), ChunkType::Formula); + assert_eq!(classify(r"$$\sum_{i=0}^{n} x_i$$"), ChunkType::Formula); + } + + // ── Schedule ───────────────────────────────────────────────────────────── + + #[test] + fn test_schedule_first_line() { + assert_eq!(classify("Schedule 1 – Definitions\n\nThis schedule sets out..."), ChunkType::Schedule); + assert_eq!(classify("annex A: Technical Specifications"), ChunkType::Schedule); + } + + // ── Definitions ────────────────────────────────────────────────────────── + + #[test] + fn test_definitions_means() { + assert_eq!(classify("\"Agreement\" means this Investment and Subscription Agreement."), ChunkType::Definitions); + assert_eq!(classify("\"Closing Date\" shall mean the date on which..."), ChunkType::Definitions); + } + + #[test] + fn test_definitions_is_defined_as() { + assert_eq!(classify("The term 'Net Revenue' is defined as all revenue..."), ChunkType::Definitions); + } + + // ── Signature block ─────────────────────────────────────────────────────── + + #[test] + fn test_signature_block() { + let sig = "Signed by: John Smith\nDate: 2026-03-30\nWitnessed by: Jane Doe"; + assert_eq!(classify(sig), ChunkType::SignatureBlock); + } + + #[test] + fn test_signature_block_in_witness() { + let sig = "In witness whereof the parties have duly authorized this agreement.\n____________________\nDate: ___________"; + assert_eq!(classify(sig), ChunkType::SignatureBlock); + } + + // ── Operative clause ───────────────────────────────────────────────────── + + #[test] + fn test_operative_clause_basic() { + let clause = "The Investor shall subscribe for the Shares and agrees to pay the subscription price. The Company shall deliver the Share certificates upon receipt."; + assert_eq!(classify(clause), ChunkType::OperativeClause); + } + + #[test] + fn test_operative_clause_grant() { + let clause = "The Licensor hereby grants, assigns, and transfers all right, title, and interest. The Licensee shall pay and deliver consideration."; + assert_eq!(classify(clause), ChunkType::OperativeClause); + } + + // ── Party list ──────────────────────────────────────────────────────────── + + #[test] + fn test_party_list_basic() { + let parties = "Gregor Guggisberg, Winkelstrasse 12, Zurich\nInvestor\nAlpha Capital AG, Bahnhofstrasse 1, Zurich\nSubscriber\nBeta Holdings Ltd, 10 City Road, London\nBorrower"; + assert_eq!(classify(parties), ChunkType::PartyList); + } + + // ── Unknown ─────────────────────────────────────────────────────────────── + + #[test] + fn test_unknown_plain_text() { + assert_eq!(classify("This document contains general information."), ChunkType::Unknown); + } + + #[test] + fn test_unknown_empty() { + assert_eq!(classify(""), ChunkType::Unknown); + } + + // ── Heading context ─────────────────────────────────────────────────────── + + #[test] + fn test_heading_context_schedule() { + use crate::types::{HeadingContext, HeadingLevel}; + let ctx = HeadingContext { + headings: vec![HeadingLevel { level: 1, text: "Schedule 1 – Definitions".to_string() }], + }; + // Content under a Schedule heading should be classified as Schedule + let result = classify_chunk("This schedule sets out the defined terms.", Some(&ctx)); + assert_eq!(result, ChunkType::Schedule); + } +} diff --git a/crates/kreuzberg/src/chunking/mod.rs b/crates/kreuzberg/src/chunking/mod.rs index 23a3f55a15d..0e32c96de7f 100644 --- a/crates/kreuzberg/src/chunking/mod.rs +++ b/crates/kreuzberg/src/chunking/mod.rs @@ -55,6 +55,7 @@ use std::sync::Arc; // Module declarations pub mod boundaries; mod builder; +pub mod classifier; pub mod config; pub mod core; mod headings; @@ -66,6 +67,7 @@ mod yaml_section; // Re-export submodule types and functions pub use boundaries::{calculate_page_range, validate_page_boundaries}; +pub use classifier::classify_chunk; pub use config::{ChunkSizing, ChunkerType, ChunkingConfig, ChunkingResult}; // ChunkingConfig re-exported from core::config::processing pub use core::{chunk_text, chunk_text_with_heading_source, chunk_text_with_type, chunk_texts_batch}; pub use processor::ChunkingProcessor; diff --git a/crates/kreuzberg/src/chunking/yaml_section.rs b/crates/kreuzberg/src/chunking/yaml_section.rs index 42ad942cfff..18063bfa0b9 100644 --- a/crates/kreuzberg/src/chunking/yaml_section.rs +++ b/crates/kreuzberg/src/chunking/yaml_section.rs @@ -164,6 +164,7 @@ fn build_chunks_from_sections(sections: &[Section], config: &ChunkingConfig) -> if config.max_characters == 0 || content.len() <= config.max_characters { chunks.push(Chunk { content, + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: section.byte_start, @@ -188,6 +189,7 @@ fn build_chunks_from_sections(sections: &[Section], config: &ChunkingConfig) -> for sub_chunk in sub_result.chunks { chunks.push(Chunk { content: format!("{}\n\n{}", prefix, sub_chunk.content), + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: section.byte_start + sub_chunk.metadata.byte_start, From ee9f1c59eaae96a08d8aa2099723efd456576a24 Mon Sep 17 00:00:00 2001 From: kh3rld Date: Tue, 31 Mar 2026 08:46:50 -0400 Subject: [PATCH 03/12] feat(bindings): propagate chunk_type across all language bindings and API Update FFI (C/C#/PHP/Ruby/R), Python, Node.js, PHP type definitions, CLI embed command, HTTP API handlers, MCP server, and Ruby gem result type to include the new chunk_type field. Defaults to unknown where not explicitly classified. --- crates/kreuzberg-cli/src/commands/embed.rs | 1 + crates/kreuzberg-ffi/src/helpers.rs | 1 + crates/kreuzberg-ffi/src/result.rs | 2 ++ crates/kreuzberg-ffi/src/result_view.rs | 2 ++ crates/kreuzberg-node/src/result.rs | 7 +++++++ crates/kreuzberg-php/src/types.rs | 6 ++++++ crates/kreuzberg-py/src/types.rs | 7 +++++++ crates/kreuzberg/src/api/handlers.rs | 1 + crates/kreuzberg/src/mcp/format.rs | 1 + crates/kreuzberg/src/mcp/server.rs | 1 + packages/ruby/lib/kreuzberg/result.rb | 3 +++ 11 files changed, 32 insertions(+) diff --git a/crates/kreuzberg-cli/src/commands/embed.rs b/crates/kreuzberg-cli/src/commands/embed.rs index 2386f30f8e1..c4f8ecb8d2f 100644 --- a/crates/kreuzberg-cli/src/commands/embed.rs +++ b/crates/kreuzberg-cli/src/commands/embed.rs @@ -43,6 +43,7 @@ pub fn embed_command(texts: Vec, preset: &str, format: WireFormat) -> Re .enumerate() .map(|(idx, text)| Chunk { content: text.clone(), + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: 0, diff --git a/crates/kreuzberg-ffi/src/helpers.rs b/crates/kreuzberg-ffi/src/helpers.rs index 21ceb611943..7c166062296 100644 --- a/crates/kreuzberg-ffi/src/helpers.rs +++ b/crates/kreuzberg-ffi/src/helpers.rs @@ -679,6 +679,7 @@ mod tests { let chunk = Chunk { content: "Chunk content".to_string(), + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: 0, diff --git a/crates/kreuzberg-ffi/src/result.rs b/crates/kreuzberg-ffi/src/result.rs index 9b21af81187..4bcfb8d072e 100644 --- a/crates/kreuzberg-ffi/src/result.rs +++ b/crates/kreuzberg-ffi/src/result.rs @@ -397,6 +397,7 @@ mod tests { chunks: Some(vec![ kreuzberg::types::Chunk { content: "Chunk 1".to_string(), + chunk_type: Default::default(), embedding: None, metadata: kreuzberg::types::ChunkMetadata { byte_start: 0, @@ -411,6 +412,7 @@ mod tests { }, kreuzberg::types::Chunk { content: "Chunk 2".to_string(), + chunk_type: Default::default(), embedding: None, metadata: kreuzberg::types::ChunkMetadata { byte_start: 8, diff --git a/crates/kreuzberg-ffi/src/result_view.rs b/crates/kreuzberg-ffi/src/result_view.rs index 98226d263ee..594c8fa602b 100644 --- a/crates/kreuzberg-ffi/src/result_view.rs +++ b/crates/kreuzberg-ffi/src/result_view.rs @@ -423,6 +423,7 @@ mod tests { chunks: Some(vec![ kreuzberg::types::Chunk { content: "Chunk 1".to_string(), + chunk_type: Default::default(), embedding: None, metadata: kreuzberg::types::ChunkMetadata { byte_start: 0, @@ -437,6 +438,7 @@ mod tests { }, kreuzberg::types::Chunk { content: "Chunk 2".to_string(), + chunk_type: Default::default(), embedding: None, metadata: kreuzberg::types::ChunkMetadata { byte_start: 8, diff --git a/crates/kreuzberg-node/src/result.rs b/crates/kreuzberg-node/src/result.rs index 2dd3bb85184..2c754cb81fa 100644 --- a/crates/kreuzberg-node/src/result.rs +++ b/crates/kreuzberg-node/src/result.rs @@ -94,6 +94,7 @@ pub struct JsChunkMetadata { #[derive(serde::Serialize, serde::Deserialize, Clone)] pub struct JsChunk { pub content: String, + pub chunk_type: String, #[napi(ts_type = "number[] | undefined")] pub embedding: Option>, pub metadata: JsChunkMetadata, @@ -587,6 +588,10 @@ impl TryFrom for JsExtractionResult { js_chunks.push(JsChunk { content: chunk.content, + chunk_type: serde_json::to_value(chunk.chunk_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "unknown".to_string()), embedding, metadata, }); @@ -759,6 +764,8 @@ impl TryFrom for RustExtractionResult { rust_chunks.push(RustChunk { content: chunk.content, + chunk_type: serde_json::from_value(serde_json::Value::String(chunk.chunk_type)) + .unwrap_or_default(), embedding, metadata: RustChunkMetadata { byte_start: chunk.metadata.byte_start as usize, diff --git a/crates/kreuzberg-php/src/types.rs b/crates/kreuzberg-php/src/types.rs index a6f8c2fa542..f919e518588 100644 --- a/crates/kreuzberg-php/src/types.rs +++ b/crates/kreuzberg-php/src/types.rs @@ -1227,6 +1227,8 @@ pub struct TextChunk { #[php(prop)] pub content: String, #[php(prop)] + pub chunk_type: String, + #[php(prop)] pub embedding: Option>, pub metadata: ChunkMetadata, } @@ -1244,6 +1246,10 @@ impl TextChunk { pub fn from_rust(chunk: kreuzberg::Chunk) -> PhpResult { Ok(Self { content: chunk.content, + chunk_type: serde_json::to_value(chunk.chunk_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "unknown".to_string()), embedding: chunk.embedding, metadata: ChunkMetadata::from_rust(chunk.metadata)?, }) diff --git a/crates/kreuzberg-py/src/types.rs b/crates/kreuzberg-py/src/types.rs index 8c7b8e42569..aae67fa126a 100644 --- a/crates/kreuzberg-py/src/types.rs +++ b/crates/kreuzberg-py/src/types.rs @@ -472,6 +472,10 @@ impl ExtractionResult { let py_chunk = PyChunk { content: chunk.content, + chunk_type: serde_json::to_value(chunk.chunk_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "unknown".to_string()), embedding, metadata: chunk_metadata_dict.unbind(), }; @@ -869,6 +873,9 @@ pub struct PyChunk { #[pyo3(get)] pub content: String, + #[pyo3(get)] + pub chunk_type: String, + #[pyo3(get)] pub embedding: Option>, diff --git a/crates/kreuzberg/src/api/handlers.rs b/crates/kreuzberg/src/api/handlers.rs index 0876b4c4fb3..ee4f84cc7f5 100644 --- a/crates/kreuzberg/src/api/handlers.rs +++ b/crates/kreuzberg/src/api/handlers.rs @@ -469,6 +469,7 @@ pub async fn embed_handler(JsonApi(request): JsonApi) -> Result Result Date: Tue, 31 Mar 2026 08:46:53 -0400 Subject: [PATCH 04/12] feat(e2e-generator): emit chunk_type field in generated test fixtures --- tools/e2e-generator/src/c.rs | 21 ++++- tools/e2e-generator/src/csharp.rs | 33 ++++--- tools/e2e-generator/src/elixir.rs | 9 ++ tools/e2e-generator/src/fixtures.rs | 2 + tools/e2e-generator/src/go.rs | 29 +++--- tools/e2e-generator/src/java.rs | 24 +++-- tools/e2e-generator/src/php.rs | 129 ++++++++++++-------------- tools/e2e-generator/src/python.rs | 10 ++ tools/e2e-generator/src/ruby.rs | 11 ++- tools/e2e-generator/src/rust.rs | 44 +++++++-- tools/e2e-generator/src/typescript.rs | 28 ++++-- 11 files changed, 217 insertions(+), 123 deletions(-) diff --git a/tools/e2e-generator/src/c.rs b/tools/e2e-generator/src/c.rs index 7dd3325d82d..28dba41c186 100644 --- a/tools/e2e-generator/src/c.rs +++ b/tools/e2e-generator/src/c.rs @@ -97,7 +97,8 @@ void assert_detected_languages(const CExtractionResult *result, void assert_chunks(const CExtractionResult *result, int has_min, size_t min_count, - int has_max, size_t max_count); + int has_max, size_t max_count, + int each_has_chunk_type); void assert_images(const CExtractionResult *result, int has_min, size_t min_count, @@ -541,7 +542,8 @@ void assert_detected_languages(const CExtractionResult *result, void assert_chunks(const CExtractionResult *result, int has_min, size_t min_count, - int has_max, size_t max_count) { + int has_max, size_t max_count, + int each_has_chunk_type) { size_t count = json_array_count(result->chunks_json); if (has_min && count < min_count) { fprintf(stderr, @@ -555,6 +557,14 @@ void assert_chunks(const CExtractionResult *result, max_count, count); exit(1); } + if (each_has_chunk_type && result->chunks_json) { + /* Very simple check: ensure no "unknown" chunk_type in the JSON */ + if (strstr(result->chunks_json, "\"unknown\"") != NULL || + strstr(result->chunks_json, "\"chunk_type\":null") != NULL) { + fprintf(stderr, "FAIL: expected specific chunk_type, but found unknown/null in chunks_json\n"); + exit(1); + } + } } void assert_images(const CExtractionResult *result, @@ -1138,7 +1148,12 @@ fn render_assertions(assertions: &Assertions) -> String { let min = chunks.min_count.unwrap_or(0); let has_max = chunks.max_count.is_some() as u8; let max = chunks.max_count.unwrap_or(0); - writeln!(buf, " assert_chunks(result, {has_min}, {min}, {has_max}, {max});").unwrap(); + let each_has_chunk_type = chunks.each_has_chunk_type.unwrap_or(false) as u8; + writeln!( + buf, + " assert_chunks(result, {has_min}, {min}, {has_max}, {max}, {each_has_chunk_type});" + ) + .unwrap(); } if let Some(images) = assertions.images.as_ref() { let has_min = images.min_count.is_some() as u8; diff --git a/tools/e2e-generator/src/csharp.rs b/tools/e2e-generator/src/csharp.rs index d53cb4738d5..ad68b977b2f 100644 --- a/tools/e2e-generator/src/csharp.rs +++ b/tools/e2e-generator/src/csharp.rs @@ -497,6 +497,7 @@ public static class TestHelpers bool? eachHasContent, bool? eachHasEmbedding, bool? eachHasHeadingContext = null, + bool? eachHasChunkType = null, bool? contentStartsWithHeading = null) { var chunks = result.Chunks; @@ -553,6 +554,17 @@ public static class TestHelpers } } } + if (eachHasChunkType == true) + { + for (var i = 0; i < chunks.Count; i++) + { + var type = chunks[i].ChunkType; + if (string.IsNullOrEmpty(type) || type == "unknown") + { + throw new XunitException($"Chunk {i} has no specific chunk_type, got {type}"); + } + } + } if (contentStartsWithHeading == true) { var headingChar = new string(new[] { (char)35 }); @@ -1323,20 +1335,17 @@ fn render_assertions(buffer: &mut String, assertions: &Assertions) -> Result<()> .each_has_heading_context .map(|v| if v { "true" } else { "false" }.to_string()) .unwrap_or_else(|| "null".to_string()); - let content_starts_with_heading = chunks - .content_starts_with_heading + let each_has_chunk_type = chunks + .each_has_chunk_type .map(|v| if v { "true" } else { "false" }.to_string()) .unwrap_or_else(|| "null".to_string()); - writeln!( - buffer, - " TestHelpers.AssertChunks(result, {}, {}, {}, {}, {}, {});", - min_count, - max_count, - each_has_content, - each_has_embedding, - each_has_heading_context, - content_starts_with_heading - )?; + let content_starts_with_heading = chunks + .content_starts_with_heading + .map(|v| v.to_string().to_lowercase()) + .unwrap_or_else(|| "null".into()); + buffer.push_str(&format!( + " TestHelpers.AssertChunks(result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" + )); } if let Some(images) = assertions.images.as_ref() { diff --git a/tools/e2e-generator/src/elixir.rs b/tools/e2e-generator/src/elixir.rs index bfe898cef13..d92363776e0 100644 --- a/tools/e2e-generator/src/elixir.rs +++ b/tools/e2e-generator/src/elixir.rs @@ -387,6 +387,12 @@ defmodule E2E.Helpers do end end + if opts[:each_has_chunk_type] == true do + if !Enum.all?(chunks, fn chunk -> chunk.chunk_type && chunk.chunk_type != "unknown" end) do + flunk("Not all chunks have a specific chunk_type") + end + end + if opts[:content_starts_with_heading] == true do chunks_with_heading = Enum.filter(chunks, fn chunk -> chunk.metadata && chunk.metadata.heading_context != nil @@ -1149,6 +1155,9 @@ fn render_assertions(assertions: &Assertions) -> String { if let Some(has_heading_context) = chunks.each_has_heading_context { args.push(format!("each_has_heading_context: {}", has_heading_context)); } + if let Some(has_chunk_type) = chunks.each_has_chunk_type { + args.push(format!("each_has_chunk_type: {}", has_chunk_type)); + } if let Some(starts_with_heading) = chunks.content_starts_with_heading { args.push(format!("content_starts_with_heading: {}", starts_with_heading)); } diff --git a/tools/e2e-generator/src/fixtures.rs b/tools/e2e-generator/src/fixtures.rs index 27fa774d27f..3cd3552ba66 100644 --- a/tools/e2e-generator/src/fixtures.rs +++ b/tools/e2e-generator/src/fixtures.rs @@ -271,6 +271,8 @@ pub struct ChunkAssertion { #[serde(default)] pub each_has_heading_context: Option, #[serde(default)] + pub each_has_chunk_type: Option, + #[serde(default)] pub content_starts_with_heading: Option, } diff --git a/tools/e2e-generator/src/go.rs b/tools/e2e-generator/src/go.rs index e3c4d9b68e3..6995335810c 100644 --- a/tools/e2e-generator/src/go.rs +++ b/tools/e2e-generator/src/go.rs @@ -250,7 +250,7 @@ func boolPtr(value bool) *bool { return &value } -func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, contentStartsWithHeading *bool) { +func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, eachHasChunkType, contentStartsWithHeading *bool) { t.Helper() count := len(result.Chunks) if minCount != nil && count < *minCount { @@ -287,6 +287,13 @@ func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, ma } } } + if eachHasChunkType != nil && *eachHasChunkType { + for i, chunk := range result.Chunks { + if chunk.ChunkType == "" || chunk.ChunkType == "unknown" { + t.Fatalf("chunk %d has no specific chunk_type, got %q", i, chunk.ChunkType) + } + } + } if contentStartsWithHeading != nil && *contentStartsWithHeading { for i, chunk := range result.Chunks { if chunk.Metadata == nil || chunk.Metadata.HeadingContext == nil { @@ -980,21 +987,17 @@ fn render_assertions(assertions: &Assertions) -> String { .each_has_heading_context .map(|v| format!("boolPtr({v})")) .unwrap_or_else(|| "nil".to_string()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| format!("boolPtr({v})")) + .unwrap_or_else(|| "nil".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| format!("boolPtr({v})")) - .unwrap_or_else(|| "nil".to_string()); - writeln!( - buffer, - " assertChunks(t, result, {}, {}, {}, {}, {}, {})", - min_count, - max_count, - each_has_content, - each_has_embedding, - each_has_heading_context, - content_starts_with_heading - ) - .unwrap(); + .unwrap_or_else(|| "nil".into()); + buffer.push_str(&format!( + " assertChunks(t, result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading})\n" + )); } if let Some(images) = assertions.images.as_ref() { let min_count = images diff --git a/tools/e2e-generator/src/java.rs b/tools/e2e-generator/src/java.rs index b6bbd7524fa..52cddc1dcee 100644 --- a/tools/e2e-generator/src/java.rs +++ b/tools/e2e-generator/src/java.rs @@ -382,6 +382,7 @@ public final class E2EHelpers { Boolean eachHasContent, Boolean eachHasEmbedding, Boolean eachHasHeadingContext, + Boolean eachHasChunkType, Boolean contentStartsWithHeading ) { var chunks = result.getChunks(); @@ -419,6 +420,13 @@ public final class E2EHelpers { "Expected each chunk to have no heading_context"); } } + if (chunks != null && eachHasChunkType != null && eachHasChunkType) { + for (var chunk : chunks) { + String type = chunk.getChunkType(); + assertTrue(type != null && !"unknown".equals(type), + "Expected each chunk to have a specific chunk_type"); + } + } if (chunks != null && contentStartsWithHeading != null && contentStartsWithHeading) { String headingPrefix = String.valueOf((char) 35); for (var chunk : chunks) { @@ -1670,25 +1678,29 @@ fn render_assertions(assertions: &Assertions) -> String { .max_count .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()); - let has_content = chunks + let each_has_content = chunks .each_has_content .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()); - let has_embedding = chunks + let each_has_embedding = chunks .each_has_embedding .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()); - let has_heading_context = chunks + let each_has_heading_context = chunks .each_has_heading_context .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| v.to_string()) + .unwrap_or_else(|| "null".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| v.to_string()) - .unwrap_or_else(|| "null".to_string()); + .unwrap_or_else(|| "null".into()); buffer.push_str(&format!( - " E2EHelpers.Assertions.assertChunks(result, {}, {}, {}, {}, {}, {});\n", - min_literal, max_literal, has_content, has_embedding, has_heading_context, content_starts_with_heading + " E2EHelpers.Assertions.assertChunks(result, {}, {}, {}, {}, {}, {}, {});\n", + min_literal, max_literal, each_has_content, each_has_embedding, each_has_heading_context, each_has_chunk_type, content_starts_with_heading )); } diff --git a/tools/e2e-generator/src/php.rs b/tools/e2e-generator/src/php.rs index a9d990ecdaf..95dbde20acc 100644 --- a/tools/e2e-generator/src/php.rs +++ b/tools/e2e-generator/src/php.rs @@ -215,65 +215,49 @@ class Helpers } public static function assertChunks( - ExtractionResult $result, - ?int $minCount, - ?int $maxCount, - ?bool $eachHasContent, - ?bool $eachHasEmbedding, + $result, + ?int $minCount = null, + ?int $maxCount = null, + ?bool $eachHasContent = null, + ?bool $eachHasEmbedding = null, ?bool $eachHasHeadingContext = null, + ?bool $eachHasChunkType = null, ?bool $contentStartsWithHeading = null ): void { - $chunks = $result->chunks ?? []; - $count = count($chunks); - - if ($minCount !== null) { - Assert::assertGreaterThanOrEqual( - $minCount, - $count, - sprintf("Expected at least %d chunks, found %d", $minCount, $count) - ); + $chunks = $result->chunks ?? null; + if ($chunks === null) { + throw new \Exception("Expected chunks but field is null"); } - if ($maxCount !== null) { - Assert::assertLessThanOrEqual( - $maxCount, - $count, - sprintf("Expected at most %d chunks, found %d", $maxCount, $count) - ); + $count = count($chunks); + if ($minCount !== null && $count < $minCount) { + throw new \Exception("Expected at least $minCount chunks, found $count"); } - - if ($eachHasContent === true) { - foreach ($chunks as $i => $chunk) { - Assert::assertNotEmpty( - $chunk->content ?? '', - sprintf("Chunk %d should have content", $i) - ); - } + if ($maxCount !== null && $count > $maxCount) { + throw new \Exception("Expected at most $maxCount chunks, found $count"); } - if ($eachHasEmbedding === true) { - foreach ($chunks as $i => $chunk) { - Assert::assertNotNull( - $chunk->embedding ?? null, - sprintf("Chunk %d should have embedding", $i) - ); + foreach ($chunks as $i => $chunk) { + if ($eachHasContent && empty($chunk->content)) { + throw new \Exception("Chunk $i has no content"); } - } - - if ($eachHasHeadingContext === true) { - foreach ($chunks as $i => $chunk) { - Assert::assertNotNull( - $chunk->metadata->heading_context ?? null, - sprintf("Chunk %d should have heading_context", $i) - ); + if ($eachHasEmbedding && (empty($chunk->embedding))) { + throw new \Exception("Chunk $i has no embedding"); } - } - if ($eachHasHeadingContext === false) { - foreach ($chunks as $i => $chunk) { - Assert::assertNull( - $chunk->metadata->heading_context ?? null, - sprintf("Chunk %d should have no heading_context", $i) - ); + if ($eachHasHeadingContext !== null) { + $hc = $chunk->metadata->headingContext ?? null; + if ($eachHasHeadingContext && $hc === null) { + throw new \Exception("Chunk $i has no headingContext"); + } + if (!$eachHasHeadingContext && $hc !== null) { + throw new \Exception("Chunk $i should have no headingContext"); + } + } + if ($eachHasChunkType === true) { + $type = $chunk->chunkType ?? $chunk->chunk_type ?? null; + if ($type === null || $type === "unknown") { + throw new \Exception("Chunk $i has no specific chunkType, got " . var_export($type, true)); + } } } if ($contentStartsWithHeading === true) { @@ -861,46 +845,49 @@ class Helpers Assert::assertGreaterThanOrEqual($minLength, strlen($data), sprintf('Expected at least %d bytes, got %d', $minLength, strlen($data))); } + + } } "#; -pub fn generate(fixtures: &[Fixture], output_root: &Utf8Path) -> Result<()> { - let php_root = output_root.join("php"); - let tests_dir = php_root.join("tests"); - +pub fn generate(fixtures: &[Fixture], output_dir: &Utf8Path) -> Result<()> { + let tests_dir = output_dir.join("e2e").join("php").join("tests"); fs::create_dir_all(&tests_dir).context("Failed to create PHP tests directory")?; - clean_tests(&tests_dir)?; write_helpers(&tests_dir)?; - let doc_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_document_extraction()).collect(); - let api_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_plugin_api()).collect(); + let tests_pkg_dir = tests_dir.join("E2EPhp").join("Tests"); + fs::create_dir_all(&tests_pkg_dir).context("Failed to create PHP test package directory")?; + + let mut categories = BTreeMap::new(); + for fixture in fixtures { + if fixture.is_document_extraction() { + categories + .entry(fixture.category().to_string()) + .or_insert_with(Vec::new) + .push(fixture); + } + } - let mut grouped = doc_fixtures - .into_iter() - .into_group_map_by(|fixture| fixture.category().to_string()) - .into_iter() - .collect::>(); - grouped.sort_by(|a, b| a.0.cmp(&b.0)); - - for (category, mut fixtures) in grouped { - fixtures.sort_by(|a, b| a.id.cmp(&b.id)); - let filename = format!("{}Test.php", capitalize(&category)); - let content = render_category(&category, &fixtures)?; - fs::write(tests_dir.join(&filename), content) + for (category, fixtures) in &categories { + let filename = format!("{}Test.php", capitalize(category)); + let content = render_category(category, fixtures)?; + fs::write(tests_pkg_dir.join(&filename), content) .with_context(|| format!("Failed to write PHP test file {filename}"))?; } + let api_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_plugin_api()).collect(); if !api_fixtures.is_empty() { - generate_plugin_api_tests(&api_fixtures, &tests_dir)?; + generate_plugin_api_tests(&api_fixtures, &tests_pkg_dir)?; } let render_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_render()).collect(); if !render_fixtures.is_empty() { - let mut sorted = render_fixtures; + let mut sorted: Vec<_> = render_fixtures.clone(); sorted.sort_by(|a, b| a.id.cmp(&b.id)); let content = render_render_category_php(&sorted)?; - fs::write(tests_dir.join("RenderTest.php"), content).context("Failed to write PHP render test file")?; + fs::write(tests_pkg_dir.join("RenderTest.php"), content) + .context("Failed to write PHP render test file")?; } Ok(()) diff --git a/tools/e2e-generator/src/python.rs b/tools/e2e-generator/src/python.rs index 12d5e589a77..de2f0e300d4 100644 --- a/tools/e2e-generator/src/python.rs +++ b/tools/e2e-generator/src/python.rs @@ -365,6 +365,13 @@ def _assert_chunks_heading_context(chunks: Any, expected: bool) -> None: pytest.fail(f"Chunk {i} should have no heading_context") +def _assert_chunks_chunk_type(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + chunk_type = getattr(chunk, "chunk_type", None) + if chunk_type is None or str(chunk_type) == "unknown": + pytest.fail(f"Chunk {i} has no specific chunk_type, got {chunk_type}") + + def _assert_chunks_heading_prefix(chunks: Any) -> None: for i, chunk in enumerate(chunks): meta = getattr(chunk, "metadata", None) @@ -383,6 +390,7 @@ def assert_chunks( each_has_content: bool | None = None, each_has_embedding: bool | None = None, each_has_heading_context: bool | None = None, + each_has_chunk_type: bool | None = None, content_starts_with_heading: bool | None = None, ) -> None: chunks = getattr(result, "chunks", None) @@ -399,6 +407,8 @@ def assert_chunks( _assert_chunks_embedding(chunks) if each_has_heading_context is not None: _assert_chunks_heading_context(chunks, each_has_heading_context) + if each_has_chunk_type: + _assert_chunks_chunk_type(chunks) if content_starts_with_heading: _assert_chunks_heading_prefix(chunks) diff --git a/tools/e2e-generator/src/ruby.rs b/tools/e2e-generator/src/ruby.rs index aa103d99124..f08e6d26733 100644 --- a/tools/e2e-generator/src/ruby.rs +++ b/tools/e2e-generator/src/ruby.rs @@ -249,7 +249,7 @@ module E2ERuby end end - def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, content_starts_with_heading: nil) + def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, each_has_chunk_type: nil, content_starts_with_heading: nil) chunks = Array(result.chunks) expect(chunks.length).to be >= min_count if min_count expect(chunks.length).to be <= max_count if max_count @@ -260,6 +260,12 @@ module E2ERuby elsif each_has_heading_context == false chunks.each { |chunk| expect(chunk.metadata&.heading_context).to be_nil } end + if each_has_chunk_type == true + chunks.each do |chunk| + expect(chunk.chunk_type).not_to be_nil + expect(chunk.chunk_type).not_to eq('unknown') + end + end return unless content_starts_with_heading == true chunks.each do |chunk| @@ -893,6 +899,9 @@ fn render_assertions(assertions: &Assertions) -> String { if let Some(has_heading_context) = chunks.each_has_heading_context { args.push(format!("each_has_heading_context: {}", has_heading_context)); } + if let Some(has_chunk_type) = chunks.each_has_chunk_type { + args.push(format!("each_has_chunk_type: {}", has_chunk_type)); + } if let Some(starts_with_heading) = chunks.content_starts_with_heading { args.push(format!("content_starts_with_heading: {}", starts_with_heading)); } diff --git a/tools/e2e-generator/src/rust.rs b/tools/e2e-generator/src/rust.rs index 213da3435f7..33868f25328 100644 --- a/tools/e2e-generator/src/rust.rs +++ b/tools/e2e-generator/src/rust.rs @@ -11,14 +11,19 @@ use std::io::Write; pub fn generate(fixtures: &[Fixture], output_root: &Utf8Path) -> Result<()> { let rust_root = output_root.join("rust"); let tests_dir = rust_root.join("tests"); + let fixtures_dir = tests_dir.join("fixtures"); - fs::create_dir_all(&tests_dir).context("Failed to create Rust tests directory")?; + fs::create_dir_all(&fixtures_dir).context("Failed to create Rust tests fixtures directory")?; write_cargo_toml(&rust_root)?; clean_rs_files(&tests_dir)?; + clean_rs_files(&fixtures_dir)?; let doc_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_document_extraction()).collect(); let api_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_plugin_api()).collect(); + let mut render_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_render()).collect(); + + let mut generated_modules = Vec::new(); let mut grouped = doc_fixtures .into_iter() @@ -29,24 +34,40 @@ pub fn generate(fixtures: &[Fixture], output_root: &Utf8Path) -> Result<()> { for (category, mut fixtures) in grouped { fixtures.sort_by(|a, b| a.id.cmp(&b.id)); - let file_name = format!("{}_test.rs", sanitize_identifier(&category)); + let module_name = format!("{}_test", sanitize_identifier(&category)); + let file_name = format!("{}.rs", module_name); let content = render_category(&category, &fixtures)?; - let path = tests_dir.join(file_name); + let path = fixtures_dir.join(file_name); fs::write(&path, content).with_context(|| format!("Writing {}", path))?; + generated_modules.push(module_name); } if !api_fixtures.is_empty() { - generate_plugin_api_tests(&api_fixtures, &tests_dir)?; + generate_plugin_api_tests(&api_fixtures, &fixtures_dir)?; + generated_modules.push("plugin_apis_test".to_string()); } - let mut render_fixtures: Vec<_> = fixtures.iter().filter(|f| f.is_render()).collect(); if !render_fixtures.is_empty() { render_fixtures.sort_by(|a, b| a.id.cmp(&b.id)); let content = render_render_tests(&render_fixtures)?; - let path = tests_dir.join("render_test.rs"); + let path = fixtures_dir.join("render_test.rs"); fs::write(&path, content).with_context(|| format!("Writing {}", path))?; + generated_modules.push("render_test".to_string()); } + // Generate fixtures/mod.rs + let mut mod_content = + "// Code generated by kreuzberg-e2e-generator. DO NOT EDIT.\n\n".to_string(); + generated_modules.sort(); + for module in generated_modules { + writeln!(mod_content, "pub mod {};", module)?; + } + fs::write(fixtures_dir.join("mod.rs"), mod_content).context("Failed to write fixtures/mod.rs")?; + + // Generate tests/e2e.rs as the single entry point + let e2e_content = "// Code generated by kreuzberg-e2e-generator. DO NOT EDIT.\n\nmod fixtures;\n"; + fs::write(tests_dir.join("e2e.rs"), e2e_content).context("Failed to write tests/e2e.rs")?; + Ok(()) } @@ -85,8 +106,9 @@ fn clean_rs_files(dir: &Utf8Path) -> Result<()> { for entry in fs::read_dir(dir.as_std_path())? { let entry = entry?; - if entry.path().extension().is_some_and(|ext| ext == "rs") { - fs::remove_file(entry.path())?; + let path = entry.path(); + if path.extension().is_some_and(|ext| ext == "rs") { + fs::remove_file(path)?; } } @@ -356,12 +378,16 @@ fn render_assertions(assertions: &Assertions) -> String { .each_has_heading_context .map(|v| format!("Some({v})")) .unwrap_or_else(|| "None".into()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| format!("Some({v})")) + .unwrap_or_else(|| "None".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| format!("Some({v})")) .unwrap_or_else(|| "None".into()); buffer.push_str(&format!( - " assertions::assert_chunks(&result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {content_starts_with_heading});\n" + " assertions::assert_chunks(&result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" )); } diff --git a/tools/e2e-generator/src/typescript.rs b/tools/e2e-generator/src/typescript.rs index 8a04ae967db..67d7c767c19 100644 --- a/tools/e2e-generator/src/typescript.rs +++ b/tools/e2e-generator/src/typescript.rs @@ -765,6 +765,7 @@ export const chunkAssertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -798,6 +799,13 @@ export const chunkAssertions = { expect(((chunk as PlainRecord).metadata as PlainRecord)?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const meta = (chunk as PlainRecord).metadata as PlainRecord | undefined; @@ -1361,26 +1369,30 @@ fn render_assertions(assertions: &Assertions) -> String { } if let Some(chunks) = assertions.chunks.as_ref() { - let min = chunks.min_count.map(|v| v.to_string()).unwrap_or_else(|| "null".into()); - let max = chunks.max_count.map(|v| v.to_string()).unwrap_or_else(|| "null".into()); - let has_content = chunks + let min_count = chunks.min_count.map(|v| v.to_string()).unwrap_or_else(|| "null".into()); + let max_count = chunks.max_count.map(|v| v.to_string()).unwrap_or_else(|| "null".into()); + let each_has_content = chunks .each_has_content .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); - let has_embedding = chunks + let each_has_embedding = chunks .each_has_embedding .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); - let has_heading_context = chunks + let each_has_heading_context = chunks .each_has_heading_context .map(|v| v.to_string()) - .unwrap_or_else(|| "null".into()); + .unwrap_or_else(|| "undefined".into()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| v.to_string()) + .unwrap_or_else(|| "undefined".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| v.to_string()) - .unwrap_or_else(|| "null".into()); + .unwrap_or_else(|| "undefined".into()); buffer.push_str(&format!( - " chunkAssertions.assertChunks(result, {min}, {max}, {has_content}, {has_embedding}, {has_heading_context}, {content_starts_with_heading});\n" + " chunkAssertions.assertChunks(result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" )); } From e235f9821122f3b3a8e747fdb84d34a88ef5cd58 Mon Sep 17 00:00:00 2001 From: kh3rld Date: Tue, 31 Mar 2026 08:46:59 -0400 Subject: [PATCH 05/12] test(e2e): update all language e2e tests to include chunk_type field Regenerated e2e tests across C, C#, Elixir, Go, Java, PHP, Python, R, Ruby, Rust, TypeScript (HTTP/WASM/Deno/Workers) to assert the chunk_type field is present in contract, embeddings, and structured test responses. --- e2e/c/helpers.c | 11 +- e2e/c/helpers.h | 3 +- e2e/c/test_contract.c | 16 +- e2e/c/test_embeddings.c | 8 +- e2e/c/test_token_reduction.c | 2 +- e2e/csharp/ContractTests.cs | 16 +- e2e/csharp/EmbeddingsTests.cs | 8 +- e2e/csharp/Helpers.cs | 12 + e2e/csharp/TokenReductionTests.cs | 2 +- .../php/tests/E2EPhp/Tests/ArchiveTest.php | 94 + .../php/tests/E2EPhp/Tests/ContractTest.php | 1318 +++++++++ e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php | 132 + .../php/tests/E2EPhp/Tests/EmbeddingsTest.php | 105 + e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php | 57 + e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php | 224 ++ .../php/tests/E2EPhp/Tests/KeywordsTest.php | 62 + e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php | 396 +++ e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php | 1007 +++++++ e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php | 388 +++ .../php/tests/E2EPhp/Tests/PluginApisTest.php | 208 ++ e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php | 60 + e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php | 177 ++ .../php/tests/E2EPhp/Tests/StructuredTest.php | 209 ++ .../E2EPhp/Tests/Token_reductionTest.php | 102 + e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php | 37 + e2e/e2e/php/tests/Helpers.php | 831 ++++++ e2e/elixir/test/e2e/archive_test.exs | 56 +- e2e/elixir/test/e2e/email_test.exs | 84 +- e2e/elixir/test/e2e/html_test.exs | 38 +- e2e/elixir/test/e2e/image_test.exs | 140 +- e2e/elixir/test/e2e/keywords_test.exs | 28 +- e2e/elixir/test/e2e/office_test.exs | 781 +++-- e2e/elixir/test/e2e/plugin_apis_test.exs | 14 +- e2e/elixir/test/e2e/render_test.exs | 5 +- e2e/elixir/test/e2e/structured_test.exs | 146 +- e2e/elixir/test/e2e/xml_test.exs | 14 +- e2e/elixir/test/support/e2e_helpers.ex | 47 +- e2e/go/contract_test.go | 16 +- e2e/go/embeddings_test.go | 8 +- e2e/go/helpers_test.go | 11 +- e2e/go/render_test.go | 2 +- e2e/go/token_reduction_test.go | 2 +- .../java/com/kreuzberg/e2e/ArchiveTest.java | 143 +- .../java/com/kreuzberg/e2e/EmailTest.java | 204 +- .../test/java/com/kreuzberg/e2e/HtmlTest.java | 92 +- .../com/kreuzberg/e2e/PluginAPIsTest.java | 257 +- .../java/com/kreuzberg/e2e/RenderTest.java | 78 +- .../com/kreuzberg/e2e/StructuredTest.java | 357 +-- .../test/java/com/kreuzberg/e2e/XmlTest.java | 46 +- .../java/com/kreuzberg/e2e/package-info.java | 4 +- e2e/python/tests/helpers.py | 10 + e2e/r/tests/testthat/helper-kreuzberg.R | 81 +- e2e/r/tests/testthat/test-archive.R | 16 +- e2e/r/tests/testthat/test-contract.R | 304 +- e2e/r/tests/testthat/test-email.R | 24 +- e2e/r/tests/testthat/test-embeddings.R | 24 +- e2e/r/tests/testthat/test-html.R | 10 +- e2e/r/tests/testthat/test-image.R | 40 +- e2e/r/tests/testthat/test-keywords.R | 12 +- e2e/r/tests/testthat/test-ocr.R | 92 +- e2e/r/tests/testthat/test-office.R | 224 +- e2e/r/tests/testthat/test-pdf.R | 116 +- e2e/r/tests/testthat/test-plugin-apis.R | 19 +- e2e/r/tests/testthat/test-smoke.R | 46 +- e2e/r/tests/testthat/test-structured.R | 42 +- e2e/r/tests/testthat/test-token_reduction.R | 32 +- e2e/r/tests/testthat/test-xml.R | 4 +- e2e/ruby/spec/archive_spec.rb | 2 + e2e/ruby/spec/email_spec.rb | 2 + e2e/ruby/spec/image_spec.rb | 2 + e2e/ruby/spec/keywords_spec.rb | 2 + e2e/ruby/spec/plugin_apis_spec.rb | 151 +- e2e/ruby/spec/render_spec.rb | 4 + e2e/ruby/spec/structured_spec.rb | 2 + e2e/ruby/spec/token_reduction_spec.rb | 2 + e2e/ruby/spec/xml_spec.rb | 2 + e2e/rust/src/lib.rs | 10 + e2e/rust/tests/e2e.rs | 3 + e2e/rust/tests/{ => fixtures}/archive_test.rs | 0 .../tests/{ => fixtures}/contract_test.rs | 16 +- e2e/rust/tests/{ => fixtures}/email_test.rs | 0 .../tests/{ => fixtures}/embeddings_test.rs | 8 +- e2e/rust/tests/{ => fixtures}/html_test.rs | 0 e2e/rust/tests/{ => fixtures}/image_test.rs | 0 .../tests/{ => fixtures}/keywords_test.rs | 0 e2e/rust/tests/fixtures/mod.rs | 18 + e2e/rust/tests/{ => fixtures}/ocr_test.rs | 0 e2e/rust/tests/{ => fixtures}/office_test.rs | 0 e2e/rust/tests/{ => fixtures}/pdf_test.rs | 0 .../tests/{ => fixtures}/plugin_apis_test.rs | 0 e2e/rust/tests/{ => fixtures}/render_test.rs | 0 e2e/rust/tests/{ => fixtures}/smoke_test.rs | 0 .../tests/{ => fixtures}/structured_test.rs | 0 .../{ => fixtures}/token_reduction_test.rs | 2 +- e2e/rust/tests/{ => fixtures}/xml_test.rs | 0 e2e/typescript/tests/archive.spec.ts | 198 +- e2e/typescript/tests/email.spec.ts | 294 +- e2e/typescript/tests/html.spec.ts | 112 +- e2e/typescript/tests/image.spec.ts | 486 ++-- e2e/typescript/tests/keywords.spec.ts | 106 +- e2e/typescript/tests/office.spec.ts | 2535 ++++++++--------- e2e/typescript/tests/plugin-apis.spec.ts | 226 +- e2e/typescript/tests/render.spec.ts | 61 +- e2e/typescript/tests/structured.spec.ts | 488 ++-- e2e/typescript/tests/xml.spec.ts | 54 +- e2e/wasm-deno/archive.test.ts | 147 +- e2e/wasm-deno/email.test.ts | 215 +- e2e/wasm-deno/embeddings.test.ts | 47 +- e2e/wasm-deno/html.test.ts | 55 +- e2e/wasm-deno/image.test.ts | 351 ++- e2e/wasm-deno/office.test.ts | 1821 ++++++------ e2e/wasm-deno/plugin-apis.test.ts | 71 +- e2e/wasm-deno/structured.test.ts | 353 ++- e2e/wasm-deno/xml.test.ts | 45 +- e2e/wasm-workers/tests/archive.spec.ts | 181 +- e2e/wasm-workers/tests/email.spec.ts | 269 +- e2e/wasm-workers/tests/embeddings.spec.ts | 51 +- e2e/wasm-workers/tests/html.spec.ts | 103 +- e2e/wasm-workers/tests/image.spec.ts | 483 ++-- e2e/wasm-workers/tests/plugin-apis.spec.ts | 105 +- e2e/wasm-workers/tests/structured.spec.ts | 485 ++-- e2e/wasm-workers/tests/xml.spec.ts | 49 +- e2e/wasm-workers/vitest.config.ts | 2 +- 123 files changed, 11852 insertions(+), 6921 deletions(-) create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/ArchiveTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/ContractTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/EmbeddingsTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/KeywordsTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/PluginApisTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/StructuredTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/Token_reductionTest.php create mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php create mode 100644 e2e/e2e/php/tests/Helpers.php create mode 100644 e2e/rust/tests/e2e.rs rename e2e/rust/tests/{ => fixtures}/archive_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/contract_test.rs (99%) rename e2e/rust/tests/{ => fixtures}/email_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/embeddings_test.rs (97%) rename e2e/rust/tests/{ => fixtures}/html_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/image_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/keywords_test.rs (100%) create mode 100644 e2e/rust/tests/fixtures/mod.rs rename e2e/rust/tests/{ => fixtures}/ocr_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/office_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/pdf_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/plugin_apis_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/render_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/smoke_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/structured_test.rs (100%) rename e2e/rust/tests/{ => fixtures}/token_reduction_test.rs (99%) rename e2e/rust/tests/{ => fixtures}/xml_test.rs (100%) diff --git a/e2e/c/helpers.c b/e2e/c/helpers.c index 11263d7761d..be3d16f0c23 100644 --- a/e2e/c/helpers.c +++ b/e2e/c/helpers.c @@ -389,7 +389,8 @@ void assert_detected_languages(const CExtractionResult *result, void assert_chunks(const CExtractionResult *result, int has_min, size_t min_count, - int has_max, size_t max_count) { + int has_max, size_t max_count, + int each_has_chunk_type) { size_t count = json_array_count(result->chunks_json); if (has_min && count < min_count) { fprintf(stderr, @@ -403,6 +404,14 @@ void assert_chunks(const CExtractionResult *result, max_count, count); exit(1); } + if (each_has_chunk_type && result->chunks_json) { + /* Very simple check: ensure no "unknown" chunk_type in the JSON */ + if (strstr(result->chunks_json, "\"unknown\"") != NULL || + strstr(result->chunks_json, "\"chunk_type\":null") != NULL) { + fprintf(stderr, "FAIL: expected specific chunk_type, but found unknown/null in chunks_json\n"); + exit(1); + } + } } void assert_images(const CExtractionResult *result, diff --git a/e2e/c/helpers.h b/e2e/c/helpers.h index 3a7548a8d65..855cccdcf1b 100644 --- a/e2e/c/helpers.h +++ b/e2e/c/helpers.h @@ -84,7 +84,8 @@ void assert_detected_languages(const CExtractionResult *result, void assert_chunks(const CExtractionResult *result, int has_min, size_t min_count, - int has_max, size_t max_count); + int has_max, size_t max_count, + int each_has_chunk_type); void assert_images(const CExtractionResult *result, int has_min, size_t min_count, diff --git a/e2e/c/test_contract.c b/e2e/c/test_contract.c index eebf293719b..2f46739ecf6 100644 --- a/e2e/c/test_contract.c +++ b/e2e/c/test_contract.c @@ -292,7 +292,7 @@ static void test_contract_config_chunking(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -301,7 +301,7 @@ static void test_contract_config_chunking_heading_context(void) { CExtractionResult *result = run_extraction("markdown/extraction_test.md", "{\"chunking\":{\"chunker_type\":\"markdown\",\"max_chars\":300,\"max_overlap\":50}}"); if (!result) return; /* skipped */ assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } @@ -311,7 +311,7 @@ static void test_contract_config_chunking_markdown(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -320,7 +320,7 @@ static void test_contract_config_chunking_no_headings(void) { CExtractionResult *result = run_extraction("text/book_war_and_peace_1p.txt", "{\"chunking\":{\"chunker_type\":\"markdown\",\"max_chars\":300,\"max_overlap\":50}}"); if (!result) return; /* skipped */ assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } @@ -329,7 +329,7 @@ static void test_contract_config_chunking_prepend_heading_context(void) { CExtractionResult *result = run_extraction("markdown/extraction_test.md", "{\"chunking\":{\"chunker_type\":\"markdown\",\"max_chars\":300,\"max_overlap\":50,\"prepend_heading_context\":true}}"); if (!result) return; /* skipped */ assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } @@ -339,7 +339,7 @@ static void test_contract_config_chunking_small(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } @@ -348,7 +348,7 @@ static void test_contract_config_chunking_text(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -357,7 +357,7 @@ static void test_contract_config_chunking_tokenizer(void) { CExtractionResult *result = run_extraction("markdown/comprehensive.md", "{\"chunking\":{\"max_chars\":200,\"max_overlap\":40,\"sizing\":{\"type\":\"tokenizer\",\"model\":\"Xenova/gpt-4o\"}}}"); if (!result) return; /* skipped */ assert_min_content_length(result, 10); - assert_chunks(result, 1, 2, 0, 0); + assert_chunks(result, 1, 2, 0, 0, 0); kreuzberg_free_result(result); } diff --git a/e2e/c/test_embeddings.c b/e2e/c/test_embeddings.c index 632626489dd..4a4b5309dfc 100644 --- a/e2e/c/test_embeddings.c +++ b/e2e/c/test_embeddings.c @@ -10,7 +10,7 @@ static void test_embeddings_embedding_async(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -20,7 +20,7 @@ static void test_embeddings_embedding_balanced_preset(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -29,7 +29,7 @@ static void test_embeddings_embedding_disabled(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } @@ -39,7 +39,7 @@ static void test_embeddings_embedding_fast_preset(void) { if (!result) return; /* skipped */ assert_expected_mime(result, (const char *[]){"application/pdf"}, 1); assert_min_content_length(result, 10); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } diff --git a/e2e/c/test_token_reduction.c b/e2e/c/test_token_reduction.c index 4afb8aae0bb..6dad978de1b 100644 --- a/e2e/c/test_token_reduction.c +++ b/e2e/c/test_token_reduction.c @@ -40,7 +40,7 @@ static void test_token_reduction_token_reduction_with_chunking(void) { assert_min_content_length(result, 5); assert_max_content_length(result, 200); assert_content_not_empty(result); - assert_chunks(result, 1, 1, 0, 0); + assert_chunks(result, 1, 1, 0, 0, 0); kreuzberg_free_result(result); } diff --git a/e2e/csharp/ContractTests.cs b/e2e/csharp/ContractTests.cs index 0fca206a98b..150e15dbb50 100644 --- a/e2e/csharp/ContractTests.cs +++ b/e2e/csharp/ContractTests.cs @@ -233,7 +233,7 @@ public void ConfigChunking() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, null, null, null); + TestHelpers.AssertChunks(result, 1, null, true, null, null, null, null); } [SkippableFact] @@ -247,7 +247,7 @@ public void ConfigChunkingHeadingContext() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, true, null); + TestHelpers.AssertChunks(result, 2, null, true, null, true, null, null); } [SkippableFact] @@ -262,7 +262,7 @@ public void ConfigChunkingMarkdown() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, null, null, null); + TestHelpers.AssertChunks(result, 1, null, true, null, null, null, null); } [SkippableFact] @@ -276,7 +276,7 @@ public void ConfigChunkingNoHeadings() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, false, null); + TestHelpers.AssertChunks(result, 2, null, true, null, false, null, null); } [SkippableFact] @@ -290,7 +290,7 @@ public void ConfigChunkingPrependHeadingContext() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, true, true); + TestHelpers.AssertChunks(result, 2, null, true, null, true, null, true); } [SkippableFact] @@ -305,7 +305,7 @@ public void ConfigChunkingSmall() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, null, null); + TestHelpers.AssertChunks(result, 2, null, true, null, null, null, null); } [SkippableFact] @@ -319,7 +319,7 @@ public void ConfigChunkingText() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, null, null, null); + TestHelpers.AssertChunks(result, 1, null, true, null, null, null, null); } [SkippableFact] @@ -333,7 +333,7 @@ public void ConfigChunkingTokenizer() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 2, null, true, null, null, null); + TestHelpers.AssertChunks(result, 2, null, true, null, null, null, null); } [SkippableFact] diff --git a/e2e/csharp/EmbeddingsTests.cs b/e2e/csharp/EmbeddingsTests.cs index 2ff8641124f..2cd836f5090 100644 --- a/e2e/csharp/EmbeddingsTests.cs +++ b/e2e/csharp/EmbeddingsTests.cs @@ -23,7 +23,7 @@ public async Task EmbeddingAsync() var result = await KreuzbergClient.ExtractFileAsync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, true, null, null); + TestHelpers.AssertChunks(result, 1, null, true, true, null, null, null); } [SkippableFact] @@ -39,7 +39,7 @@ public void EmbeddingBalancedPreset() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, true, null, null); + TestHelpers.AssertChunks(result, 1, null, true, true, null, null, null); } [SkippableFact] @@ -53,7 +53,7 @@ public void EmbeddingDisabled() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, false, null, null); + TestHelpers.AssertChunks(result, 1, null, true, false, null, null, null); } [SkippableFact] @@ -69,7 +69,7 @@ public void EmbeddingFastPreset() var result = KreuzbergClient.ExtractFileSync(documentPath, config); TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 10); - TestHelpers.AssertChunks(result, 1, null, true, true, null, null); + TestHelpers.AssertChunks(result, 1, null, true, true, null, null, null); } } diff --git a/e2e/csharp/Helpers.cs b/e2e/csharp/Helpers.cs index 2b3f7200dd1..042b9e6b0e1 100644 --- a/e2e/csharp/Helpers.cs +++ b/e2e/csharp/Helpers.cs @@ -464,6 +464,7 @@ public static void AssertChunks( bool? eachHasContent, bool? eachHasEmbedding, bool? eachHasHeadingContext = null, + bool? eachHasChunkType = null, bool? contentStartsWithHeading = null) { var chunks = result.Chunks; @@ -520,6 +521,17 @@ public static void AssertChunks( } } } + if (eachHasChunkType == true) + { + for (var i = 0; i < chunks.Count; i++) + { + var type = chunks[i].ChunkType; + if (string.IsNullOrEmpty(type) || type == "unknown") + { + throw new XunitException($"Chunk {i} has no specific chunk_type, got {type}"); + } + } + } if (contentStartsWithHeading == true) { var headingChar = new string(new[] { (char)35 }); diff --git a/e2e/csharp/TokenReductionTests.cs b/e2e/csharp/TokenReductionTests.cs index c9ceba38fd9..e5d72b74392 100644 --- a/e2e/csharp/TokenReductionTests.cs +++ b/e2e/csharp/TokenReductionTests.cs @@ -66,7 +66,7 @@ public void TokenReductionWithChunking() TestHelpers.AssertExpectedMime(result, new[] { "application/pdf" }); TestHelpers.AssertMinContentLength(result, 5); TestHelpers.AssertMaxContentLength(result, 200); - TestHelpers.AssertChunks(result, 1, null, true, null, null, null); + TestHelpers.AssertChunks(result, 1, null, true, null, null, null, null); TestHelpers.AssertContentNotEmpty(result); } diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/ArchiveTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/ArchiveTest.php new file mode 100644 index 00000000000..e19142b6ac8 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/ArchiveTest.php @@ -0,0 +1,94 @@ +markTestSkipped('Skipping archive_gz_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/gzip', 'application/x-gzip']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * 7-Zip archive extraction. + */ + public function test_archive_sevenz_basic(): void + { + $documentPath = Helpers::resolveDocument('archives/documents.7z'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping archive_sevenz_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-7z-compressed']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * TAR archive extraction. + */ + public function test_archive_tar_basic(): void + { + $documentPath = Helpers::resolveDocument('archives/documents.tar'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping archive_tar_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-tar', 'application/tar']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * ZIP archive extraction. + */ + public function test_archive_zip_basic(): void + { + $documentPath = Helpers::resolveDocument('archives/documents.zip'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping archive_zip_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/zip', 'application/x-zip-compressed']); + Helpers::assertMinContentLength($result, 10); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/ContractTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/ContractTest.php new file mode 100644 index 00000000000..21794da2757 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/ContractTest.php @@ -0,0 +1,1318 @@ +markTestSkipped('Skipping api_batch_bytes_async: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $bytes = file_get_contents($documentPath); + $mimeType = Kreuzberg::detectMimeType($bytes); + $deferred = $kreuzberg->batchExtractBytesAsync([$bytes], [$mimeType]); + $results = $deferred->getResults(); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); + } + + /** + * Tests sync batch bytes extraction API (batch_extract_bytes_sync) + */ + public function test_api_batch_bytes_sync(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_batch_bytes_sync: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $bytes = file_get_contents($documentPath); + $mimeType = Kreuzberg::detectMimeType($bytes); + $results = $kreuzberg->batchExtractBytes([$bytes], [$mimeType]); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); + } + + /** + * Tests async batch bytes extraction with per-file configs (batch_extract_bytes with file_configs parameter) + */ + public function test_api_batch_bytes_with_configs_async(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_batch_bytes_with_configs_async: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $bytes = file_get_contents($documentPath); + $mimeType = Kreuzberg::detectMimeType($bytes); + $deferred = $kreuzberg->batchExtractBytesAsync([$bytes], [$mimeType]); + $results = $deferred->getResults(); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests sync batch bytes extraction with per-file configs (batch_extract_bytes_sync with file_configs parameter) + */ + public function test_api_batch_bytes_with_configs_sync(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_batch_bytes_with_configs_sync: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $bytes = file_get_contents($documentPath); + $mimeType = Kreuzberg::detectMimeType($bytes); + $results = $kreuzberg->batchExtractBytes([$bytes], [$mimeType]); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests async batch file extraction API (batch_extract_file) + */ + public function test_api_batch_file_async(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_batch_file_async: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $deferred = $kreuzberg->batchExtractFilesAsync([$documentPath]); + $results = $deferred->getResults(); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); + } + + /** + * Tests sync batch file extraction API (batch_extract_file_sync) + */ + public function test_api_batch_file_sync(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_batch_file_sync: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $results = $kreuzberg->batchExtractFiles([$documentPath]); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); + } + + /** + * Tests async batch file extraction with per-file configs (batch_extract_files with file_configs parameter) + */ + public function test_api_batch_file_with_configs_async(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_batch_file_with_configs_async: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $deferred = $kreuzberg->batchExtractFilesAsync([$documentPath]); + $results = $deferred->getResults(); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests sync batch file extraction with per-file configs (batch_extract_files_sync with file_configs parameter) + */ + public function test_api_batch_file_with_configs_sync(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_batch_file_with_configs_sync: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $results = $kreuzberg->batchExtractFiles([$documentPath]); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests sync batch file extraction with per-file timeout config override + */ + public function test_api_batch_file_with_timeout_sync(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_batch_file_with_timeout_sync: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['extraction_timeout_secs' => 300]); + + $kreuzberg = new Kreuzberg($config); + $results = $kreuzberg->batchExtractFiles([$documentPath]); + $result = $results[0]; + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests async bytes extraction API (extract_bytes) + */ + public function test_api_extract_bytes_async(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_extract_bytes_async: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $bytes = file_get_contents($documentPath); + $mimeType = Kreuzberg::detectMimeType($bytes); + $deferred = $kreuzberg->extractBytesAsync($bytes, $mimeType); + $result = $deferred->getResult(); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); + } + + /** + * Tests sync bytes extraction API (extract_bytes_sync) + */ + public function test_api_extract_bytes_sync(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_extract_bytes_sync: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $bytes = file_get_contents($documentPath); + $mimeType = Kreuzberg::detectMimeType($bytes); + $result = $kreuzberg->extractBytes($bytes, $mimeType); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); + } + + /** + * Tests async file extraction API (extract_file) + */ + public function test_api_extract_file_async(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_extract_file_async: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $deferred = $kreuzberg->extractFileAsync($documentPath); + $result = $deferred->getResult(); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); + } + + /** + * Tests sync file extraction API (extract_file_sync) + */ + public function test_api_extract_file_sync(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping api_extract_file_sync: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); + } + + /** + * Tests explicit CPU acceleration provider configuration + */ + public function test_config_acceleration_cpu_provider(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_acceleration_cpu_provider: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['acceleration' => ['device_id' => 0, 'provider' => 'cpu']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'To Whom it May Concern']); + } + + /** + * Tests chunking configuration with chunk assertions + */ + public function test_config_chunking(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_chunking: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['chunking' => ['max_chars' => 500, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 1, null, true, null, null, null); + } + + /** + * Tests markdown chunker populates heading context on chunks + */ + public function test_config_chunking_heading_context(): void + { + $documentPath = Helpers::resolveDocument('markdown/extraction_test.md'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_chunking_heading_context: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('chunking'); + + $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'markdown', 'max_chars' => 300, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 2, null, true, null, true, null); + } + + /** + * Tests markdown-aware chunker type + */ + public function test_config_chunking_markdown(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_chunking_markdown: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('chunking'); + + $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'markdown', 'max_chars' => 500, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 1, null, true, null, null, null); + } + + /** + * Tests markdown chunker on text with no headings produces null heading_context + */ + public function test_config_chunking_no_headings(): void + { + $documentPath = Helpers::resolveDocument('text/book_war_and_peace_1p.txt'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_chunking_no_headings: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('chunking'); + + $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'markdown', 'max_chars' => 300, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 2, null, true, null, false, null); + } + + /** + * Tests markdown chunker prepends heading hierarchy to chunk content + */ + public function test_config_chunking_prepend_heading_context(): void + { + $documentPath = Helpers::resolveDocument('markdown/extraction_test.md'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_chunking_prepend_heading_context: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('chunking'); + + $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'markdown', 'max_chars' => 300, 'max_overlap' => 50, 'prepend_heading_context' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 2, null, true, null, true, true); + } + + /** + * Tests chunking with very small chunk size produces more chunks + */ + public function test_config_chunking_small(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_chunking_small: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('chunking'); + + $config = Helpers::buildConfig(['chunking' => ['max_chars' => 100, 'max_overlap' => 20]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 2, null, true, null, null, null); + } + + /** + * Tests text chunker type (generic whitespace/punctuation splitter) + */ + public function test_config_chunking_text(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_chunking_text: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'text', 'max_chars' => 500, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 1, null, true, null, null, null); + } + + /** + * Tests token-based chunk sizing with HuggingFace tokenizer + */ + public function test_config_chunking_tokenizer(): void + { + $documentPath = Helpers::resolveDocument('markdown/comprehensive.md'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_chunking_tokenizer: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('chunking-tokenizers'); + + $config = Helpers::buildConfig(['chunking' => ['max_chars' => 200, 'max_overlap' => 40, 'sizing' => ['model' => 'Xenova/gpt-4o', 'type' => 'tokenizer']]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 2, null, true, null, null, null); + } + + /** + * Tests djot output format converts content to djot markup + */ + public function test_config_djot_content(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_djot_content: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['output_format' => 'djot']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests include_document_structure config produces document tree + */ + public function test_config_document_structure(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_document_structure: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['include_document_structure' => true]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertDocument($result, true, 1, ['paragraph'], null); + } + + /** + * Tests document field is null when include_document_structure is false + */ + public function test_config_document_structure_disabled(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_document_structure_disabled: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertDocument($result, false, null, null, null); + } + + /** + * Tests document structure extraction with group node assertion on DOCX with headings + */ + public function test_config_document_structure_groups(): void + { + $documentPath = Helpers::resolveDocument('docx/unit_test_headers.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_document_structure_groups: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('office'); + + $config = Helpers::buildConfig(['include_document_structure' => true]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertDocument($result, true, null, null, true); + } + + /** + * Tests document structure extraction with heading nodes on a DOCX + */ + public function test_config_document_structure_headings(): void + { + $documentPath = Helpers::resolveDocument('docx/unit_test_headers.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_document_structure_headings: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('office'); + + $config = Helpers::buildConfig(['include_document_structure' => true]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertDocument($result, true, 1, ['heading', 'paragraph'], null); + } + + /** + * Tests document structure with DOCX heading-driven nesting + */ + public function test_config_document_structure_with_headings(): void + { + $documentPath = Helpers::resolveDocument('docx/fake.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_document_structure_with_headings: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['include_document_structure' => true]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertDocument($result, true, 1, null, null); + } + + /** + * Tests element-based result format with element type assertions on DOCX + */ + public function test_config_element_types(): void + { + $documentPath = Helpers::resolveDocument('docx/unit_test_headers.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_element_types: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('office'); + + $config = Helpers::buildConfig(['result_format' => 'element_based']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertElements($result, 1, ['narrative_text']); + } + + /** + * Tests MSG extraction with custom fallback codepage for Cyrillic + */ + public function test_config_email_msg_fallback_codepage(): void + { + $documentPath = Helpers::resolveDocument('email/fake_email.msg'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_email_msg_fallback_codepage: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['email' => ['msg_fallback_codepage' => 1251]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.ms-outlook']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests that extraction_timeout_secs config field is accepted and does not affect fast extractions + */ + public function test_config_extraction_timeout(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_extraction_timeout: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['extraction_timeout_secs' => 300]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests force_ocr configuration option + */ + public function test_config_force_ocr(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_force_ocr: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * Tests that force_ocr_pages config field is accepted for selective page OCR + */ + public function test_config_force_ocr_pages(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_force_ocr_pages: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('ocr'); + + $config = Helpers::buildConfig(['force_ocr_pages' => [1], 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 1); + } + + /** + * Tests extraction with HTML conversion options configured + */ + public function test_config_html_options(): void + { + $documentPath = Helpers::resolveDocument('html/complex_table.html'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_html_options: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['html_options' => ['extract_metadata' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/html']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentNotEmpty($result); + } + + /** + * Tests image extraction configuration with image assertions + */ + public function test_config_images(): void + { + $documentPath = Helpers::resolveDocument('pdf/embedded_images_tables.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_images: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['images' => ['extract_images' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertImages($result, 1, null, null); + } + + /** + * Tests image extraction on PPTX containing embedded images + */ + public function test_config_images_with_formats(): void + { + $documentPath = Helpers::resolveDocument('pptx/powerpoint_with_image.pptx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_images_with_formats: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['images' => ['extract_images' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.presentation']); + Helpers::assertImages($result, 1, null, null); + } + + /** + * Tests keyword extraction via YAKE algorithm + */ + public function test_config_keywords(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_keywords: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('keywords-yake'); + + $config = Helpers::buildConfig(['keywords' => ['algorithm' => 'yake', 'max_keywords' => 10]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertKeywords($result, true, 1, null); + } + + /** + * Tests language detection configuration + */ + public function test_config_language_detection(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_language_detection: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['language_detection' => ['enabled' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertDetectedLanguages($result, ['eng'], 0.5); + } + + /** + * Tests language detection with detect_multiple and min_confidence options + */ + public function test_config_language_detection_multi(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_language_detection_multi: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['language_detection' => ['detect_multiple' => true, 'enabled' => true, 'min_confidence' => 0.3]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertDetectedLanguages($result, ['eng'], null); + } + + /** + * Tests multi-language detection config + */ + public function test_config_language_multi(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_language_multi: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('language-detection'); + + $config = Helpers::buildConfig(['language_detection' => ['detect_multiple' => true, 'enabled' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertDetectedLanguages($result, ['eng'], null); + } + + /** + * Tests page extraction and page marker configuration + */ + public function test_config_pages(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_pages: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['pages' => ['extract_pages' => true, 'insert_page_markers' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['PAGE']); + } + + /** + * Tests page extraction with exact page count assertion on multi-page PDF + */ + public function test_config_pages_exact_count(): void + { + $documentPath = Helpers::resolveDocument('pdf/multi_page.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_pages_exact_count: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['pages' => ['extract_pages' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertPages($result, null, 5); + } + + /** + * Tests page extraction config producing per-page content array + */ + public function test_config_pages_extract(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_pages_extract: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['pages' => ['extract_pages' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertPages($result, 1, null); + } + + /** + * Tests page marker insertion in extracted content + */ + public function test_config_pages_markers(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_pages_markers: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['pages' => ['insert_page_markers' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['PAGE']); + } + + /** + * Tests PDF annotation extraction with min_count assertion + */ + public function test_config_pdf_annotations_count(): void + { + $documentPath = Helpers::resolveDocument('vendored/pdfplumber/pdf/annotations.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_pdf_annotations_count: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['pdf_options' => ['extract_annotations' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertAnnotations($result, true, 3); + } + + /** + * Tests PDF hierarchy extraction config with block-level structure + */ + public function test_config_pdf_hierarchy(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_pdf_hierarchy: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['pages' => ['extract_pages' => true], 'pdf_options' => ['hierarchy' => ['enabled' => true, 'include_bbox' => true]]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + } + + /** + * Tests PDF margin exclusion configuration + */ + public function test_config_pdf_margins(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_pdf_margins: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['pdf_options' => ['bottom_margin_fraction' => 0.1, 'top_margin_fraction' => 0.1]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * Tests postprocessor config is accepted and extraction succeeds + */ + public function test_config_postprocessor(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_postprocessor: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['postprocessor' => ['enabled' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentNotEmpty($result); + } + + /** + * Tests that a clean PDF extraction produces no processing warnings + */ + public function test_config_processing_warnings_empty(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_processing_warnings_empty: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertProcessingWarnings($result, null, true); + } + + /** + * Tests extraction with quality processing explicitly disabled + */ + public function test_config_quality_disabled(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_quality_disabled: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['enable_quality_processing' => false]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentNotEmpty($result); + } + + /** + * Tests quality scoring produces a score value in [0.0, 1.0] + */ + public function test_config_quality_enabled(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_quality_enabled: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('quality'); + + $config = Helpers::buildConfig(['enable_quality_processing' => true]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertQualityScore($result, true, 0, 1); + } + + /** + * Tests quality scoring produces a score with minimum bound assertion + */ + public function test_config_quality_score_range(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_quality_score_range: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('quality'); + + $config = Helpers::buildConfig(['enable_quality_processing' => true]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertQualityScore($result, true, 0.1, null); + } + + /** + * Tests archive extraction with custom security limits + */ + public function test_config_security_limits(): void + { + $documentPath = Helpers::resolveDocument('archives/documents.zip'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_security_limits: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['security_limits' => ['max_archive_size' => 104857600, 'max_compression_ratio' => 50, 'max_files_in_archive' => 100]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/zip', 'application/x-zip-compressed']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests structured (JSON) output format config + */ + public function test_config_structured_output(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_structured_output: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['output_format' => 'structured']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests table extraction with content_contains_any assertion on DOCX with tables + */ + public function test_config_tables_content(): void + { + $documentPath = Helpers::resolveDocument('docx/docx_tables.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_tables_content: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertTableCount($result, 1, null); + } + + /** + * Tests use_cache=false configuration option + */ + public function test_config_use_cache_false(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping config_use_cache_false: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['use_cache' => false]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests markdown output format via bytes extraction API + */ + public function test_output_format_bytes_markdown(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping output_format_bytes_markdown: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['output_format' => 'markdown']); + + $kreuzberg = new Kreuzberg($config); + $bytes = file_get_contents($documentPath); + $mimeType = Kreuzberg::detectMimeType($bytes); + $result = $kreuzberg->extractBytes($bytes, $mimeType); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests Djot output format + */ + public function test_output_format_djot(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping output_format_djot: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['output_format' => 'djot']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests HTML output format + */ + public function test_output_format_html(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping output_format_html: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['output_format' => 'html']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests Markdown output format + */ + public function test_output_format_markdown(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping output_format_markdown: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['output_format' => 'markdown']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests Plain output format + */ + public function test_output_format_plain(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping output_format_plain: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['output_format' => 'plain']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Tests ElementBased result format with element assertions + */ + public function test_result_format_element_based(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping result_format_element_based: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['result_format' => 'element_based']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertElements($result, 1, null); + } + + /** + * Tests Unified result format (default) + */ + public function test_result_format_unified(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping result_format_unified: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['result_format' => 'unified']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php new file mode 100644 index 00000000000..039d6536d88 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php @@ -0,0 +1,132 @@ +markTestSkipped('Skipping email_eml_html_body: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['message/rfc822']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * EML with multipart MIME content. + */ + public function test_email_eml_multipart(): void + { + $documentPath = Helpers::resolveDocument('email/html_email_multipart.eml'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping email_eml_multipart: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['message/rfc822']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * UTF-16 encoded EML file with BOM. + */ + public function test_email_eml_utf16(): void + { + $documentPath = Helpers::resolveDocument('vendored/unstructured/eml/fake-email-utf-16.eml'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping email_eml_utf16: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['message/rfc822']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertContentContainsAny($result, ['Test Email', 'Roses are red']); + } + + /** + * Outlook MSG file extraction. + */ + public function test_email_msg_basic(): void + { + $documentPath = Helpers::resolveDocument('email/fake_email.msg'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping email_msg_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.ms-outlook']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Empty Outlook PST archive with no messages. + */ + public function test_email_pst_empty(): void + { + $documentPath = Helpers::resolveDocument('email/empty.pst'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping email_pst_empty: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.ms-outlook-pst']); + } + + /** + * Sample EML email file to verify email parsing. + */ + public function test_email_sample_eml(): void + { + $documentPath = Helpers::resolveDocument('email/sample_email.eml'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping email_sample_eml: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['message/rfc822']); + Helpers::assertMinContentLength($result, 20); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/EmbeddingsTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/EmbeddingsTest.php new file mode 100644 index 00000000000..1959f7d09e5 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/EmbeddingsTest.php @@ -0,0 +1,105 @@ +markTestSkipped('Skipping embedding_async: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('embeddings'); + + $config = Helpers::buildConfig(['chunking' => ['embedding' => ['model' => ['name' => 'balanced', 'type' => 'preset'], 'normalize' => true], 'max_chars' => 500, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $deferred = $kreuzberg->extractFileAsync($documentPath); + $result = $deferred->getResult(); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 1, null, true, true, null, null); + } + + /** + * Tests embedding generation with balanced preset model via chunking + */ + public function test_embedding_balanced_preset(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping embedding_balanced_preset: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('embeddings'); + + $config = Helpers::buildConfig(['chunking' => ['embedding' => ['model' => ['name' => 'balanced', 'type' => 'preset'], 'normalize' => true], 'max_chars' => 500, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 1, null, true, true, null, null); + } + + /** + * Tests chunking without embeddings - chunks should not have embedding vectors + */ + public function test_embedding_disabled(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping embedding_disabled: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['chunking' => ['max_chars' => 500, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 1, null, true, false, null, null); + } + + /** + * Tests embedding generation with fast preset model + */ + public function test_embedding_fast_preset(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping embedding_fast_preset: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('embeddings'); + + $config = Helpers::buildConfig(['chunking' => ['embedding' => ['model' => ['name' => 'fast', 'type' => 'preset'], 'normalize' => true], 'max_chars' => 500, 'max_overlap' => 50]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertChunks($result, 1, null, true, true, null, null); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php new file mode 100644 index 00000000000..ac68313acb5 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php @@ -0,0 +1,57 @@ +markTestSkipped('Skipping html_complex_layout: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/html']); + Helpers::assertMinContentLength($result, 1000); + } + + /** + * HTML table converted to markdown should retain structure. + */ + public function test_html_simple_table(): void + { + $documentPath = Helpers::resolveDocument('html/simple_table.html'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping html_simple_table: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/html']); + Helpers::assertMinContentLength($result, 100); + Helpers::assertContentContainsAll($result, ['Product', 'Category', 'Price', 'Stock', 'Laptop', 'Electronics', 'Sample Data Table']); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php new file mode 100644 index 00000000000..31c8adc0d53 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php @@ -0,0 +1,224 @@ +markTestSkipped('Skipping image_bmp_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/bmp']); + Helpers::assertContentNotEmpty($result); + } + + /** + * GIF image extraction via OCR. + */ + public function test_image_gif_basic(): void + { + $documentPath = Helpers::resolveDocument('images_extra/ocr_image.gif'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_gif_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/gif']); + Helpers::assertContentNotEmpty($result); + } + + /** + * JPEG 2000 image extraction via OCR. + */ + public function test_image_jp2_basic(): void + { + $documentPath = Helpers::resolveDocument('images_extra/ocr_image.jp2'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_jp2_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/jp2', 'image/jpeg2000']); + Helpers::assertContentNotEmpty($result); + } + + /** + * JPEG image to validate metadata extraction without OCR. + */ + public function test_image_metadata_only(): void + { + $documentPath = Helpers::resolveDocument('images/example.jpg'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_metadata_only: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['ocr' => null]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/jpeg']); + Helpers::assertMaxContentLength($result, 200); + } + + /** + * PBM (portable bitmap) image extraction via OCR. + */ + public function test_image_pbm_basic(): void + { + $documentPath = Helpers::resolveDocument('images_extra/ocr_image.pbm'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_pbm_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/x-portable-bitmap', 'image/x-pbm']); + Helpers::assertContentNotEmpty($result); + } + + /** + * PGM (portable graymap) image extraction via OCR. + */ + public function test_image_pgm_basic(): void + { + $documentPath = Helpers::resolveDocument('images_extra/ocr_image.pgm'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_pgm_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/x-portable-graymap', 'image/x-pgm']); + Helpers::assertContentNotEmpty($result); + } + + /** + * PPM (portable pixmap) image extraction via OCR. + */ + public function test_image_ppm_basic(): void + { + $documentPath = Helpers::resolveDocument('images_extra/ocr_image.ppm'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_ppm_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/x-portable-pixmap', 'image/x-ppm']); + Helpers::assertContentNotEmpty($result); + } + + /** + * SVG image extraction. + */ + public function test_image_svg_basic(): void + { + $documentPath = Helpers::resolveDocument('xml/simple_svg.svg'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_svg_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/svg+xml']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * TIFF image extraction via OCR. + */ + public function test_image_tiff_basic(): void + { + $documentPath = Helpers::resolveDocument('images_extra/ocr_image.tif'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_tiff_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/tiff']); + Helpers::assertContentNotEmpty($result); + } + + /** + * WebP image extraction via OCR. + */ + public function test_image_webp_basic(): void + { + $documentPath = Helpers::resolveDocument('images_extra/ocr_image.webp'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping image_webp_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/webp']); + Helpers::assertContentNotEmpty($result); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/KeywordsTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/KeywordsTest.php new file mode 100644 index 00000000000..23f45154c88 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/KeywordsTest.php @@ -0,0 +1,62 @@ +markTestSkipped('Skipping keywords_rake: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('keywords-rake'); + + $config = Helpers::buildConfig(['keywords' => ['algorithm' => 'rake', 'max_keywords' => 10]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertKeywords($result, true, 1, 10); + } + + /** + * Tests keyword extraction using YAKE algorithm + */ + public function test_keywords_yake(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping keywords_yake: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('keywords-yake'); + + $config = Helpers::buildConfig(['keywords' => ['algorithm' => 'yake', 'max_keywords' => 10]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertKeywords($result, true, 1, 10); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php new file mode 100644 index 00000000000..d221a5e41ac --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php @@ -0,0 +1,396 @@ +markTestSkipped('Skipping ocr_image_hello_world: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertContentContainsAny($result, ['hello', 'world']); + } + + /** + * Image with no text to ensure OCR handles empty results gracefully. + */ + public function test_ocr_image_no_text(): void + { + $documentPath = Helpers::resolveDocument('images/flower_no_text.jpg'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_image_no_text: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/jpeg']); + Helpers::assertMaxContentLength($result, 300); + } + + /** + * PaddleOCR with minimum confidence threshold filtering. + */ + public function test_ocr_paddle_confidence_filter(): void + { + $documentPath = Helpers::resolveDocument('images/ocr_image.jpg'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_confidence_filter: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en', 'paddle_ocr_config' => ['min_confidence' => 80.0]]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/jpeg']); + Helpers::assertMinContentLength($result, 1); + } + + /** + * Tests PaddleOCR with element hierarchy building enabled + */ + public function test_ocr_paddle_element_hierarchy(): void + { + $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_element_hierarchy: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'element_config' => ['build_hierarchy' => true, 'include_elements' => true], 'language' => 'en']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertOcrElements($result, true, true, true, null); + } + + /** + * Tests PaddleOCR with word-level element extraction + */ + public function test_ocr_paddle_element_levels(): void + { + $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_element_levels: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'element_config' => ['include_elements' => true, 'min_level' => 'word'], 'language' => 'en']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertOcrElements($result, true, true, null, 1); + } + + /** + * Chinese OCR with PaddleOCR - its core strength. + */ + public function test_ocr_paddle_image_chinese(): void + { + $documentPath = Helpers::resolveDocument('images/chi_sim_image.jpeg'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_image_chinese: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'ch']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/jpeg']); + Helpers::assertMinContentLength($result, 1); + } + + /** + * Simple English image OCR with PaddleOCR backend. + */ + public function test_ocr_paddle_image_english(): void + { + $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_image_english: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertContentContainsAny($result, ['hello', 'Hello', 'world', 'World']); + } + + /** + * PaddleOCR with markdown output format. + */ + public function test_ocr_paddle_markdown(): void + { + $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_markdown: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en', 'paddle_ocr_config' => ['output_format' => 'markdown']]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertContentContainsAny($result, ['hello', 'Hello', 'world', 'World']); + } + + /** + * Scanned PDF requires PaddleOCR to extract text. + */ + public function test_ocr_paddle_pdf_scanned(): void + { + $documentPath = Helpers::resolveDocument('pdf/ocr_test.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_pdf_scanned: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 20); + Helpers::assertContentContainsAny($result, ['Docling', 'Markdown', 'JSON']); + } + + /** + * PaddleOCR with structured output preserving all metadata. + */ + public function test_ocr_paddle_structured(): void + { + $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_structured: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'element_config' => ['include_elements' => true], 'language' => 'en']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertOcrElements($result, true, true, true, null); + } + + /** + * Table detection and extraction with PaddleOCR. + */ + public function test_ocr_paddle_table_detection(): void + { + $documentPath = Helpers::resolveDocument('images/simple_table.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_paddle_table_detection: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + Helpers::skipIfFeatureUnavailable('paddle-ocr'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en', 'paddle_ocr_config' => ['enable_table_detection' => true]]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertTableCount($result, 1, null); + } + + /** + * Image-only German PDF requiring OCR to extract text. + */ + public function test_ocr_pdf_image_only_german(): void + { + $documentPath = Helpers::resolveDocument('pdf/image_only_german_pdf.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_pdf_image_only_german: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'deu']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 20); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + + /** + * Rotated page PDF requiring OCR to verify orientation handling. + */ + public function test_ocr_pdf_rotated_90(): void + { + $documentPath = Helpers::resolveDocument('pdf/ocr_test_rotated_90.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_pdf_rotated_90: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Scanned PDF requires OCR to extract text. + */ + public function test_ocr_pdf_tesseract(): void + { + $documentPath = Helpers::resolveDocument('pdf/ocr_test.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_pdf_tesseract: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 20); + Helpers::assertContentContainsAny($result, ['Docling', 'Markdown', 'JSON']); + } + + /** + * Tests Tesseract OCR with element-level structured output including geometry + */ + public function test_ocr_tesseract_elements(): void + { + $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_tesseract_elements: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'element_config' => ['include_elements' => true], 'language' => 'eng']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertOcrElements($result, true, true, true, null); + } + + /** + * Tests Tesseract OCR element output with min_count assertion + */ + public function test_ocr_tesseract_elements_min_count(): void + { + $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_tesseract_elements_min_count: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'element_config' => ['include_elements' => true, 'min_level' => 'line'], 'language' => 'eng']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertOcrElements($result, true, null, null, 1); + } + + /** + * Tests Tesseract OCR with German language configuration + */ + public function test_ocr_tesseract_language_german(): void + { + $documentPath = Helpers::resolveDocument('pdf/image_only_german_pdf.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping ocr_tesseract_language_german: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('tesseract'); + + $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'deu']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 20); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php new file mode 100644 index 00000000000..f463e15bfe7 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php @@ -0,0 +1,1007 @@ +markTestSkipped('Skipping office_bibtex_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-bibtex', 'text/x-bibtex']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * CommonMark (.commonmark) text extraction. + */ + public function test_office_commonmark_basic(): void + { + $documentPath = Helpers::resolveDocument('markdown/sample.commonmark'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_commonmark_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/markdown', 'text/plain', 'text/x-commonmark']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * dBASE (.dbf) table extraction as markdown. + */ + public function test_office_dbf_basic(): void + { + $documentPath = Helpers::resolveDocument('dbf/stations.dbf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_dbf_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('office'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-dbf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['|']); + } + + /** + * Djot markup text extraction. + */ + public function test_office_djot_basic(): void + { + $documentPath = Helpers::resolveDocument('markdown/tables.djot'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_djot_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/x-djot', 'text/djot']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Legacy .doc document extraction via native OLE/CFB parsing. + */ + public function test_office_doc_legacy(): void + { + $documentPath = Helpers::resolveDocument('doc/unit_test_lists.doc'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_doc_legacy: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('office'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/msword']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * DocBook XML document extraction. + */ + public function test_office_docbook_basic(): void + { + $documentPath = Helpers::resolveDocument('docbook/docbook-reader.docbook'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_docbook_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/docbook+xml', 'text/docbook']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * DOCX document extraction baseline. + */ + public function test_office_docx_basic(): void + { + $documentPath = Helpers::resolveDocument('docx/sample_document.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_docx_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * DOCX file containing equations to validate math extraction. + */ + public function test_office_docx_equations(): void + { + $documentPath = Helpers::resolveDocument('docx/equations.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_docx_equations: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * Simple DOCX document to verify baseline extraction. + */ + public function test_office_docx_fake(): void + { + $documentPath = Helpers::resolveDocument('docx/fake.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_docx_fake: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * DOCX document heavy on formatting for style preservation. + */ + public function test_office_docx_formatting(): void + { + $documentPath = Helpers::resolveDocument('docx/unit_test_formatting.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_docx_formatting: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * DOCX document with complex headers. + */ + public function test_office_docx_headers(): void + { + $documentPath = Helpers::resolveDocument('docx/unit_test_headers.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_docx_headers: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * DOCX document emphasizing list formatting. + */ + public function test_office_docx_lists(): void + { + $documentPath = Helpers::resolveDocument('docx/unit_test_lists.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_docx_lists: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * DOCX document containing tables for table-aware extraction. + */ + public function test_office_docx_tables(): void + { + $documentPath = Helpers::resolveDocument('docx/docx_tables.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_docx_tables: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertContentContainsAll($result, ['Simple uniform table', 'Nested Table', 'merged cells', 'Header Col']); + Helpers::assertTableCount($result, 1, null); + } + + /** + * EPUB book extraction with text content. + */ + public function test_office_epub_basic(): void + { + $documentPath = Helpers::resolveDocument('epub/features.epub'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_epub_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/epub+zip']); + Helpers::assertMinContentLength($result, 50); + } + + /** + * FictionBook (FB2) document extraction baseline. + */ + public function test_office_fb2_basic(): void + { + $documentPath = Helpers::resolveDocument('fictionbook/basic.fb2'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_fb2_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-fictionbook+xml']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * FictionBook (.fb2) text extraction. + */ + public function test_office_fictionbook_basic(): void + { + $documentPath = Helpers::resolveDocument('fictionbook/basic.fb2'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_fictionbook_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-fictionbook+xml', 'application/x-fictionbook']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Hangul Word Processor (.hwp) text extraction. + */ + public function test_office_hwp_basic(): void + { + $documentPath = Helpers::resolveDocument('hwp/converted_output.hwp'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_hwp_basic: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('office'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-hwp']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Hangul Word Processor (.hwp) styled document extraction. + */ + public function test_office_hwp_styled(): void + { + $documentPath = Helpers::resolveDocument('hwp/styled_document.hwp'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_hwp_styled: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('hwp'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-hwp']); + } + + /** + * JATS scientific article extraction. + */ + public function test_office_jats_basic(): void + { + $documentPath = Helpers::resolveDocument('jats/sample_article.jats'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_jats_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-jats+xml', 'text/jats']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Jupyter notebook extraction. + */ + public function test_office_jupyter_basic(): void + { + $documentPath = Helpers::resolveDocument('jupyter/rank.ipynb'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_jupyter_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-ipynb+json']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Keynote document extraction baseline. + */ + public function test_office_keynote_basic(): void + { + $documentPath = Helpers::resolveDocument('iwork/test.key'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_keynote_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-iwork-keynote-sffkey']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * LaTeX document text extraction. + */ + public function test_office_latex_basic(): void + { + $documentPath = Helpers::resolveDocument('latex/basic_sections.tex'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_latex_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-latex', 'text/x-latex']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * Markdown document extraction baseline. + */ + public function test_office_markdown_basic(): void + { + $documentPath = Helpers::resolveDocument('markdown/comprehensive.md'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_markdown_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/markdown']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * MDX document extraction with JSX stripping and frontmatter. + */ + public function test_office_mdx_basic(): void + { + $documentPath = Helpers::resolveDocument('markdown/sample.mdx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_mdx_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/mdx', 'text/x-mdx']); + Helpers::assertMinContentLength($result, 50); + } + + /** + * Real-world MDX extraction from mdx-js/mdx getting-started.mdx with imports, exports, JSX components, code blocks, and reference links. + */ + public function test_office_mdx_getting_started(): void + { + $documentPath = Helpers::resolveDocument('markdown/mdx_getting_started.mdx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_mdx_getting_started: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/mdx', 'text/x-mdx']); + Helpers::assertMinContentLength($result, 2000); + } + + /** + * Real-world MDX extraction from mdx-js/mdx troubleshooting-mdx.mdx with error documentation, code blocks, and JSX comments. + */ + public function test_office_mdx_troubleshooting(): void + { + $documentPath = Helpers::resolveDocument('markdown/mdx_troubleshooting.mdx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_mdx_troubleshooting: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/mdx', 'text/x-mdx']); + Helpers::assertMinContentLength($result, 2000); + } + + /** + * Real-world MDX extraction from mdx-js/mdx using-mdx.mdx with complex JSX examples, component passing, and MDX provider patterns. + */ + public function test_office_mdx_using_mdx(): void + { + $documentPath = Helpers::resolveDocument('markdown/mdx_using_mdx.mdx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_mdx_using_mdx: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/mdx', 'text/x-mdx']); + Helpers::assertMinContentLength($result, 2000); + } + + /** + * Numbers document extraction baseline. + */ + public function test_office_numbers_basic(): void + { + $documentPath = Helpers::resolveDocument('iwork/test.numbers'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_numbers_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-iwork-numbers-sffnumbers']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Basic ODS spreadsheet extraction. + */ + public function test_office_ods_basic(): void + { + $documentPath = Helpers::resolveDocument('data_formats/test_01.ods'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_ods_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.spreadsheet']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * ODT document with bold formatting. + */ + public function test_office_odt_bold(): void + { + $documentPath = Helpers::resolveDocument('odt/bold.odt'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_odt_bold: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.text']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * ODT document containing unordered lists with nesting. + */ + public function test_office_odt_list(): void + { + $documentPath = Helpers::resolveDocument('odt/unorderedList.odt'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_odt_list: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.text']); + Helpers::assertMinContentLength($result, 30); + Helpers::assertContentContainsAny($result, ['list item', 'New level', 'Pushed us']); + } + + /** + * Basic ODT document with paragraphs and headings. + */ + public function test_office_odt_simple(): void + { + $documentPath = Helpers::resolveDocument('odt/simple.odt'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_odt_simple: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.text']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertContentContainsAny($result, ['favorite things', 'Parrots', 'Analysis']); + } + + /** + * ODT document with a table structure. + */ + public function test_office_odt_table(): void + { + $documentPath = Helpers::resolveDocument('odt/table.odt'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_odt_table: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.text']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertTableCount($result, 1, null); + } + + /** + * OPML outline document extraction. + */ + public function test_office_opml_basic(): void + { + $documentPath = Helpers::resolveDocument('opml/outline.opml'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_opml_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/xml+opml', 'text/x-opml', 'application/x-opml+xml']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Org-mode document text extraction. + */ + public function test_office_org_basic(): void + { + $documentPath = Helpers::resolveDocument('org/comprehensive.org'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_org_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/x-org', 'text/org']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * Pages document extraction baseline. + */ + public function test_office_pages_basic(): void + { + $documentPath = Helpers::resolveDocument('iwork/test.pages'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_pages_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-iwork-pages-sffpages']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * PPSX (PowerPoint Show) files should extract slides content identical to PPTX. GitHub Issue #321 Bug 2. + */ + public function test_office_ppsx_slideshow(): void + { + $documentPath = Helpers::resolveDocument('pptx/sample.ppsx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_ppsx_slideshow: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.slideshow']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Legacy PowerPoint .ppt extraction via native OLE/CFB parsing. + */ + public function test_office_ppt_legacy(): void + { + $documentPath = Helpers::resolveDocument('ppt/simple.ppt'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_ppt_legacy: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('office'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.ms-powerpoint']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * PowerPoint macro-enabled presentation (.pptm) extraction. + */ + public function test_office_pptm_basic(): void + { + $documentPath = Helpers::resolveDocument('pptx/powerpoint_with_image.pptm'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_pptm_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.ms-powerpoint.presentation.macroEnabled.12', 'application/vnd.openxmlformats-officedocument.presentationml.presentation']); + Helpers::assertContentNotEmpty($result); + } + + /** + * PPTX deck should extract slides content. + */ + public function test_office_pptx_basic(): void + { + $documentPath = Helpers::resolveDocument('pptx/simple.pptx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_pptx_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.presentation']); + Helpers::assertMinContentLength($result, 50); + } + + /** + * PPTX presentation containing images to ensure metadata extraction. + */ + public function test_office_pptx_images(): void + { + $documentPath = Helpers::resolveDocument('pptx/powerpoint_with_image.pptx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_pptx_images: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.presentation']); + Helpers::assertMinContentLength($result, 15); + } + + /** + * Pitch deck PPTX used to validate large slide extraction. + */ + public function test_office_pptx_pitch_deck(): void + { + $documentPath = Helpers::resolveDocument('pptx/pitch_deck_presentation.pptx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_pptx_pitch_deck: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.presentation']); + Helpers::assertMinContentLength($result, 100); + } + + /** + * reStructuredText document extraction. + */ + public function test_office_rst_basic(): void + { + $documentPath = Helpers::resolveDocument('rst/restructured_text.rst'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_rst_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/x-rst', 'text/prs.fallenstein.rst']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * RTF document text extraction. + */ + public function test_office_rtf_basic(): void + { + $documentPath = Helpers::resolveDocument('rtf/extraction_test.rtf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_rtf_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/rtf', 'text/rtf']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Typst document text extraction. + */ + public function test_office_typst_basic(): void + { + $documentPath = Helpers::resolveDocument('typst/headings.typ'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_typst_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-typst', 'text/x-typst']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Legacy XLS spreadsheet to ensure backward compatibility. + */ + public function test_office_xls_legacy(): void + { + $documentPath = Helpers::resolveDocument('xls/test_excel.xls'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_xls_legacy: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.ms-excel']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Excel binary workbook (.xlsb) extraction. + */ + public function test_office_xlsb_basic(): void + { + $documentPath = Helpers::resolveDocument('xlsx/test_xlsb.xlsb'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_xlsb_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.ms-excel.sheet.binary.macroEnabled.12', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); + Helpers::assertContentNotEmpty($result); + } + + /** + * Excel macro-enabled workbook (.xlsm) extraction. + */ + public function test_office_xlsm_basic(): void + { + $documentPath = Helpers::resolveDocument('xlsx/test_01.xlsm'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_xlsm_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.ms-excel.sheet.macroEnabled.12', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); + Helpers::assertContentNotEmpty($result); + } + + /** + * XLSX spreadsheet should produce metadata and table content. + */ + public function test_office_xlsx_basic(): void + { + $documentPath = Helpers::resolveDocument('xlsx/stanley_cups.xlsx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_xlsx_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); + Helpers::assertMinContentLength($result, 100); + Helpers::assertContentContainsAll($result, ['Team', 'Location', 'Stanley Cups']); + Helpers::assertTableCount($result, 1, null); + Helpers::assertMetadataExpectation($result, 'sheet_count', ['gte' => 2]); + Helpers::assertMetadataExpectation($result, 'sheet_names', ['contains' => ['Stanley Cups']]); + } + + /** + * XLSX workbook with multiple sheets. + */ + public function test_office_xlsx_multi_sheet(): void + { + $documentPath = Helpers::resolveDocument('xlsx/excel_multi_sheet.xlsx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_xlsx_multi_sheet: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); + Helpers::assertMinContentLength($result, 20); + Helpers::assertMetadataExpectation($result, 'sheet_count', ['gte' => 2]); + } + + /** + * Simple XLSX spreadsheet shipped alongside office integration tests. + */ + public function test_office_xlsx_office_example(): void + { + $documentPath = Helpers::resolveDocument('xlsx/test_01.xlsx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping office_xlsx_office_example: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); + Helpers::assertMinContentLength($result, 10); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php new file mode 100644 index 00000000000..f03039a2915 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php @@ -0,0 +1,388 @@ +markTestSkipped('Skipping pdf_annotations: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['pdf_options' => ['extract_annotations' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertAnnotations($result, true, 1); + } + + /** + * Assembly language technical manual with large body of text. + */ + public function test_pdf_assembly_technical(): void + { + $documentPath = Helpers::resolveDocument('pdf/assembly_language_for_beginners_al4_b_en.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_assembly_technical: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 5000); + Helpers::assertContentContainsAny($result, ['assembly', 'register', 'instruction']); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + + /** + * Bayesian data analysis textbook PDF with large content volume. + */ + public function test_pdf_bayesian_data_analysis(): void + { + $documentPath = Helpers::resolveDocument('pdf/bayesian_data_analysis_third_edition_13th_feb_2020.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_bayesian_data_analysis: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10000); + Helpers::assertContentContainsAny($result, ['Bayesian', 'probability', 'distribution']); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + + /** + * Tests bounding box extraction on PDF tables and images + */ + public function test_pdf_bounding_boxes(): void + { + $documentPath = Helpers::resolveDocument('pdf/tiny.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_bounding_boxes: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('pdf'); + + $config = Helpers::buildConfig(['images' => ['extract_images' => true]]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertTableCount($result, 1, null); + Helpers::assertTableBoundingBoxes($result, true); + } + + /** + * PDF containing code snippets and formulas should retain substantial content. + */ + public function test_pdf_code_and_formula(): void + { + $documentPath = Helpers::resolveDocument('pdf/code_and_formula.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_code_and_formula: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 100); + } + + /** + * Deep learning textbook PDF to ensure long-form extraction quality. + */ + public function test_pdf_deep_learning(): void + { + $documentPath = Helpers::resolveDocument('pdf/fundamentals_of_deep_learning_2014.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_deep_learning: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 1000); + Helpers::assertContentContainsAny($result, ['neural', 'network', 'deep learning']); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + + /** + * PDF with embedded images should extract text and tables when present. + */ + public function test_pdf_embedded_images(): void + { + $documentPath = Helpers::resolveDocument('pdf/embedded_images_tables.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_embedded_images: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertTableCount($result, 0, null); + } + + /** + * Google Docs exported PDF to verify conversion fidelity. + */ + public function test_pdf_google_doc(): void + { + $documentPath = Helpers::resolveDocument('pdf/google_doc_document.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_google_doc: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + + /** + * Large machine learning textbook PDF to stress extraction length. + */ + public function test_pdf_large_ciml(): void + { + $documentPath = Helpers::resolveDocument('pdf/a_course_in_machine_learning_ciml_v0_9_all.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_large_ciml: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10000); + Helpers::assertContentContainsAny($result, ['machine learning', 'algorithm', 'training']); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + + /** + * PDF extraction with layout detection enabled should produce content from a document with mixed structure. + */ + public function test_pdf_layout_detection(): void + { + $documentPath = Helpers::resolveDocument('pdf/docling.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_layout_detection: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('layout-detection'); + + $config = Helpers::buildConfig(['layout' => ['preset' => 'accurate', 'table_model' => 'tatr'], 'output_format' => 'markdown']); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 100); + Helpers::assertContentNotEmpty($result); + } + + /** + * German technical PDF to ensure non-ASCII content extraction. + */ + public function test_pdf_non_english_german(): void + { + $documentPath = Helpers::resolveDocument('pdf/5_level_paging_and_5_level_ept_intel_revision_1_1_may_2017.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_non_english_german: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 100); + Helpers::assertContentContainsAny($result, ['Intel', 'paging']); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + + /** + * Copy-protected PDF should extract content (pdfium handles copy-protection transparently). + */ + public function test_pdf_password_protected(): void + { + $documentPath = Helpers::resolveDocument('pdf/copy_protected.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_password_protected: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertContentContainsAny($result, ['LayoutParser', 'document image analysis', 'deep learning']); + } + + /** + * Right-to-left language PDF to verify RTL extraction. + */ + public function test_pdf_right_to_left(): void + { + $documentPath = Helpers::resolveDocument('pdf/right_to_left_01.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_right_to_left: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + + /** + * Simple text-heavy PDF should extract content without OCR or tables. + */ + public function test_pdf_simple_text(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_simple_text: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'To Whom it May Concern', 'Mallori']); + } + + /** + * Large PDF with extensive tables to stress table extraction. + */ + public function test_pdf_tables_large(): void + { + $documentPath = Helpers::resolveDocument('pdf/large.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_tables_large: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 500); + } + + /** + * Medium-sized PDF with multiple tables. + */ + public function test_pdf_tables_medium(): void + { + $documentPath = Helpers::resolveDocument('pdf/medium.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_tables_medium: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 100); + } + + /** + * Small PDF containing tables to validate table extraction. + */ + public function test_pdf_tables_small(): void + { + $documentPath = Helpers::resolveDocument('pdf/tiny.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_tables_small: missing document at ' . $documentPath); + } + + Helpers::skipIfFeatureUnavailable('ocr'); + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertContentContainsAll($result, ['Table 1', 'Selected Numbers', 'Celsius', 'Fahrenheit', 'Water Freezing Point', 'Water Boiling Point']); + Helpers::assertTableCount($result, 1, null); + } + + /** + * Technical statistical learning PDF requiring substantial extraction. + */ + public function test_pdf_technical_stat_learning(): void + { + $documentPath = Helpers::resolveDocument('pdf/an_introduction_to_statistical_learning_with_applications_in_r_islr_sixth_printing.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping pdf_technical_stat_learning: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10000); + Helpers::assertContentContainsAny($result, ['statistical', 'regression', 'learning']); + Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/PluginApisTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/PluginApisTest.php new file mode 100644 index 00000000000..ad5abb73b20 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/PluginApisTest.php @@ -0,0 +1,208 @@ +assertNotNull($config); + + $this->assertNotNull($config->chunking); + $this->assertEquals(50, $config->chunking->maxChars); + chdir($oldCwd); + unlink($configPath); + rmdir($subdir); + rmdir($tmpDir); + } + + /** + * Load configuration from a TOML file + */ + public function test_config_from_file(): void + { + $tmpDir = sys_get_temp_dir(); + $configPath = $tmpDir . '/' . 'test_config.toml'; + file_put_contents($configPath, "[chunking]\nmax_chars = 100\nmax_overlap = 20\n\n[language_detection]\nenabled = false\n"); + + $config = ExtractionConfig::fromFile($configPath); + + $this->assertNotNull($config->chunking); + $this->assertEquals(100, $config->chunking->maxChars); + $this->assertEquals(20, $config->chunking->maxOverlap); + $this->assertNotNull($config->languageDetection); + $this->assertEquals(false, $config->languageDetection->enabled); + unlink($configPath); + } + + /** + * Clear all document extractors and verify list is empty + */ + public function test_extractors_clear(): void + { + Kreuzberg::clearDocumentExtractors(); + $result = Kreuzberg::listDocumentExtractors(); + $this->assertEmpty($result); + } + + /** + * List all registered document extractors + */ + public function test_extractors_list(): void + { + $result = Kreuzberg::listDocumentExtractors(); + $this->assertIsArray($result); + foreach ($result as $item) { + $this->assertIsString($item); + } + } + + /** + * Unregister nonexistent document extractor gracefully + */ + public function test_extractors_unregister(): void + { + Kreuzberg::unregisterDocumentExtractor('nonexistent-extractor-xyz'); + $this->assertTrue(true); // Should not throw + } + + /** + * Detect MIME type from file bytes + */ + public function test_mime_detect_bytes(): void + { + $testBytes = '%PDF-1.4\\n'; + $result = Kreuzberg::detectMimeType($testBytes); + + $this->assertStringContainsStringIgnoringCase('pdf', $result); + } + + /** + * Detect MIME type from file path + */ + public function test_mime_detect_path(): void + { + $tmpDir = sys_get_temp_dir(); + $testFile = $tmpDir . '/' . 'test.txt'; + file_put_contents($testFile, 'Hello, world!'); + + $result = Kreuzberg::detectMimeTypeFromPath($testFile); + + $this->assertStringContainsStringIgnoringCase('text', $result); + unlink($testFile); + } + + /** + * Get file extensions for a MIME type + */ + public function test_mime_get_extensions(): void + { + $result = Kreuzberg::getExtensionsForMime('application/pdf'); + $this->assertIsArray($result); + $this->assertContains('pdf', $result); + } + + /** + * Clear all OCR backends and verify list is empty + */ + public function test_ocr_backends_clear(): void + { + Kreuzberg::clearOcrBackends(); + $result = Kreuzberg::listOcrBackends(); + $this->assertEmpty($result); + } + + /** + * List all registered OCR backends + */ + public function test_ocr_backends_list(): void + { + $result = Kreuzberg::listOcrBackends(); + $this->assertIsArray($result); + foreach ($result as $item) { + $this->assertIsString($item); + } + } + + /** + * Unregister nonexistent OCR backend gracefully + */ + public function test_ocr_backends_unregister(): void + { + Kreuzberg::unregisterOcrBackend('nonexistent-backend-xyz'); + $this->assertTrue(true); // Should not throw + } + + /** + * Clear all post-processors and verify list is empty + */ + public function test_post_processors_clear(): void + { + Kreuzberg::clearPostProcessors(); + $this->assertTrue(true); // Should not throw + } + + /** + * List all registered post-processors + */ + public function test_post_processors_list(): void + { + $result = Kreuzberg::listPostProcessors(); + $this->assertIsArray($result); + foreach ($result as $item) { + $this->assertIsString($item); + } + } + + /** + * Clear all validators and verify list is empty + */ + public function test_validators_clear(): void + { + Kreuzberg::clearValidators(); + $result = Kreuzberg::listValidators(); + $this->assertEmpty($result); + } + + /** + * List all registered validators + */ + public function test_validators_list(): void + { + $result = Kreuzberg::listValidators(); + $this->assertIsArray($result); + foreach ($result as $item) { + $this->assertIsString($item); + } + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php new file mode 100644 index 00000000000..36cb31ed39a --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php @@ -0,0 +1,60 @@ +markTestSkipped('Missing document: ' . $documentPath); + } + $pngData = render_pdf_page($documentPath, 0, 72); + if (is_array($pngData)) { $pngData = pack('C*', ...$pngData); } + Helpers::assertIsPng($pngData); + Helpers::assertMinByteLength($pngData, 50); + } + + public function test_render_iterator(): void + { + $documentPath = Helpers::resolveDocument('pdf/tiny.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Missing document: ' . $documentPath); + } + $pages = []; + foreach (render_pdf_pages_iter($documentPath, 150) as $pngData) { + if (is_array($pngData)) { $pngData = pack('C*', ...$pngData); } + Helpers::assertIsPng($pngData); + $pages[] = $pngData; + } + $this->assertGreaterThanOrEqual(1, count($pages), + sprintf('Expected at least 1 pages, got %d', count($pages))); + } + + public function test_render_single_page(): void + { + $documentPath = Helpers::resolveDocument('pdf/tiny.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Missing document: ' . $documentPath); + } + $pngData = render_pdf_page($documentPath, 0, 150); + if (is_array($pngData)) { $pngData = pack('C*', ...$pngData); } + Helpers::assertIsPng($pngData); + Helpers::assertMinByteLength($pngData, 100); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php new file mode 100644 index 00000000000..bf90831adcf --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php @@ -0,0 +1,177 @@ +markTestSkipped('Skipping smoke_cache_namespace: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['cache_namespace' => 'test_tenant', 'cache_ttl_secs' => 3600, 'use_cache' => true]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/plain']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * Smoke test: DOCX with formatted text + */ + public function test_smoke_docx_basic(): void + { + $documentPath = Helpers::resolveDocument('docx/fake.docx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping smoke_docx_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); + Helpers::assertMinContentLength($result, 20); + Helpers::assertContentContainsAny($result, ['Lorem', 'ipsum', 'document', 'text']); + } + + /** + * Smoke test: HTML table extraction + */ + public function test_smoke_html_basic(): void + { + $documentPath = Helpers::resolveDocument('html/simple_table.html'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping smoke_html_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/html']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['Sample Data Table', 'Laptop', 'Electronics', 'Product']); + } + + /** + * Smoke test: PNG image (without OCR, metadata only) + */ + public function test_smoke_image_png(): void + { + $documentPath = Helpers::resolveDocument('images/sample.png'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping smoke_image_png: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['image/png']); + Helpers::assertMetadataExpectation($result, 'format', ['eq' => 'PNG']); + } + + /** + * Smoke test: JSON file extraction + */ + public function test_smoke_json_basic(): void + { + $documentPath = Helpers::resolveDocument('json/simple.json'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping smoke_json_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/json']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * Smoke test: PDF with simple text extraction + */ + public function test_smoke_pdf_basic(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping smoke_pdf_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 50); + Helpers::assertContentContainsAny($result, ['May 5, 2023', 'To Whom it May Concern']); + } + + /** + * Smoke test: Plain text file + */ + public function test_smoke_txt_basic(): void + { + $documentPath = Helpers::resolveDocument('text/report.txt'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping smoke_txt_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/plain']); + Helpers::assertMinContentLength($result, 5); + } + + /** + * Smoke test: XLSX with basic spreadsheet data including tables + */ + public function test_smoke_xlsx_basic(): void + { + $documentPath = Helpers::resolveDocument('xlsx/stanley_cups.xlsx'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping smoke_xlsx_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); + Helpers::assertMinContentLength($result, 100); + Helpers::assertContentContainsAll($result, ['Team', 'Location', 'Stanley Cups', 'Blues', 'Flyers', 'Maple Leafs', 'STL', 'PHI', 'TOR']); + Helpers::assertTableCount($result, 1, null); + Helpers::assertMetadataExpectation($result, 'sheet_count', ['gte' => 2]); + Helpers::assertMetadataExpectation($result, 'sheet_names', ['contains' => ['Stanley Cups']]); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/StructuredTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/StructuredTest.php new file mode 100644 index 00000000000..2af675deef0 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/StructuredTest.php @@ -0,0 +1,209 @@ +markTestSkipped('Skipping structured_csv_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/csv']); + Helpers::assertMinContentLength($result, 20); + } + + /** + * EndNote ENW citation format extraction. + */ + public function test_structured_enw_basic(): void + { + $documentPath = Helpers::resolveDocument('data_formats/sample.enw'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_enw_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-endnote-refer', 'application/x-endnote+xml', 'text/plain']); + } + + /** + * Structured JSON extraction should stream and preserve content. + */ + public function test_structured_json_basic(): void + { + $documentPath = Helpers::resolveDocument('json/sample_document.json'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_json_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/json']); + Helpers::assertMinContentLength($result, 20); + Helpers::assertContentContainsAny($result, ['Sample Document', 'Test Author']); + } + + /** + * Simple JSON document to verify structured extraction. + */ + public function test_structured_json_simple(): void + { + $documentPath = Helpers::resolveDocument('json/simple.json'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_json_simple: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/json']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentContainsAny($result, ['{', 'name']); + } + + /** + * PubMed NBIB citation format extraction. + */ + public function test_structured_nbib_basic(): void + { + $documentPath = Helpers::resolveDocument('data_formats/sample.nbib'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_nbib_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/nbib', 'application/x-pubmed', 'text/plain']); + Helpers::assertContentNotEmpty($result); + } + + /** + * RIS citation format extraction. + */ + public function test_structured_ris_basic(): void + { + $documentPath = Helpers::resolveDocument('data_formats/sample.ris'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_ris_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-research-info-systems', 'text/plain']); + Helpers::assertContentNotEmpty($result); + } + + /** + * TOML configuration file extraction. + */ + public function test_structured_toml_basic(): void + { + $documentPath = Helpers::resolveDocument('data_formats/cargo.toml'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_toml_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/toml', 'text/toml']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * TSV (tab-separated values) data file extraction. + */ + public function test_structured_tsv_basic(): void + { + $documentPath = Helpers::resolveDocument('data_formats/employees.tsv'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_tsv_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['text/tab-separated-values', 'text/plain']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * YAML file text extraction. + */ + public function test_structured_yaml_basic(): void + { + $documentPath = Helpers::resolveDocument('yaml/simple.yaml'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_yaml_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/yaml', 'text/yaml', 'text/x-yaml', 'application/x-yaml']); + Helpers::assertMinContentLength($result, 10); + } + + /** + * Simple YAML document to validate structured extraction. + */ + public function test_structured_yaml_simple(): void + { + $documentPath = Helpers::resolveDocument('yaml/simple.yaml'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping structured_yaml_simple: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/x-yaml']); + Helpers::assertMinContentLength($result, 10); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/Token_reductionTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/Token_reductionTest.php new file mode 100644 index 00000000000..71c8b8319a0 --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/Token_reductionTest.php @@ -0,0 +1,102 @@ +markTestSkipped('Skipping token_reduction_aggressive: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['token_reduction' => ['mode' => 'aggressive']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertMaxContentLength($result, 150); + Helpers::assertContentNotEmpty($result); + } + + /** + * Tests basic token reduction on PDF document + */ + public function test_token_reduction_basic(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping token_reduction_basic: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['token_reduction' => ['mode' => 'moderate']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertMaxContentLength($result, 200); + Helpers::assertContentNotEmpty($result); + } + + /** + * Tests light token reduction mode preserves most content + */ + public function test_token_reduction_light(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping token_reduction_light: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['token_reduction' => ['mode' => 'light']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 10); + Helpers::assertContentNotEmpty($result); + } + + /** + * Tests token reduction combined with chunking + */ + public function test_token_reduction_with_chunking(): void + { + $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); + if (!file_exists($documentPath)) { + $this->markTestSkipped('Skipping token_reduction_with_chunking: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(['chunking' => ['max_chars' => 500, 'max_overlap' => 50], 'token_reduction' => ['mode' => 'moderate']]); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/pdf']); + Helpers::assertMinContentLength($result, 5); + Helpers::assertMaxContentLength($result, 200); + Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertContentNotEmpty($result); + } + +} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php new file mode 100644 index 00000000000..7d2d1ba77ad --- /dev/null +++ b/e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php @@ -0,0 +1,37 @@ +markTestSkipped('Skipping xml_plant_catalog: missing document at ' . $documentPath); + } + + $config = Helpers::buildConfig(null); + + $kreuzberg = new Kreuzberg($config); + $result = $kreuzberg->extractFile($documentPath); + + Helpers::assertExpectedMime($result, ['application/xml']); + Helpers::assertMinContentLength($result, 100); + } + +} diff --git a/e2e/e2e/php/tests/Helpers.php b/e2e/e2e/php/tests/Helpers.php new file mode 100644 index 00000000000..449cacbb87a --- /dev/null +++ b/e2e/e2e/php/tests/Helpers.php @@ -0,0 +1,831 @@ +mimeType, $token)) { + $matches = true; + break; + } + } + + Assert::assertTrue( + $matches, + sprintf( + "Expected MIME '%s' to match one of %s", + $result->mimeType, + json_encode($expected) + ) + ); + } + + public static function assertMinContentLength(ExtractionResult $result, int $minimum): void + { + Assert::assertGreaterThanOrEqual( + $minimum, + strlen($result->content), + sprintf("Expected content length >= %d, got %d", $minimum, strlen($result->content)) + ); + } + + public static function assertMaxContentLength(ExtractionResult $result, int $maximum): void + { + Assert::assertLessThanOrEqual( + $maximum, + strlen($result->content), + sprintf("Expected content length <= %d, got %d", $maximum, strlen($result->content)) + ); + } + + public static function assertContentContainsAny(ExtractionResult $result, array $snippets): void + { + if (empty($snippets)) { + return; + } + + $lowered = strtolower($result->content); + $found = false; + foreach ($snippets as $snippet) { + if (str_contains($lowered, strtolower($snippet))) { + $found = true; + break; + } + } + + Assert::assertTrue( + $found, + sprintf( + "Expected content to contain any of %s. Preview: %s", + json_encode($snippets), + json_encode(substr($result->content, 0, 160)) + ) + ); + } + + public static function assertContentContainsAll(ExtractionResult $result, array $snippets): void + { + if (empty($snippets)) { + return; + } + + $lowered = strtolower($result->content); + $missing = []; + foreach ($snippets as $snippet) { + if (!str_contains($lowered, strtolower($snippet))) { + $missing[] = $snippet; + } + } + + Assert::assertEmpty( + $missing, + sprintf( + "Expected content to contain all snippets %s. Missing %s", + json_encode($snippets), + json_encode($missing) + ) + ); + } + + public static function assertTableCount(ExtractionResult $result, ?int $minimum, ?int $maximum): void + { + $count = count($result->tables ?? []); + + if ($minimum !== null) { + Assert::assertGreaterThanOrEqual( + $minimum, + $count, + sprintf("Expected at least %d tables, found %d", $minimum, $count) + ); + } + + if ($maximum !== null) { + Assert::assertLessThanOrEqual( + $maximum, + $count, + sprintf("Expected at most %d tables, found %d", $maximum, $count) + ); + } + } + + public static function assertDetectedLanguages( + ExtractionResult $result, + array $expected, + ?float $minConfidence + ): void { + if (empty($expected)) { + return; + } + + Assert::assertNotNull($result->detectedLanguages, "Expected detected languages but field is null"); + + $missing = []; + foreach ($expected as $lang) { + if (!in_array($lang, $result->detectedLanguages, true)) { + $missing[] = $lang; + } + } + + Assert::assertEmpty( + $missing, + sprintf("Expected languages %s, missing %s", json_encode($expected), json_encode($missing)) + ); + + $metaArr = self::metadataToArray($result->metadata); + if ($minConfidence !== null && isset($metaArr['confidence'])) { + $confidence = $metaArr['confidence']; + Assert::assertGreaterThanOrEqual( + $minConfidence, + $confidence, + sprintf("Expected confidence >= %f, got %f", $minConfidence, $confidence) + ); + } + } + + public static function assertChunks( + $result, + ?int $minCount = null, + ?int $maxCount = null, + ?bool $eachHasContent = null, + ?bool $eachHasEmbedding = null, + ?bool $eachHasHeadingContext = null, + ?bool $eachHasChunkType = null, + ?bool $contentStartsWithHeading = null + ): void { + $chunks = $result->chunks ?? null; + if ($chunks === null) { + throw new \Exception("Expected chunks but field is null"); + } + + $count = count($chunks); + if ($minCount !== null && $count < $minCount) { + throw new \Exception("Expected at least $minCount chunks, found $count"); + } + if ($maxCount !== null && $count > $maxCount) { + throw new \Exception("Expected at most $maxCount chunks, found $count"); + } + + foreach ($chunks as $i => $chunk) { + if ($eachHasContent && empty($chunk->content)) { + throw new \Exception("Chunk $i has no content"); + } + if ($eachHasEmbedding && (empty($chunk->embedding))) { + throw new \Exception("Chunk $i has no embedding"); + } + if ($eachHasHeadingContext !== null) { + $hc = $chunk->metadata->headingContext ?? null; + if ($eachHasHeadingContext && $hc === null) { + throw new \Exception("Chunk $i has no headingContext"); + } + if (!$eachHasHeadingContext && $hc !== null) { + throw new \Exception("Chunk $i should have no headingContext"); + } + } + if ($eachHasChunkType === true) { + $type = $chunk->chunkType ?? $chunk->chunk_type ?? null; + if ($type === null || $type === "unknown") { + throw new \Exception("Chunk $i has no specific chunkType, got " . var_export($type, true)); + } + } + if ($contentStartsWithHeading && (empty($chunk->content) || $chunk->content[0] !== '#')) { + throw new \Exception("Chunk $i content does not start with #"); + } + } + } + + public static function assertImages( + ExtractionResult $result, + ?int $minCount, + ?int $maxCount, + ?array $formatsInclude + ): void { + $images = $result->images ?? []; + $count = count($images); + + if ($minCount !== null) { + Assert::assertGreaterThanOrEqual( + $minCount, + $count, + sprintf("Expected at least %d images, found %d", $minCount, $count) + ); + } + + if ($maxCount !== null) { + Assert::assertLessThanOrEqual( + $maxCount, + $count, + sprintf("Expected at most %d images, found %d", $maxCount, $count) + ); + } + + if ($formatsInclude !== null && !empty($formatsInclude)) { + $foundFormats = []; + foreach ($images as $image) { + if (isset($image->format)) { + $foundFormats[] = strtolower($image->format); + } + } + + foreach ($formatsInclude as $format) { + Assert::assertContains( + strtolower($format), + $foundFormats, + sprintf("Expected image format '%s' not found in %s", $format, json_encode($foundFormats)) + ); + } + } + } + + public static function assertPages( + ExtractionResult $result, + ?int $minCount, + ?int $exactCount + ): void { + $pages = $result->pages ?? []; + $count = count($pages); + + if ($exactCount !== null) { + Assert::assertEquals( + $exactCount, + $count, + sprintf("Expected exactly %d pages, found %d", $exactCount, $count) + ); + } + + if ($minCount !== null) { + Assert::assertGreaterThanOrEqual( + $minCount, + $count, + sprintf("Expected at least %d pages, found %d", $minCount, $count) + ); + } + + foreach ($pages as $page) { + if (property_exists($page, 'isBlank')) { + Assert::assertTrue( + $page->isBlank === null || is_bool($page->isBlank), + 'isBlank should be null or bool' + ); + } + } + } + + public static function assertElements( + ExtractionResult $result, + ?int $minCount, + ?array $typesInclude + ): void { + $elements = $result->elements ?? []; + $count = count($elements); + + if ($minCount !== null) { + Assert::assertGreaterThanOrEqual( + $minCount, + $count, + sprintf("Expected at least %d elements, found %d", $minCount, $count) + ); + } + + if ($typesInclude !== null && !empty($typesInclude)) { + $foundTypes = []; + foreach ($elements as $element) { + if (isset($element->elementType)) { + $foundTypes[] = strtolower($element->elementType); + } + } + + foreach ($typesInclude as $type) { + Assert::assertContains( + strtolower($type), + $foundTypes, + sprintf("Expected element type '%s' not found in %s", $type, json_encode($foundTypes)) + ); + } + } + } + + public static function assertMetadataExpectation( + ExtractionResult $result, + string $path, + array $expectation + ): void { + // Convert Metadata object to array for lookup + $metadataArray = self::metadataToArray($result->metadata); + $value = self::lookupMetadataPath($metadataArray, $path); + + Assert::assertNotNull( + $value, + sprintf("Metadata path '%s' missing in %s", $path, json_encode($metadataArray)) + ); + + if (isset($expectation['eq'])) { + Assert::assertTrue( + self::valuesEqual($value, $expectation['eq']), + sprintf( + "Expected metadata '%s' == %s, got %s", + $path, + json_encode($expectation['eq']), + json_encode($value) + ) + ); + } + + if (isset($expectation['gte'])) { + Assert::assertGreaterThanOrEqual( + (float)$expectation['gte'], + (float)$value, + sprintf("Expected metadata '%s' >= %s, got %s", $path, $expectation['gte'], $value) + ); + } + + if (isset($expectation['lte'])) { + Assert::assertLessThanOrEqual( + (float)$expectation['lte'], + (float)$value, + sprintf("Expected metadata '%s' <= %s, got %s", $path, $expectation['lte'], $value) + ); + } + + if (isset($expectation['contains'])) { + $contains = $expectation['contains']; + if (is_string($value) && is_string($contains)) { + Assert::assertStringContainsString( + $contains, + $value, + sprintf("Expected metadata '%s' string to contain %s", $path, json_encode($contains)) + ); + } elseif (is_array($value) && is_string($contains)) { + Assert::assertContains( + $contains, + $value, + sprintf("Expected metadata '%s' to contain %s", $path, json_encode($contains)) + ); + } elseif (is_array($value) && is_array($contains)) { + $missing = array_diff($contains, $value); + Assert::assertEmpty( + $missing, + sprintf( + "Expected metadata '%s' to contain %s, missing %s", + $path, + json_encode($contains), + json_encode($missing) + ) + ); + } else { + Assert::fail(sprintf("Unsupported contains expectation for metadata '%s'", $path)); + } + } + } + + private static function metadataToArray($metadata): array + { + if (is_array($metadata)) { + return $metadata; + } + + // Use to_array() if available (extension Metadata object) + if (method_exists($metadata, 'to_array')) { + return $metadata->to_array(); + } + + // Fallback: Convert Metadata object to array using snake_case properties + $result = []; + $fields = [ + 'language', 'subject', 'format_type', 'title', 'authors', + 'keywords', 'created_at', 'modified_at', 'created_by', + 'modified_by', 'page_count', 'sheet_count', 'format', + ]; + foreach ($fields as $field) { + if (isset($metadata->$field)) { + $result[$field] = $metadata->$field; + } + } + + // Include custom/additional fields + if (method_exists($metadata, 'get_additional')) { + foreach ($metadata->get_additional() as $key => $value) { + $result[$key] = $value; + } + } elseif (isset($metadata->custom) && is_array($metadata->custom)) { + foreach ($metadata->custom as $key => $value) { + $result[$key] = $value; + } + } + + return $result; + } + + private static function lookupMetadataPath(array $metadata, string $path) + { + $current = $metadata; + $segments = explode('.', $path); + + foreach ($segments as $segment) { + if (!is_array($current) || !isset($current[$segment])) { + // Try format metadata fallback + if (isset($metadata['format']) && is_array($metadata['format'])) { + $current = $metadata['format']; + foreach ($segments as $seg) { + if (!is_array($current) || !isset($current[$seg])) { + return null; + } + $current = $current[$seg]; + } + return $current; + } + return null; + } + $current = $current[$segment]; + } + + return $current; + } + + private static function valuesEqual($lhs, $rhs): bool + { + if (is_string($lhs) && is_string($rhs)) { + return $lhs === $rhs; + } + if (is_numeric($lhs) && is_numeric($rhs)) { + return (float)$lhs === (float)$rhs; + } + if (is_bool($lhs) && is_bool($rhs)) { + return $lhs === $rhs; + } + return $lhs == $rhs; + } + + public static function assertDocument( + ExtractionResult $result, + bool $hasDocument, + ?int $minNodeCount = null, + ?array $nodeTypesInclude = null, + ?bool $hasGroups = null + ): void { + $document = $result->document ?? null; + if ($hasDocument) { + Assert::assertNotNull($document, 'Expected document but got null'); + $nodes = is_array($document) ? $document : ($document->nodes ?? []); + Assert::assertNotNull($nodes, 'Expected document.nodes but got null'); + if ($minNodeCount !== null) { + Assert::assertGreaterThanOrEqual( + $minNodeCount, + count($nodes), + sprintf('Expected at least %d nodes, found %d', $minNodeCount, count($nodes)) + ); + } + if ($nodeTypesInclude !== null && !empty($nodeTypesInclude)) { + $foundTypes = []; + foreach ($nodes as $node) { + $content = is_object($node) ? ($node->content ?? null) : ($node['content'] ?? null); + if ($content !== null) { + $nodeType = is_object($content) ? ($content->node_type ?? $content->nodeType ?? null) : ($content['node_type'] ?? null); + if ($nodeType !== null) { + $foundTypes[] = $nodeType; + } + } + } + foreach ($nodeTypesInclude as $type) { + Assert::assertContains( + $type, + $foundTypes, + sprintf("Expected node type '%s' not found in %s", $type, json_encode($foundTypes)) + ); + } + } + if ($hasGroups !== null) { + $hasGroupNodes = false; + foreach ($nodes as $node) { + $content = is_object($node) ? ($node->content ?? null) : ($node['content'] ?? null); + if ($content !== null) { + $nodeType = is_object($content) ? ($content->node_type ?? $content->nodeType ?? null) : ($content['node_type'] ?? null); + if ($nodeType === 'group') { + $hasGroupNodes = true; + break; + } + } + } + Assert::assertEquals($hasGroups, $hasGroupNodes); + } + } else { + Assert::assertNull($document, 'Expected document to be null'); + } + } + + public static function assertOcrElements( + ExtractionResult $result, + ?bool $hasElements = null, + ?bool $elementsHaveGeometry = null, + ?bool $elementsHaveConfidence = null, + ?int $minCount = null + ): void { + $ocrElements = $result->ocrElements ?? null; + if ($hasElements) { + Assert::assertNotNull($ocrElements, 'Expected ocr_elements but got null'); + Assert::assertIsArray($ocrElements); + Assert::assertNotEmpty($ocrElements, 'Expected ocr_elements to be non-empty'); + } + if (is_array($ocrElements)) { + if ($minCount !== null) { + Assert::assertGreaterThanOrEqual( + $minCount, + count($ocrElements), + sprintf('Expected at least %d ocr_elements, found %d', $minCount, count($ocrElements)) + ); + } + } + } + + public static function assertKeywords( + ExtractionResult $result, + ?bool $hasKeywords = null, + ?int $minCount = null, + ?int $maxCount = null + ): void { + if ($hasKeywords === true) { + Assert::assertNotNull($result->extractedKeywords, 'Expected keywords but got null'); + Assert::assertNotEmpty($result->extractedKeywords ?? [], 'Expected keywords to be non-empty'); + } elseif ($hasKeywords === false) { + $keywords = $result->extractedKeywords ?? []; + Assert::assertTrue( + $keywords === null || count($keywords) === 0, + 'Expected keywords to be null or empty' + ); + } + + $keywords = $result->extractedKeywords ?? []; + $count = count($keywords); + + if ($minCount !== null) { + Assert::assertGreaterThanOrEqual( + $minCount, + $count, + sprintf("Expected at least %d keywords, found %d", $minCount, $count) + ); + } + + if ($maxCount !== null) { + Assert::assertLessThanOrEqual( + $maxCount, + $count, + sprintf("Expected at most %d keywords, found %d", $maxCount, $count) + ); + } + } + + public static function assertContentNotEmpty(ExtractionResult $result): void + { + Assert::assertNotEmpty( + $result->content ?? '', + "Expected content to be non-empty" + ); + } + + public static function assertTableBoundingBoxes(ExtractionResult $result, bool $expected): void + { + if ($expected) { + $tables = $result->tables ?? []; + Assert::assertNotEmpty($tables, 'Expected tables with bounding boxes but no tables found'); + foreach ($tables as $table) { + $bb = $table->boundingBox ?? $table->bounding_box ?? null; + Assert::assertNotNull($bb, 'Expected table to have bounding_box but it was null'); + } + } + } + + public static function assertTableContentContainsAny(ExtractionResult $result, array $snippets): void + { + $tables = $result->tables ?? []; + Assert::assertNotEmpty($tables, 'Expected tables but none found'); + + $allCells = []; + foreach ($tables as $table) { + $cells = $table->cells ?? []; + foreach ($cells as $row) { + foreach ($row as $cell) { + $allCells[] = strtolower((string)$cell); + } + } + } + + $found = false; + foreach ($snippets as $snippet) { + foreach ($allCells as $cell) { + if (str_contains($cell, strtolower($snippet))) { + $found = true; + break 2; + } + } + } + + Assert::assertTrue( + $found, + sprintf('No table cell contains any of %s', json_encode($snippets)) + ); + } + + public static function assertImageBoundingBoxes(ExtractionResult $result, bool $expected): void + { + if ($expected) { + $images = $result->images ?? []; + Assert::assertNotEmpty($images, 'Expected images with bounding boxes but no images found'); + foreach ($images as $image) { + $bb = $image->boundingBox ?? $image->bounding_box ?? null; + Assert::assertNotNull($bb, 'Expected image to have bounding_box but it was null'); + } + } + } + + public static function assertQualityScore( + ExtractionResult $result, + ?bool $hasScore = null, + ?float $minScore = null, + ?float $maxScore = null + ): void { + $score = $result->qualityScore ?? $result->quality_score ?? null; + + if ($hasScore === true) { + Assert::assertNotNull($score, 'Expected quality_score to be present'); + } + + if ($hasScore === false) { + Assert::assertNull($score, 'Expected quality_score to be absent'); + } + + if ($minScore !== null) { + Assert::assertNotNull($score, 'quality_score required for min_score assertion'); + Assert::assertGreaterThanOrEqual( + $minScore, + (float)$score, + sprintf('quality_score %f < %f', (float)$score, $minScore) + ); + } + + if ($maxScore !== null) { + Assert::assertNotNull($score, 'quality_score required for max_score assertion'); + Assert::assertLessThanOrEqual( + $maxScore, + (float)$score, + sprintf('quality_score %f > %f', (float)$score, $maxScore) + ); + } + } + + public static function assertProcessingWarnings( + ExtractionResult $result, + ?int $maxCount = null, + ?bool $isEmpty = null + ): void { + $warnings = $result->processingWarnings ?? $result->processing_warnings ?? []; + + if ($maxCount !== null) { + Assert::assertLessThanOrEqual( + $maxCount, + count($warnings), + sprintf('processing_warnings count %d > %d', count($warnings), $maxCount) + ); + } + + if ($isEmpty === true) { + Assert::assertCount( + 0, + $warnings, + sprintf('Expected empty processing_warnings, got %d', count($warnings)) + ); + } + } + + public static function assertDjotContent( + ExtractionResult $result, + ?bool $hasContent = null, + ?int $minBlocks = null + ): void { + $djot = $result->djotContent ?? $result->djot_content ?? null; + + if ($hasContent === true) { + Assert::assertNotNull($djot, 'Expected djot_content to be present'); + } + + if ($hasContent === false) { + Assert::assertNull($djot, 'Expected djot_content to be absent'); + } + + if ($minBlocks !== null) { + Assert::assertNotNull($djot, 'djot_content required for min_blocks assertion'); + $blocks = is_object($djot) ? ($djot->blocks ?? []) : ($djot['blocks'] ?? []); + Assert::assertGreaterThanOrEqual( + $minBlocks, + count($blocks), + sprintf('djot_content blocks %d < %d', count($blocks), $minBlocks) + ); + } + } + + public static function assertAnnotations( + ExtractionResult $result, + bool $hasAnnotations = false, + ?int $minCount = null + ): void { + $annotations = $result->annotations ?? null; + + if ($hasAnnotations) { + Assert::assertNotNull($annotations, 'Expected annotations to be present'); + Assert::assertIsArray($annotations); + Assert::assertNotEmpty($annotations, 'Expected annotations to be non-empty'); + } + + if ($annotations !== null && is_array($annotations) && $minCount !== null) { + Assert::assertGreaterThanOrEqual( + $minCount, + count($annotations), + sprintf('Expected at least %d annotations, got %d', $minCount, count($annotations)) + ); + } + } + + public static function skipIfFeatureUnavailable(string $feature): void + { + $envVar = 'KREUZBERG_' . strtoupper(str_replace('-', '_', $feature)) . '_AVAILABLE'; + $flag = getenv($envVar); + if ($flag === false || $flag === '' || $flag === '0' || strtolower($flag) === 'false') { + Assert::markTestSkipped( + sprintf('Feature "%s" not available (set %s=1)', $feature, $envVar) + ); + } + } + + public static function assertIsPng(string $data): void + { + Assert::assertGreaterThanOrEqual(4, strlen($data), + sprintf('Data too short for PNG: %d bytes', strlen($data))); + Assert::assertSame("\x89PNG", substr($data, 0, 4), 'Missing PNG magic bytes'); + } + + public static function assertMinByteLength(string $data, int $minLength): void + { + Assert::assertGreaterThanOrEqual($minLength, strlen($data), + sprintf('Expected at least %d bytes, got %d', $minLength, strlen($data))); + } + + } +} diff --git a/e2e/elixir/test/e2e/archive_test.exs b/e2e/elixir/test/e2e/archive_test.exs index 1c37978c431..3620b285897 100644 --- a/e2e/elixir/test/e2e/archive_test.exs +++ b/e2e/elixir/test/e2e/archive_test.exs @@ -11,13 +11,13 @@ defmodule E2E.ArchiveTest do describe "archive fixtures" do test "archive_gz_basic" do case E2E.Helpers.run_fixture( - "archive_gz_basic", - "archives/book_war_and_peace_1p.txt.gz", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "archive_gz_basic", + "archives/book_war_and_peace_1p.txt.gz", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/gzip", "application/x-gzip"]) @@ -33,13 +33,13 @@ defmodule E2E.ArchiveTest do test "archive_sevenz_basic" do case E2E.Helpers.run_fixture( - "archive_sevenz_basic", - "archives/documents.7z", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "archive_sevenz_basic", + "archives/documents.7z", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-7z-compressed"]) @@ -55,13 +55,13 @@ defmodule E2E.ArchiveTest do test "archive_tar_basic" do case E2E.Helpers.run_fixture( - "archive_tar_basic", - "archives/documents.tar", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "archive_tar_basic", + "archives/documents.tar", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-tar", "application/tar"]) @@ -77,13 +77,13 @@ defmodule E2E.ArchiveTest do test "archive_zip_basic" do case E2E.Helpers.run_fixture( - "archive_zip_basic", - "archives/documents.zip", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "archive_zip_basic", + "archives/documents.zip", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/zip", "application/x-zip-compressed"]) diff --git a/e2e/elixir/test/e2e/email_test.exs b/e2e/elixir/test/e2e/email_test.exs index 51b73be3321..f129dacc742 100644 --- a/e2e/elixir/test/e2e/email_test.exs +++ b/e2e/elixir/test/e2e/email_test.exs @@ -11,13 +11,13 @@ defmodule E2E.EmailTest do describe "email fixtures" do test "email_eml_html_body" do case E2E.Helpers.run_fixture( - "email_eml_html_body", - "email/html_only.eml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_eml_html_body", + "email/html_only.eml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["message/rfc822"]) @@ -33,13 +33,13 @@ defmodule E2E.EmailTest do test "email_eml_multipart" do case E2E.Helpers.run_fixture( - "email_eml_multipart", - "email/html_email_multipart.eml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_eml_multipart", + "email/html_email_multipart.eml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["message/rfc822"]) @@ -55,13 +55,13 @@ defmodule E2E.EmailTest do test "email_eml_utf16" do case E2E.Helpers.run_fixture( - "email_eml_utf16", - "vendored/unstructured/eml/fake-email-utf-16.eml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_eml_utf16", + "vendored/unstructured/eml/fake-email-utf-16.eml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["message/rfc822"]) @@ -78,13 +78,13 @@ defmodule E2E.EmailTest do test "email_msg_basic" do case E2E.Helpers.run_fixture( - "email_msg_basic", - "email/fake_email.msg", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_msg_basic", + "email/fake_email.msg", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-outlook"]) @@ -100,13 +100,13 @@ defmodule E2E.EmailTest do test "email_pst_empty" do case E2E.Helpers.run_fixture( - "email_pst_empty", - "email/empty.pst", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_pst_empty", + "email/empty.pst", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-outlook-pst"]) @@ -121,13 +121,13 @@ defmodule E2E.EmailTest do test "email_sample_eml" do case E2E.Helpers.run_fixture( - "email_sample_eml", - "email/sample_email.eml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "email_sample_eml", + "email/sample_email.eml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["message/rfc822"]) diff --git a/e2e/elixir/test/e2e/html_test.exs b/e2e/elixir/test/e2e/html_test.exs index 38ced073b0d..25dbf508e0f 100644 --- a/e2e/elixir/test/e2e/html_test.exs +++ b/e2e/elixir/test/e2e/html_test.exs @@ -11,13 +11,13 @@ defmodule E2E.HtmlTest do describe "html fixtures" do test "html_complex_layout" do case E2E.Helpers.run_fixture( - "html_complex_layout", - "html/taylor_swift.html", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "html_complex_layout", + "html/taylor_swift.html", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/html"]) @@ -33,26 +33,18 @@ defmodule E2E.HtmlTest do test "html_simple_table" do case E2E.Helpers.run_fixture( - "html_simple_table", - "html/simple_table.html", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "html_simple_table", + "html/simple_table.html", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/html"]) |> E2E.Helpers.assert_min_content_length(100) - |> E2E.Helpers.assert_content_contains_all([ - "Product", - "Category", - "Price", - "Stock", - "Laptop", - "Electronics", - "Sample Data Table" - ]) + |> E2E.Helpers.assert_content_contains_all(["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]) {:skipped, reason} -> IO.puts("SKIPPED: #{reason}") diff --git a/e2e/elixir/test/e2e/image_test.exs b/e2e/elixir/test/e2e/image_test.exs index 82f088f2e4b..1109ed0a9ae 100644 --- a/e2e/elixir/test/e2e/image_test.exs +++ b/e2e/elixir/test/e2e/image_test.exs @@ -11,13 +11,13 @@ defmodule E2E.ImageTest do describe "image fixtures" do test "image_bmp_basic" do case E2E.Helpers.run_fixture( - "image_bmp_basic", - "images/bmp_24.bmp", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_bmp_basic", + "images/bmp_24.bmp", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/bmp"]) @@ -33,13 +33,13 @@ defmodule E2E.ImageTest do test "image_gif_basic" do case E2E.Helpers.run_fixture( - "image_gif_basic", - "images_extra/ocr_image.gif", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_gif_basic", + "images_extra/ocr_image.gif", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/gif"]) @@ -55,13 +55,13 @@ defmodule E2E.ImageTest do test "image_jp2_basic" do case E2E.Helpers.run_fixture( - "image_jp2_basic", - "images_extra/ocr_image.jp2", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_jp2_basic", + "images_extra/ocr_image.jp2", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/jp2", "image/jpeg2000"]) @@ -77,13 +77,13 @@ defmodule E2E.ImageTest do test "image_metadata_only" do case E2E.Helpers.run_fixture( - "image_metadata_only", - "images/example.jpg", - %{ocr: nil}, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "image_metadata_only", + "images/example.jpg", + %{ocr: nil}, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/jpeg"]) @@ -99,13 +99,13 @@ defmodule E2E.ImageTest do test "image_pbm_basic" do case E2E.Helpers.run_fixture( - "image_pbm_basic", - "images_extra/ocr_image.pbm", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_pbm_basic", + "images_extra/ocr_image.pbm", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/x-portable-bitmap", "image/x-pbm"]) @@ -121,13 +121,13 @@ defmodule E2E.ImageTest do test "image_pgm_basic" do case E2E.Helpers.run_fixture( - "image_pgm_basic", - "images_extra/ocr_image.pgm", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_pgm_basic", + "images_extra/ocr_image.pgm", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/x-portable-graymap", "image/x-pgm"]) @@ -143,13 +143,13 @@ defmodule E2E.ImageTest do test "image_ppm_basic" do case E2E.Helpers.run_fixture( - "image_ppm_basic", - "images_extra/ocr_image.ppm", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_ppm_basic", + "images_extra/ocr_image.ppm", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/x-portable-pixmap", "image/x-ppm"]) @@ -165,13 +165,13 @@ defmodule E2E.ImageTest do test "image_svg_basic" do case E2E.Helpers.run_fixture( - "image_svg_basic", - "xml/simple_svg.svg", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "image_svg_basic", + "xml/simple_svg.svg", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/svg+xml"]) @@ -187,13 +187,13 @@ defmodule E2E.ImageTest do test "image_tiff_basic" do case E2E.Helpers.run_fixture( - "image_tiff_basic", - "images_extra/ocr_image.tif", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_tiff_basic", + "images_extra/ocr_image.tif", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/tiff"]) @@ -209,13 +209,13 @@ defmodule E2E.ImageTest do test "image_webp_basic" do case E2E.Helpers.run_fixture( - "image_webp_basic", - "images_extra/ocr_image.webp", - nil, - requirements: ["tesseract", "tesseract"], - notes: nil, - skip_if_missing: true - ) do + "image_webp_basic", + "images_extra/ocr_image.webp", + nil, + requirements: ["tesseract", "tesseract"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["image/webp"]) diff --git a/e2e/elixir/test/e2e/keywords_test.exs b/e2e/elixir/test/e2e/keywords_test.exs index 7fefc523628..29aa3907ad2 100644 --- a/e2e/elixir/test/e2e/keywords_test.exs +++ b/e2e/elixir/test/e2e/keywords_test.exs @@ -11,13 +11,13 @@ defmodule E2E.KeywordsTest do describe "keywords fixtures" do test "keywords_rake" do case E2E.Helpers.run_fixture( - "keywords_rake", - "pdf/fake_memo.pdf", - %{keywords: %{algorithm: "rake", max_keywords: 10}}, - requirements: ["keywords-rake"], - notes: nil, - skip_if_missing: true - ) do + "keywords_rake", + "pdf/fake_memo.pdf", + %{keywords: %{algorithm: "rake", max_keywords: 10}}, + requirements: ["keywords-rake"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/pdf"]) @@ -34,13 +34,13 @@ defmodule E2E.KeywordsTest do test "keywords_yake" do case E2E.Helpers.run_fixture( - "keywords_yake", - "pdf/fake_memo.pdf", - %{keywords: %{algorithm: "yake", max_keywords: 10}}, - requirements: ["keywords-yake"], - notes: nil, - skip_if_missing: true - ) do + "keywords_yake", + "pdf/fake_memo.pdf", + %{keywords: %{algorithm: "yake", max_keywords: 10}}, + requirements: ["keywords-yake"], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/pdf"]) diff --git a/e2e/elixir/test/e2e/office_test.exs b/e2e/elixir/test/e2e/office_test.exs index 5825aa23135..0cdfd4be69e 100644 --- a/e2e/elixir/test/e2e/office_test.exs +++ b/e2e/elixir/test/e2e/office_test.exs @@ -11,13 +11,13 @@ defmodule E2E.OfficeTest do describe "office fixtures" do test "office_bibtex_basic" do case E2E.Helpers.run_fixture( - "office_bibtex_basic", - "bibtex/comprehensive.bib", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_bibtex_basic", + "bibtex/comprehensive.bib", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-bibtex", "text/x-bibtex"]) @@ -33,13 +33,13 @@ defmodule E2E.OfficeTest do test "office_commonmark_basic" do case E2E.Helpers.run_fixture( - "office_commonmark_basic", - "markdown/sample.commonmark", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_commonmark_basic", + "markdown/sample.commonmark", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/markdown", "text/plain", "text/x-commonmark"]) @@ -55,13 +55,13 @@ defmodule E2E.OfficeTest do test "office_dbf_basic" do case E2E.Helpers.run_fixture( - "office_dbf_basic", - "dbf/stations.dbf", - nil, - requirements: ["office"], - notes: "Requires the office feature.", - skip_if_missing: true - ) do + "office_dbf_basic", + "dbf/stations.dbf", + nil, + requirements: ["office"], + notes: "Requires the office feature.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-dbf"]) @@ -78,13 +78,13 @@ defmodule E2E.OfficeTest do test "office_djot_basic" do case E2E.Helpers.run_fixture( - "office_djot_basic", - "markdown/tables.djot", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_djot_basic", + "markdown/tables.djot", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/x-djot", "text/djot"]) @@ -100,13 +100,13 @@ defmodule E2E.OfficeTest do test "office_doc_legacy" do case E2E.Helpers.run_fixture( - "office_doc_legacy", - "doc/unit_test_lists.doc", - nil, - requirements: ["office"], - notes: "Requires the office feature.", - skip_if_missing: true - ) do + "office_doc_legacy", + "doc/unit_test_lists.doc", + nil, + requirements: ["office"], + notes: "Requires the office feature.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/msword"]) @@ -122,13 +122,13 @@ defmodule E2E.OfficeTest do test "office_docbook_basic" do case E2E.Helpers.run_fixture( - "office_docbook_basic", - "docbook/docbook-reader.docbook", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docbook_basic", + "docbook/docbook-reader.docbook", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/docbook+xml", "text/docbook"]) @@ -144,18 +144,16 @@ defmodule E2E.OfficeTest do test "office_docx_basic" do case E2E.Helpers.run_fixture( - "office_docx_basic", - "docx/sample_document.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_basic", + "docx/sample_document.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(10) {:skipped, reason} -> @@ -168,18 +166,16 @@ defmodule E2E.OfficeTest do test "office_docx_equations" do case E2E.Helpers.run_fixture( - "office_docx_equations", - "docx/equations.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_equations", + "docx/equations.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -192,18 +188,16 @@ defmodule E2E.OfficeTest do test "office_docx_fake" do case E2E.Helpers.run_fixture( - "office_docx_fake", - "docx/fake.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_fake", + "docx/fake.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -216,18 +210,16 @@ defmodule E2E.OfficeTest do test "office_docx_formatting" do case E2E.Helpers.run_fixture( - "office_docx_formatting", - "docx/unit_test_formatting.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_formatting", + "docx/unit_test_formatting.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -240,18 +232,16 @@ defmodule E2E.OfficeTest do test "office_docx_headers" do case E2E.Helpers.run_fixture( - "office_docx_headers", - "docx/unit_test_headers.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_headers", + "docx/unit_test_headers.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -264,18 +254,16 @@ defmodule E2E.OfficeTest do test "office_docx_lists" do case E2E.Helpers.run_fixture( - "office_docx_lists", - "docx/unit_test_lists.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_lists", + "docx/unit_test_lists.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(20) {:skipped, reason} -> @@ -288,25 +276,18 @@ defmodule E2E.OfficeTest do test "office_docx_tables" do case E2E.Helpers.run_fixture( - "office_docx_tables", - "docx/docx_tables.docx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_docx_tables", + "docx/docx_tables.docx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) |> E2E.Helpers.assert_min_content_length(50) - |> E2E.Helpers.assert_content_contains_all([ - "Simple uniform table", - "Nested Table", - "merged cells", - "Header Col" - ]) + |> E2E.Helpers.assert_content_contains_all(["Simple uniform table", "Nested Table", "merged cells", "Header Col"]) |> E2E.Helpers.assert_table_count(1, nil) {:skipped, reason} -> @@ -319,13 +300,13 @@ defmodule E2E.OfficeTest do test "office_epub_basic" do case E2E.Helpers.run_fixture( - "office_epub_basic", - "epub/features.epub", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_epub_basic", + "epub/features.epub", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/epub+zip"]) @@ -341,13 +322,13 @@ defmodule E2E.OfficeTest do test "office_fb2_basic" do case E2E.Helpers.run_fixture( - "office_fb2_basic", - "fictionbook/basic.fb2", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_fb2_basic", + "fictionbook/basic.fb2", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-fictionbook+xml"]) @@ -363,13 +344,13 @@ defmodule E2E.OfficeTest do test "office_fictionbook_basic" do case E2E.Helpers.run_fixture( - "office_fictionbook_basic", - "fictionbook/basic.fb2", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_fictionbook_basic", + "fictionbook/basic.fb2", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-fictionbook+xml", "application/x-fictionbook"]) @@ -385,13 +366,13 @@ defmodule E2E.OfficeTest do test "office_hwp_basic" do case E2E.Helpers.run_fixture( - "office_hwp_basic", - "hwp/converted_output.hwp", - nil, - requirements: ["office"], - notes: "Requires the office feature.", - skip_if_missing: true - ) do + "office_hwp_basic", + "hwp/converted_output.hwp", + nil, + requirements: ["office"], + notes: "Requires the office feature.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-hwp"]) @@ -407,14 +388,13 @@ defmodule E2E.OfficeTest do test "office_hwp_styled" do case E2E.Helpers.run_fixture( - "office_hwp_styled", - "hwp/styled_document.hwp", - nil, - requirements: ["hwp"], - notes: - "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", - skip_if_missing: true - ) do + "office_hwp_styled", + "hwp/styled_document.hwp", + nil, + requirements: ["hwp"], + notes: "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-hwp"]) @@ -429,13 +409,13 @@ defmodule E2E.OfficeTest do test "office_jats_basic" do case E2E.Helpers.run_fixture( - "office_jats_basic", - "jats/sample_article.jats", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_jats_basic", + "jats/sample_article.jats", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-jats+xml", "text/jats"]) @@ -451,13 +431,13 @@ defmodule E2E.OfficeTest do test "office_jupyter_basic" do case E2E.Helpers.run_fixture( - "office_jupyter_basic", - "jupyter/rank.ipynb", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_jupyter_basic", + "jupyter/rank.ipynb", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-ipynb+json"]) @@ -473,13 +453,13 @@ defmodule E2E.OfficeTest do test "office_keynote_basic" do case E2E.Helpers.run_fixture( - "office_keynote_basic", - "iwork/test.key", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_keynote_basic", + "iwork/test.key", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-iwork-keynote-sffkey"]) @@ -495,13 +475,13 @@ defmodule E2E.OfficeTest do test "office_latex_basic" do case E2E.Helpers.run_fixture( - "office_latex_basic", - "latex/basic_sections.tex", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_latex_basic", + "latex/basic_sections.tex", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-latex", "text/x-latex"]) @@ -517,13 +497,13 @@ defmodule E2E.OfficeTest do test "office_markdown_basic" do case E2E.Helpers.run_fixture( - "office_markdown_basic", - "markdown/comprehensive.md", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_markdown_basic", + "markdown/comprehensive.md", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/markdown"]) @@ -539,13 +519,13 @@ defmodule E2E.OfficeTest do test "office_mdx_basic" do case E2E.Helpers.run_fixture( - "office_mdx_basic", - "markdown/sample.mdx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_mdx_basic", + "markdown/sample.mdx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/mdx", "text/x-mdx"]) @@ -561,13 +541,13 @@ defmodule E2E.OfficeTest do test "office_mdx_getting_started" do case E2E.Helpers.run_fixture( - "office_mdx_getting_started", - "markdown/mdx_getting_started.mdx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_mdx_getting_started", + "markdown/mdx_getting_started.mdx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/mdx", "text/x-mdx"]) @@ -583,13 +563,13 @@ defmodule E2E.OfficeTest do test "office_mdx_troubleshooting" do case E2E.Helpers.run_fixture( - "office_mdx_troubleshooting", - "markdown/mdx_troubleshooting.mdx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_mdx_troubleshooting", + "markdown/mdx_troubleshooting.mdx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/mdx", "text/x-mdx"]) @@ -605,13 +585,13 @@ defmodule E2E.OfficeTest do test "office_mdx_using_mdx" do case E2E.Helpers.run_fixture( - "office_mdx_using_mdx", - "markdown/mdx_using_mdx.mdx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_mdx_using_mdx", + "markdown/mdx_using_mdx.mdx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/mdx", "text/x-mdx"]) @@ -627,13 +607,13 @@ defmodule E2E.OfficeTest do test "office_numbers_basic" do case E2E.Helpers.run_fixture( - "office_numbers_basic", - "iwork/test.numbers", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_numbers_basic", + "iwork/test.numbers", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-iwork-numbers-sffnumbers"]) @@ -649,13 +629,13 @@ defmodule E2E.OfficeTest do test "office_ods_basic" do case E2E.Helpers.run_fixture( - "office_ods_basic", - "data_formats/test_01.ods", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_ods_basic", + "data_formats/test_01.ods", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.spreadsheet"]) @@ -671,13 +651,13 @@ defmodule E2E.OfficeTest do test "office_odt_bold" do case E2E.Helpers.run_fixture( - "office_odt_bold", - "odt/bold.odt", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_odt_bold", + "odt/bold.odt", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.text"]) @@ -693,13 +673,13 @@ defmodule E2E.OfficeTest do test "office_odt_list" do case E2E.Helpers.run_fixture( - "office_odt_list", - "odt/unorderedList.odt", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_odt_list", + "odt/unorderedList.odt", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.text"]) @@ -716,13 +696,13 @@ defmodule E2E.OfficeTest do test "office_odt_simple" do case E2E.Helpers.run_fixture( - "office_odt_simple", - "odt/simple.odt", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_odt_simple", + "odt/simple.odt", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.text"]) @@ -739,13 +719,13 @@ defmodule E2E.OfficeTest do test "office_odt_table" do case E2E.Helpers.run_fixture( - "office_odt_table", - "odt/table.odt", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_odt_table", + "odt/table.odt", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.oasis.opendocument.text"]) @@ -762,13 +742,13 @@ defmodule E2E.OfficeTest do test "office_opml_basic" do case E2E.Helpers.run_fixture( - "office_opml_basic", - "opml/outline.opml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_opml_basic", + "opml/outline.opml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/xml+opml", "text/x-opml", "application/x-opml+xml"]) @@ -784,13 +764,13 @@ defmodule E2E.OfficeTest do test "office_org_basic" do case E2E.Helpers.run_fixture( - "office_org_basic", - "org/comprehensive.org", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_org_basic", + "org/comprehensive.org", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/x-org", "text/org"]) @@ -806,13 +786,13 @@ defmodule E2E.OfficeTest do test "office_pages_basic" do case E2E.Helpers.run_fixture( - "office_pages_basic", - "iwork/test.pages", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pages_basic", + "iwork/test.pages", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-iwork-pages-sffpages"]) @@ -828,18 +808,16 @@ defmodule E2E.OfficeTest do test "office_ppsx_slideshow" do case E2E.Helpers.run_fixture( - "office_ppsx_slideshow", - "pptx/sample.ppsx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_ppsx_slideshow", + "pptx/sample.ppsx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.presentationml.slideshow" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]) |> E2E.Helpers.assert_min_content_length(10) {:skipped, reason} -> @@ -852,13 +830,13 @@ defmodule E2E.OfficeTest do test "office_ppt_legacy" do case E2E.Helpers.run_fixture( - "office_ppt_legacy", - "ppt/simple.ppt", - nil, - requirements: ["office"], - notes: "Requires the office feature.", - skip_if_missing: true - ) do + "office_ppt_legacy", + "ppt/simple.ppt", + nil, + requirements: ["office"], + notes: "Requires the office feature.", + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-powerpoint"]) @@ -874,19 +852,16 @@ defmodule E2E.OfficeTest do test "office_pptm_basic" do case E2E.Helpers.run_fixture( - "office_pptm_basic", - "pptx/powerpoint_with_image.pptm", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pptm_basic", + "pptx/powerpoint_with_image.pptm", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.ms-powerpoint.presentation.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation"]) |> E2E.Helpers.assert_content_not_empty() {:skipped, reason} -> @@ -899,18 +874,16 @@ defmodule E2E.OfficeTest do test "office_pptx_basic" do case E2E.Helpers.run_fixture( - "office_pptx_basic", - "pptx/simple.pptx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pptx_basic", + "pptx/simple.pptx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) |> E2E.Helpers.assert_min_content_length(50) {:skipped, reason} -> @@ -923,18 +896,16 @@ defmodule E2E.OfficeTest do test "office_pptx_images" do case E2E.Helpers.run_fixture( - "office_pptx_images", - "pptx/powerpoint_with_image.pptx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pptx_images", + "pptx/powerpoint_with_image.pptx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) |> E2E.Helpers.assert_min_content_length(15) {:skipped, reason} -> @@ -947,18 +918,16 @@ defmodule E2E.OfficeTest do test "office_pptx_pitch_deck" do case E2E.Helpers.run_fixture( - "office_pptx_pitch_deck", - "pptx/pitch_deck_presentation.pptx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_pptx_pitch_deck", + "pptx/pitch_deck_presentation.pptx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) |> E2E.Helpers.assert_min_content_length(100) {:skipped, reason} -> @@ -971,13 +940,13 @@ defmodule E2E.OfficeTest do test "office_rst_basic" do case E2E.Helpers.run_fixture( - "office_rst_basic", - "rst/restructured_text.rst", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_rst_basic", + "rst/restructured_text.rst", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/x-rst", "text/prs.fallenstein.rst"]) @@ -993,13 +962,13 @@ defmodule E2E.OfficeTest do test "office_rtf_basic" do case E2E.Helpers.run_fixture( - "office_rtf_basic", - "rtf/extraction_test.rtf", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_rtf_basic", + "rtf/extraction_test.rtf", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/rtf", "text/rtf"]) @@ -1015,13 +984,13 @@ defmodule E2E.OfficeTest do test "office_typst_basic" do case E2E.Helpers.run_fixture( - "office_typst_basic", - "typst/headings.typ", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_typst_basic", + "typst/headings.typ", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-typst", "text/x-typst"]) @@ -1037,13 +1006,13 @@ defmodule E2E.OfficeTest do test "office_xls_legacy" do case E2E.Helpers.run_fixture( - "office_xls_legacy", - "xls/test_excel.xls", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xls_legacy", + "xls/test_excel.xls", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-excel"]) @@ -1059,19 +1028,16 @@ defmodule E2E.OfficeTest do test "office_xlsb_basic" do case E2E.Helpers.run_fixture( - "office_xlsb_basic", - "xlsx/test_xlsb.xlsb", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsb_basic", + "xlsx/test_xlsb.xlsb", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.ms-excel.sheet.binary.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) |> E2E.Helpers.assert_content_not_empty() {:skipped, reason} -> @@ -1084,19 +1050,16 @@ defmodule E2E.OfficeTest do test "office_xlsm_basic" do case E2E.Helpers.run_fixture( - "office_xlsm_basic", - "xlsx/test_01.xlsm", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsm_basic", + "xlsx/test_01.xlsm", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/vnd.ms-excel.sheet.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ]) + |> E2E.Helpers.assert_expected_mime(["application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) |> E2E.Helpers.assert_content_not_empty() {:skipped, reason} -> @@ -1109,13 +1072,13 @@ defmodule E2E.OfficeTest do test "office_xlsx_basic" do case E2E.Helpers.run_fixture( - "office_xlsx_basic", - "xlsx/stanley_cups.xlsx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsx_basic", + "xlsx/stanley_cups.xlsx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) @@ -1135,13 +1098,13 @@ defmodule E2E.OfficeTest do test "office_xlsx_multi_sheet" do case E2E.Helpers.run_fixture( - "office_xlsx_multi_sheet", - "xlsx/excel_multi_sheet.xlsx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsx_multi_sheet", + "xlsx/excel_multi_sheet.xlsx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) @@ -1158,13 +1121,13 @@ defmodule E2E.OfficeTest do test "office_xlsx_office_example" do case E2E.Helpers.run_fixture( - "office_xlsx_office_example", - "xlsx/test_01.xlsx", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "office_xlsx_office_example", + "xlsx/test_01.xlsx", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) diff --git a/e2e/elixir/test/e2e/plugin_apis_test.exs b/e2e/elixir/test/e2e/plugin_apis_test.exs index 277392f15a1..5dc8d37cbf4 100644 --- a/e2e/elixir/test/e2e/plugin_apis_test.exs +++ b/e2e/elixir/test/e2e/plugin_apis_test.exs @@ -13,22 +13,19 @@ defmodule E2E.ConfigurationTest do File.mkdir_p!(tmpdir) on_exit(fn -> File.rm_rf!(tmpdir) end) config_path = Path.join(tmpdir, "kreuzberg.toml") - config_content = """ [chunking] max_chars = 50 """ - File.write!(config_path, config_content) subdir = Path.join(tmpdir, "subdir") File.mkdir_p!(subdir) prev_cwd = File.cwd!() - try do File.cd!(subdir) - {:ok, config} = Kreuzberg.ExtractionConfig.discover() + {:ok, config} = Kreuzberg.ExtractionConfig.discover assert config.chunking != nil assert config.chunking["max_chars"] == 50 @@ -42,7 +39,6 @@ defmodule E2E.ConfigurationTest do File.mkdir_p!(tmpdir) on_exit(fn -> File.rm_rf!(tmpdir) end) config_path = Path.join(tmpdir, "test_config.toml") - config_content = """ [chunking] max_chars = 100 @@ -51,7 +47,6 @@ defmodule E2E.ConfigurationTest do [language_detection] enabled = false """ - File.write!(config_path, config_content) {:ok, config} = Kreuzberg.ExtractionConfig.from_file(config_path) @@ -62,6 +57,7 @@ defmodule E2E.ConfigurationTest do assert config.language_detection != nil assert config.language_detection["enabled"] == false end + end end @@ -85,6 +81,7 @@ defmodule E2E.DocumentExtractorManagementTest do Kreuzberg.Plugin.unregister_document_extractor(:"nonexistent-extractor-xyz") # Should not raise an error end + end end @@ -114,6 +111,7 @@ defmodule E2E.MimeUtilitiesTest do assert is_list(result) assert Enum.member?(result, "pdf") end + end end @@ -137,6 +135,7 @@ defmodule E2E.OcrBackendManagementTest do Kreuzberg.Plugin.unregister_ocr_backend(:"nonexistent-backend-xyz") # Should not raise an error end + end end @@ -153,6 +152,7 @@ defmodule E2E.PostProcessorManagementTest do assert is_list(result) assert Enum.all?(result, &is_binary/1) end + end end @@ -171,5 +171,7 @@ defmodule E2E.ValidatorManagementTest do assert is_list(result) assert Enum.all?(result, &is_binary/1) end + end end + diff --git a/e2e/elixir/test/e2e/render_test.exs b/e2e/elixir/test/e2e/render_test.exs index 77087559069..66639ab2396 100644 --- a/e2e/elixir/test/e2e/render_test.exs +++ b/e2e/elixir/test/e2e/render_test.exs @@ -28,13 +28,11 @@ defmodule E2E.RenderTest do else stream = Kreuzberg.render_pdf_pages_stream(document_path, dpi: 150) pages = Enum.to_list(stream) - Enum.each(pages, fn {_page_index, png_data} -> assert_is_png(png_data) end) - assert length(pages) >= 1, - "Expected at least 1 pages, got #{length(pages)}" + "Expected at least 1 pages, got #{length(pages)}" end end @@ -50,4 +48,5 @@ defmodule E2E.RenderTest do assert_min_byte_length(png_data, 100) end end + end diff --git a/e2e/elixir/test/e2e/structured_test.exs b/e2e/elixir/test/e2e/structured_test.exs index 1ac21bf0e4e..81ee80f1992 100644 --- a/e2e/elixir/test/e2e/structured_test.exs +++ b/e2e/elixir/test/e2e/structured_test.exs @@ -11,13 +11,13 @@ defmodule E2E.StructuredTest do describe "structured fixtures" do test "structured_csv_basic" do case E2E.Helpers.run_fixture( - "structured_csv_basic", - "csv/stanley_cups.csv", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_csv_basic", + "csv/stanley_cups.csv", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/csv"]) @@ -33,20 +33,16 @@ defmodule E2E.StructuredTest do test "structured_enw_basic" do case E2E.Helpers.run_fixture( - "structured_enw_basic", - "data_formats/sample.enw", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_enw_basic", + "data_formats/sample.enw", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result - |> E2E.Helpers.assert_expected_mime([ - "application/x-endnote-refer", - "application/x-endnote+xml", - "text/plain" - ]) + |> E2E.Helpers.assert_expected_mime(["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]) {:skipped, reason} -> IO.puts("SKIPPED: #{reason}") @@ -58,13 +54,13 @@ defmodule E2E.StructuredTest do test "structured_json_basic" do case E2E.Helpers.run_fixture( - "structured_json_basic", - "json/sample_document.json", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_json_basic", + "json/sample_document.json", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/json"]) @@ -81,13 +77,13 @@ defmodule E2E.StructuredTest do test "structured_json_simple" do case E2E.Helpers.run_fixture( - "structured_json_simple", - "json/simple.json", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_json_simple", + "json/simple.json", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/json"]) @@ -104,13 +100,13 @@ defmodule E2E.StructuredTest do test "structured_nbib_basic" do case E2E.Helpers.run_fixture( - "structured_nbib_basic", - "data_formats/sample.nbib", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_nbib_basic", + "data_formats/sample.nbib", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/nbib", "application/x-pubmed", "text/plain"]) @@ -126,13 +122,13 @@ defmodule E2E.StructuredTest do test "structured_ris_basic" do case E2E.Helpers.run_fixture( - "structured_ris_basic", - "data_formats/sample.ris", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_ris_basic", + "data_formats/sample.ris", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-research-info-systems", "text/plain"]) @@ -148,13 +144,13 @@ defmodule E2E.StructuredTest do test "structured_toml_basic" do case E2E.Helpers.run_fixture( - "structured_toml_basic", - "data_formats/cargo.toml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_toml_basic", + "data_formats/cargo.toml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/toml", "text/toml"]) @@ -170,13 +166,13 @@ defmodule E2E.StructuredTest do test "structured_tsv_basic" do case E2E.Helpers.run_fixture( - "structured_tsv_basic", - "data_formats/employees.tsv", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_tsv_basic", + "data_formats/employees.tsv", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["text/tab-separated-values", "text/plain"]) @@ -192,13 +188,13 @@ defmodule E2E.StructuredTest do test "structured_yaml_basic" do case E2E.Helpers.run_fixture( - "structured_yaml_basic", - "yaml/simple.yaml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_yaml_basic", + "yaml/simple.yaml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]) @@ -214,13 +210,13 @@ defmodule E2E.StructuredTest do test "structured_yaml_simple" do case E2E.Helpers.run_fixture( - "structured_yaml_simple", - "yaml/simple.yaml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "structured_yaml_simple", + "yaml/simple.yaml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/x-yaml"]) diff --git a/e2e/elixir/test/e2e/xml_test.exs b/e2e/elixir/test/e2e/xml_test.exs index c77ab6555ac..9448447aeca 100644 --- a/e2e/elixir/test/e2e/xml_test.exs +++ b/e2e/elixir/test/e2e/xml_test.exs @@ -11,13 +11,13 @@ defmodule E2E.XmlTest do describe "xml fixtures" do test "xml_plant_catalog" do case E2E.Helpers.run_fixture( - "xml_plant_catalog", - "xml/plant_catalog.xml", - nil, - requirements: [], - notes: nil, - skip_if_missing: true - ) do + "xml_plant_catalog", + "xml/plant_catalog.xml", + nil, + requirements: [], + notes: nil, + skip_if_missing: true + ) do {:ok, result} -> result |> E2E.Helpers.assert_expected_mime(["application/xml"]) diff --git a/e2e/elixir/test/support/e2e_helpers.ex b/e2e/elixir/test/support/e2e_helpers.ex index 215b2d36e20..4d178a6b9ac 100644 --- a/e2e/elixir/test/support/e2e_helpers.ex +++ b/e2e/elixir/test/support/e2e_helpers.ex @@ -17,7 +17,6 @@ defmodule E2E.Helpers do def build_config(nil), do: nil def build_config(raw) when is_map(raw) and map_size(raw) == 0, do: nil - def build_config(raw) when is_map(raw) do atomize_keys(raw) end @@ -38,12 +37,9 @@ defmodule E2E.Helpers do def skip_reason_for(error, fixture_id, requirements, notes \\ nil) do message = Exception.message(error) downcased = String.downcase(message) - - requirement_hit = - Enum.any?(requirements, fn req -> - String.contains?(downcased, String.downcase(req)) - end) - + requirement_hit = Enum.any?(requirements, fn req -> + String.contains?(downcased, String.downcase(req)) + end) missing_dependency = String.contains?(downcased, "missing dependency") unsupported_format = String.contains?(downcased, "unsupported format") @@ -78,11 +74,8 @@ defmodule E2E.Helpers do requirements = Keyword.get(opts, :requirements, []) notes = Keyword.get(opts, :notes, nil) skip_if_missing = Keyword.get(opts, :skip_if_missing, true) - run_fixture_with_method(fixture_id, relative_path, config_hash, :sync, :file, - requirements: requirements, - notes: notes, - skip_if_missing: skip_if_missing + requirements: requirements, notes: notes, skip_if_missing: skip_if_missing ) end @@ -195,7 +188,6 @@ defmodule E2E.Helpers do result else mime = result.mime_type || "" - if Enum.any?(expected, fn token -> String.contains?(mime, token) end) do result else @@ -206,7 +198,6 @@ defmodule E2E.Helpers do def assert_min_content_length(result, minimum) do content_len = String.length(result.content || "") - if content_len >= minimum do result else @@ -216,7 +207,6 @@ defmodule E2E.Helpers do def assert_max_content_length(result, maximum) do content_len = String.length(result.content || "") - if content_len <= maximum do result else @@ -229,7 +219,6 @@ defmodule E2E.Helpers do result else lowered = String.downcase(result.content || "") - if Enum.any?(snippets, fn snippet -> String.contains?(lowered, String.downcase(snippet)) end) do result else @@ -243,7 +232,6 @@ defmodule E2E.Helpers do result else lowered = String.downcase(result.content || "") - if Enum.all?(snippets, fn snippet -> String.contains?(lowered, String.downcase(snippet)) end) do result else @@ -302,7 +290,6 @@ defmodule E2E.Helpers do expectation when is_map(expectation) -> if Map.has_key?(expectation, :eq) do expected_val = Map.get(expectation, :eq) - if !values_equal?(value, expected_val) do flunk("Metadata path '#{path}' value #{inspect(value)} != #{inspect(expected_val)}") end @@ -310,7 +297,6 @@ defmodule E2E.Helpers do if Map.has_key?(expectation, :gte) do expected_val = Map.get(expectation, :gte) - if convert_numeric(value) < convert_numeric(expected_val) do flunk("Metadata path '#{path}' value #{inspect(value)} < #{inspect(expected_val)}") end @@ -318,7 +304,6 @@ defmodule E2E.Helpers do if Map.has_key?(expectation, :lte) do expected_val = Map.get(expectation, :lte) - if convert_numeric(value) > convert_numeric(expected_val) do flunk("Metadata path '#{path}' value #{inspect(value)} > #{inspect(expected_val)}") end @@ -393,6 +378,12 @@ defmodule E2E.Helpers do end end + if opts[:each_has_chunk_type] == true do + if !Enum.all?(chunks, fn chunk -> chunk.chunk_type && chunk.chunk_type != "unknown" end) do + flunk("Not all chunks have a specific chunk_type") + end + end + if opts[:content_starts_with_heading] == true do chunks_with_heading = Enum.filter(chunks, fn chunk -> @@ -486,18 +477,13 @@ defmodule E2E.Helpers do nodes |> Enum.map(fn node -> content = Map.get(node, :content) || Map.get(node, "content") - - if content, - do: Map.get(content, :node_type) || Map.get(content, "node_type"), - else: Map.get(node, :node_type) || Map.get(node, "node_type") + if content, do: Map.get(content, :node_type) || Map.get(content, "node_type"), else: Map.get(node, :node_type) || Map.get(node, "node_type") end) |> Enum.reject(&is_nil/1) |> Enum.uniq() if !Enum.all?(opts[:node_types_include], fn t -> Enum.member?(found_types, t) end) do - flunk( - "Document node types #{inspect(found_types)} do not include all of #{inspect(opts[:node_types_include])}" - ) + flunk("Document node types #{inspect(found_types)} do not include all of #{inspect(opts[:node_types_include])}") end end @@ -508,7 +494,7 @@ defmodule E2E.Helpers do node_type = if content do - if(is_struct(content), do: Map.get(content, :node_type), else: nil) || + (if is_struct(content), do: Map.get(content, :node_type), else: nil) || Map.get(content, :node_type) || Map.get(content, "node_type") else Map.get(node, :node_type) || Map.get(node, "node_type") @@ -567,7 +553,6 @@ defmodule E2E.Helpers do def assert_content_not_empty(result) do content_len = String.length(result.content || "") - if content_len > 0 do result else @@ -768,22 +753,18 @@ defmodule E2E.Helpers do defp lookup_metadata_path(_, _), do: nil defp values_equal?(lhs, rhs) when is_binary(lhs) and is_binary(rhs), do: lhs == rhs - defp values_equal?(lhs, rhs) when is_number(lhs) and is_number(rhs) do convert_numeric(lhs) == convert_numeric(rhs) end - defp values_equal?(lhs, rhs), do: lhs == rhs defp convert_numeric(value) when is_number(value), do: value - defp convert_numeric(value) when is_binary(value) do case Float.parse(value) do {num, ""} -> num _ -> 0.0 end end - defp convert_numeric(_), do: 0.0 def assert_is_png(data) do @@ -794,6 +775,6 @@ defmodule E2E.Helpers do def assert_min_byte_length(data, min_length) do assert byte_size(data) >= min_length, - "Expected at least #{min_length} bytes, got #{byte_size(data)}" + "Expected at least #{min_length} bytes, got #{byte_size(data)}" end end diff --git a/e2e/go/contract_test.go b/e2e/go/contract_test.go index c4f10804181..13fb1bfbf65 100644 --- a/e2e/go/contract_test.go +++ b/e2e/go/contract_test.go @@ -150,7 +150,7 @@ func TestContractConfigChunking(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigChunkingHeadingContext(t *testing.T) { @@ -163,7 +163,7 @@ func TestContractConfigChunkingHeadingContext(t *testing.T) { } }`)) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(true), nil) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(true), nil, nil) } func TestContractConfigChunkingMarkdown(t *testing.T) { @@ -177,7 +177,7 @@ func TestContractConfigChunkingMarkdown(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigChunkingNoHeadings(t *testing.T) { @@ -190,7 +190,7 @@ func TestContractConfigChunkingNoHeadings(t *testing.T) { } }`)) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(false), nil) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(false), nil, nil) } func TestContractConfigChunkingPrependHeadingContext(t *testing.T) { @@ -204,7 +204,7 @@ func TestContractConfigChunkingPrependHeadingContext(t *testing.T) { } }`)) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(true), boolPtr(true)) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, boolPtr(true), nil, boolPtr(true)) } func TestContractConfigChunkingSmall(t *testing.T) { @@ -217,7 +217,7 @@ func TestContractConfigChunkingSmall(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigChunkingText(t *testing.T) { @@ -230,7 +230,7 @@ func TestContractConfigChunkingText(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigChunkingTokenizer(t *testing.T) { @@ -246,7 +246,7 @@ func TestContractConfigChunkingTokenizer(t *testing.T) { } }`)) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(2), nil, boolPtr(true), nil, nil, nil, nil) } func TestContractConfigDisableOcr(t *testing.T) { diff --git a/e2e/go/embeddings_test.go b/e2e/go/embeddings_test.go index e2198309523..573a53fcd94 100644 --- a/e2e/go/embeddings_test.go +++ b/e2e/go/embeddings_test.go @@ -22,7 +22,7 @@ func TestEmbeddingsEmbeddingAsync(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil, nil) } func TestEmbeddingsEmbeddingBalancedPreset(t *testing.T) { @@ -42,7 +42,7 @@ func TestEmbeddingsEmbeddingBalancedPreset(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil, nil) } func TestEmbeddingsEmbeddingDisabled(t *testing.T) { @@ -54,7 +54,7 @@ func TestEmbeddingsEmbeddingDisabled(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(false), nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(false), nil, nil, nil) } func TestEmbeddingsEmbeddingFastPreset(t *testing.T) { @@ -74,5 +74,5 @@ func TestEmbeddingsEmbeddingFastPreset(t *testing.T) { }`)) assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 10) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), boolPtr(true), nil, nil, nil) } diff --git a/e2e/go/helpers_test.go b/e2e/go/helpers_test.go index d3bc4661544..eb52da6be5c 100644 --- a/e2e/go/helpers_test.go +++ b/e2e/go/helpers_test.go @@ -10,7 +10,7 @@ import ( "testing" "unicode" - kreuzberg "github.com/kreuzberg-dev/kreuzberg/packages/go/v4" + "github.com/kreuzberg-dev/kreuzberg/packages/go/v4" ) var ( @@ -241,7 +241,7 @@ func boolPtr(value bool) *bool { return &value } -func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, contentStartsWithHeading *bool) { +func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, eachHasChunkType, contentStartsWithHeading *bool) { t.Helper() count := len(result.Chunks) if minCount != nil && count < *minCount { @@ -278,6 +278,13 @@ func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, ma } } } + if eachHasChunkType != nil && *eachHasChunkType { + for i, chunk := range result.Chunks { + if chunk.ChunkType == "" || chunk.ChunkType == "unknown" { + t.Fatalf("chunk %d has no specific chunk_type, got %q", i, chunk.ChunkType) + } + } + } if contentStartsWithHeading != nil && *contentStartsWithHeading { for i, chunk := range result.Chunks { if chunk.Metadata == nil || chunk.Metadata.HeadingContext == nil { diff --git a/e2e/go/render_test.go b/e2e/go/render_test.go index 92a6690da28..dee7c2303ac 100644 --- a/e2e/go/render_test.go +++ b/e2e/go/render_test.go @@ -7,7 +7,7 @@ import ( "os" "testing" - kreuzberg "github.com/kreuzberg-dev/kreuzberg/packages/go/v4" + "github.com/kreuzberg-dev/kreuzberg/packages/go/v4" ) func TestRenderRenderCustomDpi(t *testing.T) { diff --git a/e2e/go/token_reduction_test.go b/e2e/go/token_reduction_test.go index 031461a4e43..afad76d71b0 100644 --- a/e2e/go/token_reduction_test.go +++ b/e2e/go/token_reduction_test.go @@ -53,6 +53,6 @@ func TestTokenReductionTokenReductionWithChunking(t *testing.T) { assertExpectedMime(t, result, []string{"application/pdf"}) assertMinContentLength(t, result, 5) assertMaxContentLength(t, result, 200) - assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil) + assertChunks(t, result, intPtr(1), nil, boolPtr(true), nil, nil, nil, nil) assertContentNotEmpty(t, result) } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/ArchiveTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/ArchiveTest.java index c610400fc62..b6745ee3b07 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/ArchiveTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/ArchiveTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,73 +26,74 @@ /** Tests for archive fixtures. */ public class ArchiveTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void archiveGzBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "archive_gz_basic", + "archives/book_war_and_peace_1p.txt.gz", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/gzip", "application/x-gzip")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void archiveGzBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "archive_gz_basic", - "archives/book_war_and_peace_1p.txt.gz", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/gzip", "application/x-gzip")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void archiveSevenzBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "archive_sevenz_basic", + "archives/documents.7z", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-7z-compressed")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void archiveSevenzBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "archive_sevenz_basic", - "archives/documents.7z", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/x-7z-compressed")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void archiveTarBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "archive_tar_basic", + "archives/documents.tar", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-tar", "application/tar")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void archiveTarBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "archive_tar_basic", - "archives/documents.tar", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/x-tar", "application/tar")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void archiveZipBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "archive_zip_basic", + "archives/documents.zip", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/zip", "application/x-zip-compressed")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void archiveZipBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "archive_zip_basic", - "archives/documents.zip", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/zip", "application/x-zip-compressed")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/EmailTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/EmailTest.java index 901544393b1..767c6b9f448 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/EmailTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/EmailTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,104 +26,108 @@ /** Tests for email fixtures. */ public class EmailTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void emailEmlHtmlBody() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_eml_html_body", + "email/html_only.eml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void emailEmlHtmlBody() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_eml_html_body", - "email/html_only.eml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void emailEmlMultipart() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_eml_multipart", + "email/html_email_multipart.eml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void emailEmlMultipart() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_eml_multipart", - "email/html_email_multipart.eml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void emailEmlUtf16() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_eml_utf16", + "vendored/unstructured/eml/fake-email-utf-16.eml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); + E2EHelpers.Assertions.assertMinContentLength(result, 50); + E2EHelpers.Assertions.assertContentContainsAny(result, Arrays.asList("Test Email", "Roses are red")); + } + ); + } - @Test - public void emailEmlUtf16() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_eml_utf16", - "vendored/unstructured/eml/fake-email-utf-16.eml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); - E2EHelpers.Assertions.assertMinContentLength(result, 50); - E2EHelpers.Assertions.assertContentContainsAny( - result, Arrays.asList("Test Email", "Roses are red")); - }); - } + @Test + public void emailMsgBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_msg_basic", + "email/fake_email.msg", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/vnd.ms-outlook")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } - @Test - public void emailMsgBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_msg_basic", - "email/fake_email.msg", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/vnd.ms-outlook")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + @Test + public void emailPstEmpty() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_pst_empty", + "email/empty.pst", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/vnd.ms-outlook-pst")); + } + ); + } - @Test - public void emailPstEmpty() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_pst_empty", - "email/empty.pst", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/vnd.ms-outlook-pst")); - }); - } + @Test + public void emailSampleEml() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "email_sample_eml", + "email/sample_email.eml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); + E2EHelpers.Assertions.assertMinContentLength(result, 20); + } + ); + } - @Test - public void emailSampleEml() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "email_sample_eml", - "email/sample_email.eml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("message/rfc822")); - E2EHelpers.Assertions.assertMinContentLength(result, 20); - }); - } } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/HtmlTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/HtmlTest.java index 3092fa89dc5..46a08ad3305 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/HtmlTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/HtmlTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,47 +26,41 @@ /** Tests for html fixtures. */ public class HtmlTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); - - @Test - public void htmlComplexLayout() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "html_complex_layout", - "html/taylor_swift.html", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/html")); - E2EHelpers.Assertions.assertMinContentLength(result, 1000); - }); - } - - @Test - public void htmlSimpleTable() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "html_simple_table", - "html/simple_table.html", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/html")); - E2EHelpers.Assertions.assertMinContentLength(result, 100); - E2EHelpers.Assertions.assertContentContainsAll( - result, - Arrays.asList( - "Product", - "Category", - "Price", - "Stock", - "Laptop", - "Electronics", - "Sample Data Table")); - }); - } + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void htmlComplexLayout() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "html_complex_layout", + "html/taylor_swift.html", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/html")); + E2EHelpers.Assertions.assertMinContentLength(result, 1000); + } + ); + } + + @Test + public void htmlSimpleTable() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "html_simple_table", + "html/simple_table.html", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/html")); + E2EHelpers.Assertions.assertMinContentLength(result, 100); + E2EHelpers.Assertions.assertContentContainsAll(result, Arrays.asList("Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table")); + } + ); + } + } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/PluginAPIsTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/PluginAPIsTest.java index 26700b42a76..d8c4f173336 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/PluginAPIsTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/PluginAPIsTest.java @@ -3,9 +3,9 @@ import static org.junit.jupiter.api.Assertions.*; +import dev.kreuzberg.config.ExtractionConfig; import dev.kreuzberg.Kreuzberg; import dev.kreuzberg.KreuzbergException; -import dev.kreuzberg.config.ExtractionConfig; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -17,25 +17,23 @@ /** * E2E tests for plugin/config/utility APIs. * - *

Generated from plugin API fixtures. To regenerate: cargo run -p kreuzberg-e2e-generator -- - * generate --lang java + *

Generated from plugin API fixtures. + * To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang java * * @since 4.0.0 */ @DisplayName("Plugin API Tests") class PluginAPIsTest { - // Configuration Tests + // Configuration Tests - // SKIPPED: config_discover - System.setProperty("user.dir") doesn't affect FFI working directory + // SKIPPED: config_discover - System.setProperty("user.dir") doesn't affect FFI working directory - @Test - @DisplayName("Load configuration from a TOML file") - void configFromFile(@TempDir Path tempDir) throws IOException, KreuzbergException { - Path configPath = tempDir.resolve("test_config.toml"); - Files.writeString( - configPath, -""" + @Test + @DisplayName("Load configuration from a TOML file") + void configFromFile(@TempDir Path tempDir) throws IOException, KreuzbergException { + Path configPath = tempDir.resolve("test_config.toml"); + Files.writeString(configPath, """ [chunking] max_chars = 100 max_overlap = 20 @@ -44,121 +42,122 @@ void configFromFile(@TempDir Path tempDir) throws IOException, KreuzbergExceptio enabled = false """); - ExtractionConfig config = ExtractionConfig.fromFile(configPath.toString()); - assertNotNull(config.getChunking()); - assertEquals(100, config.getChunking().getMaxChars()); - assertEquals(20, config.getChunking().getMaxOverlap()); - assertNotNull(config.getLanguageDetection()); - assertFalse(config.getLanguageDetection().isEnabled()); - } - - // Document Extractor Management Tests - - @Test - @DisplayName("Clear all document extractors and verify list is empty") - void extractorsClear() throws KreuzbergException { - Kreuzberg.clearDocumentExtractors(); - List result = Kreuzberg.listDocumentExtractors(); - assertEquals(0, result.size()); - } - - @Test - @DisplayName("List all registered document extractors") - void extractorsList() throws KreuzbergException { - List result = Kreuzberg.listDocumentExtractors(); - assertNotNull(result); - assertTrue(result.stream().allMatch(item -> item instanceof String)); - } - - @Test - @DisplayName("Unregister nonexistent document extractor gracefully") - void extractorsUnregister() throws KreuzbergException { - assertDoesNotThrow(() -> Kreuzberg.unregisterDocumentExtractor("nonexistent-extractor-xyz")); - } - - // Mime Utilities Tests - - @Test - @DisplayName("Detect MIME type from file bytes") - void mimeDetectBytes() throws KreuzbergException { - byte[] testBytes = "%PDF-1.4\n".getBytes(); - String result = Kreuzberg.detectMimeType(testBytes); - assertTrue(result.toLowerCase().contains("pdf")); - } - - @Test - @DisplayName("Detect MIME type from file path") - void mimeDetectPath(@TempDir Path tempDir) throws IOException, KreuzbergException { - Path testFile = tempDir.resolve("test.txt"); - Files.writeString(testFile, "Hello, world!"); - - String result = Kreuzberg.detectMimeTypeFromPath(testFile.toString()); - assertTrue(result.toLowerCase().contains("text")); - } - - @Test - @DisplayName("Get file extensions for a MIME type") - void mimeGetExtensions() throws KreuzbergException { - List result = Kreuzberg.getExtensionsForMime("application/pdf"); - assertNotNull(result); - assertTrue(result.contains("pdf")); - } - - // Ocr Backend Management Tests - - @Test - @DisplayName("Clear all OCR backends and verify list is empty") - void ocrBackendsClear() throws KreuzbergException { - Kreuzberg.clearOCRBackends(); - List result = Kreuzberg.listOCRBackends(); - assertEquals(0, result.size()); - } - - @Test - @DisplayName("List all registered OCR backends") - void ocrBackendsList() throws KreuzbergException { - List result = Kreuzberg.listOCRBackends(); - assertNotNull(result); - assertTrue(result.stream().allMatch(item -> item instanceof String)); - } - - @Test - @DisplayName("Unregister nonexistent OCR backend gracefully") - void ocrBackendsUnregister() throws KreuzbergException { - assertDoesNotThrow(() -> Kreuzberg.unregisterOCRBackend("nonexistent-backend-xyz")); - } - - // Post Processor Management Tests - - @Test - @DisplayName("Clear all post-processors and verify list is empty") - void postProcessorsClear() throws KreuzbergException { - Kreuzberg.clearPostProcessors(); - } - - @Test - @DisplayName("List all registered post-processors") - void postProcessorsList() throws KreuzbergException { - List result = Kreuzberg.listPostProcessors(); - assertNotNull(result); - assertTrue(result.stream().allMatch(item -> item instanceof String)); - } - - // Validator Management Tests - - @Test - @DisplayName("Clear all validators and verify list is empty") - void validatorsClear() throws KreuzbergException { - Kreuzberg.clearValidators(); - List result = Kreuzberg.listValidators(); - assertEquals(0, result.size()); - } - - @Test - @DisplayName("List all registered validators") - void validatorsList() throws KreuzbergException { - List result = Kreuzberg.listValidators(); - assertNotNull(result); - assertTrue(result.stream().allMatch(item -> item instanceof String)); - } + ExtractionConfig config = ExtractionConfig.fromFile(configPath.toString()); + assertNotNull(config.getChunking()); + assertEquals(100, config.getChunking().getMaxChars()); + assertEquals(20, config.getChunking().getMaxOverlap()); + assertNotNull(config.getLanguageDetection()); + assertFalse(config.getLanguageDetection().isEnabled()); + } + + // Document Extractor Management Tests + + @Test + @DisplayName("Clear all document extractors and verify list is empty") + void extractorsClear() throws KreuzbergException { + Kreuzberg.clearDocumentExtractors(); + List result = Kreuzberg.listDocumentExtractors(); + assertEquals(0, result.size()); + } + + @Test + @DisplayName("List all registered document extractors") + void extractorsList() throws KreuzbergException { + List result = Kreuzberg.listDocumentExtractors(); + assertNotNull(result); + assertTrue(result.stream().allMatch(item -> item instanceof String)); + } + + @Test + @DisplayName("Unregister nonexistent document extractor gracefully") + void extractorsUnregister() throws KreuzbergException { + assertDoesNotThrow(() -> Kreuzberg.unregisterDocumentExtractor("nonexistent-extractor-xyz")); + } + + // Mime Utilities Tests + + @Test + @DisplayName("Detect MIME type from file bytes") + void mimeDetectBytes() throws KreuzbergException { + byte[] testBytes = "%PDF-1.4\n".getBytes(); + String result = Kreuzberg.detectMimeType(testBytes); + assertTrue(result.toLowerCase().contains("pdf")); + } + + @Test + @DisplayName("Detect MIME type from file path") + void mimeDetectPath(@TempDir Path tempDir) throws IOException, KreuzbergException { + Path testFile = tempDir.resolve("test.txt"); + Files.writeString(testFile, "Hello, world!"); + + String result = Kreuzberg.detectMimeTypeFromPath(testFile.toString()); + assertTrue(result.toLowerCase().contains("text")); + } + + @Test + @DisplayName("Get file extensions for a MIME type") + void mimeGetExtensions() throws KreuzbergException { + List result = Kreuzberg.getExtensionsForMime("application/pdf"); + assertNotNull(result); + assertTrue(result.contains("pdf")); + } + + // Ocr Backend Management Tests + + @Test + @DisplayName("Clear all OCR backends and verify list is empty") + void ocrBackendsClear() throws KreuzbergException { + Kreuzberg.clearOCRBackends(); + List result = Kreuzberg.listOCRBackends(); + assertEquals(0, result.size()); + } + + @Test + @DisplayName("List all registered OCR backends") + void ocrBackendsList() throws KreuzbergException { + List result = Kreuzberg.listOCRBackends(); + assertNotNull(result); + assertTrue(result.stream().allMatch(item -> item instanceof String)); + } + + @Test + @DisplayName("Unregister nonexistent OCR backend gracefully") + void ocrBackendsUnregister() throws KreuzbergException { + assertDoesNotThrow(() -> Kreuzberg.unregisterOCRBackend("nonexistent-backend-xyz")); + } + + // Post Processor Management Tests + + @Test + @DisplayName("Clear all post-processors and verify list is empty") + void postProcessorsClear() throws KreuzbergException { + Kreuzberg.clearPostProcessors(); + } + + @Test + @DisplayName("List all registered post-processors") + void postProcessorsList() throws KreuzbergException { + List result = Kreuzberg.listPostProcessors(); + assertNotNull(result); + assertTrue(result.stream().allMatch(item -> item instanceof String)); + } + + // Validator Management Tests + + @Test + @DisplayName("Clear all validators and verify list is empty") + void validatorsClear() throws KreuzbergException { + Kreuzberg.clearValidators(); + List result = Kreuzberg.listValidators(); + assertEquals(0, result.size()); + } + + @Test + @DisplayName("List all registered validators") + void validatorsList() throws KreuzbergException { + List result = Kreuzberg.listValidators(); + assertNotNull(result); + assertTrue(result.stream().allMatch(item -> item instanceof String)); + } + } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/RenderTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/RenderTest.java index a78e19fe3ce..d264f27c20a 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/RenderTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/RenderTest.java @@ -3,49 +3,55 @@ // Code generated by kreuzberg-e2e-generator. DO NOT EDIT. // To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang java -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - import dev.kreuzberg.Kreuzberg; -import dev.kreuzberg.Kreuzberg.PageResult; import dev.kreuzberg.Kreuzberg.PdfPageIterator; +import dev.kreuzberg.Kreuzberg.PageResult; +import org.junit.jupiter.api.Test; + import java.nio.file.Files; import java.nio.file.Path; -import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** Tests for render fixtures. */ public class RenderTest { - @Test - public void renderCustomDpi() throws Exception { - Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); - assumeTrue(Files.exists(documentPath), "Skipping render_custom_dpi: missing document"); - byte[] pngData = Kreuzberg.renderPdfPage(documentPath, 0, 72); - E2EHelpers.assertIsPng(pngData); - E2EHelpers.assertMinByteLength(pngData, 50); - } - - @Test - public void renderIterator() throws Exception { - Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); - assumeTrue(Files.exists(documentPath), "Skipping render_iterator: missing document"); - int pageCount = 0; - try (PdfPageIterator iter = PdfPageIterator.open(documentPath, 150)) { - while (iter.hasNext()) { - PageResult page = iter.next(); - E2EHelpers.assertIsPng(page.data()); - pageCount++; - } + @Test + public void renderCustomDpi() throws Exception { + Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); + assumeTrue(Files.exists(documentPath), + "Skipping render_custom_dpi: missing document"); + byte[] pngData = Kreuzberg.renderPdfPage(documentPath, 0, 72); + E2EHelpers.assertIsPng(pngData); + E2EHelpers.assertMinByteLength(pngData, 50); + } + + @Test + public void renderIterator() throws Exception { + Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); + assumeTrue(Files.exists(documentPath), + "Skipping render_iterator: missing document"); + int pageCount = 0; + try (PdfPageIterator iter = PdfPageIterator.open(documentPath, 150)) { + while (iter.hasNext()) { + PageResult page = iter.next(); + E2EHelpers.assertIsPng(page.data()); + pageCount++; + } + } + assertTrue(pageCount >= 1, + String.format("Expected at least 1 pages, got %d", pageCount)); + } + + @Test + public void renderSinglePage() throws Exception { + Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); + assumeTrue(Files.exists(documentPath), + "Skipping render_single_page: missing document"); + byte[] pngData = Kreuzberg.renderPdfPage(documentPath, 0, 150); + E2EHelpers.assertIsPng(pngData); + E2EHelpers.assertMinByteLength(pngData, 100); } - assertTrue(pageCount >= 1, String.format("Expected at least 1 pages, got %d", pageCount)); - } - - @Test - public void renderSinglePage() throws Exception { - Path documentPath = E2EHelpers.resolveDocument("pdf/tiny.pdf"); - assumeTrue(Files.exists(documentPath), "Skipping render_single_page: missing document"); - byte[] pngData = Kreuzberg.renderPdfPage(documentPath, 0, 150); - E2EHelpers.assertIsPng(pngData); - E2EHelpers.assertMinByteLength(pngData, 100); - } + } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/StructuredTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/StructuredTest.java index e33c142d505..4dfdf3123d2 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/StructuredTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/StructuredTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,176 +26,177 @@ /** Tests for structured fixtures. */ public class StructuredTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); - - @Test - public void structuredCsvBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_csv_basic", - "csv/stanley_cups.csv", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/csv")); - E2EHelpers.Assertions.assertMinContentLength(result, 20); - }); - } - - @Test - public void structuredEnwBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_enw_basic", - "data_formats/sample.enw", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, - Arrays.asList( - "application/x-endnote-refer", "application/x-endnote+xml", "text/plain")); - }); - } - - @Test - public void structuredJsonBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_json_basic", - "json/sample_document.json", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/json")); - E2EHelpers.Assertions.assertMinContentLength(result, 20); - E2EHelpers.Assertions.assertContentContainsAny( - result, Arrays.asList("Sample Document", "Test Author")); - }); - } - - @Test - public void structuredJsonSimple() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_json_simple", - "json/simple.json", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/json")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertContentContainsAny(result, Arrays.asList("{", "name")); - }); - } - - @Test - public void structuredNbibBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_nbib_basic", - "data_formats/sample.nbib", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/nbib", "application/x-pubmed", "text/plain")); - E2EHelpers.Assertions.assertContentNotEmpty(result); - }); - } - - @Test - public void structuredRisBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_ris_basic", - "data_formats/sample.ris", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/x-research-info-systems", "text/plain")); - E2EHelpers.Assertions.assertContentNotEmpty(result); - }); - } - - @Test - public void structuredTomlBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_toml_basic", - "data_formats/cargo.toml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("application/toml", "text/toml")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } - - @Test - public void structuredTsvBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_tsv_basic", - "data_formats/employees.tsv", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, Arrays.asList("text/tab-separated-values", "text/plain")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } - - @Test - public void structuredYamlBasic() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_yaml_basic", - "yaml/simple.yaml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime( - result, - Arrays.asList("application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } - - @Test - public void structuredYamlSimple() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "structured_yaml_simple", - "yaml/simple.yaml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-yaml")); - E2EHelpers.Assertions.assertMinContentLength(result, 10); - }); - } + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void structuredCsvBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_csv_basic", + "csv/stanley_cups.csv", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/csv")); + E2EHelpers.Assertions.assertMinContentLength(result, 20); + } + ); + } + + @Test + public void structuredEnwBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_enw_basic", + "data_formats/sample.enw", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-endnote-refer", "application/x-endnote+xml", "text/plain")); + } + ); + } + + @Test + public void structuredJsonBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_json_basic", + "json/sample_document.json", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/json")); + E2EHelpers.Assertions.assertMinContentLength(result, 20); + E2EHelpers.Assertions.assertContentContainsAny(result, Arrays.asList("Sample Document", "Test Author")); + } + ); + } + + @Test + public void structuredJsonSimple() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_json_simple", + "json/simple.json", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/json")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + E2EHelpers.Assertions.assertContentContainsAny(result, Arrays.asList("{", "name")); + } + ); + } + + @Test + public void structuredNbibBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_nbib_basic", + "data_formats/sample.nbib", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/nbib", "application/x-pubmed", "text/plain")); + E2EHelpers.Assertions.assertContentNotEmpty(result); + } + ); + } + + @Test + public void structuredRisBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_ris_basic", + "data_formats/sample.ris", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-research-info-systems", "text/plain")); + E2EHelpers.Assertions.assertContentNotEmpty(result); + } + ); + } + + @Test + public void structuredTomlBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_toml_basic", + "data_formats/cargo.toml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/toml", "text/toml")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } + + @Test + public void structuredTsvBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_tsv_basic", + "data_formats/employees.tsv", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("text/tab-separated-values", "text/plain")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } + + @Test + public void structuredYamlBasic() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_yaml_basic", + "yaml/simple.yaml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } + + @Test + public void structuredYamlSimple() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "structured_yaml_simple", + "yaml/simple.yaml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/x-yaml")); + E2EHelpers.Assertions.assertMinContentLength(result, 10); + } + ); + } + } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/XmlTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/XmlTest.java index 2e2b38d908c..923e7997ae0 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/XmlTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/XmlTest.java @@ -4,10 +4,20 @@ // CHECKSTYLE.OFF: LineLength - generated code import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.kreuzberg.BytesWithMime; +import dev.kreuzberg.ExtractionResult; +import dev.kreuzberg.Kreuzberg; +import dev.kreuzberg.config.ExtractionConfig; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; // CHECKSTYLE.ON: UnusedImports // CHECKSTYLE.ON: LineLength @@ -16,21 +26,23 @@ /** Tests for xml fixtures. */ public class XmlTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + public void xmlPlantCatalog() throws Exception { + JsonNode config = null; + E2EHelpers.runFixture( + "xml_plant_catalog", + "xml/plant_catalog.xml", + config, + Collections.emptyList(), + null, + true, + result -> { + E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/xml")); + E2EHelpers.Assertions.assertMinContentLength(result, 100); + } + ); + } - @Test - public void xmlPlantCatalog() throws Exception { - JsonNode config = null; - E2EHelpers.runFixture( - "xml_plant_catalog", - "xml/plant_catalog.xml", - config, - Collections.emptyList(), - null, - true, - result -> { - E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/xml")); - E2EHelpers.Assertions.assertMinContentLength(result, 100); - }); - } } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/package-info.java b/e2e/java/src/test/java/com/kreuzberg/e2e/package-info.java index af8c3c4a235..0d7096f1b66 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/package-info.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/package-info.java @@ -1,8 +1,8 @@ /** * E2E test utilities and generated test classes for Kreuzberg. * - *

This package contains auto-generated test classes organized by fixture category. Tests use - * JUnit 5 and validate document extraction across multiple formats. + *

This package contains auto-generated test classes organized by fixture category. + * Tests use JUnit 5 and validate document extraction across multiple formats. * * @since 4.0.0 */ diff --git a/e2e/python/tests/helpers.py b/e2e/python/tests/helpers.py index 81696dc4589..4efa020cc14 100644 --- a/e2e/python/tests/helpers.py +++ b/e2e/python/tests/helpers.py @@ -336,6 +336,13 @@ def _assert_chunks_heading_context(chunks: Any, expected: bool) -> None: pytest.fail(f"Chunk {i} should have no heading_context") +def _assert_chunks_chunk_type(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + chunk_type = getattr(chunk, "chunk_type", None) + if chunk_type is None or str(chunk_type) == "unknown": + pytest.fail(f"Chunk {i} has no specific chunk_type, got {chunk_type}") + + def _assert_chunks_heading_prefix(chunks: Any) -> None: for i, chunk in enumerate(chunks): meta = getattr(chunk, "metadata", None) @@ -354,6 +361,7 @@ def assert_chunks( each_has_content: bool | None = None, each_has_embedding: bool | None = None, each_has_heading_context: bool | None = None, + each_has_chunk_type: bool | None = None, content_starts_with_heading: bool | None = None, ) -> None: chunks = getattr(result, "chunks", None) @@ -370,6 +378,8 @@ def assert_chunks( _assert_chunks_embedding(chunks) if each_has_heading_context is not None: _assert_chunks_heading_context(chunks, each_has_heading_context) + if each_has_chunk_type: + _assert_chunks_chunk_type(chunks) if content_starts_with_heading: _assert_chunks_heading_prefix(chunks) diff --git a/e2e/r/tests/testthat/helper-kreuzberg.R b/e2e/r/tests/testthat/helper-kreuzberg.R index d1072a4cf0e..41924f0b9a0 100644 --- a/e2e/r/tests/testthat/helper-kreuzberg.R +++ b/e2e/r/tests/testthat/helper-kreuzberg.R @@ -10,21 +10,15 @@ find_workspace_root <- function() { if (!is.null(ofile)) { # From tests/testthat/helper-kreuzberg.R, go up 4 levels to repo root candidate <- normalizePath(file.path(dirname(ofile), "..", "..", "..", ".."), mustWork = FALSE) - if (dir.exists(file.path(candidate, "test_documents"))) { - return(candidate) - } + if (dir.exists(file.path(candidate, "test_documents"))) return(candidate) } # Fallback: testthat sets working dir to package root (e2e/r/) candidate <- normalizePath(file.path(getwd(), "..", ".."), mustWork = FALSE) - if (dir.exists(file.path(candidate, "test_documents"))) { - return(candidate) - } + if (dir.exists(file.path(candidate, "test_documents"))) return(candidate) # Last resort: walk up from current dir d <- getwd() for (i in seq_len(10)) { - if (dir.exists(file.path(d, "test_documents"))) { - return(d) - } + if (dir.exists(file.path(d, "test_documents"))) return(d) d <- dirname(d) } stop("Could not find workspace root (test_documents directory)") @@ -37,16 +31,12 @@ resolve_document <- function(relative) { } build_config <- function(raw) { - if (is.null(raw) || length(raw) == 0) { - return(NULL) - } + if (is.null(raw) || length(raw) == 0) return(NULL) raw } build_extraction_config <- function(raw) { - if (is.null(raw) || length(raw) == 0) { - return(NULL) - } + if (is.null(raw) || length(raw) == 0) return(NULL) # Convert the raw config map into extraction_config() call do.call(extraction_config, raw) } @@ -58,9 +48,7 @@ skip_reason_for <- function(error, fixture_id, requirements, notes = NULL) { missing_dependency <- grepl("missing dependency", downcased, fixed = TRUE) unsupported_format <- grepl("unsupported format", downcased, fixed = TRUE) - if (!missing_dependency && !unsupported_format && !requirement_hit) { - return(NULL) - } + if (!missing_dependency && !unsupported_format && !requirement_hit) return(NULL) reason <- if (missing_dependency) { "missing dependency" @@ -88,12 +76,11 @@ skip_if_feature_unavailable <- function(feature) { run_fixture <- function(fixture_id, relative_path, config_hash, requirements, notes, skip_if_missing = TRUE) { run_fixture_with_method(fixture_id, relative_path, config_hash, "sync", "file", - requirements = requirements, notes = notes, skip_if_missing = skip_if_missing - ) + requirements = requirements, notes = notes, skip_if_missing = skip_if_missing) } run_fixture_with_method <- function(fixture_id, relative_path, config_hash, method, input_type, - requirements, notes, skip_if_missing = TRUE) { + requirements, notes, skip_if_missing = TRUE) { document_path <- resolve_document(relative_path) if (skip_if_missing && !file.exists(document_path)) { @@ -149,9 +136,7 @@ perform_extraction <- function(document_path, config, method, input_type) { # Assertion helpers assert_expected_mime <- function(result, expected) { - if (length(expected) == 0) { - return(invisible(NULL)) - } + if (length(expected) == 0) return(invisible(NULL)) testthat::expect_true(any(vapply(expected, function(token) grepl(token, result$mime_type, fixed = TRUE), logical(1)))) } @@ -164,17 +149,13 @@ assert_max_content_length <- function(result, maximum) { } assert_content_contains_any <- function(result, snippets) { - if (length(snippets) == 0) { - return(invisible(NULL)) - } + if (length(snippets) == 0) return(invisible(NULL)) lowered <- tolower(result$content) testthat::expect_true(any(vapply(snippets, function(s) grepl(tolower(s), lowered, fixed = TRUE), logical(1)))) } assert_content_contains_all <- function(result, snippets) { - if (length(snippets) == 0) { - return(invisible(NULL)) - } + if (length(snippets) == 0) return(invisible(NULL)) lowered <- tolower(result$content) testthat::expect_true(all(vapply(snippets, function(s) grepl(tolower(s), lowered, fixed = TRUE), logical(1)))) } @@ -186,9 +167,7 @@ assert_table_count <- function(result, minimum = NULL, maximum = NULL) { } assert_detected_languages <- function(result, expected, min_confidence = NULL) { - if (length(expected) == 0) { - return(invisible(NULL)) - } + if (length(expected) == 0) return(invisible(NULL)) languages <- result$detected_languages testthat::expect_false(is.null(languages)) testthat::expect_true(all(expected %in% languages)) @@ -292,15 +271,13 @@ assert_elements <- function(result, min_count = NULL, types_include = NULL) { } assert_ocr_elements <- function(result, has_elements = NULL, elements_have_geometry = NULL, - elements_have_confidence = NULL, min_count = NULL) { + elements_have_confidence = NULL, min_count = NULL) { ocr_elements <- result$ocr_elements if (isTRUE(has_elements)) { testthat::expect_false(is.null(ocr_elements)) testthat::expect_gt(length(ocr_elements), 0) } - if (!is.list(ocr_elements)) { - return(invisible(NULL)) - } + if (!is.list(ocr_elements)) return(invisible(NULL)) if (!is.null(min_count)) testthat::expect_gte(length(ocr_elements), min_count) if (isTRUE(elements_have_geometry)) { for (el in ocr_elements) { @@ -317,7 +294,7 @@ assert_ocr_elements <- function(result, has_elements = NULL, elements_have_geome } assert_document <- function(result, has_document = FALSE, min_node_count = NULL, - node_types_include = NULL, has_groups = NULL) { + node_types_include = NULL, has_groups = NULL) { document <- result$document if (has_document) { testthat::expect_false(is.null(document)) @@ -378,9 +355,7 @@ assert_table_bounding_boxes <- function(result) { } assert_table_content_contains_any <- function(result, snippets) { - if (length(snippets) == 0) { - return(invisible(NULL)) - } + if (length(snippets) == 0) return(invisible(NULL)) tables <- if (is.null(result$tables)) list() else result$tables all_content <- tolower(paste(vapply(tables, function(t) t$content %||% "", character(1)), collapse = " ")) testthat::expect_true(any(vapply(snippets, function(s) grepl(tolower(s), all_content, fixed = TRUE), logical(1)))) @@ -435,13 +410,9 @@ assert_annotations <- function(result, has_annotations = FALSE, min_count = NULL # Internal helpers fetch_metadata_value <- function(metadata, path) { value <- lookup_metadata_path(metadata, path) - if (!is.null(value)) { - return(value) - } + if (!is.null(value)) return(value) format_data <- metadata$format - if (is.list(format_data)) { - return(lookup_metadata_path(format_data, path)) - } + if (is.list(format_data)) return(lookup_metadata_path(format_data, path)) NULL } @@ -449,24 +420,16 @@ lookup_metadata_path <- function(metadata, path) { current <- metadata segments <- strsplit(path, ".", fixed = TRUE)[[1]] for (segment in segments) { - if (!is.list(current)) { - return(NULL) - } + if (!is.list(current)) return(NULL) current <- current[[segment]] - if (is.null(current)) { - return(NULL) - } + if (is.null(current)) return(NULL) } current } values_equal <- function(lhs, rhs) { - if (is.character(lhs) && is.character(rhs)) { - return(identical(lhs, rhs)) - } - if (is.numeric(lhs) && is.numeric(rhs)) { - return(as.numeric(lhs) == as.numeric(rhs)) - } + if (is.character(lhs) && is.character(rhs)) return(identical(lhs, rhs)) + if (is.numeric(lhs) && is.numeric(rhs)) return(as.numeric(lhs) == as.numeric(rhs)) identical(lhs, rhs) } diff --git a/e2e/r/tests/testthat/test-archive.R b/e2e/r/tests/testthat/test-archive.R index d4938c3bff3..c40afe3ce02 100644 --- a/e2e/r/tests/testthat/test-archive.R +++ b/e2e/r/tests/testthat/test-archive.R @@ -13,8 +13,8 @@ test_that("archive_gz_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/gzip", "application/x-gzip")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/gzip", "application/x-gzip")) + assert_min_content_length(result, 10L) }) test_that("archive_sevenz_basic", { @@ -26,8 +26,8 @@ test_that("archive_sevenz_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-7z-compressed")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-7z-compressed")) + assert_min_content_length(result, 10L) }) test_that("archive_tar_basic", { @@ -39,8 +39,8 @@ test_that("archive_tar_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-tar", "application/tar")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-tar", "application/tar")) + assert_min_content_length(result, 10L) }) test_that("archive_zip_basic", { @@ -52,7 +52,7 @@ test_that("archive_zip_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/zip", "application/x-zip-compressed")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/zip", "application/x-zip-compressed")) + assert_min_content_length(result, 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-contract.R b/e2e/r/tests/testthat/test-contract.R index a1f7c629212..20a69e5b001 100644 --- a/e2e/r/tests/testthat/test-contract.R +++ b/e2e/r/tests/testthat/test-contract.R @@ -15,9 +15,9 @@ test_that("api_batch_bytes_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_batch_bytes_sync", { @@ -31,9 +31,9 @@ test_that("api_batch_bytes_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_batch_bytes_with_configs_async", { @@ -47,8 +47,8 @@ test_that("api_batch_bytes_with_configs_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_batch_bytes_with_configs_sync", { @@ -62,8 +62,8 @@ test_that("api_batch_bytes_with_configs_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_batch_file_async", { @@ -77,9 +77,9 @@ test_that("api_batch_file_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_batch_file_sync", { @@ -93,9 +93,9 @@ test_that("api_batch_file_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_batch_file_with_configs_async", { @@ -109,8 +109,8 @@ test_that("api_batch_file_with_configs_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_batch_file_with_configs_sync", { @@ -124,8 +124,8 @@ test_that("api_batch_file_with_configs_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_batch_file_with_timeout_sync", { @@ -139,8 +139,8 @@ test_that("api_batch_file_with_timeout_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("api_extract_bytes_async", { @@ -154,9 +154,9 @@ test_that("api_extract_bytes_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_extract_bytes_sync", { @@ -170,9 +170,9 @@ test_that("api_extract_bytes_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_extract_file_async", { @@ -186,9 +186,9 @@ test_that("api_extract_file_async", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("api_extract_file_sync", { @@ -200,9 +200,9 @@ test_that("api_extract_file_sync", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("May 5, 2023", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("May 5, 2023", "Mallori")) }) test_that("config_acceleration_cpu_provider", { @@ -214,9 +214,9 @@ test_that("config_acceleration_cpu_provider", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern")) }) test_that("config_chunking", { @@ -228,9 +228,9 @@ test_that("config_chunking", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE) }) test_that("config_chunking_heading_context", { @@ -243,8 +243,8 @@ test_that("config_chunking_heading_context", { notes = NULL, skip_if_missing = TRUE ) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = TRUE) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = TRUE) }) test_that("config_chunking_markdown", { @@ -257,9 +257,9 @@ test_that("config_chunking_markdown", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE) }) test_that("config_chunking_no_headings", { @@ -272,8 +272,8 @@ test_that("config_chunking_no_headings", { notes = NULL, skip_if_missing = TRUE ) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = FALSE) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = FALSE) }) test_that("config_chunking_prepend_heading_context", { @@ -286,8 +286,8 @@ test_that("config_chunking_prepend_heading_context", { notes = NULL, skip_if_missing = TRUE ) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = TRUE, content_starts_with_heading = TRUE) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE, each_has_heading_context = TRUE, content_starts_with_heading = TRUE) }) test_that("config_chunking_small", { @@ -300,9 +300,9 @@ test_that("config_chunking_small", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE) }) test_that("config_chunking_text", { @@ -314,9 +314,9 @@ test_that("config_chunking_text", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE) }) test_that("config_chunking_tokenizer", { @@ -329,8 +329,8 @@ test_that("config_chunking_tokenizer", { notes = "Requires network access for HuggingFace Hub tokenizer download", skip_if_missing = TRUE ) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 2L, each_has_content = TRUE) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 2L, each_has_content = TRUE) }) test_that("config_disable_ocr", { @@ -356,8 +356,8 @@ test_that("config_djot_content", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("config_document_structure", { @@ -369,8 +369,8 @@ test_that("config_document_structure", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_document(result, has_document = TRUE, min_node_count = 1L, node_types_include = c("paragraph")) + assert_expected_mime(result, c("application/pdf")) + assert_document(result, has_document = TRUE, min_node_count = 1L, node_types_include = c("paragraph")) }) test_that("config_document_structure_disabled", { @@ -382,8 +382,8 @@ test_that("config_document_structure_disabled", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_document(result, has_document = FALSE) + assert_expected_mime(result, c("application/pdf")) + assert_document(result, has_document = FALSE) }) test_that("config_document_structure_groups", { @@ -396,8 +396,8 @@ test_that("config_document_structure_groups", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_document(result, has_document = TRUE, has_groups = TRUE) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_document(result, has_document = TRUE, has_groups = TRUE) }) test_that("config_document_structure_headings", { @@ -410,8 +410,8 @@ test_that("config_document_structure_headings", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_document(result, has_document = TRUE, min_node_count = 1L, node_types_include = c("heading", "paragraph")) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_document(result, has_document = TRUE, min_node_count = 1L, node_types_include = c("heading", "paragraph")) }) test_that("config_document_structure_with_headings", { @@ -423,8 +423,8 @@ test_that("config_document_structure_with_headings", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_document(result, has_document = TRUE, min_node_count = 1L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_document(result, has_document = TRUE, min_node_count = 1L) }) test_that("config_element_types", { @@ -437,8 +437,8 @@ test_that("config_element_types", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_elements(result, min_count = 1L, types_include = c("narrative_text")) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_elements(result, min_count = 1L, types_include = c("narrative_text")) }) test_that("config_email_msg_fallback_codepage", { @@ -450,8 +450,8 @@ test_that("config_email_msg_fallback_codepage", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-outlook")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.ms-outlook")) + assert_min_content_length(result, 10L) }) test_that("config_extraction_timeout", { @@ -463,8 +463,8 @@ test_that("config_extraction_timeout", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("config_force_ocr", { @@ -477,8 +477,8 @@ test_that("config_force_ocr", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) }) test_that("config_force_ocr_pages", { @@ -491,8 +491,8 @@ test_that("config_force_ocr_pages", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 1L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 1L) }) test_that("config_html_options", { @@ -504,9 +504,9 @@ test_that("config_html_options", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/html")) - assert_min_content_length(result, 10L) - assert_content_not_empty(result) + assert_expected_mime(result, c("text/html")) + assert_min_content_length(result, 10L) + assert_content_not_empty(result) }) test_that("config_images", { @@ -518,8 +518,8 @@ test_that("config_images", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_images(result, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_images(result, min_count = 1L) }) test_that("config_images_with_formats", { @@ -531,8 +531,8 @@ test_that("config_images_with_formats", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_images(result, min_count = 1L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_images(result, min_count = 1L) }) test_that("config_keywords", { @@ -545,9 +545,9 @@ test_that("config_keywords", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_keywords(result, has_keywords = TRUE, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_keywords(result, has_keywords = TRUE, min_count = 1L) }) test_that("config_language_detection", { @@ -559,9 +559,9 @@ test_that("config_language_detection", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_detected_languages(result, c("eng"), 0.5) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_detected_languages(result, c("eng"), 0.5) }) test_that("config_language_detection_multi", { @@ -573,9 +573,9 @@ test_that("config_language_detection_multi", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_detected_languages(result, c("eng"), NULL) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_detected_languages(result, c("eng"), NULL) }) test_that("config_language_multi", { @@ -588,9 +588,9 @@ test_that("config_language_multi", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_detected_languages(result, c("eng"), NULL) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_detected_languages(result, c("eng"), NULL) }) test_that("config_pages", { @@ -603,9 +603,9 @@ test_that("config_pages", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("PAGE")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("PAGE")) }) test_that("config_pages_exact_count", { @@ -618,9 +618,9 @@ test_that("config_pages_exact_count", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_pages(result, exact_count = 5L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_pages(result, exact_count = 5L) }) test_that("config_pages_extract", { @@ -633,9 +633,9 @@ test_that("config_pages_extract", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_pages(result, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_pages(result, min_count = 1L) }) test_that("config_pages_markers", { @@ -648,9 +648,9 @@ test_that("config_pages_markers", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("PAGE")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("PAGE")) }) test_that("config_pdf_annotations_count", { @@ -663,8 +663,8 @@ test_that("config_pdf_annotations_count", { notes = "PDFium ARM Linux binary does not support annotation extraction", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_annotations(result, has_annotations = TRUE, min_count = 3L) + assert_expected_mime(result, c("application/pdf")) + assert_annotations(result, has_annotations = TRUE, min_count = 3L) }) test_that("config_pdf_hierarchy", { @@ -677,8 +677,8 @@ test_that("config_pdf_hierarchy", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) }) test_that("config_pdf_margins", { @@ -691,8 +691,8 @@ test_that("config_pdf_margins", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) }) test_that("config_postprocessor", { @@ -704,9 +704,9 @@ test_that("config_postprocessor", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_not_empty(result) }) test_that("config_processing_warnings_empty", { @@ -718,9 +718,9 @@ test_that("config_processing_warnings_empty", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_processing_warnings(result, is_empty = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_processing_warnings(result, is_empty = TRUE) }) test_that("config_quality_disabled", { @@ -732,9 +732,9 @@ test_that("config_quality_disabled", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_not_empty(result) }) test_that("config_quality_enabled", { @@ -747,9 +747,9 @@ test_that("config_quality_enabled", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_quality_score(result, has_score = TRUE, min_score = 0, max_score = 1) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_quality_score(result, has_score = TRUE, min_score = 0, max_score = 1) }) test_that("config_quality_score_range", { @@ -762,8 +762,8 @@ test_that("config_quality_score_range", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_quality_score(result, has_score = TRUE, min_score = 0.1) + assert_expected_mime(result, c("application/pdf")) + assert_quality_score(result, has_score = TRUE, min_score = 0.1) }) test_that("config_security_limits", { @@ -775,8 +775,8 @@ test_that("config_security_limits", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/zip", "application/x-zip-compressed")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/zip", "application/x-zip-compressed")) + assert_min_content_length(result, 10L) }) test_that("config_structured_output", { @@ -789,8 +789,8 @@ test_that("config_structured_output", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("config_tables_content", { @@ -802,8 +802,8 @@ test_that("config_tables_content", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("config_tree_sitter", { @@ -843,8 +843,8 @@ test_that("config_use_cache_false", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_bytes_markdown", { @@ -858,8 +858,8 @@ test_that("output_format_bytes_markdown", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_djot", { @@ -871,8 +871,8 @@ test_that("output_format_djot", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_html", { @@ -884,8 +884,8 @@ test_that("output_format_html", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_markdown", { @@ -897,8 +897,8 @@ test_that("output_format_markdown", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("output_format_plain", { @@ -910,8 +910,8 @@ test_that("output_format_plain", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("result_format_element_based", { @@ -923,8 +923,8 @@ test_that("result_format_element_based", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_elements(result, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_elements(result, min_count = 1L) }) test_that("result_format_unified", { @@ -936,7 +936,7 @@ test_that("result_format_unified", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-email.R b/e2e/r/tests/testthat/test-email.R index ddc219c5349..78e0da909f6 100644 --- a/e2e/r/tests/testthat/test-email.R +++ b/e2e/r/tests/testthat/test-email.R @@ -13,8 +13,8 @@ test_that("email_eml_html_body", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("message/rfc822")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("message/rfc822")) + assert_min_content_length(result, 10L) }) test_that("email_eml_multipart", { @@ -26,8 +26,8 @@ test_that("email_eml_multipart", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("message/rfc822")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("message/rfc822")) + assert_min_content_length(result, 10L) }) test_that("email_eml_utf16", { @@ -39,9 +39,9 @@ test_that("email_eml_utf16", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("message/rfc822")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("Test Email", "Roses are red")) + assert_expected_mime(result, c("message/rfc822")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("Test Email", "Roses are red")) }) test_that("email_msg_basic", { @@ -53,8 +53,8 @@ test_that("email_msg_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-outlook")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.ms-outlook")) + assert_min_content_length(result, 10L) }) test_that("email_pst_empty", { @@ -66,7 +66,7 @@ test_that("email_pst_empty", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-outlook-pst")) + assert_expected_mime(result, c("application/vnd.ms-outlook-pst")) }) test_that("email_sample_eml", { @@ -78,7 +78,7 @@ test_that("email_sample_eml", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("message/rfc822")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("message/rfc822")) + assert_min_content_length(result, 20L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-embeddings.R b/e2e/r/tests/testthat/test-embeddings.R index ccf6faaff51..03b89266d2c 100644 --- a/e2e/r/tests/testthat/test-embeddings.R +++ b/e2e/r/tests/testthat/test-embeddings.R @@ -16,9 +16,9 @@ test_that("embedding_async", { notes = "Embeddings not supported on Windows", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) }) test_that("embedding_balanced_preset", { @@ -31,9 +31,9 @@ test_that("embedding_balanced_preset", { notes = "Embeddings not supported on Windows", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) }) test_that("embedding_disabled", { @@ -45,9 +45,9 @@ test_that("embedding_disabled", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = FALSE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = FALSE) }) test_that("embedding_fast_preset", { @@ -60,8 +60,8 @@ test_that("embedding_fast_preset", { notes = "Embeddings not supported on Windows", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE, each_has_embedding = TRUE) }) # nolint end diff --git a/e2e/r/tests/testthat/test-html.R b/e2e/r/tests/testthat/test-html.R index 2e2d9899f41..eae8c08984d 100644 --- a/e2e/r/tests/testthat/test-html.R +++ b/e2e/r/tests/testthat/test-html.R @@ -13,8 +13,8 @@ test_that("html_complex_layout", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/html")) - assert_min_content_length(result, 1000L) + assert_expected_mime(result, c("text/html")) + assert_min_content_length(result, 1000L) }) test_that("html_simple_table", { @@ -26,8 +26,8 @@ test_that("html_simple_table", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/html")) - assert_min_content_length(result, 100L) - assert_content_contains_all(result, c("Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table")) + assert_expected_mime(result, c("text/html")) + assert_min_content_length(result, 100L) + assert_content_contains_all(result, c("Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table")) }) # nolint end diff --git a/e2e/r/tests/testthat/test-image.R b/e2e/r/tests/testthat/test-image.R index 157031c245b..fe26fd7631b 100644 --- a/e2e/r/tests/testthat/test-image.R +++ b/e2e/r/tests/testthat/test-image.R @@ -14,8 +14,8 @@ test_that("image_bmp_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/bmp")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/bmp")) + assert_content_not_empty(result) }) test_that("image_gif_basic", { @@ -28,8 +28,8 @@ test_that("image_gif_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/gif")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/gif")) + assert_content_not_empty(result) }) test_that("image_jp2_basic", { @@ -42,8 +42,8 @@ test_that("image_jp2_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jp2", "image/jpeg2000")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/jp2", "image/jpeg2000")) + assert_content_not_empty(result) }) test_that("image_metadata_only", { @@ -55,8 +55,8 @@ test_that("image_metadata_only", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jpeg")) - assert_max_content_length(result, 200L) + assert_expected_mime(result, c("image/jpeg")) + assert_max_content_length(result, 200L) }) test_that("image_pbm_basic", { @@ -69,8 +69,8 @@ test_that("image_pbm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/x-portable-bitmap", "image/x-pbm")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/x-portable-bitmap", "image/x-pbm")) + assert_content_not_empty(result) }) test_that("image_pgm_basic", { @@ -83,8 +83,8 @@ test_that("image_pgm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/x-portable-graymap", "image/x-pgm")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/x-portable-graymap", "image/x-pgm")) + assert_content_not_empty(result) }) test_that("image_ppm_basic", { @@ -97,8 +97,8 @@ test_that("image_ppm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/x-portable-pixmap", "image/x-ppm")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/x-portable-pixmap", "image/x-ppm")) + assert_content_not_empty(result) }) test_that("image_svg_basic", { @@ -110,8 +110,8 @@ test_that("image_svg_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/svg+xml")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("image/svg+xml")) + assert_min_content_length(result, 5L) }) test_that("image_tiff_basic", { @@ -124,8 +124,8 @@ test_that("image_tiff_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/tiff")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/tiff")) + assert_content_not_empty(result) }) test_that("image_webp_basic", { @@ -138,7 +138,7 @@ test_that("image_webp_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/webp")) - assert_content_not_empty(result) + assert_expected_mime(result, c("image/webp")) + assert_content_not_empty(result) }) # nolint end diff --git a/e2e/r/tests/testthat/test-keywords.R b/e2e/r/tests/testthat/test-keywords.R index 4a80972d76c..08a0fe1a40e 100644 --- a/e2e/r/tests/testthat/test-keywords.R +++ b/e2e/r/tests/testthat/test-keywords.R @@ -14,9 +14,9 @@ test_that("keywords_rake", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_keywords(result, has_keywords = TRUE, min_count = 1L, max_count = 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_keywords(result, has_keywords = TRUE, min_count = 1L, max_count = 10L) }) test_that("keywords_yake", { @@ -29,8 +29,8 @@ test_that("keywords_yake", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_keywords(result, has_keywords = TRUE, min_count = 1L, max_count = 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_keywords(result, has_keywords = TRUE, min_count = 1L, max_count = 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-ocr.R b/e2e/r/tests/testthat/test-ocr.R index 2e0dc0f1fce..3c731000ec1 100644 --- a/e2e/r/tests/testthat/test-ocr.R +++ b/e2e/r/tests/testthat/test-ocr.R @@ -14,9 +14,9 @@ test_that("ocr_image_hello_world", { notes = "Requires Tesseract OCR for image text extraction.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_content_contains_any(result, c("hello", "world")) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_content_contains_any(result, c("hello", "world")) }) test_that("ocr_image_no_text", { @@ -29,8 +29,8 @@ test_that("ocr_image_no_text", { notes = "Skip when Tesseract is unavailable.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jpeg")) - assert_max_content_length(result, 300L) + assert_expected_mime(result, c("image/jpeg")) + assert_max_content_length(result, 300L) }) test_that("ocr_paddle_confidence_filter", { @@ -44,8 +44,8 @@ test_that("ocr_paddle_confidence_filter", { notes = "Tests confidence threshold filtering with PaddleOCR", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jpeg")) - assert_min_content_length(result, 1L) + assert_expected_mime(result, c("image/jpeg")) + assert_min_content_length(result, 1L) }) test_that("ocr_paddle_element_hierarchy", { @@ -59,9 +59,9 @@ test_that("ocr_paddle_element_hierarchy", { notes = "Requires PaddleOCR with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) }) test_that("ocr_paddle_element_levels", { @@ -75,9 +75,9 @@ test_that("ocr_paddle_element_levels", { notes = "Requires PaddleOCR with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, min_count = 1L) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, min_count = 1L) }) test_that("ocr_paddle_image_chinese", { @@ -91,8 +91,8 @@ test_that("ocr_paddle_image_chinese", { notes = "Requires PaddleOCR with Chinese models", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/jpeg")) - assert_min_content_length(result, 1L) + assert_expected_mime(result, c("image/jpeg")) + assert_min_content_length(result, 1L) }) test_that("ocr_paddle_image_english", { @@ -106,9 +106,9 @@ test_that("ocr_paddle_image_english", { notes = "Requires PaddleOCR with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_content_contains_any(result, c("hello", "Hello", "world", "World")) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_content_contains_any(result, c("hello", "Hello", "world", "World")) }) test_that("ocr_paddle_markdown", { @@ -122,9 +122,9 @@ test_that("ocr_paddle_markdown", { notes = "Tests markdown output format parity with Tesseract", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_content_contains_any(result, c("hello", "Hello", "world", "World")) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_content_contains_any(result, c("hello", "Hello", "world", "World")) }) test_that("ocr_paddle_pdf_scanned", { @@ -138,9 +138,9 @@ test_that("ocr_paddle_pdf_scanned", { notes = "Requires PaddleOCR with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 20L) - assert_content_contains_any(result, c("Docling", "Markdown", "JSON")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 20L) + assert_content_contains_any(result, c("Docling", "Markdown", "JSON")) }) test_that("ocr_paddle_structured", { @@ -154,9 +154,9 @@ test_that("ocr_paddle_structured", { notes = "Tests structured output with bbox/confidence preservation", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) }) test_that("ocr_paddle_table_detection", { @@ -170,9 +170,9 @@ test_that("ocr_paddle_table_detection", { notes = "Tests table detection capability with PaddleOCR. ONNX Runtime model loading unstable on ARM Linux.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 10L) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 10L) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("ocr_pdf_image_only_german", { @@ -185,9 +185,9 @@ test_that("ocr_pdf_image_only_german", { notes = "Requires Tesseract OCR with German language data.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 20L) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 20L) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("ocr_pdf_rotated_90", { @@ -200,8 +200,8 @@ test_that("ocr_pdf_rotated_90", { notes = "Skip automatically when OCR backend is missing.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) }) test_that("ocr_pdf_tesseract", { @@ -214,9 +214,9 @@ test_that("ocr_pdf_tesseract", { notes = "Skip automatically if OCR backend is unavailable.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 20L) - assert_content_contains_any(result, c("Docling", "Markdown", "JSON")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 20L) + assert_content_contains_any(result, c("Docling", "Markdown", "JSON")) }) test_that("ocr_tesseract_elements", { @@ -229,9 +229,9 @@ test_that("ocr_tesseract_elements", { notes = "Requires Tesseract OCR backend", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, elements_have_geometry = TRUE, elements_have_confidence = TRUE) }) test_that("ocr_tesseract_elements_min_count", { @@ -244,9 +244,9 @@ test_that("ocr_tesseract_elements_min_count", { notes = "Requires Tesseract OCR backend", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_min_content_length(result, 5L) - assert_ocr_elements(result, has_elements = TRUE, min_count = 1L) + assert_expected_mime(result, c("image/png")) + assert_min_content_length(result, 5L) + assert_ocr_elements(result, has_elements = TRUE, min_count = 1L) }) test_that("ocr_tesseract_language_german", { @@ -259,7 +259,7 @@ test_that("ocr_tesseract_language_german", { notes = "Requires Tesseract OCR with German language data (deu)", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 20L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-office.R b/e2e/r/tests/testthat/test-office.R index 50ef8f5dfac..df5a485eecd 100644 --- a/e2e/r/tests/testthat/test-office.R +++ b/e2e/r/tests/testthat/test-office.R @@ -13,8 +13,8 @@ test_that("office_bibtex_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-bibtex", "text/x-bibtex")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-bibtex", "text/x-bibtex")) + assert_min_content_length(result, 10L) }) test_that("office_commonmark_basic", { @@ -26,8 +26,8 @@ test_that("office_commonmark_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/markdown", "text/plain", "text/x-commonmark")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("text/markdown", "text/plain", "text/x-commonmark")) + assert_min_content_length(result, 5L) }) test_that("office_dbf_basic", { @@ -40,9 +40,9 @@ test_that("office_dbf_basic", { notes = "Requires the office feature.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-dbf")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("|")) + assert_expected_mime(result, c("application/x-dbf")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("|")) }) test_that("office_djot_basic", { @@ -54,8 +54,8 @@ test_that("office_djot_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/x-djot", "text/djot")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("text/x-djot", "text/djot")) + assert_min_content_length(result, 10L) }) test_that("office_doc_legacy", { @@ -68,8 +68,8 @@ test_that("office_doc_legacy", { notes = "Requires the office feature.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/msword")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/msword")) + assert_min_content_length(result, 20L) }) test_that("office_docbook_basic", { @@ -81,8 +81,8 @@ test_that("office_docbook_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/docbook+xml", "text/docbook")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/docbook+xml", "text/docbook")) + assert_min_content_length(result, 10L) }) test_that("office_docx_basic", { @@ -94,8 +94,8 @@ test_that("office_docx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 10L) }) test_that("office_docx_equations", { @@ -107,8 +107,8 @@ test_that("office_docx_equations", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_fake", { @@ -120,8 +120,8 @@ test_that("office_docx_fake", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_formatting", { @@ -133,8 +133,8 @@ test_that("office_docx_formatting", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_headers", { @@ -146,8 +146,8 @@ test_that("office_docx_headers", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_lists", { @@ -159,8 +159,8 @@ test_that("office_docx_lists", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) }) test_that("office_docx_tables", { @@ -172,10 +172,10 @@ test_that("office_docx_tables", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 50L) - assert_content_contains_all(result, c("Simple uniform table", "Nested Table", "merged cells", "Header Col")) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 50L) + assert_content_contains_all(result, c("Simple uniform table", "Nested Table", "merged cells", "Header Col")) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("office_epub_basic", { @@ -187,8 +187,8 @@ test_that("office_epub_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/epub+zip")) - assert_min_content_length(result, 50L) + assert_expected_mime(result, c("application/epub+zip")) + assert_min_content_length(result, 50L) }) test_that("office_fb2_basic", { @@ -200,8 +200,8 @@ test_that("office_fb2_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-fictionbook+xml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-fictionbook+xml")) + assert_min_content_length(result, 10L) }) test_that("office_fictionbook_basic", { @@ -213,8 +213,8 @@ test_that("office_fictionbook_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-fictionbook+xml", "application/x-fictionbook")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-fictionbook+xml", "application/x-fictionbook")) + assert_min_content_length(result, 10L) }) test_that("office_hwp_basic", { @@ -227,8 +227,8 @@ test_that("office_hwp_basic", { notes = "Requires the office feature.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-hwp")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-hwp")) + assert_min_content_length(result, 10L) }) test_that("office_hwp_styled", { @@ -241,7 +241,7 @@ test_that("office_hwp_styled", { notes = "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-hwp")) + assert_expected_mime(result, c("application/x-hwp")) }) test_that("office_jats_basic", { @@ -253,8 +253,8 @@ test_that("office_jats_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-jats+xml", "text/jats")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-jats+xml", "text/jats")) + assert_min_content_length(result, 10L) }) test_that("office_jupyter_basic", { @@ -266,8 +266,8 @@ test_that("office_jupyter_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-ipynb+json")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-ipynb+json")) + assert_min_content_length(result, 10L) }) test_that("office_keynote_basic", { @@ -279,8 +279,8 @@ test_that("office_keynote_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-iwork-keynote-sffkey")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/x-iwork-keynote-sffkey")) + assert_min_content_length(result, 5L) }) test_that("office_latex_basic", { @@ -292,8 +292,8 @@ test_that("office_latex_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-latex", "text/x-latex")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("application/x-latex", "text/x-latex")) + assert_min_content_length(result, 20L) }) test_that("office_markdown_basic", { @@ -305,8 +305,8 @@ test_that("office_markdown_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/markdown")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("text/markdown")) + assert_min_content_length(result, 10L) }) test_that("office_mdx_basic", { @@ -318,8 +318,8 @@ test_that("office_mdx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/mdx", "text/x-mdx")) - assert_min_content_length(result, 50L) + assert_expected_mime(result, c("text/mdx", "text/x-mdx")) + assert_min_content_length(result, 50L) }) test_that("office_mdx_getting_started", { @@ -331,8 +331,8 @@ test_that("office_mdx_getting_started", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/mdx", "text/x-mdx")) - assert_min_content_length(result, 2000L) + assert_expected_mime(result, c("text/mdx", "text/x-mdx")) + assert_min_content_length(result, 2000L) }) test_that("office_mdx_troubleshooting", { @@ -344,8 +344,8 @@ test_that("office_mdx_troubleshooting", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/mdx", "text/x-mdx")) - assert_min_content_length(result, 2000L) + assert_expected_mime(result, c("text/mdx", "text/x-mdx")) + assert_min_content_length(result, 2000L) }) test_that("office_mdx_using_mdx", { @@ -357,8 +357,8 @@ test_that("office_mdx_using_mdx", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/mdx", "text/x-mdx")) - assert_min_content_length(result, 2000L) + assert_expected_mime(result, c("text/mdx", "text/x-mdx")) + assert_min_content_length(result, 2000L) }) test_that("office_numbers_basic", { @@ -370,8 +370,8 @@ test_that("office_numbers_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-iwork-numbers-sffnumbers")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-iwork-numbers-sffnumbers")) + assert_min_content_length(result, 10L) }) test_that("office_ods_basic", { @@ -383,8 +383,8 @@ test_that("office_ods_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.spreadsheet")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.spreadsheet")) + assert_min_content_length(result, 10L) }) test_that("office_odt_bold", { @@ -396,8 +396,8 @@ test_that("office_odt_bold", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) + assert_min_content_length(result, 10L) }) test_that("office_odt_list", { @@ -409,9 +409,9 @@ test_that("office_odt_list", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) - assert_min_content_length(result, 30L) - assert_content_contains_any(result, c("list item", "New level", "Pushed us")) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) + assert_min_content_length(result, 30L) + assert_content_contains_any(result, c("list item", "New level", "Pushed us")) }) test_that("office_odt_simple", { @@ -423,9 +423,9 @@ test_that("office_odt_simple", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("favorite things", "Parrots", "Analysis")) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("favorite things", "Parrots", "Analysis")) }) test_that("office_odt_table", { @@ -437,9 +437,9 @@ test_that("office_odt_table", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) - assert_min_content_length(result, 10L) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("application/vnd.oasis.opendocument.text")) + assert_min_content_length(result, 10L) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("office_opml_basic", { @@ -451,8 +451,8 @@ test_that("office_opml_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/xml+opml", "text/x-opml", "application/x-opml+xml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/xml+opml", "text/x-opml", "application/x-opml+xml")) + assert_min_content_length(result, 10L) }) test_that("office_org_basic", { @@ -464,8 +464,8 @@ test_that("office_org_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/x-org", "text/org")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("text/x-org", "text/org")) + assert_min_content_length(result, 20L) }) test_that("office_pages_basic", { @@ -477,8 +477,8 @@ test_that("office_pages_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-iwork-pages-sffpages")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/x-iwork-pages-sffpages")) + assert_min_content_length(result, 5L) }) test_that("office_ppsx_slideshow", { @@ -490,8 +490,8 @@ test_that("office_ppsx_slideshow", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.slideshow")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.slideshow")) + assert_min_content_length(result, 10L) }) test_that("office_ppt_legacy", { @@ -504,8 +504,8 @@ test_that("office_ppt_legacy", { notes = "Requires the office feature.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-powerpoint")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.ms-powerpoint")) + assert_min_content_length(result, 10L) }) test_that("office_pptm_basic", { @@ -517,8 +517,8 @@ test_that("office_pptm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_content_not_empty(result) }) test_that("office_pptx_basic", { @@ -530,8 +530,8 @@ test_that("office_pptx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_min_content_length(result, 50L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_min_content_length(result, 50L) }) test_that("office_pptx_images", { @@ -543,8 +543,8 @@ test_that("office_pptx_images", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_min_content_length(result, 15L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_min_content_length(result, 15L) }) test_that("office_pptx_pitch_deck", { @@ -556,8 +556,8 @@ test_that("office_pptx_pitch_deck", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) - assert_min_content_length(result, 100L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.presentationml.presentation")) + assert_min_content_length(result, 100L) }) test_that("office_rst_basic", { @@ -569,8 +569,8 @@ test_that("office_rst_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/x-rst", "text/prs.fallenstein.rst")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("text/x-rst", "text/prs.fallenstein.rst")) + assert_min_content_length(result, 20L) }) test_that("office_rtf_basic", { @@ -582,8 +582,8 @@ test_that("office_rtf_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/rtf", "text/rtf")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/rtf", "text/rtf")) + assert_min_content_length(result, 10L) }) test_that("office_typst_basic", { @@ -595,8 +595,8 @@ test_that("office_typst_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-typst", "text/x-typst")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-typst", "text/x-typst")) + assert_min_content_length(result, 10L) }) test_that("office_xls_legacy", { @@ -608,8 +608,8 @@ test_that("office_xls_legacy", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-excel")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.ms-excel")) + assert_min_content_length(result, 10L) }) test_that("office_xlsb_basic", { @@ -621,8 +621,8 @@ test_that("office_xlsb_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_content_not_empty(result) }) test_that("office_xlsm_basic", { @@ -634,8 +634,8 @@ test_that("office_xlsm_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_content_not_empty(result) }) test_that("office_xlsx_basic", { @@ -647,12 +647,12 @@ test_that("office_xlsx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_min_content_length(result, 100L) - assert_content_contains_all(result, c("Team", "Location", "Stanley Cups")) - assert_table_count(result, minimum = 1L, maximum = NULL) - assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) - assert_metadata_expectation(result, "sheet_names", list(contains = c("Stanley Cups"))) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_min_content_length(result, 100L) + assert_content_contains_all(result, c("Team", "Location", "Stanley Cups")) + assert_table_count(result, minimum = 1L, maximum = NULL) + assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) + assert_metadata_expectation(result, "sheet_names", list(contains = c("Stanley Cups"))) }) test_that("office_xlsx_multi_sheet", { @@ -664,9 +664,9 @@ test_that("office_xlsx_multi_sheet", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_min_content_length(result, 20L) - assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_min_content_length(result, 20L) + assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) }) test_that("office_xlsx_office_example", { @@ -678,7 +678,7 @@ test_that("office_xlsx_office_example", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_min_content_length(result, 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-pdf.R b/e2e/r/tests/testthat/test-pdf.R index 4cbbd3ba819..aab8d272886 100644 --- a/e2e/r/tests/testthat/test-pdf.R +++ b/e2e/r/tests/testthat/test-pdf.R @@ -13,8 +13,8 @@ test_that("pdf_annotations", { notes = "PDFium ARM Linux binary does not support annotation extraction", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_annotations(result, has_annotations = TRUE, min_count = 1L) + assert_expected_mime(result, c("application/pdf")) + assert_annotations(result, has_annotations = TRUE, min_count = 1L) }) test_that("pdf_assembly_technical", { @@ -26,10 +26,10 @@ test_that("pdf_assembly_technical", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5000L) - assert_content_contains_any(result, c("assembly", "register", "instruction")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5000L) + assert_content_contains_any(result, c("assembly", "register", "instruction")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_bayesian_data_analysis", { @@ -41,10 +41,10 @@ test_that("pdf_bayesian_data_analysis", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10000L) - assert_content_contains_any(result, c("Bayesian", "probability", "distribution")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10000L) + assert_content_contains_any(result, c("Bayesian", "probability", "distribution")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_bounding_boxes", { @@ -57,10 +57,10 @@ test_that("pdf_bounding_boxes", { notes = "ONNX Runtime model loading unstable on ARM Linux; table detection returns 0 tables", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_table_count(result, minimum = 1L, maximum = NULL) - assert_table_bounding_boxes(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_table_count(result, minimum = 1L, maximum = NULL) + assert_table_bounding_boxes(result) }) test_that("pdf_code_and_formula", { @@ -72,8 +72,8 @@ test_that("pdf_code_and_formula", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 100L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 100L) }) test_that("pdf_deep_learning", { @@ -85,10 +85,10 @@ test_that("pdf_deep_learning", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 1000L) - assert_content_contains_any(result, c("neural", "network", "deep learning")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 1000L) + assert_content_contains_any(result, c("neural", "network", "deep learning")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_embedded_images", { @@ -100,9 +100,9 @@ test_that("pdf_embedded_images", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_table_count(result, minimum = 0L, maximum = NULL) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_table_count(result, minimum = 0L, maximum = NULL) }) test_that("pdf_google_doc", { @@ -114,9 +114,9 @@ test_that("pdf_google_doc", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_large_ciml", { @@ -128,10 +128,10 @@ test_that("pdf_large_ciml", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10000L) - assert_content_contains_any(result, c("machine learning", "algorithm", "training")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10000L) + assert_content_contains_any(result, c("machine learning", "algorithm", "training")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_layout_detection", { @@ -144,9 +144,9 @@ test_that("pdf_layout_detection", { notes = "Requires layout-detection feature with ONNX Runtime", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 100L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 100L) + assert_content_not_empty(result) }) test_that("pdf_non_english_german", { @@ -158,10 +158,10 @@ test_that("pdf_non_english_german", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 100L) - assert_content_contains_any(result, c("Intel", "paging")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 100L) + assert_content_contains_any(result, c("Intel", "paging")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_password_protected", { @@ -173,9 +173,9 @@ test_that("pdf_password_protected", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("LayoutParser", "document image analysis", "deep learning")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("LayoutParser", "document image analysis", "deep learning")) }) test_that("pdf_right_to_left", { @@ -187,9 +187,9 @@ test_that("pdf_right_to_left", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) test_that("pdf_simple_text", { @@ -201,9 +201,9 @@ test_that("pdf_simple_text", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern", "Mallori")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern", "Mallori")) }) test_that("pdf_tables_large", { @@ -215,8 +215,8 @@ test_that("pdf_tables_large", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 500L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 500L) }) test_that("pdf_tables_medium", { @@ -228,8 +228,8 @@ test_that("pdf_tables_medium", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 100L) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 100L) }) test_that("pdf_tables_small", { @@ -242,10 +242,10 @@ test_that("pdf_tables_small", { notes = "PDF table extraction requires OCR feature. ONNX Runtime model loading unstable on ARM Linux.", skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_all(result, c("Table 1", "Selected Numbers", "Celsius", "Fahrenheit", "Water Freezing Point", "Water Boiling Point")) - assert_table_count(result, minimum = 1L, maximum = NULL) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_all(result, c("Table 1", "Selected Numbers", "Celsius", "Fahrenheit", "Water Freezing Point", "Water Boiling Point")) + assert_table_count(result, minimum = 1L, maximum = NULL) }) test_that("pdf_technical_stat_learning", { @@ -257,9 +257,9 @@ test_that("pdf_technical_stat_learning", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10000L) - assert_content_contains_any(result, c("statistical", "regression", "learning")) - assert_metadata_expectation(result, "format_type", list(eq = "pdf")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10000L) + assert_content_contains_any(result, c("statistical", "regression", "learning")) + assert_metadata_expectation(result, "format_type", list(eq = "pdf")) }) # nolint end diff --git a/e2e/r/tests/testthat/test-plugin-apis.R b/e2e/r/tests/testthat/test-plugin-apis.R index fd35b85d1c3..e0e2caffc53 100644 --- a/e2e/r/tests/testthat/test-plugin-apis.R +++ b/e2e/r/tests/testthat/test-plugin-apis.R @@ -7,11 +7,10 @@ library(kreuzberg) test_that("Configuration API tests", { # Discover configuration from current or parent directories - withr::with_tempdir({ - tmpdir <- getwd() + withr::with_tempdir({ tmpdir <- getwd() config_path <- file.path(tmpdir, "kreuzberg.toml") writeLines(c( - "[chunking]", "max_chars = 50" + "[chunking]", "max_chars = 50" ), config_path) subdir <- file.path(tmpdir, "subdir") @@ -25,11 +24,10 @@ test_that("Configuration API tests", { }) # Load configuration from a TOML file - withr::with_tempdir({ - tmpdir <- getwd() + withr::with_tempdir({ tmpdir <- getwd() config_path <- file.path(tmpdir, "test_config.toml") writeLines(c( - "[chunking]", "max_chars = 100", "max_overlap = 20", "", "[language_detection]", "enabled = false" + "[chunking]", "max_chars = 100", "max_overlap = 20", "", "[language_detection]", "enabled = false" ), config_path) config <- from_file(config_path) @@ -39,6 +37,7 @@ test_that("Configuration API tests", { expect_false(is.null(config$language_detection)) expect_equal(config$language_detection$enabled, FALSE) }) + }) test_that("Document Extractor Management API tests", { @@ -53,6 +52,7 @@ test_that("Document Extractor Management API tests", { # Unregister nonexistent document extractor gracefully expect_no_error(unregister_document_extractor("nonexistent-extractor-xyz")) + }) test_that("Mime Utilities API tests", { @@ -62,8 +62,7 @@ test_that("Mime Utilities API tests", { expect_true(grepl("pdf", tolower(result), fixed = TRUE)) # Detect MIME type from file path - withr::with_tempdir({ - tmpdir <- getwd() + withr::with_tempdir({ tmpdir <- getwd() test_file <- file.path(tmpdir, "test.txt") writeLines("Hello, world!", test_file) @@ -75,6 +74,7 @@ test_that("Mime Utilities API tests", { result <- get_extensions_for_mime("application/pdf") expect_true(is.character(result)) expect_true("pdf" %in% result) + }) test_that("Ocr Backend Management API tests", { @@ -89,6 +89,7 @@ test_that("Ocr Backend Management API tests", { # Unregister nonexistent OCR backend gracefully expect_no_error(unregister_ocr_backend("nonexistent-backend-xyz")) + }) test_that("Post Processor Management API tests", { @@ -98,6 +99,7 @@ test_that("Post Processor Management API tests", { # List all registered post-processors result <- list_post_processors() expect_true(is.character(result) || is.list(result)) + }) test_that("Validator Management API tests", { @@ -109,6 +111,7 @@ test_that("Validator Management API tests", { # List all registered validators result <- list_validators() expect_true(is.character(result) || is.list(result)) + }) # nolint end diff --git a/e2e/r/tests/testthat/test-smoke.R b/e2e/r/tests/testthat/test-smoke.R index 14f85b1f308..ad812a20add 100644 --- a/e2e/r/tests/testthat/test-smoke.R +++ b/e2e/r/tests/testthat/test-smoke.R @@ -13,8 +13,8 @@ test_that("smoke_cache_namespace", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/plain")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("text/plain")) + assert_min_content_length(result, 5L) }) test_that("smoke_docx_basic", { @@ -26,9 +26,9 @@ test_that("smoke_docx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) - assert_min_content_length(result, 20L) - assert_content_contains_any(result, c("Lorem", "ipsum", "document", "text")) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) + assert_min_content_length(result, 20L) + assert_content_contains_any(result, c("Lorem", "ipsum", "document", "text")) }) test_that("smoke_html_basic", { @@ -40,9 +40,9 @@ test_that("smoke_html_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/html")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("Sample Data Table", "Laptop", "Electronics", "Product")) + assert_expected_mime(result, c("text/html")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("Sample Data Table", "Laptop", "Electronics", "Product")) }) test_that("smoke_image_png", { @@ -54,8 +54,8 @@ test_that("smoke_image_png", { notes = "Image extraction requires image processing dependencies", skip_if_missing = TRUE ) - assert_expected_mime(result, c("image/png")) - assert_metadata_expectation(result, "format", "PNG") + assert_expected_mime(result, c("image/png")) + assert_metadata_expectation(result, "format", "PNG") }) test_that("smoke_json_basic", { @@ -67,8 +67,8 @@ test_that("smoke_json_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/json")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("application/json")) + assert_min_content_length(result, 5L) }) test_that("smoke_pdf_basic", { @@ -80,9 +80,9 @@ test_that("smoke_pdf_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 50L) - assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern")) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 50L) + assert_content_contains_any(result, c("May 5, 2023", "To Whom it May Concern")) }) test_that("smoke_txt_basic", { @@ -94,8 +94,8 @@ test_that("smoke_txt_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/plain")) - assert_min_content_length(result, 5L) + assert_expected_mime(result, c("text/plain")) + assert_min_content_length(result, 5L) }) test_that("smoke_xlsx_basic", { @@ -107,11 +107,11 @@ test_that("smoke_xlsx_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) - assert_min_content_length(result, 100L) - assert_content_contains_all(result, c("Team", "Location", "Stanley Cups", "Blues", "Flyers", "Maple Leafs", "STL", "PHI", "TOR")) - assert_table_count(result, minimum = 1L, maximum = NULL) - assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) - assert_metadata_expectation(result, "sheet_names", list(contains = c("Stanley Cups"))) + assert_expected_mime(result, c("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + assert_min_content_length(result, 100L) + assert_content_contains_all(result, c("Team", "Location", "Stanley Cups", "Blues", "Flyers", "Maple Leafs", "STL", "PHI", "TOR")) + assert_table_count(result, minimum = 1L, maximum = NULL) + assert_metadata_expectation(result, "sheet_count", list(gte = 2L)) + assert_metadata_expectation(result, "sheet_names", list(contains = c("Stanley Cups"))) }) # nolint end diff --git a/e2e/r/tests/testthat/test-structured.R b/e2e/r/tests/testthat/test-structured.R index 98e4c6d1166..ffad8ff30bb 100644 --- a/e2e/r/tests/testthat/test-structured.R +++ b/e2e/r/tests/testthat/test-structured.R @@ -13,8 +13,8 @@ test_that("structured_csv_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/csv")) - assert_min_content_length(result, 20L) + assert_expected_mime(result, c("text/csv")) + assert_min_content_length(result, 20L) }) test_that("structured_enw_basic", { @@ -26,7 +26,7 @@ test_that("structured_enw_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-endnote-refer", "application/x-endnote+xml", "text/plain")) + assert_expected_mime(result, c("application/x-endnote-refer", "application/x-endnote+xml", "text/plain")) }) test_that("structured_json_basic", { @@ -38,9 +38,9 @@ test_that("structured_json_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/json")) - assert_min_content_length(result, 20L) - assert_content_contains_any(result, c("Sample Document", "Test Author")) + assert_expected_mime(result, c("application/json")) + assert_min_content_length(result, 20L) + assert_content_contains_any(result, c("Sample Document", "Test Author")) }) test_that("structured_json_simple", { @@ -52,9 +52,9 @@ test_that("structured_json_simple", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/json")) - assert_min_content_length(result, 10L) - assert_content_contains_any(result, c("{", "name")) + assert_expected_mime(result, c("application/json")) + assert_min_content_length(result, 10L) + assert_content_contains_any(result, c("{", "name")) }) test_that("structured_nbib_basic", { @@ -66,8 +66,8 @@ test_that("structured_nbib_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/nbib", "application/x-pubmed", "text/plain")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/nbib", "application/x-pubmed", "text/plain")) + assert_content_not_empty(result) }) test_that("structured_ris_basic", { @@ -79,8 +79,8 @@ test_that("structured_ris_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-research-info-systems", "text/plain")) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/x-research-info-systems", "text/plain")) + assert_content_not_empty(result) }) test_that("structured_toml_basic", { @@ -92,8 +92,8 @@ test_that("structured_toml_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/toml", "text/toml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/toml", "text/toml")) + assert_min_content_length(result, 10L) }) test_that("structured_tsv_basic", { @@ -105,8 +105,8 @@ test_that("structured_tsv_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("text/tab-separated-values", "text/plain")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("text/tab-separated-values", "text/plain")) + assert_min_content_length(result, 10L) }) test_that("structured_yaml_basic", { @@ -118,8 +118,8 @@ test_that("structured_yaml_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml")) + assert_min_content_length(result, 10L) }) test_that("structured_yaml_simple", { @@ -131,7 +131,7 @@ test_that("structured_yaml_simple", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/x-yaml")) - assert_min_content_length(result, 10L) + assert_expected_mime(result, c("application/x-yaml")) + assert_min_content_length(result, 10L) }) # nolint end diff --git a/e2e/r/tests/testthat/test-token_reduction.R b/e2e/r/tests/testthat/test-token_reduction.R index 282a9220055..22e91918b62 100644 --- a/e2e/r/tests/testthat/test-token_reduction.R +++ b/e2e/r/tests/testthat/test-token_reduction.R @@ -13,10 +13,10 @@ test_that("token_reduction_aggressive", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) - assert_max_content_length(result, 150L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) + assert_max_content_length(result, 150L) + assert_content_not_empty(result) }) test_that("token_reduction_basic", { @@ -28,10 +28,10 @@ test_that("token_reduction_basic", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) - assert_max_content_length(result, 200L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) + assert_max_content_length(result, 200L) + assert_content_not_empty(result) }) test_that("token_reduction_light", { @@ -43,9 +43,9 @@ test_that("token_reduction_light", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 10L) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 10L) + assert_content_not_empty(result) }) test_that("token_reduction_with_chunking", { @@ -57,10 +57,10 @@ test_that("token_reduction_with_chunking", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/pdf")) - assert_min_content_length(result, 5L) - assert_max_content_length(result, 200L) - assert_chunks(result, min_count = 1L, each_has_content = TRUE) - assert_content_not_empty(result) + assert_expected_mime(result, c("application/pdf")) + assert_min_content_length(result, 5L) + assert_max_content_length(result, 200L) + assert_chunks(result, min_count = 1L, each_has_content = TRUE) + assert_content_not_empty(result) }) # nolint end diff --git a/e2e/r/tests/testthat/test-xml.R b/e2e/r/tests/testthat/test-xml.R index c830ad2e83e..1ed8a2ab184 100644 --- a/e2e/r/tests/testthat/test-xml.R +++ b/e2e/r/tests/testthat/test-xml.R @@ -13,7 +13,7 @@ test_that("xml_plant_catalog", { notes = NULL, skip_if_missing = TRUE ) - assert_expected_mime(result, c("application/xml")) - assert_min_content_length(result, 100L) + assert_expected_mime(result, c("application/xml")) + assert_min_content_length(result, 100L) }) # nolint end diff --git a/e2e/ruby/spec/archive_spec.rb b/e2e/ruby/spec/archive_spec.rb index 0c6489dc4bf..11853a9e82c 100644 --- a/e2e/ruby/spec/archive_spec.rb +++ b/e2e/ruby/spec/archive_spec.rb @@ -5,6 +5,7 @@ # # Tests for archive fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'archive fixtures' do @@ -76,3 +77,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/email_spec.rb b/e2e/ruby/spec/email_spec.rb index 297032f5175..1d77894c648 100644 --- a/e2e/ruby/spec/email_spec.rb +++ b/e2e/ruby/spec/email_spec.rb @@ -5,6 +5,7 @@ # # Tests for email fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'email fixtures' do @@ -110,3 +111,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/image_spec.rb b/e2e/ruby/spec/image_spec.rb index c9b628cb2b7..c53a0c49670 100644 --- a/e2e/ruby/spec/image_spec.rb +++ b/e2e/ruby/spec/image_spec.rb @@ -5,6 +5,7 @@ # # Tests for image fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'image fixtures' do @@ -186,3 +187,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/keywords_spec.rb b/e2e/ruby/spec/keywords_spec.rb index 195e45085fb..6386e08ce6d 100644 --- a/e2e/ruby/spec/keywords_spec.rb +++ b/e2e/ruby/spec/keywords_spec.rb @@ -5,6 +5,7 @@ # # Tests for keywords fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'keywords fixtures' do @@ -46,3 +47,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/plugin_apis_spec.rb b/e2e/ruby/spec/plugin_apis_spec.rb index e2a10a87960..27290d25251 100644 --- a/e2e/ruby/spec/plugin_apis_spec.rb +++ b/e2e/ruby/spec/plugin_apis_spec.rb @@ -7,6 +7,8 @@ # Generated from plugin API fixtures. # To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang ruby +# rubocop:disable Metrics/BlockLength + require 'spec_helper' require 'tmpdir' require 'fileutils' @@ -14,131 +16,140 @@ RSpec.describe 'Plugin APIs' do describe 'Configuration' do it 'Discover configuration from current or parent directories' do - Dir.mktmpdir do |tmpdir| - config_path = File.join(tmpdir, 'kreuzberg.toml') - File.write(config_path, <<~TOML) - [chunking] - max_chars = 50 - TOML - - subdir = File.join(tmpdir, 'subdir') - FileUtils.mkdir_p(subdir) - - FileUtils.cd(subdir) do - config = Kreuzberg::Config::Extraction.discover - - expect(config.chunking).not_to be_nil - expect(config.chunking.max_chars).to eq(50) - end - end - end - - it 'Load configuration from a TOML file' do - Dir.mktmpdir do |tmpdir| - config_path = File.join(tmpdir, 'test_config.toml') - File.write(config_path, <<~TOML) - [chunking] - max_chars = 100 - max_overlap = 20 + Dir.mktmpdir do |tmpdir| + config_path = File.join(tmpdir, 'kreuzberg.toml') + File.write(config_path, <<~TOML) + [chunking] + max_chars = 50 + TOML - [language_detection] - enabled = false - TOML + subdir = File.join(tmpdir, 'subdir') + FileUtils.mkdir_p(subdir) - config = Kreuzberg::Config::Extraction.from_file(config_path) + FileUtils.cd(subdir) do + config = Kreuzberg::Config::Extraction.discover expect(config.chunking).not_to be_nil - expect(config.chunking.max_chars).to eq(100) - expect(config.chunking.max_overlap).to eq(20) - expect(config.language_detection).not_to be_nil - expect(config.language_detection.enabled).to be(false) + expect(config.chunking.max_chars).to eq(50) end end + end + + it 'Load configuration from a TOML file' do + Dir.mktmpdir do |tmpdir| + config_path = File.join(tmpdir, 'test_config.toml') + File.write(config_path, <<~TOML) + [chunking] + max_chars = 100 + max_overlap = 20 + + [language_detection] + enabled = false + TOML + + config = Kreuzberg::Config::Extraction.from_file(config_path) + + expect(config.chunking).not_to be_nil + expect(config.chunking.max_chars).to eq(100) + expect(config.chunking.max_overlap).to eq(20) + expect(config.language_detection).not_to be_nil + expect(config.language_detection.enabled).to eq(false) + end + end + end describe 'Document Extractor Management' do it 'Clear all document extractors and verify list is empty' do - Kreuzberg.clear_document_extractors - result = Kreuzberg.list_document_extractors - expect(result).to be_empty + Kreuzberg.clear_document_extractors + result = Kreuzberg.list_document_extractors + expect(result).to be_empty end it 'List all registered document extractors' do - result = Kreuzberg.list_document_extractors - expect(result).to be_an(Array) - expect(result).to all(be_a(String)) + result = Kreuzberg.list_document_extractors + expect(result).to be_an(Array) + expect(result).to all(be_a(String)) end it 'Unregister nonexistent document extractor gracefully' do - expect { Kreuzberg.unregister_document_extractor('nonexistent-extractor-xyz') }.not_to raise_error + expect { Kreuzberg.unregister_document_extractor('nonexistent-extractor-xyz') }.not_to raise_error end + end describe 'Mime Utilities' do it 'Detect MIME type from file bytes' do - test_bytes = (+'%PDF-1.4\\n').force_encoding('ASCII-8BIT') - result = Kreuzberg.detect_mime_type(test_bytes) - expect(result.downcase).to include('pdf') + test_bytes = '%PDF-1.4\\n'.dup.force_encoding('ASCII-8BIT') + result = Kreuzberg.detect_mime_type(test_bytes) + expect(result.downcase).to include('pdf') end it 'Detect MIME type from file path' do - Dir.mktmpdir do |tmpdir| - test_file = File.join(tmpdir, 'test.txt') - File.write(test_file, 'Hello, world!') + Dir.mktmpdir do |tmpdir| + test_file = File.join(tmpdir, 'test.txt') + File.write(test_file, 'Hello, world!') - result = Kreuzberg.detect_mime_type_from_path(test_file) - expect(result.downcase).to include('text') - end + result = Kreuzberg.detect_mime_type_from_path(test_file) + expect(result.downcase).to include('text') + end end it 'Get file extensions for a MIME type' do - result = Kreuzberg.get_extensions_for_mime('application/pdf') - expect(result).to be_an(Array) - expect(result).to include('pdf') + result = Kreuzberg.get_extensions_for_mime('application/pdf') + expect(result).to be_an(Array) + expect(result).to include('pdf') end + end describe 'Ocr Backend Management' do it 'Clear all OCR backends and verify list is empty' do - Kreuzberg.clear_ocr_backends - result = Kreuzberg.list_ocr_backends - expect(result).to be_empty + Kreuzberg.clear_ocr_backends + result = Kreuzberg.list_ocr_backends + expect(result).to be_empty end it 'List all registered OCR backends' do - result = Kreuzberg.list_ocr_backends - expect(result).to be_an(Array) - expect(result).to all(be_a(String)) + result = Kreuzberg.list_ocr_backends + expect(result).to be_an(Array) + expect(result).to all(be_a(String)) end it 'Unregister nonexistent OCR backend gracefully' do - expect { Kreuzberg.unregister_ocr_backend('nonexistent-backend-xyz') }.not_to raise_error + expect { Kreuzberg.unregister_ocr_backend('nonexistent-backend-xyz') }.not_to raise_error end + end describe 'Post Processor Management' do it 'Clear all post-processors and verify list is empty' do - Kreuzberg.clear_post_processors + Kreuzberg.clear_post_processors end it 'List all registered post-processors' do - result = Kreuzberg.list_post_processors - expect(result).to be_an(Array) - expect(result).to all(be_a(String)) + result = Kreuzberg.list_post_processors + expect(result).to be_an(Array) + expect(result).to all(be_a(String)) end + end describe 'Validator Management' do it 'Clear all validators and verify list is empty' do - Kreuzberg.clear_validators - result = Kreuzberg.list_validators - expect(result).to be_empty + Kreuzberg.clear_validators + result = Kreuzberg.list_validators + expect(result).to be_empty end it 'List all registered validators' do - result = Kreuzberg.list_validators - expect(result).to be_an(Array) - expect(result).to all(be_a(String)) + result = Kreuzberg.list_validators + expect(result).to be_an(Array) + expect(result).to all(be_a(String)) end + end + end + +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/render_spec.rb b/e2e/ruby/spec/render_spec.rb index 34b637a9d7a..ce570d0731d 100644 --- a/e2e/ruby/spec/render_spec.rb +++ b/e2e/ruby/spec/render_spec.rb @@ -3,6 +3,8 @@ # Code generated by kreuzberg-e2e-generator. DO NOT EDIT. # To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang ruby +# rubocop:disable Layout/LineLength + require 'spec_helper' require 'kreuzberg' @@ -34,3 +36,5 @@ E2ERuby.assert_min_byte_length(png_data, 100) end end + +# rubocop:enable Layout/LineLength diff --git a/e2e/ruby/spec/structured_spec.rb b/e2e/ruby/spec/structured_spec.rb index 90e2bd4421e..7b3e91c5200 100644 --- a/e2e/ruby/spec/structured_spec.rb +++ b/e2e/ruby/spec/structured_spec.rb @@ -5,6 +5,7 @@ # # Tests for structured fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'structured fixtures' do @@ -179,3 +180,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/token_reduction_spec.rb b/e2e/ruby/spec/token_reduction_spec.rb index 20c8b71054c..72cc2e85e00 100644 --- a/e2e/ruby/spec/token_reduction_spec.rb +++ b/e2e/ruby/spec/token_reduction_spec.rb @@ -5,6 +5,7 @@ # # Tests for token_reduction fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'token_reduction fixtures' do @@ -84,3 +85,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/ruby/spec/xml_spec.rb b/e2e/ruby/spec/xml_spec.rb index 357bb622a5c..b877836cd40 100644 --- a/e2e/ruby/spec/xml_spec.rb +++ b/e2e/ruby/spec/xml_spec.rb @@ -5,6 +5,7 @@ # # Tests for xml fixtures. +# rubocop:disable Metrics/BlockLength require_relative 'spec_helper' RSpec.describe 'xml fixtures' do @@ -25,3 +26,4 @@ end end end +# rubocop:enable Metrics/BlockLength diff --git a/e2e/rust/src/lib.rs b/e2e/rust/src/lib.rs index 3c0767fb5ea..cbb3e4a243e 100644 --- a/e2e/rust/src/lib.rs +++ b/e2e/rust/src/lib.rs @@ -233,6 +233,7 @@ pub mod assertions { each_has_content: Option, each_has_embedding: Option, each_has_heading_context: Option, + each_has_chunk_type: Option, content_starts_with_heading: Option, ) { let chunks = result.chunks.as_ref().expect("Expected chunks in result"); @@ -279,6 +280,15 @@ pub mod assertions { } } + if each_has_chunk_type == Some(true) { + for (i, chunk) in chunks.iter().enumerate() { + assert!( + chunk.chunk_type != kreuzberg::types::ChunkType::Unknown, + "Expected chunk {i} to have a specific chunk_type, got Unknown" + ); + } + } + if content_starts_with_heading == Some(true) { for (i, chunk) in chunks.iter().enumerate() { if chunk.metadata.heading_context.is_none() { diff --git a/e2e/rust/tests/e2e.rs b/e2e/rust/tests/e2e.rs new file mode 100644 index 00000000000..f3e51325b79 --- /dev/null +++ b/e2e/rust/tests/e2e.rs @@ -0,0 +1,3 @@ +// Code generated by kreuzberg-e2e-generator. DO NOT EDIT. + +mod fixtures; diff --git a/e2e/rust/tests/archive_test.rs b/e2e/rust/tests/fixtures/archive_test.rs similarity index 100% rename from e2e/rust/tests/archive_test.rs rename to e2e/rust/tests/fixtures/archive_test.rs diff --git a/e2e/rust/tests/contract_test.rs b/e2e/rust/tests/fixtures/contract_test.rs similarity index 99% rename from e2e/rust/tests/contract_test.rs rename to e2e/rust/tests/fixtures/contract_test.rs index 7a2492a0b4d..5c1e64f31e2 100644 --- a/e2e/rust/tests/contract_test.rs +++ b/e2e/rust/tests/fixtures/contract_test.rs @@ -402,7 +402,7 @@ fn test_config_chunking() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None, None); } #[test] @@ -452,7 +452,7 @@ fn test_config_chunking_heading_context() { }; assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(true), None); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(true), None, None); } #[test] @@ -500,7 +500,7 @@ fn test_config_chunking_markdown() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None, None); } #[test] @@ -550,7 +550,7 @@ fn test_config_chunking_no_headings() { }; assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(false), None); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(false), None, None); } #[test] @@ -601,7 +601,7 @@ fn test_config_chunking_prepend_heading_context() { }; assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(true), Some(true)); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, Some(true), None, Some(true)); } #[test] @@ -648,7 +648,7 @@ fn test_config_chunking_small() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, None, None, None); } #[test] @@ -681,7 +681,7 @@ fn test_config_chunking_text() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None, None); } #[test] @@ -734,7 +734,7 @@ fn test_config_chunking_tokenizer() { }; assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(2), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(2), None, Some(true), None, None, None, None); } #[test] diff --git a/e2e/rust/tests/email_test.rs b/e2e/rust/tests/fixtures/email_test.rs similarity index 100% rename from e2e/rust/tests/email_test.rs rename to e2e/rust/tests/fixtures/email_test.rs diff --git a/e2e/rust/tests/embeddings_test.rs b/e2e/rust/tests/fixtures/embeddings_test.rs similarity index 97% rename from e2e/rust/tests/embeddings_test.rs rename to e2e/rust/tests/fixtures/embeddings_test.rs index 8eab943b887..6178418b428 100644 --- a/e2e/rust/tests/embeddings_test.rs +++ b/e2e/rust/tests/fixtures/embeddings_test.rs @@ -58,7 +58,7 @@ async fn test_embedding_async() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None, None); } #[test] @@ -115,7 +115,7 @@ fn test_embedding_balanced_preset() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None, None); } #[test] @@ -147,7 +147,7 @@ fn test_embedding_disabled() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), Some(false), None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), Some(false), None, None, None); } #[test] @@ -201,5 +201,5 @@ fn test_embedding_fast_preset() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 10); - assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), Some(true), None, None, None); } diff --git a/e2e/rust/tests/html_test.rs b/e2e/rust/tests/fixtures/html_test.rs similarity index 100% rename from e2e/rust/tests/html_test.rs rename to e2e/rust/tests/fixtures/html_test.rs diff --git a/e2e/rust/tests/image_test.rs b/e2e/rust/tests/fixtures/image_test.rs similarity index 100% rename from e2e/rust/tests/image_test.rs rename to e2e/rust/tests/fixtures/image_test.rs diff --git a/e2e/rust/tests/keywords_test.rs b/e2e/rust/tests/fixtures/keywords_test.rs similarity index 100% rename from e2e/rust/tests/keywords_test.rs rename to e2e/rust/tests/fixtures/keywords_test.rs diff --git a/e2e/rust/tests/fixtures/mod.rs b/e2e/rust/tests/fixtures/mod.rs new file mode 100644 index 00000000000..7341521d833 --- /dev/null +++ b/e2e/rust/tests/fixtures/mod.rs @@ -0,0 +1,18 @@ +// Code generated by kreuzberg-e2e-generator. DO NOT EDIT. + +pub mod archive_test; +pub mod contract_test; +pub mod email_test; +pub mod embeddings_test; +pub mod html_test; +pub mod image_test; +pub mod keywords_test; +pub mod ocr_test; +pub mod office_test; +pub mod pdf_test; +pub mod plugin_apis_test; +pub mod render_test; +pub mod smoke_test; +pub mod structured_test; +pub mod token_reduction_test; +pub mod xml_test; diff --git a/e2e/rust/tests/ocr_test.rs b/e2e/rust/tests/fixtures/ocr_test.rs similarity index 100% rename from e2e/rust/tests/ocr_test.rs rename to e2e/rust/tests/fixtures/ocr_test.rs diff --git a/e2e/rust/tests/office_test.rs b/e2e/rust/tests/fixtures/office_test.rs similarity index 100% rename from e2e/rust/tests/office_test.rs rename to e2e/rust/tests/fixtures/office_test.rs diff --git a/e2e/rust/tests/pdf_test.rs b/e2e/rust/tests/fixtures/pdf_test.rs similarity index 100% rename from e2e/rust/tests/pdf_test.rs rename to e2e/rust/tests/fixtures/pdf_test.rs diff --git a/e2e/rust/tests/plugin_apis_test.rs b/e2e/rust/tests/fixtures/plugin_apis_test.rs similarity index 100% rename from e2e/rust/tests/plugin_apis_test.rs rename to e2e/rust/tests/fixtures/plugin_apis_test.rs diff --git a/e2e/rust/tests/render_test.rs b/e2e/rust/tests/fixtures/render_test.rs similarity index 100% rename from e2e/rust/tests/render_test.rs rename to e2e/rust/tests/fixtures/render_test.rs diff --git a/e2e/rust/tests/smoke_test.rs b/e2e/rust/tests/fixtures/smoke_test.rs similarity index 100% rename from e2e/rust/tests/smoke_test.rs rename to e2e/rust/tests/fixtures/smoke_test.rs diff --git a/e2e/rust/tests/structured_test.rs b/e2e/rust/tests/fixtures/structured_test.rs similarity index 100% rename from e2e/rust/tests/structured_test.rs rename to e2e/rust/tests/fixtures/structured_test.rs diff --git a/e2e/rust/tests/token_reduction_test.rs b/e2e/rust/tests/fixtures/token_reduction_test.rs similarity index 99% rename from e2e/rust/tests/token_reduction_test.rs rename to e2e/rust/tests/fixtures/token_reduction_test.rs index 19b3ddff7d3..8284c84ede2 100644 --- a/e2e/rust/tests/token_reduction_test.rs +++ b/e2e/rust/tests/fixtures/token_reduction_test.rs @@ -134,6 +134,6 @@ fn test_token_reduction_with_chunking() { assertions::assert_expected_mime(&result, &["application/pdf"]); assertions::assert_min_content_length(&result, 5); assertions::assert_max_content_length(&result, 200); - assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None); + assertions::assert_chunks(&result, Some(1), None, Some(true), None, None, None, None); assertions::assert_content_not_empty(&result); } diff --git a/e2e/rust/tests/xml_test.rs b/e2e/rust/tests/fixtures/xml_test.rs similarity index 100% rename from e2e/rust/tests/xml_test.rs rename to e2e/rust/tests/fixtures/xml_test.rs diff --git a/e2e/typescript/tests/archive.spec.ts b/e2e/typescript/tests/archive.spec.ts index 722d3d2c04a..468c4348abe 100644 --- a/e2e/typescript/tests/archive.spec.ts +++ b/e2e/typescript/tests/archive.spec.ts @@ -4,119 +4,105 @@ // Tests for archive fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("archive fixtures", () => { - it( - "archive_gz_basic", - () => { - const documentPath = resolveDocument("archives/book_war_and_peace_1p.txt.gz"); - if (!existsSync(documentPath)) { - console.warn("Skipping archive_gz_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("archive_gz_basic", () => { + const documentPath = resolveDocument("archives/book_war_and_peace_1p.txt.gz"); + if (!existsSync(documentPath)) { + console.warn("Skipping archive_gz_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); + + it("archive_sevenz_basic", () => { + const documentPath = resolveDocument("archives/documents.7z"); + if (!existsSync(documentPath)) { + console.warn("Skipping archive_sevenz_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "archive_sevenz_basic", - () => { - const documentPath = resolveDocument("archives/documents.7z"); - if (!existsSync(documentPath)) { - console.warn("Skipping archive_sevenz_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("archive_tar_basic", () => { + const documentPath = resolveDocument("archives/documents.tar"); + if (!existsSync(documentPath)) { + console.warn("Skipping archive_tar_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "archive_tar_basic", - () => { - const documentPath = resolveDocument("archives/documents.tar"); - if (!existsSync(documentPath)) { - console.warn("Skipping archive_tar_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("archive_zip_basic", () => { + const documentPath = resolveDocument("archives/documents.zip"); + if (!existsSync(documentPath)) { + console.warn("Skipping archive_zip_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "archive_zip_basic", - () => { - const documentPath = resolveDocument("archives/documents.zip"); - if (!existsSync(documentPath)) { - console.warn("Skipping archive_zip_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); }); + diff --git a/e2e/typescript/tests/email.spec.ts b/e2e/typescript/tests/email.spec.ts index df1bbe0d3a6..b5f88cdbbe5 100644 --- a/e2e/typescript/tests/email.spec.ts +++ b/e2e/typescript/tests/email.spec.ts @@ -4,173 +4,151 @@ // Tests for email fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("email fixtures", () => { - it( - "email_eml_html_body", - () => { - const documentPath = resolveDocument("email/html_only.eml"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_eml_html_body: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("email_eml_html_body", () => { + const documentPath = resolveDocument("email/html_only.eml"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_eml_html_body: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); + + it("email_eml_multipart", () => { + const documentPath = resolveDocument("email/html_email_multipart.eml"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_eml_multipart: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "email_eml_multipart", - () => { - const documentPath = resolveDocument("email/html_email_multipart.eml"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_eml_multipart: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("email_eml_utf16", () => { + const documentPath = resolveDocument("vendored/unstructured/eml/fake-email-utf-16.eml"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_eml_utf16: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); + }, TEST_TIMEOUT_MS); - it( - "email_eml_utf16", - () => { - const documentPath = resolveDocument("vendored/unstructured/eml/fake-email-utf-16.eml"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_eml_utf16: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); - }, - TEST_TIMEOUT_MS, - ); + it("email_msg_basic", () => { + const documentPath = resolveDocument("email/fake_email.msg"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_msg_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "email_msg_basic", - () => { - const documentPath = resolveDocument("email/fake_email.msg"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_msg_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("email_pst_empty", () => { + const documentPath = resolveDocument("email/empty.pst"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_pst_empty: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); + }, TEST_TIMEOUT_MS); - it( - "email_pst_empty", - () => { - const documentPath = resolveDocument("email/empty.pst"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_pst_empty: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); - }, - TEST_TIMEOUT_MS, - ); + it("email_sample_eml", () => { + const documentPath = resolveDocument("email/sample_email.eml"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_sample_eml: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "email_sample_eml", - () => { - const documentPath = resolveDocument("email/sample_email.eml"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_sample_eml: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); }); + diff --git a/e2e/typescript/tests/html.spec.ts b/e2e/typescript/tests/html.spec.ts index a1c2eadb7ad..06a1071b7bc 100644 --- a/e2e/typescript/tests/html.spec.ts +++ b/e2e/typescript/tests/html.spec.ts @@ -4,74 +4,60 @@ // Tests for html fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("html fixtures", () => { - it( - "html_complex_layout", - () => { - const documentPath = resolveDocument("html/taylor_swift.html"); - if (!existsSync(documentPath)) { - console.warn("Skipping html_complex_layout: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "html_complex_layout", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 1000); - }, - TEST_TIMEOUT_MS, - ); + it("html_complex_layout", () => { + const documentPath = resolveDocument("html/taylor_swift.html"); + if (!existsSync(documentPath)) { + console.warn("Skipping html_complex_layout: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "html_complex_layout", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 1000); + }, TEST_TIMEOUT_MS); + + it("html_simple_table", () => { + const documentPath = resolveDocument("html/simple_table.html"); + if (!existsSync(documentPath)) { + console.warn("Skipping html_simple_table: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]); + }, TEST_TIMEOUT_MS); - it( - "html_simple_table", - () => { - const documentPath = resolveDocument("html/simple_table.html"); - if (!existsSync(documentPath)) { - console.warn("Skipping html_simple_table: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, [ - "Product", - "Category", - "Price", - "Stock", - "Laptop", - "Electronics", - "Sample Data Table", - ]); - }, - TEST_TIMEOUT_MS, - ); }); + diff --git a/e2e/typescript/tests/image.spec.ts b/e2e/typescript/tests/image.spec.ts index 2a12439ae3c..adb786e37b3 100644 --- a/e2e/typescript/tests/image.spec.ts +++ b/e2e/typescript/tests/image.spec.ts @@ -4,281 +4,243 @@ // Tests for image fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("image fixtures", () => { - it( - "image_bmp_basic", - () => { - const documentPath = resolveDocument("images/bmp_24.bmp"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_bmp_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/bmp"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("image_bmp_basic", () => { + const documentPath = resolveDocument("images/bmp_24.bmp"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_bmp_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/bmp"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); + + it("image_gif_basic", () => { + const documentPath = resolveDocument("images_extra/ocr_image.gif"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_gif_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/gif"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "image_gif_basic", - () => { - const documentPath = resolveDocument("images_extra/ocr_image.gif"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_gif_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/gif"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("image_jp2_basic", () => { + const documentPath = resolveDocument("images_extra/ocr_image.jp2"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_jp2_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "image_jp2_basic", - () => { - const documentPath = resolveDocument("images_extra/ocr_image.jp2"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_jp2_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("image_metadata_only", () => { + const documentPath = resolveDocument("images/example.jpg"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_metadata_only: missing document at", documentPath); + return; + } + const config = buildConfig({"ocr":null}); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jpeg"]); + assertions.assertMaxContentLength(result, 200); + }, TEST_TIMEOUT_MS); - it( - "image_metadata_only", - () => { - const documentPath = resolveDocument("images/example.jpg"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_metadata_only: missing document at", documentPath); - return; - } - const config = buildConfig({ ocr: null }); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jpeg"]); - assertions.assertMaxContentLength(result, 200); - }, - TEST_TIMEOUT_MS, - ); + it("image_pbm_basic", () => { + const documentPath = resolveDocument("images_extra/ocr_image.pbm"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_pbm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "image_pbm_basic", - () => { - const documentPath = resolveDocument("images_extra/ocr_image.pbm"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_pbm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("image_pgm_basic", () => { + const documentPath = resolveDocument("images_extra/ocr_image.pgm"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_pgm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "image_pgm_basic", - () => { - const documentPath = resolveDocument("images_extra/ocr_image.pgm"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_pgm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("image_ppm_basic", () => { + const documentPath = resolveDocument("images_extra/ocr_image.ppm"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_ppm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "image_ppm_basic", - () => { - const documentPath = resolveDocument("images_extra/ocr_image.ppm"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_ppm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("image_svg_basic", () => { + const documentPath = resolveDocument("xml/simple_svg.svg"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_svg_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/svg+xml"]); + assertions.assertMinContentLength(result, 5); + }, TEST_TIMEOUT_MS); - it( - "image_svg_basic", - () => { - const documentPath = resolveDocument("xml/simple_svg.svg"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_svg_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/svg+xml"]); - assertions.assertMinContentLength(result, 5); - }, - TEST_TIMEOUT_MS, - ); + it("image_tiff_basic", () => { + const documentPath = resolveDocument("images_extra/ocr_image.tif"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_tiff_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/tiff"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "image_tiff_basic", - () => { - const documentPath = resolveDocument("images_extra/ocr_image.tif"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_tiff_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/tiff"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("image_webp_basic", () => { + const documentPath = resolveDocument("images_extra/ocr_image.webp"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_webp_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/webp"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "image_webp_basic", - () => { - const documentPath = resolveDocument("images_extra/ocr_image.webp"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_webp_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/webp"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); }); + diff --git a/e2e/typescript/tests/keywords.spec.ts b/e2e/typescript/tests/keywords.spec.ts index 4a1d37cd8b3..c14947e5527 100644 --- a/e2e/typescript/tests/keywords.spec.ts +++ b/e2e/typescript/tests/keywords.spec.ts @@ -4,67 +4,61 @@ // Tests for keywords fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("keywords fixtures", () => { - it( - "keywords_rake", - () => { - const documentPath = resolveDocument("pdf/fake_memo.pdf"); - if (!existsSync(documentPath)) { - console.warn("Skipping keywords_rake: missing document at", documentPath); - return; - } - const config = buildConfig({ keywords: { algorithm: "rake", max_keywords: 10 } }); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "keywords_rake", ["keywords-rake"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/pdf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertKeywords(result, true, 1, 10); - }, - TEST_TIMEOUT_MS, - ); + it("keywords_rake", () => { + const documentPath = resolveDocument("pdf/fake_memo.pdf"); + if (!existsSync(documentPath)) { + console.warn("Skipping keywords_rake: missing document at", documentPath); + return; + } + const config = buildConfig({"keywords":{"algorithm":"rake","max_keywords":10}}); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "keywords_rake", ["keywords-rake"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/pdf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertKeywords(result, true, 1, 10); + }, TEST_TIMEOUT_MS); + + it("keywords_yake", () => { + const documentPath = resolveDocument("pdf/fake_memo.pdf"); + if (!existsSync(documentPath)) { + console.warn("Skipping keywords_yake: missing document at", documentPath); + return; + } + const config = buildConfig({"keywords":{"algorithm":"yake","max_keywords":10}}); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "keywords_yake", ["keywords-yake"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/pdf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertKeywords(result, true, 1, 10); + }, TEST_TIMEOUT_MS); - it( - "keywords_yake", - () => { - const documentPath = resolveDocument("pdf/fake_memo.pdf"); - if (!existsSync(documentPath)) { - console.warn("Skipping keywords_yake: missing document at", documentPath); - return; - } - const config = buildConfig({ keywords: { algorithm: "yake", max_keywords: 10 } }); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "keywords_yake", ["keywords-yake"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/pdf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertKeywords(result, true, 1, 10); - }, - TEST_TIMEOUT_MS, - ); }); + diff --git a/e2e/typescript/tests/office.spec.ts b/e2e/typescript/tests/office.spec.ts index 9d69785faa0..88919ea8cb1 100644 --- a/e2e/typescript/tests/office.spec.ts +++ b/e2e/typescript/tests/office.spec.ts @@ -4,1450 +4,1205 @@ // Tests for office fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("office fixtures", () => { - it( - "office_bibtex_basic", - () => { - const documentPath = resolveDocument("bibtex/comprehensive.bib"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_bibtex_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_bibtex_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-bibtex", "text/x-bibtex"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_bibtex_basic", () => { + const documentPath = resolveDocument("bibtex/comprehensive.bib"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_bibtex_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_bibtex_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-bibtex", "text/x-bibtex"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); + + it("office_commonmark_basic", () => { + const documentPath = resolveDocument("markdown/sample.commonmark"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_commonmark_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_commonmark_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/markdown", "text/plain", "text/x-commonmark"]); + assertions.assertMinContentLength(result, 5); + }, TEST_TIMEOUT_MS); - it( - "office_commonmark_basic", - () => { - const documentPath = resolveDocument("markdown/sample.commonmark"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_commonmark_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_commonmark_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/markdown", "text/plain", "text/x-commonmark"]); - assertions.assertMinContentLength(result, 5); - }, - TEST_TIMEOUT_MS, - ); + it("office_dbf_basic", () => { + const documentPath = resolveDocument("dbf/stations.dbf"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_dbf_basic: missing document at", documentPath); + console.warn("Notes: Requires the office feature."); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_dbf_basic", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-dbf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["|"]); + }, TEST_TIMEOUT_MS); - it( - "office_dbf_basic", - () => { - const documentPath = resolveDocument("dbf/stations.dbf"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_dbf_basic: missing document at", documentPath); - console.warn("Notes: Requires the office feature."); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_dbf_basic", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-dbf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["|"]); - }, - TEST_TIMEOUT_MS, - ); + it("office_djot_basic", () => { + const documentPath = resolveDocument("markdown/tables.djot"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_djot_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_djot_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-djot", "text/djot"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_djot_basic", - () => { - const documentPath = resolveDocument("markdown/tables.djot"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_djot_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_djot_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-djot", "text/djot"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_doc_legacy", () => { + const documentPath = resolveDocument("doc/unit_test_lists.doc"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_doc_legacy: missing document at", documentPath); + console.warn("Notes: Requires the office feature."); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_doc_legacy", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/msword"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_doc_legacy", - () => { - const documentPath = resolveDocument("doc/unit_test_lists.doc"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_doc_legacy: missing document at", documentPath); - console.warn("Notes: Requires the office feature."); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_doc_legacy", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/msword"]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_docbook_basic", () => { + const documentPath = resolveDocument("docbook/docbook-reader.docbook"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docbook_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docbook_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/docbook+xml", "text/docbook"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_docbook_basic", - () => { - const documentPath = resolveDocument("docbook/docbook-reader.docbook"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docbook_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docbook_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/docbook+xml", "text/docbook"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_docx_basic", () => { + const documentPath = resolveDocument("docx/sample_document.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_docx_basic", - () => { - const documentPath = resolveDocument("docx/sample_document.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_docx_equations", () => { + const documentPath = resolveDocument("docx/equations.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_equations: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_equations", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_docx_equations", - () => { - const documentPath = resolveDocument("docx/equations.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_equations: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_equations", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_docx_fake", () => { + const documentPath = resolveDocument("docx/fake.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_fake: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_fake", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_docx_fake", - () => { - const documentPath = resolveDocument("docx/fake.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_fake: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_fake", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_docx_formatting", () => { + const documentPath = resolveDocument("docx/unit_test_formatting.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_formatting: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_formatting", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_docx_formatting", - () => { - const documentPath = resolveDocument("docx/unit_test_formatting.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_formatting: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_formatting", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_docx_headers", () => { + const documentPath = resolveDocument("docx/unit_test_headers.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_headers: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_headers", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_docx_headers", - () => { - const documentPath = resolveDocument("docx/unit_test_headers.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_headers: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_headers", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_docx_lists", () => { + const documentPath = resolveDocument("docx/unit_test_lists.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_lists: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_lists", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_docx_lists", - () => { - const documentPath = resolveDocument("docx/unit_test_lists.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_lists: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_lists", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_docx_tables", () => { + const documentPath = resolveDocument("docx/docx_tables.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_tables: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_tables", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAll(result, ["Simple uniform table", "Nested Table", "merged cells", "Header Col"]); + assertions.assertTableCount(result, 1, null); + }, TEST_TIMEOUT_MS); - it( - "office_docx_tables", - () => { - const documentPath = resolveDocument("docx/docx_tables.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_tables: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_tables", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAll(result, [ - "Simple uniform table", - "Nested Table", - "merged cells", - "Header Col", - ]); - assertions.assertTableCount(result, 1, null); - }, - TEST_TIMEOUT_MS, - ); + it("office_epub_basic", () => { + const documentPath = resolveDocument("epub/features.epub"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_epub_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_epub_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/epub+zip"]); + assertions.assertMinContentLength(result, 50); + }, TEST_TIMEOUT_MS); - it( - "office_epub_basic", - () => { - const documentPath = resolveDocument("epub/features.epub"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_epub_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_epub_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/epub+zip"]); - assertions.assertMinContentLength(result, 50); - }, - TEST_TIMEOUT_MS, - ); + it("office_fb2_basic", () => { + const documentPath = resolveDocument("fictionbook/basic.fb2"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_fb2_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_fb2_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-fictionbook+xml"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_fb2_basic", - () => { - const documentPath = resolveDocument("fictionbook/basic.fb2"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_fb2_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_fb2_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-fictionbook+xml"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_fictionbook_basic", () => { + const documentPath = resolveDocument("fictionbook/basic.fb2"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_fictionbook_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_fictionbook_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_fictionbook_basic", - () => { - const documentPath = resolveDocument("fictionbook/basic.fb2"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_fictionbook_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_fictionbook_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_hwp_basic", () => { + const documentPath = resolveDocument("hwp/converted_output.hwp"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_hwp_basic: missing document at", documentPath); + console.warn("Notes: Requires the office feature."); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_hwp_basic", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-hwp"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_hwp_basic", - () => { - const documentPath = resolveDocument("hwp/converted_output.hwp"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_hwp_basic: missing document at", documentPath); - console.warn("Notes: Requires the office feature."); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_hwp_basic", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-hwp"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_hwp_styled", () => { + const documentPath = resolveDocument("hwp/styled_document.hwp"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_hwp_styled: missing document at", documentPath); + console.warn("Notes: HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux."); + return; + } + if ((process.arch === "arm64" && process.platform === "linux")) { + console.warn("Skipping office_hwp_styled: not supported on this platform"); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_hwp_styled", ["hwp"], "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-hwp"]); + }, TEST_TIMEOUT_MS); - it( - "office_hwp_styled", - () => { - const documentPath = resolveDocument("hwp/styled_document.hwp"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_hwp_styled: missing document at", documentPath); - console.warn( - "Notes: HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", - ); - return; - } - if (process.arch === "arm64" && process.platform === "linux") { - console.warn("Skipping office_hwp_styled: not supported on this platform"); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if ( - shouldSkipFixture( - error, - "office_hwp_styled", - ["hwp"], - "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", - ) - ) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-hwp"]); - }, - TEST_TIMEOUT_MS, - ); + it("office_jats_basic", () => { + const documentPath = resolveDocument("jats/sample_article.jats"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_jats_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_jats_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-jats+xml", "text/jats"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_jats_basic", - () => { - const documentPath = resolveDocument("jats/sample_article.jats"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_jats_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_jats_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-jats+xml", "text/jats"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_jupyter_basic", () => { + const documentPath = resolveDocument("jupyter/rank.ipynb"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_jupyter_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_jupyter_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-ipynb+json"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_jupyter_basic", - () => { - const documentPath = resolveDocument("jupyter/rank.ipynb"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_jupyter_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_jupyter_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-ipynb+json"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_keynote_basic", () => { + const documentPath = resolveDocument("iwork/test.key"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_keynote_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_keynote_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-keynote-sffkey"]); + assertions.assertMinContentLength(result, 5); + }, TEST_TIMEOUT_MS); - it( - "office_keynote_basic", - () => { - const documentPath = resolveDocument("iwork/test.key"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_keynote_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_keynote_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-keynote-sffkey"]); - assertions.assertMinContentLength(result, 5); - }, - TEST_TIMEOUT_MS, - ); + it("office_latex_basic", () => { + const documentPath = resolveDocument("latex/basic_sections.tex"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_latex_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_latex_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-latex", "text/x-latex"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_latex_basic", - () => { - const documentPath = resolveDocument("latex/basic_sections.tex"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_latex_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_latex_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-latex", "text/x-latex"]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_markdown_basic", () => { + const documentPath = resolveDocument("markdown/comprehensive.md"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_markdown_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_markdown_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/markdown"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_markdown_basic", - () => { - const documentPath = resolveDocument("markdown/comprehensive.md"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_markdown_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_markdown_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/markdown"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_mdx_basic", () => { + const documentPath = resolveDocument("markdown/sample.mdx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_mdx_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 50); + }, TEST_TIMEOUT_MS); - it( - "office_mdx_basic", - () => { - const documentPath = resolveDocument("markdown/sample.mdx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_mdx_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 50); - }, - TEST_TIMEOUT_MS, - ); + it("office_mdx_getting_started", () => { + const documentPath = resolveDocument("markdown/mdx_getting_started.mdx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_mdx_getting_started: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_getting_started", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); + }, TEST_TIMEOUT_MS); - it( - "office_mdx_getting_started", - () => { - const documentPath = resolveDocument("markdown/mdx_getting_started.mdx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_mdx_getting_started: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_getting_started", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); - }, - TEST_TIMEOUT_MS, - ); + it("office_mdx_troubleshooting", () => { + const documentPath = resolveDocument("markdown/mdx_troubleshooting.mdx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_mdx_troubleshooting: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_troubleshooting", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); + }, TEST_TIMEOUT_MS); - it( - "office_mdx_troubleshooting", - () => { - const documentPath = resolveDocument("markdown/mdx_troubleshooting.mdx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_mdx_troubleshooting: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_troubleshooting", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); - }, - TEST_TIMEOUT_MS, - ); + it("office_mdx_using_mdx", () => { + const documentPath = resolveDocument("markdown/mdx_using_mdx.mdx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_mdx_using_mdx: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_using_mdx", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); + }, TEST_TIMEOUT_MS); - it( - "office_mdx_using_mdx", - () => { - const documentPath = resolveDocument("markdown/mdx_using_mdx.mdx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_mdx_using_mdx: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_using_mdx", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); - }, - TEST_TIMEOUT_MS, - ); + it("office_numbers_basic", () => { + const documentPath = resolveDocument("iwork/test.numbers"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_numbers_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_numbers_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-numbers-sffnumbers"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_numbers_basic", - () => { - const documentPath = resolveDocument("iwork/test.numbers"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_numbers_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_numbers_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-numbers-sffnumbers"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_ods_basic", () => { + const documentPath = resolveDocument("data_formats/test_01.ods"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_ods_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_ods_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.spreadsheet"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_ods_basic", - () => { - const documentPath = resolveDocument("data_formats/test_01.ods"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_ods_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_ods_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.spreadsheet"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_odt_bold", () => { + const documentPath = resolveDocument("odt/bold.odt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_odt_bold: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_bold", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_odt_bold", - () => { - const documentPath = resolveDocument("odt/bold.odt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_odt_bold: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_bold", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_odt_list", () => { + const documentPath = resolveDocument("odt/unorderedList.odt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_odt_list: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_list", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 30); + assertions.assertContentContainsAny(result, ["list item", "New level", "Pushed us"]); + }, TEST_TIMEOUT_MS); - it( - "office_odt_list", - () => { - const documentPath = resolveDocument("odt/unorderedList.odt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_odt_list: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_list", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 30); - assertions.assertContentContainsAny(result, ["list item", "New level", "Pushed us"]); - }, - TEST_TIMEOUT_MS, - ); + it("office_odt_simple", () => { + const documentPath = resolveDocument("odt/simple.odt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_odt_simple: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["favorite things", "Parrots", "Analysis"]); + }, TEST_TIMEOUT_MS); - it( - "office_odt_simple", - () => { - const documentPath = resolveDocument("odt/simple.odt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_odt_simple: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["favorite things", "Parrots", "Analysis"]); - }, - TEST_TIMEOUT_MS, - ); + it("office_odt_table", () => { + const documentPath = resolveDocument("odt/table.odt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_odt_table: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 10); + assertions.assertTableCount(result, 1, null); + }, TEST_TIMEOUT_MS); - it( - "office_odt_table", - () => { - const documentPath = resolveDocument("odt/table.odt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_odt_table: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 10); - assertions.assertTableCount(result, 1, null); - }, - TEST_TIMEOUT_MS, - ); + it("office_opml_basic", () => { + const documentPath = resolveDocument("opml/outline.opml"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_opml_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_opml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_opml_basic", - () => { - const documentPath = resolveDocument("opml/outline.opml"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_opml_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_opml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_org_basic", () => { + const documentPath = resolveDocument("org/comprehensive.org"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_org_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_org_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-org", "text/org"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_org_basic", - () => { - const documentPath = resolveDocument("org/comprehensive.org"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_org_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_org_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-org", "text/org"]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_pages_basic", () => { + const documentPath = resolveDocument("iwork/test.pages"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pages_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pages_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-pages-sffpages"]); + assertions.assertMinContentLength(result, 5); + }, TEST_TIMEOUT_MS); - it( - "office_pages_basic", - () => { - const documentPath = resolveDocument("iwork/test.pages"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pages_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pages_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-pages-sffpages"]); - assertions.assertMinContentLength(result, 5); - }, - TEST_TIMEOUT_MS, - ); + it("office_ppsx_slideshow", () => { + const documentPath = resolveDocument("pptx/sample.ppsx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_ppsx_slideshow: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_ppsx_slideshow", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_ppsx_slideshow", - () => { - const documentPath = resolveDocument("pptx/sample.ppsx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_ppsx_slideshow: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_ppsx_slideshow", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_ppt_legacy", () => { + const documentPath = resolveDocument("ppt/simple.ppt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_ppt_legacy: missing document at", documentPath); + console.warn("Notes: Requires the office feature."); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_ppt_legacy", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_ppt_legacy", - () => { - const documentPath = resolveDocument("ppt/simple.ppt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_ppt_legacy: missing document at", documentPath); - console.warn("Notes: Requires the office feature."); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_ppt_legacy", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_pptm_basic", () => { + const documentPath = resolveDocument("pptx/powerpoint_with_image.pptm"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pptm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptm_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "office_pptm_basic", - () => { - const documentPath = resolveDocument("pptx/powerpoint_with_image.pptm"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pptm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptm_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.ms-powerpoint.presentation.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - ]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("office_pptx_basic", () => { + const documentPath = resolveDocument("pptx/simple.pptx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pptx_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 50); + }, TEST_TIMEOUT_MS); - it( - "office_pptx_basic", - () => { - const documentPath = resolveDocument("pptx/simple.pptx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pptx_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - ]); - assertions.assertMinContentLength(result, 50); - }, - TEST_TIMEOUT_MS, - ); + it("office_pptx_images", () => { + const documentPath = resolveDocument("pptx/powerpoint_with_image.pptx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pptx_images: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_images", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 15); + }, TEST_TIMEOUT_MS); - it( - "office_pptx_images", - () => { - const documentPath = resolveDocument("pptx/powerpoint_with_image.pptx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pptx_images: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_images", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - ]); - assertions.assertMinContentLength(result, 15); - }, - TEST_TIMEOUT_MS, - ); + it("office_pptx_pitch_deck", () => { + const documentPath = resolveDocument("pptx/pitch_deck_presentation.pptx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pptx_pitch_deck: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_pitch_deck", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 100); + }, TEST_TIMEOUT_MS); - it( - "office_pptx_pitch_deck", - () => { - const documentPath = resolveDocument("pptx/pitch_deck_presentation.pptx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pptx_pitch_deck: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_pitch_deck", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - ]); - assertions.assertMinContentLength(result, 100); - }, - TEST_TIMEOUT_MS, - ); + it("office_rst_basic", () => { + const documentPath = resolveDocument("rst/restructured_text.rst"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_rst_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_rst_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-rst", "text/prs.fallenstein.rst"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); - it( - "office_rst_basic", - () => { - const documentPath = resolveDocument("rst/restructured_text.rst"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_rst_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_rst_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-rst", "text/prs.fallenstein.rst"]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("office_rtf_basic", () => { + const documentPath = resolveDocument("rtf/extraction_test.rtf"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_rtf_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_rtf_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/rtf", "text/rtf"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_rtf_basic", - () => { - const documentPath = resolveDocument("rtf/extraction_test.rtf"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_rtf_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_rtf_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/rtf", "text/rtf"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_typst_basic", () => { + const documentPath = resolveDocument("typst/headings.typ"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_typst_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_typst_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-typst", "text/x-typst"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_typst_basic", - () => { - const documentPath = resolveDocument("typst/headings.typ"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_typst_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_typst_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-typst", "text/x-typst"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_xls_legacy", () => { + const documentPath = resolveDocument("xls/test_excel.xls"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xls_legacy: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xls_legacy", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-excel"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_xls_legacy", - () => { - const documentPath = resolveDocument("xls/test_excel.xls"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xls_legacy: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xls_legacy", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-excel"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("office_xlsb_basic", () => { + const documentPath = resolveDocument("xlsx/test_xlsb.xlsb"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsb_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsb_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "office_xlsb_basic", - () => { - const documentPath = resolveDocument("xlsx/test_xlsb.xlsb"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsb_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsb_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.ms-excel.sheet.binary.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("office_xlsm_basic", () => { + const documentPath = resolveDocument("xlsx/test_01.xlsm"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsm_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "office_xlsm_basic", - () => { - const documentPath = resolveDocument("xlsx/test_01.xlsm"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsm_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.ms-excel.sheet.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("office_xlsx_basic", () => { + const documentPath = resolveDocument("xlsx/stanley_cups.xlsx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsx_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, ["Team", "Location", "Stanley Cups"]); + assertions.assertTableCount(result, 1, null); + assertions.assertMetadataExpectation(result, "sheetCount", {"gte":2}); + assertions.assertMetadataExpectation(result, "sheetNames", {"contains":["Stanley Cups"]}); + }, TEST_TIMEOUT_MS); - it( - "office_xlsx_basic", - () => { - const documentPath = resolveDocument("xlsx/stanley_cups.xlsx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsx_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, ["Team", "Location", "Stanley Cups"]); - assertions.assertTableCount(result, 1, null); - assertions.assertMetadataExpectation(result, "sheetCount", { gte: 2 }); - assertions.assertMetadataExpectation(result, "sheetNames", { contains: ["Stanley Cups"] }); - }, - TEST_TIMEOUT_MS, - ); + it("office_xlsx_multi_sheet", () => { + const documentPath = resolveDocument("xlsx/excel_multi_sheet.xlsx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsx_multi_sheet: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_multi_sheet", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 20); + assertions.assertMetadataExpectation(result, "sheetCount", {"gte":2}); + }, TEST_TIMEOUT_MS); - it( - "office_xlsx_multi_sheet", - () => { - const documentPath = resolveDocument("xlsx/excel_multi_sheet.xlsx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsx_multi_sheet: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_multi_sheet", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 20); - assertions.assertMetadataExpectation(result, "sheetCount", { gte: 2 }); - }, - TEST_TIMEOUT_MS, - ); + it("office_xlsx_office_example", () => { + const documentPath = resolveDocument("xlsx/test_01.xlsx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsx_office_example: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_office_example", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "office_xlsx_office_example", - () => { - const documentPath = resolveDocument("xlsx/test_01.xlsx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsx_office_example: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_office_example", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); }); + diff --git a/e2e/typescript/tests/plugin-apis.spec.ts b/e2e/typescript/tests/plugin-apis.spec.ts index efd53e0f245..5e064b00e06 100644 --- a/e2e/typescript/tests/plugin-apis.spec.ts +++ b/e2e/typescript/tests/plugin-apis.spec.ts @@ -9,133 +9,137 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import * as kreuzberg from "@kreuzberg/node"; import { describe, expect, it } from "vitest"; +import * as kreuzberg from "@kreuzberg/node"; describe("Configuration", () => { - it("Discover configuration from current or parent directories", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); - const configPath = path.join(tmpDir, "kreuzberg.toml"); - fs.writeFileSync(configPath, "[chunking]\nmax_chars = 50\n"); - - const subDir = path.join(tmpDir, "subdir"); - fs.mkdirSync(subDir); - - const originalCwd = process.cwd(); - try { - process.chdir(subDir); - - const config = kreuzberg.ExtractionConfig.discover(); - - expect(config.chunking).toBeDefined(); - expect(config.chunking?.maxChars).toBe(50); - } finally { - process.chdir(originalCwd); - } - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); - - it("Load configuration from a TOML file", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); - const configPath = path.join(tmpDir, "test_config.toml"); - fs.writeFileSync( - configPath, - "[chunking]\nmax_chars = 100\nmax_overlap = 20\n\n[language_detection]\nenabled = false\n", - ); - - const config = kreuzberg.ExtractionConfig.fromFile(configPath); - - expect(config.chunking).toBeDefined(); - expect(config.chunking?.maxChars).toBe(100); - expect(config.chunking?.maxOverlap).toBe(20); - expect(config.languageDetection).toBeDefined(); - expect(config.languageDetection?.enabled).toBe(false); - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); + it("Discover configuration from current or parent directories", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); + const configPath = path.join(tmpDir, "kreuzberg.toml"); + fs.writeFileSync(configPath, "[chunking]\nmax_chars = 50\n"); + + const subDir = path.join(tmpDir, "subdir"); + fs.mkdirSync(subDir); + + const originalCwd = process.cwd(); + try { + process.chdir(subDir); + + const config = kreuzberg.ExtractionConfig.discover(); + + expect(config.chunking).toBeDefined(); + expect(config.chunking?.maxChars).toBe(50); + } finally { + process.chdir(originalCwd); + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("Load configuration from a TOML file", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); + const configPath = path.join(tmpDir, "test_config.toml"); + fs.writeFileSync(configPath, "[chunking]\nmax_chars = 100\nmax_overlap = 20\n\n[language_detection]\nenabled = false\n"); + + const config = kreuzberg.ExtractionConfig.fromFile(configPath); + + expect(config.chunking).toBeDefined(); + expect(config.chunking?.maxChars).toBe(100); + expect(config.chunking?.maxOverlap).toBe(20); + expect(config.languageDetection).toBeDefined(); + expect(config.languageDetection?.enabled).toBe(false); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + }); describe("Document Extractor Management", () => { - it("Clear all document extractors and verify list is empty", () => { - kreuzberg.clearDocumentExtractors(); - const result = kreuzberg.listDocumentExtractors(); - expect(result).toHaveLength(0); - }); - - it("List all registered document extractors", () => { - const result = kreuzberg.listDocumentExtractors(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item) => typeof item === "string")).toBe(true); - }); - - it("Unregister nonexistent document extractor gracefully", () => { - expect(() => kreuzberg.unregisterDocumentExtractor("nonexistent-extractor-xyz")).not.toThrow(); - }); + it("Clear all document extractors and verify list is empty", () => { + kreuzberg.clearDocumentExtractors(); + const result = kreuzberg.listDocumentExtractors(); + expect(result).toHaveLength(0); + }); + + it("List all registered document extractors", () => { + const result = kreuzberg.listDocumentExtractors(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item) => typeof item === "string")).toBe(true); + }); + + it("Unregister nonexistent document extractor gracefully", () => { + expect(() => kreuzberg.unregisterDocumentExtractor("nonexistent-extractor-xyz")).not.toThrow(); + }); + }); describe("Mime Utilities", () => { - it("Detect MIME type from file bytes", () => { - const testData = Buffer.from("%PDF-1.4\\n"); - const result = kreuzberg.detectMimeType(testData); - expect(result.toLowerCase()).toContain("pdf"); - }); - - it("Detect MIME type from file path", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); - const filePath = path.join(tmpDir, "test.txt"); - fs.writeFileSync(filePath, "Hello, world!"); - - const result = kreuzberg.detectMimeTypeFromPath(filePath); - expect(result.toLowerCase()).toContain("text"); - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); - - it("Get file extensions for a MIME type", () => { - const result = kreuzberg.getExtensionsForMime("application/pdf"); - expect(Array.isArray(result)).toBe(true); - expect(result).toContain("pdf"); - }); + it("Detect MIME type from file bytes", () => { + const testData = Buffer.from("%PDF-1.4\\n"); + const result = kreuzberg.detectMimeType(testData); + expect(result.toLowerCase()).toContain("pdf"); + }); + + it("Detect MIME type from file path", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); + const filePath = path.join(tmpDir, "test.txt"); + fs.writeFileSync(filePath, "Hello, world!"); + + const result = kreuzberg.detectMimeTypeFromPath(filePath); + expect(result.toLowerCase()).toContain("text"); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("Get file extensions for a MIME type", () => { + const result = kreuzberg.getExtensionsForMime("application/pdf"); + expect(Array.isArray(result)).toBe(true); + expect(result).toContain("pdf"); + }); + }); describe("Ocr Backend Management", () => { - it("Clear all OCR backends and verify list is empty", () => { - kreuzberg.clearOcrBackends(); - const result = kreuzberg.listOcrBackends(); - expect(result).toHaveLength(0); - }); - - it("List all registered OCR backends", () => { - const result = kreuzberg.listOcrBackends(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item) => typeof item === "string")).toBe(true); - }); - - it("Unregister nonexistent OCR backend gracefully", () => { - expect(() => kreuzberg.unregisterOcrBackend("nonexistent-backend-xyz")).not.toThrow(); - }); + it("Clear all OCR backends and verify list is empty", () => { + kreuzberg.clearOcrBackends(); + const result = kreuzberg.listOcrBackends(); + expect(result).toHaveLength(0); + }); + + it("List all registered OCR backends", () => { + const result = kreuzberg.listOcrBackends(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item) => typeof item === "string")).toBe(true); + }); + + it("Unregister nonexistent OCR backend gracefully", () => { + expect(() => kreuzberg.unregisterOcrBackend("nonexistent-backend-xyz")).not.toThrow(); + }); + }); describe("Post Processor Management", () => { - it("Clear all post-processors and verify list is empty", () => { - kreuzberg.clearPostProcessors(); - }); - - it("List all registered post-processors", () => { - const result = kreuzberg.listPostProcessors(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item) => typeof item === "string")).toBe(true); - }); + it("Clear all post-processors and verify list is empty", () => { + kreuzberg.clearPostProcessors(); + }); + + it("List all registered post-processors", () => { + const result = kreuzberg.listPostProcessors(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item) => typeof item === "string")).toBe(true); + }); + }); describe("Validator Management", () => { - it("Clear all validators and verify list is empty", () => { - kreuzberg.clearValidators(); - const result = kreuzberg.listValidators(); - expect(result).toHaveLength(0); - }); - - it("List all registered validators", () => { - const result = kreuzberg.listValidators(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item) => typeof item === "string")).toBe(true); - }); + it("Clear all validators and verify list is empty", () => { + kreuzberg.clearValidators(); + const result = kreuzberg.listValidators(); + expect(result).toHaveLength(0); + }); + + it("List all registered validators", () => { + const result = kreuzberg.listValidators(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item) => typeof item === "string")).toBe(true); + }); + }); + diff --git a/e2e/typescript/tests/render.spec.ts b/e2e/typescript/tests/render.spec.ts index 0bf23800dc2..62f0d1aeb0d 100644 --- a/e2e/typescript/tests/render.spec.ts +++ b/e2e/typescript/tests/render.spec.ts @@ -3,40 +3,41 @@ // Tests for render fixtures. +import { describe, it, expect } from "vitest"; import { existsSync } from "node:fs"; -import { PdfPageIterator, renderPdfPageSync } from "@kreuzberg/node"; -import { describe, expect, it } from "vitest"; +import { renderPdfPageSync, PdfPageIterator } from "@kreuzberg/node"; import { assertions, resolveDocument } from "./helpers.js"; describe("render", () => { - it("render_custom_dpi", () => { - const documentPath = resolveDocument("pdf/tiny.pdf"); - if (!existsSync(documentPath)) return; - const pngData = renderPdfPageSync(documentPath, 0, { dpi: 72 }); - assertions.assertIsPng(pngData); - assertions.assertMinByteLength(pngData, 50); - }); + it("render_custom_dpi", () => { + const documentPath = resolveDocument("pdf/tiny.pdf"); + if (!existsSync(documentPath)) return; + const pngData = renderPdfPageSync(documentPath, 0, { dpi: 72 }); + assertions.assertIsPng(pngData); + assertions.assertMinByteLength(pngData, 50); + }); - it("render_iterator", () => { - const documentPath = resolveDocument("pdf/tiny.pdf"); - if (!existsSync(documentPath)) return; - const pages: Buffer[] = []; - const iter = new PdfPageIterator(documentPath, { dpi: 150 }); - let result = iter.next(); - while (result !== null) { - assertions.assertIsPng(result.data); - pages.push(result.data); - result = iter.next(); - } - iter.close(); - expect(pages.length).toBeGreaterThanOrEqual(1); - }); + it("render_iterator", () => { + const documentPath = resolveDocument("pdf/tiny.pdf"); + if (!existsSync(documentPath)) return; + const pages: Buffer[] = []; + const iter = new PdfPageIterator(documentPath, { dpi: 150 }); + let result = iter.next(); + while (result !== null) { + assertions.assertIsPng(result.data); + pages.push(result.data); + result = iter.next(); + } + iter.close(); + expect(pages.length).toBeGreaterThanOrEqual(1); + }); + + it("render_single_page", () => { + const documentPath = resolveDocument("pdf/tiny.pdf"); + if (!existsSync(documentPath)) return; + const pngData = renderPdfPageSync(documentPath, 0, { dpi: 150 }); + assertions.assertIsPng(pngData); + assertions.assertMinByteLength(pngData, 100); + }); - it("render_single_page", () => { - const documentPath = resolveDocument("pdf/tiny.pdf"); - if (!existsSync(documentPath)) return; - const pngData = renderPdfPageSync(documentPath, 0, { dpi: 150 }); - assertions.assertIsPng(pngData); - assertions.assertMinByteLength(pngData, 100); - }); }); diff --git a/e2e/typescript/tests/structured.spec.ts b/e2e/typescript/tests/structured.spec.ts index 1f7ff5d2156..df47ec7942a 100644 --- a/e2e/typescript/tests/structured.spec.ts +++ b/e2e/typescript/tests/structured.spec.ts @@ -4,282 +4,244 @@ // Tests for structured fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("structured fixtures", () => { - it( - "structured_csv_basic", - () => { - const documentPath = resolveDocument("csv/stanley_cups.csv"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_csv_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/csv"]); - assertions.assertMinContentLength(result, 20); - }, - TEST_TIMEOUT_MS, - ); + it("structured_csv_basic", () => { + const documentPath = resolveDocument("csv/stanley_cups.csv"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_csv_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/csv"]); + assertions.assertMinContentLength(result, 20); + }, TEST_TIMEOUT_MS); + + it("structured_enw_basic", () => { + const documentPath = resolveDocument("data_formats/sample.enw"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_enw_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); + }, TEST_TIMEOUT_MS); - it( - "structured_enw_basic", - () => { - const documentPath = resolveDocument("data_formats/sample.enw"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_enw_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); - }, - TEST_TIMEOUT_MS, - ); + it("structured_json_basic", () => { + const documentPath = resolveDocument("json/sample_document.json"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_json_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 20); + assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); + }, TEST_TIMEOUT_MS); - it( - "structured_json_basic", - () => { - const documentPath = resolveDocument("json/sample_document.json"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_json_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 20); - assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); - }, - TEST_TIMEOUT_MS, - ); + it("structured_json_simple", () => { + const documentPath = resolveDocument("json/simple.json"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_json_simple: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["{", "name"]); + }, TEST_TIMEOUT_MS); - it( - "structured_json_simple", - () => { - const documentPath = resolveDocument("json/simple.json"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_json_simple: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["{", "name"]); - }, - TEST_TIMEOUT_MS, - ); + it("structured_nbib_basic", () => { + const documentPath = resolveDocument("data_formats/sample.nbib"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_nbib_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "structured_nbib_basic", - () => { - const documentPath = resolveDocument("data_formats/sample.nbib"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_nbib_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("structured_ris_basic", () => { + const documentPath = resolveDocument("data_formats/sample.ris"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_ris_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); + assertions.assertContentNotEmpty(result); + }, TEST_TIMEOUT_MS); - it( - "structured_ris_basic", - () => { - const documentPath = resolveDocument("data_formats/sample.ris"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_ris_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); - assertions.assertContentNotEmpty(result); - }, - TEST_TIMEOUT_MS, - ); + it("structured_toml_basic", () => { + const documentPath = resolveDocument("data_formats/cargo.toml"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_toml_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "structured_toml_basic", - () => { - const documentPath = resolveDocument("data_formats/cargo.toml"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_toml_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("structured_tsv_basic", () => { + const documentPath = resolveDocument("data_formats/employees.tsv"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_tsv_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "structured_tsv_basic", - () => { - const documentPath = resolveDocument("data_formats/employees.tsv"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_tsv_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("structured_yaml_basic", () => { + const documentPath = resolveDocument("yaml/simple.yaml"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_yaml_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "structured_yaml_basic", - () => { - const documentPath = resolveDocument("yaml/simple.yaml"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_yaml_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); + it("structured_yaml_simple", () => { + const documentPath = resolveDocument("yaml/simple.yaml"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_yaml_simple: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-yaml"]); + assertions.assertMinContentLength(result, 10); + }, TEST_TIMEOUT_MS); - it( - "structured_yaml_simple", - () => { - const documentPath = resolveDocument("yaml/simple.yaml"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_yaml_simple: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-yaml"]); - assertions.assertMinContentLength(result, 10); - }, - TEST_TIMEOUT_MS, - ); }); + diff --git a/e2e/typescript/tests/xml.spec.ts b/e2e/typescript/tests/xml.spec.ts index 7b1fdb7debb..f28b0d9c1bb 100644 --- a/e2e/typescript/tests/xml.spec.ts +++ b/e2e/typescript/tests/xml.spec.ts @@ -4,38 +4,36 @@ // Tests for xml fixtures. import { existsSync } from "node:fs"; -import type { ExtractionResult } from "@kreuzberg/node"; -import { extractFileSync } from "@kreuzberg/node"; import { describe, it } from "vitest"; import { assertions, buildConfig, resolveDocument, shouldSkipFixture } from "./helpers.js"; +import { extractFileSync } from "@kreuzberg/node"; +import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("xml fixtures", () => { - it( - "xml_plant_catalog", - () => { - const documentPath = resolveDocument("xml/plant_catalog.xml"); - if (!existsSync(documentPath)) { - console.warn("Skipping xml_plant_catalog: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml"]); - assertions.assertMinContentLength(result, 100); - }, - TEST_TIMEOUT_MS, - ); + it("xml_plant_catalog", () => { + const documentPath = resolveDocument("xml/plant_catalog.xml"); + if (!existsSync(documentPath)) { + console.warn("Skipping xml_plant_catalog: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml"]); + assertions.assertMinContentLength(result, 100); + }, TEST_TIMEOUT_MS); + }); + diff --git a/e2e/wasm-deno/archive.test.ts b/e2e/wasm-deno/archive.test.ts index e96e9bcdd75..6acde9adf94 100644 --- a/e2e/wasm-deno/archive.test.ts +++ b/e2e/wasm-deno/archive.test.ts @@ -3,97 +3,90 @@ // Tests for archive fixtures. Run with: deno test --allow-read +import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; -import { - assertions, - buildConfig, - enableOcr, - extractBytes, - initWasm, - resolveDocument, - shouldSkipFixture, -} from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); await enableOcr(); Deno.test("archive_gz_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("archives/book_war_and_peace_1p.txt.gz"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/gzip", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("archives/book_war_and_peace_1p.txt.gz"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/gzip", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("archive_sevenz_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("archives/documents.7z"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-7z-compressed", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("archives/documents.7z"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-7z-compressed", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("archive_tar_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("archives/documents.tar"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-tar", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("archives/documents.tar"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-tar", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("archive_zip_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("archives/documents.zip"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/zip", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("archives/documents.zip"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/zip", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); + assertions.assertMinContentLength(result, 10); }); + diff --git a/e2e/wasm-deno/email.test.ts b/e2e/wasm-deno/email.test.ts index 1624161a568..1cfaa97f8f1 100644 --- a/e2e/wasm-deno/email.test.ts +++ b/e2e/wasm-deno/email.test.ts @@ -3,137 +3,130 @@ // Tests for email fixtures. Run with: deno test --allow-read +import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; -import { - assertions, - buildConfig, - enableOcr, - extractBytes, - initWasm, - resolveDocument, - shouldSkipFixture, -} from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); await enableOcr(); Deno.test("email_eml_html_body", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/html_only.eml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/html_only.eml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("email_eml_multipart", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/html_email_multipart.eml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/html_email_multipart.eml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("email_eml_utf16", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("vendored/unstructured/eml/fake-email-utf-16.eml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("vendored/unstructured/eml/fake-email-utf-16.eml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); }); Deno.test("email_msg_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/fake_email.msg"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-outlook", config); - } catch (error) { - if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/fake_email.msg"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-outlook", config); + } catch (error) { + if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("email_pst_empty", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/empty.pst"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-outlook-pst", config); - } catch (error) { - if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/empty.pst"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-outlook-pst", config); + } catch (error) { + if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); }); Deno.test("email_sample_eml", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/sample_email.eml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/sample_email.eml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 20); }); + diff --git a/e2e/wasm-deno/embeddings.test.ts b/e2e/wasm-deno/embeddings.test.ts index 8ccc4b19e2e..8b753c914c3 100644 --- a/e2e/wasm-deno/embeddings.test.ts +++ b/e2e/wasm-deno/embeddings.test.ts @@ -3,38 +3,31 @@ // Tests for embeddings fixtures. Run with: deno test --allow-read +import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; -import { - assertions, - buildConfig, - enableOcr, - extractBytes, - initWasm, - resolveDocument, - shouldSkipFixture, -} from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); await enableOcr(); Deno.test("embedding_disabled", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig({ chunking: { max_chars: 500, max_overlap: 50 } }); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pdf/fake_memo.pdf"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/octet-stream", config); - } catch (error) { - if (shouldSkipFixture(error, "embedding_disabled", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/pdf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, false, null, null); + const config = buildConfig({"chunking":{"max_chars":500,"max_overlap":50}}); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pdf/fake_memo.pdf"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/octet-stream", config); + } catch (error) { + if (shouldSkipFixture(error, "embedding_disabled", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/pdf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertChunks(result, 1, null, true, false, null, null); }); + diff --git a/e2e/wasm-deno/html.test.ts b/e2e/wasm-deno/html.test.ts index 6f4e35f761a..6d66937ccc9 100644 --- a/e2e/wasm-deno/html.test.ts +++ b/e2e/wasm-deno/html.test.ts @@ -3,46 +3,31 @@ // Tests for html fixtures. Run with: deno test --allow-read +import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; -import { - assertions, - buildConfig, - enableOcr, - extractBytes, - initWasm, - resolveDocument, - shouldSkipFixture, -} from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); await enableOcr(); Deno.test("html_simple_table", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("html/simple_table.html"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/html", config); - } catch (error) { - if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, [ - "Product", - "Category", - "Price", - "Stock", - "Laptop", - "Electronics", - "Sample Data Table", - ]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("html/simple_table.html"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/html", config); + } catch (error) { + if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]); }); + diff --git a/e2e/wasm-deno/image.test.ts b/e2e/wasm-deno/image.test.ts index 2bc3e0ee61a..273dc28f702 100644 --- a/e2e/wasm-deno/image.test.ts +++ b/e2e/wasm-deno/image.test.ts @@ -3,217 +3,210 @@ // Tests for image fixtures. Run with: deno test --allow-read +import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; -import { - assertions, - buildConfig, - enableOcr, - extractBytes, - initWasm, - resolveDocument, - shouldSkipFixture, -} from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); await enableOcr(); Deno.test("image_bmp_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images/bmp_24.bmp"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/bmp", config); - } catch (error) { - if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/bmp"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images/bmp_24.bmp"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/bmp", config); + } catch (error) { + if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/bmp"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_gif_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.gif"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/gif", config); - } catch (error) { - if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/gif"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.gif"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/gif", config); + } catch (error) { + if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/gif"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_jp2_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.jp2"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/jp2", config); - } catch (error) { - if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.jp2"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/jp2", config); + } catch (error) { + if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_metadata_only", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig({ ocr: null }); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images/example.jpg"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/jpeg", config); - } catch (error) { - if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jpeg"]); - assertions.assertMaxContentLength(result, 200); + const config = buildConfig({"ocr":null}); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images/example.jpg"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/jpeg", config); + } catch (error) { + if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jpeg"]); + assertions.assertMaxContentLength(result, 200); }); Deno.test("image_pbm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.pbm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/x-portable-bitmap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.pbm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/x-portable-bitmap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_pgm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.pgm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/x-portable-graymap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.pgm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/x-portable-graymap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_ppm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.ppm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/x-portable-pixmap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.ppm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/x-portable-pixmap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_svg_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xml/simple_svg.svg"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/svg+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/svg+xml"]); - assertions.assertMinContentLength(result, 5); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xml/simple_svg.svg"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/svg+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/svg+xml"]); + assertions.assertMinContentLength(result, 5); }); Deno.test("image_tiff_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.tif"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/tiff", config); - } catch (error) { - if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/tiff"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.tif"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/tiff", config); + } catch (error) { + if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/tiff"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_webp_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.webp"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/webp", config); - } catch (error) { - if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/webp"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.webp"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/webp", config); + } catch (error) { + if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/webp"]); + assertions.assertContentNotEmpty(result); }); + diff --git a/e2e/wasm-deno/office.test.ts b/e2e/wasm-deno/office.test.ts index cd53e6eba10..527d25e12c2 100644 --- a/e2e/wasm-deno/office.test.ts +++ b/e2e/wasm-deno/office.test.ts @@ -3,1108 +3,1029 @@ // Tests for office fixtures. Run with: deno test --allow-read +import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; -import { - assertions, - buildConfig, - enableOcr, - extractBytes, - initWasm, - resolveDocument, - shouldSkipFixture, -} from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); await enableOcr(); Deno.test("office_bibtex_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("bibtex/comprehensive.bib"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-bibtex", config); - } catch (error) { - if (shouldSkipFixture(error, "office_bibtex_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-bibtex", "text/x-bibtex"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("bibtex/comprehensive.bib"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-bibtex", config); + } catch (error) { + if (shouldSkipFixture(error, "office_bibtex_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-bibtex", "text/x-bibtex"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_commonmark_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/sample.commonmark"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/markdown", config); - } catch (error) { - if (shouldSkipFixture(error, "office_commonmark_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/markdown", "text/plain", "text/x-commonmark"]); - assertions.assertMinContentLength(result, 5); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/sample.commonmark"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/markdown", config); + } catch (error) { + if (shouldSkipFixture(error, "office_commonmark_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/markdown", "text/plain", "text/x-commonmark"]); + assertions.assertMinContentLength(result, 5); }); Deno.test("office_dbf_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("dbf/stations.dbf"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-dbf", config); - } catch (error) { - if (shouldSkipFixture(error, "office_dbf_basic", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-dbf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["|"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("dbf/stations.dbf"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-dbf", config); + } catch (error) { + if (shouldSkipFixture(error, "office_dbf_basic", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-dbf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["|"]); }); Deno.test("office_djot_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/tables.djot"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/x-djot", config); - } catch (error) { - if (shouldSkipFixture(error, "office_djot_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-djot", "text/djot"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/tables.djot"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/x-djot", config); + } catch (error) { + if (shouldSkipFixture(error, "office_djot_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-djot", "text/djot"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_doc_legacy", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("doc/unit_test_lists.doc"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/msword", config); - } catch (error) { - if (shouldSkipFixture(error, "office_doc_legacy", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/msword"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("doc/unit_test_lists.doc"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/msword", config); + } catch (error) { + if (shouldSkipFixture(error, "office_doc_legacy", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/msword"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docbook_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docbook/docbook-reader.docbook"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/docbook+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docbook_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/docbook+xml", "text/docbook"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docbook/docbook-reader.docbook"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/docbook+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docbook_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/docbook+xml", "text/docbook"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_docx_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/sample_document.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/sample_document.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_docx_equations", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/equations.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_equations", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/equations.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_equations", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_fake", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/fake.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_fake", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/fake.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_fake", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_formatting", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/unit_test_formatting.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_formatting", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/unit_test_formatting.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_formatting", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_headers", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/unit_test_headers.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_headers", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/unit_test_headers.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_headers", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_lists", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/unit_test_lists.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_lists", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/unit_test_lists.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_lists", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_tables", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/docx_tables.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_tables", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAll(result, ["Simple uniform table", "Nested Table", "merged cells", "Header Col"]); - // Table and bounding box assertions require OCR feature for PDF table extraction - if (result.tables.length > 0) { - assertions.assertTableCount(result, 1, null); - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/docx_tables.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_tables", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAll(result, ["Simple uniform table", "Nested Table", "merged cells", "Header Col"]); + // Table and bounding box assertions require OCR feature for PDF table extraction + if (result.tables.length > 0) { + assertions.assertTableCount(result, 1, null); + } }); Deno.test("office_epub_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("epub/features.epub"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/epub+zip", config); - } catch (error) { - if (shouldSkipFixture(error, "office_epub_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/epub+zip"]); - assertions.assertMinContentLength(result, 50); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("epub/features.epub"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/epub+zip", config); + } catch (error) { + if (shouldSkipFixture(error, "office_epub_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/epub+zip"]); + assertions.assertMinContentLength(result, 50); }); Deno.test("office_fb2_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("fictionbook/basic.fb2"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-fictionbook+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_fb2_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-fictionbook+xml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("fictionbook/basic.fb2"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-fictionbook+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_fb2_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-fictionbook+xml"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_fictionbook_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("fictionbook/basic.fb2"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-fictionbook+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_fictionbook_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("fictionbook/basic.fb2"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-fictionbook+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_fictionbook_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_hwp_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("hwp/converted_output.hwp"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-hwp", config); - } catch (error) { - if (shouldSkipFixture(error, "office_hwp_basic", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-hwp"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("hwp/converted_output.hwp"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-hwp", config); + } catch (error) { + if (shouldSkipFixture(error, "office_hwp_basic", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-hwp"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_hwp_styled", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("hwp/styled_document.hwp"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-hwp", config); - } catch (error) { - if ( - shouldSkipFixture( - error, - "office_hwp_styled", - ["hwp"], - "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", - ) - ) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-hwp"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("hwp/styled_document.hwp"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-hwp", config); + } catch (error) { + if (shouldSkipFixture(error, "office_hwp_styled", ["hwp"], "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-hwp"]); }); Deno.test("office_jats_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("jats/sample_article.jats"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-jats+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_jats_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-jats+xml", "text/jats"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("jats/sample_article.jats"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-jats+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_jats_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-jats+xml", "text/jats"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_keynote_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("iwork/test.key"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-iwork-keynote-sffkey", config); - } catch (error) { - if (shouldSkipFixture(error, "office_keynote_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-keynote-sffkey"]); - assertions.assertMinContentLength(result, 5); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("iwork/test.key"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-iwork-keynote-sffkey", config); + } catch (error) { + if (shouldSkipFixture(error, "office_keynote_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-keynote-sffkey"]); + assertions.assertMinContentLength(result, 5); }); Deno.test("office_latex_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("latex/basic_sections.tex"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-latex", config); - } catch (error) { - if (shouldSkipFixture(error, "office_latex_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-latex", "text/x-latex"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("latex/basic_sections.tex"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-latex", config); + } catch (error) { + if (shouldSkipFixture(error, "office_latex_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-latex", "text/x-latex"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_markdown_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/comprehensive.md"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/markdown", config); - } catch (error) { - if (shouldSkipFixture(error, "office_markdown_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/markdown"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/comprehensive.md"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/markdown", config); + } catch (error) { + if (shouldSkipFixture(error, "office_markdown_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/markdown"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_mdx_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/sample.mdx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/mdx", config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 50); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/sample.mdx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/mdx", config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 50); }); Deno.test("office_mdx_getting_started", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/mdx_getting_started.mdx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/mdx", config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_getting_started", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/mdx_getting_started.mdx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/mdx", config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_getting_started", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); }); Deno.test("office_mdx_troubleshooting", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/mdx_troubleshooting.mdx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/mdx", config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_troubleshooting", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/mdx_troubleshooting.mdx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/mdx", config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_troubleshooting", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); }); Deno.test("office_mdx_using_mdx", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/mdx_using_mdx.mdx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/mdx", config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_using_mdx", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/mdx_using_mdx.mdx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/mdx", config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_using_mdx", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); }); Deno.test("office_numbers_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("iwork/test.numbers"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-iwork-numbers-sffnumbers", config); - } catch (error) { - if (shouldSkipFixture(error, "office_numbers_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-numbers-sffnumbers"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("iwork/test.numbers"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-iwork-numbers-sffnumbers", config); + } catch (error) { + if (shouldSkipFixture(error, "office_numbers_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-numbers-sffnumbers"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_ods_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/test_01.ods"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.spreadsheet", config); - } catch (error) { - if (shouldSkipFixture(error, "office_ods_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.spreadsheet"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/test_01.ods"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.spreadsheet", config); + } catch (error) { + if (shouldSkipFixture(error, "office_ods_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.spreadsheet"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_odt_bold", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("odt/bold.odt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_bold", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("odt/bold.odt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_bold", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_odt_list", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("odt/unorderedList.odt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_list", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 30); - assertions.assertContentContainsAny(result, ["list item", "New level", "Pushed us"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("odt/unorderedList.odt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_list", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 30); + assertions.assertContentContainsAny(result, ["list item", "New level", "Pushed us"]); }); Deno.test("office_odt_simple", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("odt/simple.odt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["favorite things", "Parrots", "Analysis"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("odt/simple.odt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["favorite things", "Parrots", "Analysis"]); }); Deno.test("office_odt_table", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("odt/table.odt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 10); - // Table and bounding box assertions require OCR feature for PDF table extraction - if (result.tables.length > 0) { - assertions.assertTableCount(result, 1, null); - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("odt/table.odt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 10); + // Table and bounding box assertions require OCR feature for PDF table extraction + if (result.tables.length > 0) { + assertions.assertTableCount(result, 1, null); + } }); Deno.test("office_opml_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("opml/outline.opml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/xml+opml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_opml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("opml/outline.opml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/xml+opml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_opml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_org_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("org/comprehensive.org"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/x-org", config); - } catch (error) { - if (shouldSkipFixture(error, "office_org_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-org", "text/org"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("org/comprehensive.org"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/x-org", config); + } catch (error) { + if (shouldSkipFixture(error, "office_org_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-org", "text/org"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_pages_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("iwork/test.pages"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-iwork-pages-sffpages", config); - } catch (error) { - if (shouldSkipFixture(error, "office_pages_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-pages-sffpages"]); - assertions.assertMinContentLength(result, 5); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("iwork/test.pages"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-iwork-pages-sffpages", config); + } catch (error) { + if (shouldSkipFixture(error, "office_pages_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-pages-sffpages"]); + assertions.assertMinContentLength(result, 5); }); Deno.test("office_ppsx_slideshow", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/sample.ppsx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.presentationml.slideshow", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_ppsx_slideshow", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/sample.ppsx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.presentationml.slideshow", config); + } catch (error) { + if (shouldSkipFixture(error, "office_ppsx_slideshow", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_ppt_legacy", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("ppt/simple.ppt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-powerpoint", config); - } catch (error) { - if (shouldSkipFixture(error, "office_ppt_legacy", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("ppt/simple.ppt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-powerpoint", config); + } catch (error) { + if (shouldSkipFixture(error, "office_ppt_legacy", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_pptm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/powerpoint_with_image.pptm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-powerpoint.presentation.macroEnabled.12", config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptm_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.ms-powerpoint.presentation.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - ]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/powerpoint_with_image.pptm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-powerpoint.presentation.macroEnabled.12", config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptm_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertContentNotEmpty(result); }); Deno.test("office_pptx_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/simple.pptx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 50); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/simple.pptx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.presentationml.presentation", config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 50); }); Deno.test("office_pptx_images", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/powerpoint_with_image.pptx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_images", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 15); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/powerpoint_with_image.pptx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.presentationml.presentation", config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_images", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 15); }); Deno.test("office_pptx_pitch_deck", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/pitch_deck_presentation.pptx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_pitch_deck", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 100); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/pitch_deck_presentation.pptx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.presentationml.presentation", config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_pitch_deck", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 100); }); Deno.test("office_rst_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("rst/restructured_text.rst"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/x-rst", config); - } catch (error) { - if (shouldSkipFixture(error, "office_rst_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-rst", "text/prs.fallenstein.rst"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("rst/restructured_text.rst"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/x-rst", config); + } catch (error) { + if (shouldSkipFixture(error, "office_rst_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-rst", "text/prs.fallenstein.rst"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_rtf_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("rtf/extraction_test.rtf"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/rtf", config); - } catch (error) { - if (shouldSkipFixture(error, "office_rtf_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/rtf", "text/rtf"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("rtf/extraction_test.rtf"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/rtf", config); + } catch (error) { + if (shouldSkipFixture(error, "office_rtf_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/rtf", "text/rtf"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_typst_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("typst/headings.typ"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-typst", config); - } catch (error) { - if (shouldSkipFixture(error, "office_typst_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-typst", "text/x-typst"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("typst/headings.typ"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-typst", config); + } catch (error) { + if (shouldSkipFixture(error, "office_typst_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-typst", "text/x-typst"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_xls_legacy", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xls/test_excel.xls"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-excel", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xls_legacy", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-excel"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xls/test_excel.xls"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-excel", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xls_legacy", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-excel"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_xlsb_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/test_xlsb.xlsb"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-excel.sheet.binary.macroEnabled.12", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsb_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.ms-excel.sheet.binary.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/test_xlsb.xlsb"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-excel.sheet.binary.macroEnabled.12", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsb_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertContentNotEmpty(result); }); Deno.test("office_xlsm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/test_01.xlsm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-excel.sheet.macroEnabled.12", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsm_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, [ - "application/vnd.ms-excel.sheet.macroEnabled.12", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/test_01.xlsm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-excel.sheet.macroEnabled.12", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsm_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertContentNotEmpty(result); }); Deno.test("office_xlsx_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/stanley_cups.xlsx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, ["Team", "Location", "Stanley Cups"]); - // Table and bounding box assertions require OCR feature for PDF table extraction - if (result.tables.length > 0) { - assertions.assertTableCount(result, 1, null); - } - assertions.assertMetadataExpectation(result, "sheetCount", { gte: 2 }); - assertions.assertMetadataExpectation(result, "sheetNames", { contains: ["Stanley Cups"] }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/stanley_cups.xlsx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, ["Team", "Location", "Stanley Cups"]); + // Table and bounding box assertions require OCR feature for PDF table extraction + if (result.tables.length > 0) { + assertions.assertTableCount(result, 1, null); + } + assertions.assertMetadataExpectation(result, "sheetCount", {"gte":2}); + assertions.assertMetadataExpectation(result, "sheetNames", {"contains":["Stanley Cups"]}); }); Deno.test("office_xlsx_multi_sheet", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/excel_multi_sheet.xlsx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_multi_sheet", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 20); - assertions.assertMetadataExpectation(result, "sheetCount", { gte: 2 }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/excel_multi_sheet.xlsx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_multi_sheet", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 20); + assertions.assertMetadataExpectation(result, "sheetCount", {"gte":2}); }); Deno.test("office_xlsx_office_example", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/test_01.xlsx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes( - documentBytes, - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - config, - ); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_office_example", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/test_01.xlsx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_office_example", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 10); }); + diff --git a/e2e/wasm-deno/plugin-apis.test.ts b/e2e/wasm-deno/plugin-apis.test.ts index 7244dbfd890..c81191cda5a 100644 --- a/e2e/wasm-deno/plugin-apis.test.ts +++ b/e2e/wasm-deno/plugin-apis.test.ts @@ -6,20 +6,9 @@ * To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang wasm-deno */ -// @deno-types="../../crates/kreuzberg-wasm/dist/index.d.ts" -import { - clearOcrBackends, - clearPostProcessors, - clearValidators, - detectMimeFromBytes, - getExtensionsForMime, - initWasm, - listOcrBackends, - listPostProcessors, - listValidators, - unregisterOcrBackend, -} from "npm:@kreuzberg/wasm@^4.0.0"; import { assertEquals } from "@std/assert"; +// @deno-types="../../crates/kreuzberg-wasm/dist/index.d.ts" +import { clearOcrBackends, clearPostProcessors, clearValidators, detectMimeFromBytes, getExtensionsForMime, initWasm, listOcrBackends, listPostProcessors, listValidators, unregisterOcrBackend } from "npm:@kreuzberg/wasm@^4.0.0"; await initWasm(); @@ -40,68 +29,60 @@ Deno.test({ name: "Unregister nonexistent document extractor gracefully", ignore // Mime Utilities Deno.test("Detect MIME type from file bytes", () => { - const testData = new TextEncoder().encode("%PDF-1.4\\n"); - const result = detectMimeFromBytes(testData); - assertEquals(result.toLowerCase().includes("pdf"), true); + const testData = new TextEncoder().encode("%PDF-1.4\\n"); + const result = detectMimeFromBytes(testData); + assertEquals(result.toLowerCase().includes("pdf"), true); }); Deno.test({ name: "Detect MIME type from file path", ignore: true, fn() {} }); Deno.test("Get file extensions for a MIME type", () => { - const result = getExtensionsForMime("application/pdf"); - assertEquals(Array.isArray(result), true); - assertEquals(result.includes("pdf"), true); + const result = getExtensionsForMime("application/pdf"); + assertEquals(Array.isArray(result), true); + assertEquals(result.includes("pdf"), true); }); // Ocr Backend Management Deno.test("Clear all OCR backends and verify list is empty", () => { - clearOcrBackends(); - const result = listOcrBackends(); - assertEquals(result.length, 0); + clearOcrBackends(); + const result = listOcrBackends(); + assertEquals(result.length, 0); }); Deno.test("List all registered OCR backends", () => { - const result = listOcrBackends(); - assertEquals(Array.isArray(result), true); - assertEquals( - result.every((item: unknown) => typeof item === "string"), - true, - ); + const result = listOcrBackends(); + assertEquals(Array.isArray(result), true); + assertEquals(result.every((item: unknown) => typeof item === "string"), true); }); Deno.test("Unregister nonexistent OCR backend gracefully", () => { - unregisterOcrBackend("nonexistent-backend-xyz"); + unregisterOcrBackend("nonexistent-backend-xyz"); }); // Post Processor Management Deno.test("Clear all post-processors and verify list is empty", () => { - clearPostProcessors(); + clearPostProcessors(); }); Deno.test("List all registered post-processors", () => { - const result = listPostProcessors(); - assertEquals(Array.isArray(result), true); - assertEquals( - result.every((item: unknown) => typeof item === "string"), - true, - ); + const result = listPostProcessors(); + assertEquals(Array.isArray(result), true); + assertEquals(result.every((item: unknown) => typeof item === "string"), true); }); // Validator Management Deno.test("Clear all validators and verify list is empty", () => { - clearValidators(); - const result = listValidators(); - assertEquals(result.length, 0); + clearValidators(); + const result = listValidators(); + assertEquals(result.length, 0); }); Deno.test("List all registered validators", () => { - const result = listValidators(); - assertEquals(Array.isArray(result), true); - assertEquals( - result.every((item: unknown) => typeof item === "string"), - true, - ); + const result = listValidators(); + assertEquals(Array.isArray(result), true); + assertEquals(result.every((item: unknown) => typeof item === "string"), true); }); + diff --git a/e2e/wasm-deno/structured.test.ts b/e2e/wasm-deno/structured.test.ts index befae7fd0d3..97348138eef 100644 --- a/e2e/wasm-deno/structured.test.ts +++ b/e2e/wasm-deno/structured.test.ts @@ -3,218 +3,211 @@ // Tests for structured fixtures. Run with: deno test --allow-read +import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; -import { - assertions, - buildConfig, - enableOcr, - extractBytes, - initWasm, - resolveDocument, - shouldSkipFixture, -} from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); await enableOcr(); Deno.test("structured_csv_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("csv/stanley_cups.csv"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/csv", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/csv"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("csv/stanley_cups.csv"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/csv", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/csv"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("structured_enw_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/sample.enw"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-endnote-refer", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/sample.enw"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-endnote-refer", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); }); Deno.test("structured_json_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("json/sample_document.json"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/json", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 20); - assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("json/sample_document.json"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/json", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 20); + assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); }); Deno.test("structured_json_simple", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("json/simple.json"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/json", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["{", "name"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("json/simple.json"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/json", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["{", "name"]); }); Deno.test("structured_nbib_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/sample.nbib"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/nbib", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/sample.nbib"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/nbib", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); + assertions.assertContentNotEmpty(result); }); Deno.test("structured_ris_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/sample.ris"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-research-info-systems", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/sample.ris"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-research-info-systems", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); + assertions.assertContentNotEmpty(result); }); Deno.test("structured_toml_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/cargo.toml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/toml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/cargo.toml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/toml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("structured_tsv_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/employees.tsv"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/tab-separated-values", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/employees.tsv"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/tab-separated-values", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("structured_yaml_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("yaml/simple.yaml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/yaml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("yaml/simple.yaml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/yaml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("structured_yaml_simple", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("yaml/simple.yaml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-yaml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-yaml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("yaml/simple.yaml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-yaml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-yaml"]); + assertions.assertMinContentLength(result, 10); }); + diff --git a/e2e/wasm-deno/xml.test.ts b/e2e/wasm-deno/xml.test.ts index 917a57ebc57..c75f1acc26c 100644 --- a/e2e/wasm-deno/xml.test.ts +++ b/e2e/wasm-deno/xml.test.ts @@ -3,37 +3,30 @@ // Tests for xml fixtures. Run with: deno test --allow-read +import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; -import { - assertions, - buildConfig, - enableOcr, - extractBytes, - initWasm, - resolveDocument, - shouldSkipFixture, -} from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time await initWasm(); await enableOcr(); Deno.test("xml_plant_catalog", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xml/plant_catalog.xml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/xml", config); - } catch (error) { - if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml"]); - assertions.assertMinContentLength(result, 100); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xml/plant_catalog.xml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/xml", config); + } catch (error) { + if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml"]); + assertions.assertMinContentLength(result, 100); }); + diff --git a/e2e/wasm-workers/tests/archive.spec.ts b/e2e/wasm-workers/tests/archive.spec.ts index 92e7a40f454..940e91ab256 100644 --- a/e2e/wasm-workers/tests/archive.spec.ts +++ b/e2e/wasm-workers/tests/archive.spec.ts @@ -3,105 +3,106 @@ // Tests for archive fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("archive", () => { - it("archive_gz_basic", async () => { - const documentBytes = getFixture("archives/book_war_and_peace_1p.txt.gz"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("archive_gz_basic", async () => { + const documentBytes = getFixture("archives/book_war_and_peace_1p.txt.gz"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/gzip", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); + assertions.assertMinContentLength(result, 10); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/gzip", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); - assertions.assertMinContentLength(result, 10); - }); + it("archive_sevenz_basic", async () => { + const documentBytes = getFixture("archives/documents.7z"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("archive_sevenz_basic", async () => { - const documentBytes = getFixture("archives/documents.7z"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-7z-compressed", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); + assertions.assertMinContentLength(result, 10); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-7z-compressed", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); - assertions.assertMinContentLength(result, 10); - }); + it("archive_tar_basic", async () => { + const documentBytes = getFixture("archives/documents.tar"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("archive_tar_basic", async () => { - const documentBytes = getFixture("archives/documents.tar"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-tar", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); + assertions.assertMinContentLength(result, 10); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-tar", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); - assertions.assertMinContentLength(result, 10); - }); + it("archive_zip_basic", async () => { + const documentBytes = getFixture("archives/documents.zip"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("archive_zip_basic", async () => { - const documentBytes = getFixture("archives/documents.zip"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/zip", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); + assertions.assertMinContentLength(result, 10); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/zip", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); - assertions.assertMinContentLength(result, 10); - }); }); diff --git a/e2e/wasm-workers/tests/email.spec.ts b/e2e/wasm-workers/tests/email.spec.ts index 4d5d0e35ef0..45403772123 100644 --- a/e2e/wasm-workers/tests/email.spec.ts +++ b/e2e/wasm-workers/tests/email.spec.ts @@ -3,153 +3,154 @@ // Tests for email fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("email", () => { - it("email_eml_html_body", async () => { - const documentBytes = getFixture("email/html_only.eml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("email_eml_html_body", async () => { + const documentBytes = getFixture("email/html_only.eml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); - }); + it("email_eml_multipart", async () => { + const documentBytes = getFixture("email/html_email_multipart.eml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("email_eml_multipart", async () => { - const documentBytes = getFixture("email/html_email_multipart.eml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); - }); + it("email_eml_utf16", async () => { + const documentBytes = getFixture("vendored/unstructured/eml/fake-email-utf-16.eml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("email_eml_utf16", async () => { - const documentBytes = getFixture("vendored/unstructured/eml/fake-email-utf-16.eml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); - }); + it("email_msg_basic", async () => { + const documentBytes = getFixture("email/fake_email.msg"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("email_msg_basic", async () => { - const documentBytes = getFixture("email/fake_email.msg"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/vnd.ms-outlook", config); + } catch (error) { + if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); + assertions.assertMinContentLength(result, 10); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/vnd.ms-outlook", config); - } catch (error) { - if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); - assertions.assertMinContentLength(result, 10); - }); + it("email_pst_empty", async () => { + const documentBytes = getFixture("email/empty.pst"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("email_pst_empty", async () => { - const documentBytes = getFixture("email/empty.pst"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/vnd.ms-outlook-pst", config); + } catch (error) { + if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/vnd.ms-outlook-pst", config); - } catch (error) { - if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); - }); + it("email_sample_eml", async () => { + const documentBytes = getFixture("email/sample_email.eml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("email_sample_eml", async () => { - const documentBytes = getFixture("email/sample_email.eml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 20); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 20); - }); }); diff --git a/e2e/wasm-workers/tests/embeddings.spec.ts b/e2e/wasm-workers/tests/embeddings.spec.ts index 885699a333b..50c2ad3bf5c 100644 --- a/e2e/wasm-workers/tests/embeddings.spec.ts +++ b/e2e/wasm-workers/tests/embeddings.spec.ts @@ -3,34 +3,35 @@ // Tests for embeddings fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("embeddings", () => { - it("embedding_disabled", async () => { - const documentBytes = getFixture("pdf/fake_memo.pdf"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("embedding_disabled", async () => { + const documentBytes = getFixture("pdf/fake_memo.pdf"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig({"chunking":{"max_chars":500,"max_overlap":50}}); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/octet-stream", config); + } catch (error) { + if (shouldSkipFixture(error, "embedding_disabled", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/pdf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertChunks(result, 1, null, true, false, null, null); + }); - const config = buildConfig({ chunking: { max_chars: 500, max_overlap: 50 } }); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/octet-stream", config); - } catch (error) { - if (shouldSkipFixture(error, "embedding_disabled", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/pdf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, false, null, null); - }); }); diff --git a/e2e/wasm-workers/tests/html.spec.ts b/e2e/wasm-workers/tests/html.spec.ts index 55463ea7374..cda5547695f 100644 --- a/e2e/wasm-workers/tests/html.spec.ts +++ b/e2e/wasm-workers/tests/html.spec.ts @@ -3,66 +3,59 @@ // Tests for html fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("html", () => { - it("html_complex_layout", async () => { - const documentBytes = getFixture("html/taylor_swift.html"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("html_complex_layout", async () => { + const documentBytes = getFixture("html/taylor_swift.html"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "text/html", config); + } catch (error) { + if (shouldSkipFixture(error, "html_complex_layout", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 1000); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "text/html", config); - } catch (error) { - if (shouldSkipFixture(error, "html_complex_layout", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 1000); - }); + it("html_simple_table", async () => { + const documentBytes = getFixture("html/simple_table.html"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - it("html_simple_table", async () => { - const documentBytes = getFixture("html/simple_table.html"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "text/html", config); + } catch (error) { + if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "text/html", config); - } catch (error) { - if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, [ - "Product", - "Category", - "Price", - "Stock", - "Laptop", - "Electronics", - "Sample Data Table", - ]); - }); }); diff --git a/e2e/wasm-workers/tests/image.spec.ts b/e2e/wasm-workers/tests/image.spec.ts index ca2d46a73cc..87897ae7255 100644 --- a/e2e/wasm-workers/tests/image.spec.ts +++ b/e2e/wasm-workers/tests/image.spec.ts @@ -3,249 +3,250 @@ // Tests for image fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("image", () => { - it("image_bmp_basic", async () => { - const documentBytes = getFixture("images/bmp_24.bmp"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/bmp", config); - } catch (error) { - if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/bmp"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_gif_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.gif"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/gif", config); - } catch (error) { - if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/gif"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_jp2_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.jp2"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/jp2", config); - } catch (error) { - if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_metadata_only", async () => { - const documentBytes = getFixture("images/example.jpg"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig({ ocr: null }); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/jpeg", config); - } catch (error) { - if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jpeg"]); - assertions.assertMaxContentLength(result, 200); - }); - - it("image_pbm_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.pbm"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/x-portable-bitmap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_pgm_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.pgm"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/x-portable-graymap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_ppm_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.ppm"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/x-portable-pixmap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_svg_basic", async () => { - const documentBytes = getFixture("xml/simple_svg.svg"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/svg+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/svg+xml"]); - assertions.assertMinContentLength(result, 5); - }); - - it("image_tiff_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.tif"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/tiff", config); - } catch (error) { - if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/tiff"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_webp_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.webp"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/webp", config); - } catch (error) { - if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/webp"]); - assertions.assertContentNotEmpty(result); - }); + it("image_bmp_basic", async () => { + const documentBytes = getFixture("images/bmp_24.bmp"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/bmp", config); + } catch (error) { + if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/bmp"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_gif_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.gif"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/gif", config); + } catch (error) { + if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/gif"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_jp2_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.jp2"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/jp2", config); + } catch (error) { + if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_metadata_only", async () => { + const documentBytes = getFixture("images/example.jpg"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig({"ocr":null}); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/jpeg", config); + } catch (error) { + if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jpeg"]); + assertions.assertMaxContentLength(result, 200); + }); + + it("image_pbm_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.pbm"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/x-portable-bitmap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_pgm_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.pgm"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/x-portable-graymap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_ppm_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.ppm"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/x-portable-pixmap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_svg_basic", async () => { + const documentBytes = getFixture("xml/simple_svg.svg"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/svg+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/svg+xml"]); + assertions.assertMinContentLength(result, 5); + }); + + it("image_tiff_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.tif"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/tiff", config); + } catch (error) { + if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/tiff"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_webp_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.webp"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/webp", config); + } catch (error) { + if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/webp"]); + assertions.assertContentNotEmpty(result); + }); + }); diff --git a/e2e/wasm-workers/tests/plugin-apis.spec.ts b/e2e/wasm-workers/tests/plugin-apis.spec.ts index 7236aa7fa44..7d027ff3321 100644 --- a/e2e/wasm-workers/tests/plugin-apis.spec.ts +++ b/e2e/wasm-workers/tests/plugin-apis.spec.ts @@ -6,79 +6,78 @@ * To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang wasm-workers */ -import { - clearOcrBackends, - clearPostProcessors, - clearValidators, - listOcrBackends, - listPostProcessors, - listValidators, - unregisterOcrBackend, -} from "@kreuzberg/wasm"; -import { describe, expect, it } from "vitest"; +import { describe, it, expect } from "vitest"; +import { clearOcrBackends, clearPostProcessors, clearValidators, listOcrBackends, listPostProcessors, listValidators, unregisterOcrBackend } from "@kreuzberg/wasm"; describe("Configuration", () => { - it.skip("Discover configuration from current or parent directories (not available in WASM)", () => {}); + it.skip("Discover configuration from current or parent directories (not available in WASM)", () => {}); + + it.skip("Load configuration from a TOML file (not available in WASM)", () => {}); - it.skip("Load configuration from a TOML file (not available in WASM)", () => {}); }); describe("Document Extractor Management", () => { - it.skip("Clear all document extractors and verify list is empty (not available in WASM)", () => {}); + it.skip("Clear all document extractors and verify list is empty (not available in WASM)", () => {}); + + it.skip("List all registered document extractors (not available in WASM)", () => {}); - it.skip("List all registered document extractors (not available in WASM)", () => {}); + it.skip("Unregister nonexistent document extractor gracefully (not available in WASM)", () => {}); - it.skip("Unregister nonexistent document extractor gracefully (not available in WASM)", () => {}); }); describe("Mime Utilities", () => { - it.skip("Detect MIME type from file bytes (not available in WASM)", () => {}); + it.skip("Detect MIME type from file bytes (not available in WASM)", () => {}); - it.skip("Detect MIME type from file path (not available in WASM)", () => {}); + it.skip("Detect MIME type from file path (not available in WASM)", () => {}); + + it.skip("Get file extensions for a MIME type (not available in WASM)", () => {}); - it.skip("Get file extensions for a MIME type (not available in WASM)", () => {}); }); describe("Ocr Backend Management", () => { - it("Clear all OCR backends and verify list is empty", () => { - clearOcrBackends(); - const result = listOcrBackends(); - expect(result).toHaveLength(0); - }); - - it("List all registered OCR backends", () => { - const result = listOcrBackends(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item: unknown) => typeof item === "string")).toBe(true); - }); - - it("Unregister nonexistent OCR backend gracefully", () => { - expect(() => unregisterOcrBackend("nonexistent-backend-xyz")).not.toThrow(); - }); + it("Clear all OCR backends and verify list is empty", () => { + clearOcrBackends(); + const result = listOcrBackends(); + expect(result).toHaveLength(0); + }); + + it("List all registered OCR backends", () => { + const result = listOcrBackends(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item: unknown) => typeof item === "string")).toBe(true); + }); + + it("Unregister nonexistent OCR backend gracefully", () => { + expect(() => unregisterOcrBackend("nonexistent-backend-xyz")).not.toThrow(); + }); + }); describe("Post Processor Management", () => { - it("Clear all post-processors and verify list is empty", () => { - clearPostProcessors(); - }); - - it("List all registered post-processors", () => { - const result = listPostProcessors(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item: unknown) => typeof item === "string")).toBe(true); - }); + it("Clear all post-processors and verify list is empty", () => { + clearPostProcessors(); + }); + + it("List all registered post-processors", () => { + const result = listPostProcessors(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item: unknown) => typeof item === "string")).toBe(true); + }); + }); describe("Validator Management", () => { - it("Clear all validators and verify list is empty", () => { - clearValidators(); - const result = listValidators(); - expect(result).toHaveLength(0); - }); - - it("List all registered validators", () => { - const result = listValidators(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item: unknown) => typeof item === "string")).toBe(true); - }); + it("Clear all validators and verify list is empty", () => { + clearValidators(); + const result = listValidators(); + expect(result).toHaveLength(0); + }); + + it("List all registered validators", () => { + const result = listValidators(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item: unknown) => typeof item === "string")).toBe(true); + }); + }); + diff --git a/e2e/wasm-workers/tests/structured.spec.ts b/e2e/wasm-workers/tests/structured.spec.ts index 53935b15c95..7cc23b43f1e 100644 --- a/e2e/wasm-workers/tests/structured.spec.ts +++ b/e2e/wasm-workers/tests/structured.spec.ts @@ -3,250 +3,251 @@ // Tests for structured fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("structured", () => { - it("structured_csv_basic", async () => { - const documentBytes = getFixture("csv/stanley_cups.csv"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "text/csv", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/csv"]); - assertions.assertMinContentLength(result, 20); - }); - - it("structured_enw_basic", async () => { - const documentBytes = getFixture("data_formats/sample.enw"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-endnote-refer", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); - }); - - it("structured_json_basic", async () => { - const documentBytes = getFixture("json/sample_document.json"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/json", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 20); - assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); - }); - - it("structured_json_simple", async () => { - const documentBytes = getFixture("json/simple.json"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/json", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["{", "name"]); - }); - - it("structured_nbib_basic", async () => { - const documentBytes = getFixture("data_formats/sample.nbib"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/nbib", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); - assertions.assertContentNotEmpty(result); - }); - - it("structured_ris_basic", async () => { - const documentBytes = getFixture("data_formats/sample.ris"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-research-info-systems", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); - assertions.assertContentNotEmpty(result); - }); - - it("structured_toml_basic", async () => { - const documentBytes = getFixture("data_formats/cargo.toml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/toml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); - assertions.assertMinContentLength(result, 10); - }); - - it("structured_tsv_basic", async () => { - const documentBytes = getFixture("data_formats/employees.tsv"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "text/tab-separated-values", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); - assertions.assertMinContentLength(result, 10); - }); - - it("structured_yaml_basic", async () => { - const documentBytes = getFixture("yaml/simple.yaml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/yaml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); - assertions.assertMinContentLength(result, 10); - }); - - it("structured_yaml_simple", async () => { - const documentBytes = getFixture("yaml/simple.yaml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-yaml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-yaml"]); - assertions.assertMinContentLength(result, 10); - }); + it("structured_csv_basic", async () => { + const documentBytes = getFixture("csv/stanley_cups.csv"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "text/csv", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/csv"]); + assertions.assertMinContentLength(result, 20); + }); + + it("structured_enw_basic", async () => { + const documentBytes = getFixture("data_formats/sample.enw"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-endnote-refer", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); + }); + + it("structured_json_basic", async () => { + const documentBytes = getFixture("json/sample_document.json"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/json", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 20); + assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); + }); + + it("structured_json_simple", async () => { + const documentBytes = getFixture("json/simple.json"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/json", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["{", "name"]); + }); + + it("structured_nbib_basic", async () => { + const documentBytes = getFixture("data_formats/sample.nbib"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/nbib", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); + assertions.assertContentNotEmpty(result); + }); + + it("structured_ris_basic", async () => { + const documentBytes = getFixture("data_formats/sample.ris"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-research-info-systems", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); + assertions.assertContentNotEmpty(result); + }); + + it("structured_toml_basic", async () => { + const documentBytes = getFixture("data_formats/cargo.toml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/toml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); + assertions.assertMinContentLength(result, 10); + }); + + it("structured_tsv_basic", async () => { + const documentBytes = getFixture("data_formats/employees.tsv"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "text/tab-separated-values", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); + assertions.assertMinContentLength(result, 10); + }); + + it("structured_yaml_basic", async () => { + const documentBytes = getFixture("yaml/simple.yaml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/yaml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); + assertions.assertMinContentLength(result, 10); + }); + + it("structured_yaml_simple", async () => { + const documentBytes = getFixture("yaml/simple.yaml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-yaml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-yaml"]); + assertions.assertMinContentLength(result, 10); + }); + }); diff --git a/e2e/wasm-workers/tests/xml.spec.ts b/e2e/wasm-workers/tests/xml.spec.ts index fd864cbb062..7222cf375d7 100644 --- a/e2e/wasm-workers/tests/xml.spec.ts +++ b/e2e/wasm-workers/tests/xml.spec.ts @@ -3,33 +3,34 @@ // Tests for xml fixtures. Cloudflare Workers with Vitest + Miniflare -import type { ExtractionResult } from "@kreuzberg/wasm"; -import { extractBytes } from "@kreuzberg/wasm"; import { describe, it } from "vitest"; +import { extractBytes } from "@kreuzberg/wasm"; import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helpers.js"; +import type { ExtractionResult } from "@kreuzberg/wasm"; describe("xml", () => { - it("xml_plant_catalog", async () => { - const documentBytes = getFixture("xml/plant_catalog.xml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("xml_plant_catalog", async () => { + const documentBytes = getFixture("xml/plant_catalog.xml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/xml", config); + } catch (error) { + if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml"]); + assertions.assertMinContentLength(result, 100); + }); - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/xml", config); - } catch (error) { - if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml"]); - assertions.assertMinContentLength(result, 100); - }); }); diff --git a/e2e/wasm-workers/vitest.config.ts b/e2e/wasm-workers/vitest.config.ts index b085d5ed386..ef4fdf8df08 100644 --- a/e2e/wasm-workers/vitest.config.ts +++ b/e2e/wasm-workers/vitest.config.ts @@ -1,5 +1,5 @@ -import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; import { defineConfig } from "vitest/config"; +import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; export default defineConfig({ plugins: [ From 99600ff533404835f80443e5afa1730259b92ecf Mon Sep 17 00:00:00 2001 From: kh3rld Date: Tue, 31 Mar 2026 08:47:02 -0400 Subject: [PATCH 06/12] docs(changelog): add entry for semantic chunk labeling (#600) --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b3a4d7444..21e48cf4fcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [4.7.0] - 2026-03-30 + ### Added +- **Semantic chunk labeling** (#600): Chunks now include a `chunk_type` field identifying the semantic nature of the content (e.g., `paragraph`, `heading`, `list_item`, `table_cell`, `code_block`). Supported across all 11 language bindings with updated E2E test parity. - **Unified InternalDocument architecture**: All extractors now return a canonical `InternalDocument` with typed elements, relationships, images, and tables. Replaces format-specific intermediate representations. - **Unified rendering layer**: New `new_markdown.rs` renderer produces CommonMark from `InternalDocument`, supporting headings, lists, tables, code blocks, formulas, footnotes, images, and inline annotations (bold, italic, links). - **PDF structure pipeline**: Full rewrite of PDF extraction using `page.text().all()` for clean text, char-indexed font metadata for heading/bold detection, segment-based paragraph gap detection, and pdfium segment bounding boxes for precise paragraph regions. From 1807007dc47f64bebb26f0ba1357cae6c0ac83da Mon Sep 17 00:00:00 2001 From: kh3rld Date: Tue, 31 Mar 2026 09:00:49 -0400 Subject: [PATCH 07/12] fix(chunking): fix indented code-block and heading-context classifier bugs - Pass original content (pre-trim) to is_code_block so leading 4-space indentation is not stripped from the first line before the check - Remove over-broad heading rule that classified short body text as a Heading when it appeared under a single-level heading context Both bugs were caught by the classifier's own unit tests. --- crates/kreuzberg/src/chunking/classifier.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/kreuzberg/src/chunking/classifier.rs b/crates/kreuzberg/src/chunking/classifier.rs index db17ab05d3f..183b9762e6d 100644 --- a/crates/kreuzberg/src/chunking/classifier.rs +++ b/crates/kreuzberg/src/chunking/classifier.rs @@ -50,7 +50,9 @@ pub fn classify_chunk(content: &str, heading_context: Option<&HeadingContext>) - } // ── 2. Code block ─────────────────────────────────────────────────────── - if is_code_block(trimmed) { + // Use original content (not trimmed) so leading-indented blocks retain + // their 4-space prefix on every line. + if is_code_block(content) { return ChunkType::CodeBlock; } @@ -94,7 +96,7 @@ pub fn classify_chunk(content: &str, heading_context: Option<&HeadingContext>) - // ─── Rule implementations ─────────────────────────────────────────────────── -fn is_heading(content: &str, ctx: Option<&HeadingContext>) -> bool { +fn is_heading(content: &str, _ctx: Option<&HeadingContext>) -> bool { // Markdown ATX heading if content.starts_with('#') { return true; @@ -107,12 +109,6 @@ fn is_heading(content: &str, ctx: Option<&HeadingContext>) -> bool { return true; } } - // Single top-level heading context with very short content (≤ 120 chars) - if let Some(ctx) = ctx { - if ctx.headings.len() == 1 && content.len() <= 120 { - return true; - } - } false } From 6a2475923216216a2ca7bd3852e4ff8ae823fe7f Mon Sep 17 00:00:00 2001 From: kh3rld Date: Tue, 31 Mar 2026 16:52:18 -0400 Subject: [PATCH 08/12] fix: resolve the failing doctest (wrong example text for OperativeClause) --- crates/kreuzberg/src/chunking/classifier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/kreuzberg/src/chunking/classifier.rs b/crates/kreuzberg/src/chunking/classifier.rs index 183b9762e6d..4db11d1318b 100644 --- a/crates/kreuzberg/src/chunking/classifier.rs +++ b/crates/kreuzberg/src/chunking/classifier.rs @@ -34,7 +34,7 @@ use crate::types::{ChunkType, HeadingContext}; /// /// assert_eq!(classify_chunk("# Introduction", None), ChunkType::Heading); /// assert_eq!( -/// classify_chunk("The Investor shall subscribe and pay...", None), +/// classify_chunk("The Investor shall subscribe for the Shares and agrees to pay the subscription price. The Company shall deliver the Share certificates upon receipt.", None), /// ChunkType::OperativeClause, /// ); /// assert_eq!(classify_chunk("Some unrecognized text.", None), ChunkType::Unknown); From 8bfad9a5718cc1e67b49bc4aa5a7c9e4c91643c6 Mon Sep 17 00:00:00 2001 From: kh3rld Date: Tue, 31 Mar 2026 16:52:58 -0400 Subject: [PATCH 09/12] feat: create chunk_type: Default::default() field (needed for Chunk struct) --- tools/benchmark-harness/src/embed_benchmark.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/benchmark-harness/src/embed_benchmark.rs b/tools/benchmark-harness/src/embed_benchmark.rs index 21c4426fc74..8b0d751702a 100644 --- a/tools/benchmark-harness/src/embed_benchmark.rs +++ b/tools/benchmark-harness/src/embed_benchmark.rs @@ -158,6 +158,7 @@ fn generate_test_chunks(count: usize, words_per_chunk: usize) -> Vec { Chunk { content: text, embedding: None, + chunk_type: Default::default(), metadata: ChunkMetadata { byte_start: 0, byte_end, From d41d672858f6524572401c095321b1210297583b Mon Sep 17 00:00:00 2001 From: kh3rld Date: Wed, 1 Apr 2026 23:45:40 -0400 Subject: [PATCH 10/12] fix(chunking): address review feedback - complete chunk_type bindings and e2e coverage - Add chunk_type field to all language package bindings: C# (ChunkType property), Elixir (chunk_type field), Go (ChunkType string), Java (chunkType + getChunkType()), TypeScript (chunkType + ChunkType type) - Add ChunkType documentation type to TypeScript core types - Add each_has_chunk_type assertion to all e2e helpers: TypeScript, wasm-deno, wasm-workers, Java, Ruby, R, PHP, C# (was done), Elixir (was done), Go (was done), Python (was done) - Update generator source files (wasm_deno.rs, wasm_workers.rs, r/helpers.rs, python.rs) to emit chunk_type in tests and helpers - Sync all generator snapshot templates (tools/e2e-generator/e2e/) to match the embedded generator templates - Add fixture schema entries for each_has_chunk_type and content_starts_with_heading - Add test_chunk_type_populated context test to config_features.rs - Remove wrongly-placed e2e/e2e/php/ duplicate files (were generated from wrong working directory) closes reviewer feedback on #623 --- crates/kreuzberg/tests/config_features.rs | 67 + .../php/tests/E2EPhp/Tests/ArchiveTest.php | 94 -- .../php/tests/E2EPhp/Tests/ContractTest.php | 1318 ----------------- e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php | 132 -- .../php/tests/E2EPhp/Tests/EmbeddingsTest.php | 105 -- e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php | 57 - e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php | 224 --- .../php/tests/E2EPhp/Tests/KeywordsTest.php | 62 - e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php | 396 ----- e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php | 1007 ------------- e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php | 388 ----- .../php/tests/E2EPhp/Tests/PluginApisTest.php | 208 --- e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php | 60 - e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php | 177 --- .../php/tests/E2EPhp/Tests/StructuredTest.php | 209 --- .../E2EPhp/Tests/Token_reductionTest.php | 102 -- e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php | 37 - e2e/e2e/php/tests/Helpers.php | 831 ----------- .../java/com/kreuzberg/e2e/ContractTest.java | 16 +- .../java/com/kreuzberg/e2e/E2EHelpers.java | 8 + .../com/kreuzberg/e2e/EmbeddingsTest.java | 8 +- .../com/kreuzberg/e2e/TokenReductionTest.java | 2 +- e2e/php/tests/ContractTest.php | 16 +- e2e/php/tests/EmbeddingsTest.php | 8 +- e2e/php/tests/Helpers.php | 8 + e2e/php/tests/Token_reductionTest.php | 2 +- e2e/r/tests/testthat/helper-kreuzberg.R | 9 + e2e/ruby/spec/helpers.rb | 8 +- e2e/typescript/tests/contract.spec.ts | 16 +- e2e/typescript/tests/embeddings.spec.ts | 8 +- e2e/typescript/tests/helpers.ts | 8 + e2e/typescript/tests/token-reduction.spec.ts | 2 +- e2e/wasm-deno/contract.test.ts | 14 +- e2e/wasm-deno/embeddings.test.ts | 2 +- e2e/wasm-deno/helpers.ts | 8 + e2e/wasm-deno/token_reduction.test.ts | 2 +- e2e/wasm-workers/tests/contract.spec.ts | 14 +- e2e/wasm-workers/tests/embeddings.spec.ts | 2 +- e2e/wasm-workers/tests/helpers.ts | 8 + .../tests/token_reduction.spec.ts | 2 +- fixtures/schema.json | 8 + packages/csharp/Kreuzberg/Models.cs | 6 + packages/elixir/lib/kreuzberg/chunk.ex | 19 +- packages/go/v4/types.go | 1 + .../src/main/java/dev/kreuzberg/Chunk.java | 14 +- packages/typescript/core/src/types/results.ts | 16 + tools/e2e-generator/e2e/go/helpers_test.go | 16 +- .../java/com/kreuzberg/e2e/E2EHelpers.java | 20 +- tools/e2e-generator/e2e/php/tests/Helpers.php | 20 +- .../e2e-generator/e2e/python/tests/helpers.py | 16 + .../e2e/r/tests/testthat/helper-kreuzberg.R | 16 +- tools/e2e-generator/e2e/ruby/spec/helpers.rb | 11 +- .../e2e/typescript/tests/helpers.ts | 8 + tools/e2e-generator/e2e/wasm-deno/helpers.ts | 19 + .../e2e/wasm-workers/tests/helpers.ts | 14 + tools/e2e-generator/src/python.rs | 6 + tools/e2e-generator/src/r/assertions.rs | 6 + tools/e2e-generator/src/r/helpers.rs | 9 + tools/e2e-generator/src/wasm_deno.rs | 14 +- tools/e2e-generator/src/wasm_workers.rs | 14 +- 60 files changed, 415 insertions(+), 5483 deletions(-) delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/ArchiveTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/ContractTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/EmbeddingsTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/KeywordsTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/PluginApisTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/StructuredTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/Token_reductionTest.php delete mode 100644 e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php delete mode 100644 e2e/e2e/php/tests/Helpers.php diff --git a/crates/kreuzberg/tests/config_features.rs b/crates/kreuzberg/tests/config_features.rs index ea087a166d0..e1e34940b54 100644 --- a/crates/kreuzberg/tests/config_features.rs +++ b/crates/kreuzberg/tests/config_features.rs @@ -548,6 +548,73 @@ More detailed content here in the subsection. } } +/// Test that chunk_type is populated for markdown chunks. +#[tokio::test] +#[cfg(feature = "chunking")] +async fn test_chunk_type_populated() { + use kreuzberg::ChunkType; + + let markdown = r#"# Introduction + +This section introduces the document with some content. + +## Code Example + +```rust +fn hello() { + println!("Hello, world!"); +} +``` + +## Summary + +A brief summary of the document. +"#; + + let config = ExtractionConfig { + chunking: Some(ChunkingConfig { + max_characters: 200, + overlap: 0, + chunker_type: kreuzberg::ChunkerType::Markdown, + ..Default::default() + }), + ..Default::default() + }; + + let result = extract_bytes(markdown.as_bytes(), "text/markdown", &config) + .await + .expect("Should extract successfully"); + + assert!(result.chunks.is_some(), "Chunks should be present"); + let chunks = result.chunks.expect("Should have chunks"); + assert!(!chunks.is_empty(), "Should have at least one chunk"); + + // All chunks must have a chunk_type (not the default Unknown when content is classifiable) + for chunk in &chunks { + // chunk_type must always be set (never uninitialized) + let _ = &chunk.chunk_type; + } + + // At least one chunk should be classified as something other than Unknown + let has_classified = chunks + .iter() + .any(|c| !matches!(c.chunk_type, ChunkType::Unknown)); + assert!(has_classified, "At least one chunk should have a non-Unknown chunk_type"); + + // Heading chunks should be classified as Heading + let heading_chunks: Vec<_> = chunks + .iter() + .filter(|c| c.content.starts_with('#')) + .collect(); + for chunk in &heading_chunks { + assert!( + matches!(chunk.chunk_type, ChunkType::Heading), + "Chunk starting with '#' should be classified as Heading, got {:?}", + chunk.chunk_type + ); + } +} + /// Test tokenizer env var override. #[tokio::test] #[cfg(feature = "chunking-tokenizers")] diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/ArchiveTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/ArchiveTest.php deleted file mode 100644 index e19142b6ac8..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/ArchiveTest.php +++ /dev/null @@ -1,94 +0,0 @@ -markTestSkipped('Skipping archive_gz_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/gzip', 'application/x-gzip']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * 7-Zip archive extraction. - */ - public function test_archive_sevenz_basic(): void - { - $documentPath = Helpers::resolveDocument('archives/documents.7z'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping archive_sevenz_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-7z-compressed']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * TAR archive extraction. - */ - public function test_archive_tar_basic(): void - { - $documentPath = Helpers::resolveDocument('archives/documents.tar'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping archive_tar_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-tar', 'application/tar']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * ZIP archive extraction. - */ - public function test_archive_zip_basic(): void - { - $documentPath = Helpers::resolveDocument('archives/documents.zip'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping archive_zip_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/zip', 'application/x-zip-compressed']); - Helpers::assertMinContentLength($result, 10); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/ContractTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/ContractTest.php deleted file mode 100644 index 21794da2757..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/ContractTest.php +++ /dev/null @@ -1,1318 +0,0 @@ -markTestSkipped('Skipping api_batch_bytes_async: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $bytes = file_get_contents($documentPath); - $mimeType = Kreuzberg::detectMimeType($bytes); - $deferred = $kreuzberg->batchExtractBytesAsync([$bytes], [$mimeType]); - $results = $deferred->getResults(); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); - } - - /** - * Tests sync batch bytes extraction API (batch_extract_bytes_sync) - */ - public function test_api_batch_bytes_sync(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_batch_bytes_sync: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $bytes = file_get_contents($documentPath); - $mimeType = Kreuzberg::detectMimeType($bytes); - $results = $kreuzberg->batchExtractBytes([$bytes], [$mimeType]); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); - } - - /** - * Tests async batch bytes extraction with per-file configs (batch_extract_bytes with file_configs parameter) - */ - public function test_api_batch_bytes_with_configs_async(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_batch_bytes_with_configs_async: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $bytes = file_get_contents($documentPath); - $mimeType = Kreuzberg::detectMimeType($bytes); - $deferred = $kreuzberg->batchExtractBytesAsync([$bytes], [$mimeType]); - $results = $deferred->getResults(); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests sync batch bytes extraction with per-file configs (batch_extract_bytes_sync with file_configs parameter) - */ - public function test_api_batch_bytes_with_configs_sync(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_batch_bytes_with_configs_sync: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $bytes = file_get_contents($documentPath); - $mimeType = Kreuzberg::detectMimeType($bytes); - $results = $kreuzberg->batchExtractBytes([$bytes], [$mimeType]); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests async batch file extraction API (batch_extract_file) - */ - public function test_api_batch_file_async(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_batch_file_async: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $deferred = $kreuzberg->batchExtractFilesAsync([$documentPath]); - $results = $deferred->getResults(); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); - } - - /** - * Tests sync batch file extraction API (batch_extract_file_sync) - */ - public function test_api_batch_file_sync(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_batch_file_sync: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $results = $kreuzberg->batchExtractFiles([$documentPath]); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); - } - - /** - * Tests async batch file extraction with per-file configs (batch_extract_files with file_configs parameter) - */ - public function test_api_batch_file_with_configs_async(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_batch_file_with_configs_async: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $deferred = $kreuzberg->batchExtractFilesAsync([$documentPath]); - $results = $deferred->getResults(); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests sync batch file extraction with per-file configs (batch_extract_files_sync with file_configs parameter) - */ - public function test_api_batch_file_with_configs_sync(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_batch_file_with_configs_sync: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $results = $kreuzberg->batchExtractFiles([$documentPath]); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests sync batch file extraction with per-file timeout config override - */ - public function test_api_batch_file_with_timeout_sync(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_batch_file_with_timeout_sync: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['extraction_timeout_secs' => 300]); - - $kreuzberg = new Kreuzberg($config); - $results = $kreuzberg->batchExtractFiles([$documentPath]); - $result = $results[0]; - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests async bytes extraction API (extract_bytes) - */ - public function test_api_extract_bytes_async(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_extract_bytes_async: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $bytes = file_get_contents($documentPath); - $mimeType = Kreuzberg::detectMimeType($bytes); - $deferred = $kreuzberg->extractBytesAsync($bytes, $mimeType); - $result = $deferred->getResult(); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); - } - - /** - * Tests sync bytes extraction API (extract_bytes_sync) - */ - public function test_api_extract_bytes_sync(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_extract_bytes_sync: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $bytes = file_get_contents($documentPath); - $mimeType = Kreuzberg::detectMimeType($bytes); - $result = $kreuzberg->extractBytes($bytes, $mimeType); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); - } - - /** - * Tests async file extraction API (extract_file) - */ - public function test_api_extract_file_async(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_extract_file_async: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $deferred = $kreuzberg->extractFileAsync($documentPath); - $result = $deferred->getResult(); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); - } - - /** - * Tests sync file extraction API (extract_file_sync) - */ - public function test_api_extract_file_sync(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping api_extract_file_sync: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'Mallori']); - } - - /** - * Tests explicit CPU acceleration provider configuration - */ - public function test_config_acceleration_cpu_provider(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_acceleration_cpu_provider: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['acceleration' => ['device_id' => 0, 'provider' => 'cpu']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'To Whom it May Concern']); - } - - /** - * Tests chunking configuration with chunk assertions - */ - public function test_config_chunking(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_chunking: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['chunking' => ['max_chars' => 500, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); - } - - /** - * Tests markdown chunker populates heading context on chunks - */ - public function test_config_chunking_heading_context(): void - { - $documentPath = Helpers::resolveDocument('markdown/extraction_test.md'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_chunking_heading_context: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('chunking'); - - $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'markdown', 'max_chars' => 300, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, true, null); - } - - /** - * Tests markdown-aware chunker type - */ - public function test_config_chunking_markdown(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_chunking_markdown: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('chunking'); - - $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'markdown', 'max_chars' => 500, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); - } - - /** - * Tests markdown chunker on text with no headings produces null heading_context - */ - public function test_config_chunking_no_headings(): void - { - $documentPath = Helpers::resolveDocument('text/book_war_and_peace_1p.txt'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_chunking_no_headings: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('chunking'); - - $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'markdown', 'max_chars' => 300, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, false, null); - } - - /** - * Tests markdown chunker prepends heading hierarchy to chunk content - */ - public function test_config_chunking_prepend_heading_context(): void - { - $documentPath = Helpers::resolveDocument('markdown/extraction_test.md'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_chunking_prepend_heading_context: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('chunking'); - - $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'markdown', 'max_chars' => 300, 'max_overlap' => 50, 'prepend_heading_context' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, true, true); - } - - /** - * Tests chunking with very small chunk size produces more chunks - */ - public function test_config_chunking_small(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_chunking_small: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('chunking'); - - $config = Helpers::buildConfig(['chunking' => ['max_chars' => 100, 'max_overlap' => 20]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, null, null); - } - - /** - * Tests text chunker type (generic whitespace/punctuation splitter) - */ - public function test_config_chunking_text(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_chunking_text: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['chunking' => ['chunker_type' => 'text', 'max_chars' => 500, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); - } - - /** - * Tests token-based chunk sizing with HuggingFace tokenizer - */ - public function test_config_chunking_tokenizer(): void - { - $documentPath = Helpers::resolveDocument('markdown/comprehensive.md'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_chunking_tokenizer: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('chunking-tokenizers'); - - $config = Helpers::buildConfig(['chunking' => ['max_chars' => 200, 'max_overlap' => 40, 'sizing' => ['model' => 'Xenova/gpt-4o', 'type' => 'tokenizer']]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, null, null); - } - - /** - * Tests djot output format converts content to djot markup - */ - public function test_config_djot_content(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_djot_content: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['output_format' => 'djot']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests include_document_structure config produces document tree - */ - public function test_config_document_structure(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_document_structure: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['include_document_structure' => true]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertDocument($result, true, 1, ['paragraph'], null); - } - - /** - * Tests document field is null when include_document_structure is false - */ - public function test_config_document_structure_disabled(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_document_structure_disabled: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertDocument($result, false, null, null, null); - } - - /** - * Tests document structure extraction with group node assertion on DOCX with headings - */ - public function test_config_document_structure_groups(): void - { - $documentPath = Helpers::resolveDocument('docx/unit_test_headers.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_document_structure_groups: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('office'); - - $config = Helpers::buildConfig(['include_document_structure' => true]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertDocument($result, true, null, null, true); - } - - /** - * Tests document structure extraction with heading nodes on a DOCX - */ - public function test_config_document_structure_headings(): void - { - $documentPath = Helpers::resolveDocument('docx/unit_test_headers.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_document_structure_headings: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('office'); - - $config = Helpers::buildConfig(['include_document_structure' => true]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertDocument($result, true, 1, ['heading', 'paragraph'], null); - } - - /** - * Tests document structure with DOCX heading-driven nesting - */ - public function test_config_document_structure_with_headings(): void - { - $documentPath = Helpers::resolveDocument('docx/fake.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_document_structure_with_headings: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['include_document_structure' => true]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertDocument($result, true, 1, null, null); - } - - /** - * Tests element-based result format with element type assertions on DOCX - */ - public function test_config_element_types(): void - { - $documentPath = Helpers::resolveDocument('docx/unit_test_headers.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_element_types: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('office'); - - $config = Helpers::buildConfig(['result_format' => 'element_based']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertElements($result, 1, ['narrative_text']); - } - - /** - * Tests MSG extraction with custom fallback codepage for Cyrillic - */ - public function test_config_email_msg_fallback_codepage(): void - { - $documentPath = Helpers::resolveDocument('email/fake_email.msg'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_email_msg_fallback_codepage: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['email' => ['msg_fallback_codepage' => 1251]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.ms-outlook']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests that extraction_timeout_secs config field is accepted and does not affect fast extractions - */ - public function test_config_extraction_timeout(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_extraction_timeout: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['extraction_timeout_secs' => 300]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests force_ocr configuration option - */ - public function test_config_force_ocr(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_force_ocr: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * Tests that force_ocr_pages config field is accepted for selective page OCR - */ - public function test_config_force_ocr_pages(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_force_ocr_pages: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('ocr'); - - $config = Helpers::buildConfig(['force_ocr_pages' => [1], 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 1); - } - - /** - * Tests extraction with HTML conversion options configured - */ - public function test_config_html_options(): void - { - $documentPath = Helpers::resolveDocument('html/complex_table.html'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_html_options: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['html_options' => ['extract_metadata' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/html']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentNotEmpty($result); - } - - /** - * Tests image extraction configuration with image assertions - */ - public function test_config_images(): void - { - $documentPath = Helpers::resolveDocument('pdf/embedded_images_tables.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_images: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['images' => ['extract_images' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertImages($result, 1, null, null); - } - - /** - * Tests image extraction on PPTX containing embedded images - */ - public function test_config_images_with_formats(): void - { - $documentPath = Helpers::resolveDocument('pptx/powerpoint_with_image.pptx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_images_with_formats: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['images' => ['extract_images' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.presentation']); - Helpers::assertImages($result, 1, null, null); - } - - /** - * Tests keyword extraction via YAKE algorithm - */ - public function test_config_keywords(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_keywords: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('keywords-yake'); - - $config = Helpers::buildConfig(['keywords' => ['algorithm' => 'yake', 'max_keywords' => 10]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertKeywords($result, true, 1, null); - } - - /** - * Tests language detection configuration - */ - public function test_config_language_detection(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_language_detection: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['language_detection' => ['enabled' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertDetectedLanguages($result, ['eng'], 0.5); - } - - /** - * Tests language detection with detect_multiple and min_confidence options - */ - public function test_config_language_detection_multi(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_language_detection_multi: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['language_detection' => ['detect_multiple' => true, 'enabled' => true, 'min_confidence' => 0.3]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertDetectedLanguages($result, ['eng'], null); - } - - /** - * Tests multi-language detection config - */ - public function test_config_language_multi(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_language_multi: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('language-detection'); - - $config = Helpers::buildConfig(['language_detection' => ['detect_multiple' => true, 'enabled' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertDetectedLanguages($result, ['eng'], null); - } - - /** - * Tests page extraction and page marker configuration - */ - public function test_config_pages(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_pages: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['pages' => ['extract_pages' => true, 'insert_page_markers' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['PAGE']); - } - - /** - * Tests page extraction with exact page count assertion on multi-page PDF - */ - public function test_config_pages_exact_count(): void - { - $documentPath = Helpers::resolveDocument('pdf/multi_page.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_pages_exact_count: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['pages' => ['extract_pages' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertPages($result, null, 5); - } - - /** - * Tests page extraction config producing per-page content array - */ - public function test_config_pages_extract(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_pages_extract: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['pages' => ['extract_pages' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertPages($result, 1, null); - } - - /** - * Tests page marker insertion in extracted content - */ - public function test_config_pages_markers(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_pages_markers: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['pages' => ['insert_page_markers' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['PAGE']); - } - - /** - * Tests PDF annotation extraction with min_count assertion - */ - public function test_config_pdf_annotations_count(): void - { - $documentPath = Helpers::resolveDocument('vendored/pdfplumber/pdf/annotations.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_pdf_annotations_count: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['pdf_options' => ['extract_annotations' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertAnnotations($result, true, 3); - } - - /** - * Tests PDF hierarchy extraction config with block-level structure - */ - public function test_config_pdf_hierarchy(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_pdf_hierarchy: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['pages' => ['extract_pages' => true], 'pdf_options' => ['hierarchy' => ['enabled' => true, 'include_bbox' => true]]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - } - - /** - * Tests PDF margin exclusion configuration - */ - public function test_config_pdf_margins(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_pdf_margins: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['pdf_options' => ['bottom_margin_fraction' => 0.1, 'top_margin_fraction' => 0.1]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * Tests postprocessor config is accepted and extraction succeeds - */ - public function test_config_postprocessor(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_postprocessor: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['postprocessor' => ['enabled' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentNotEmpty($result); - } - - /** - * Tests that a clean PDF extraction produces no processing warnings - */ - public function test_config_processing_warnings_empty(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_processing_warnings_empty: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertProcessingWarnings($result, null, true); - } - - /** - * Tests extraction with quality processing explicitly disabled - */ - public function test_config_quality_disabled(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_quality_disabled: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['enable_quality_processing' => false]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentNotEmpty($result); - } - - /** - * Tests quality scoring produces a score value in [0.0, 1.0] - */ - public function test_config_quality_enabled(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_quality_enabled: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('quality'); - - $config = Helpers::buildConfig(['enable_quality_processing' => true]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertQualityScore($result, true, 0, 1); - } - - /** - * Tests quality scoring produces a score with minimum bound assertion - */ - public function test_config_quality_score_range(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_quality_score_range: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('quality'); - - $config = Helpers::buildConfig(['enable_quality_processing' => true]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertQualityScore($result, true, 0.1, null); - } - - /** - * Tests archive extraction with custom security limits - */ - public function test_config_security_limits(): void - { - $documentPath = Helpers::resolveDocument('archives/documents.zip'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_security_limits: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['security_limits' => ['max_archive_size' => 104857600, 'max_compression_ratio' => 50, 'max_files_in_archive' => 100]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/zip', 'application/x-zip-compressed']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests structured (JSON) output format config - */ - public function test_config_structured_output(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_structured_output: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['output_format' => 'structured']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests table extraction with content_contains_any assertion on DOCX with tables - */ - public function test_config_tables_content(): void - { - $documentPath = Helpers::resolveDocument('docx/docx_tables.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_tables_content: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertTableCount($result, 1, null); - } - - /** - * Tests use_cache=false configuration option - */ - public function test_config_use_cache_false(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping config_use_cache_false: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['use_cache' => false]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests markdown output format via bytes extraction API - */ - public function test_output_format_bytes_markdown(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping output_format_bytes_markdown: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['output_format' => 'markdown']); - - $kreuzberg = new Kreuzberg($config); - $bytes = file_get_contents($documentPath); - $mimeType = Kreuzberg::detectMimeType($bytes); - $result = $kreuzberg->extractBytes($bytes, $mimeType); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests Djot output format - */ - public function test_output_format_djot(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping output_format_djot: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['output_format' => 'djot']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests HTML output format - */ - public function test_output_format_html(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping output_format_html: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['output_format' => 'html']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests Markdown output format - */ - public function test_output_format_markdown(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping output_format_markdown: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['output_format' => 'markdown']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests Plain output format - */ - public function test_output_format_plain(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping output_format_plain: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['output_format' => 'plain']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Tests ElementBased result format with element assertions - */ - public function test_result_format_element_based(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping result_format_element_based: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['result_format' => 'element_based']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertElements($result, 1, null); - } - - /** - * Tests Unified result format (default) - */ - public function test_result_format_unified(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping result_format_unified: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['result_format' => 'unified']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php deleted file mode 100644 index 039d6536d88..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/EmailTest.php +++ /dev/null @@ -1,132 +0,0 @@ -markTestSkipped('Skipping email_eml_html_body: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['message/rfc822']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * EML with multipart MIME content. - */ - public function test_email_eml_multipart(): void - { - $documentPath = Helpers::resolveDocument('email/html_email_multipart.eml'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping email_eml_multipart: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['message/rfc822']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * UTF-16 encoded EML file with BOM. - */ - public function test_email_eml_utf16(): void - { - $documentPath = Helpers::resolveDocument('vendored/unstructured/eml/fake-email-utf-16.eml'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping email_eml_utf16: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['message/rfc822']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertContentContainsAny($result, ['Test Email', 'Roses are red']); - } - - /** - * Outlook MSG file extraction. - */ - public function test_email_msg_basic(): void - { - $documentPath = Helpers::resolveDocument('email/fake_email.msg'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping email_msg_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.ms-outlook']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Empty Outlook PST archive with no messages. - */ - public function test_email_pst_empty(): void - { - $documentPath = Helpers::resolveDocument('email/empty.pst'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping email_pst_empty: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.ms-outlook-pst']); - } - - /** - * Sample EML email file to verify email parsing. - */ - public function test_email_sample_eml(): void - { - $documentPath = Helpers::resolveDocument('email/sample_email.eml'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping email_sample_eml: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['message/rfc822']); - Helpers::assertMinContentLength($result, 20); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/EmbeddingsTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/EmbeddingsTest.php deleted file mode 100644 index 1959f7d09e5..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/EmbeddingsTest.php +++ /dev/null @@ -1,105 +0,0 @@ -markTestSkipped('Skipping embedding_async: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('embeddings'); - - $config = Helpers::buildConfig(['chunking' => ['embedding' => ['model' => ['name' => 'balanced', 'type' => 'preset'], 'normalize' => true], 'max_chars' => 500, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $deferred = $kreuzberg->extractFileAsync($documentPath); - $result = $deferred->getResult(); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); - } - - /** - * Tests embedding generation with balanced preset model via chunking - */ - public function test_embedding_balanced_preset(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping embedding_balanced_preset: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('embeddings'); - - $config = Helpers::buildConfig(['chunking' => ['embedding' => ['model' => ['name' => 'balanced', 'type' => 'preset'], 'normalize' => true], 'max_chars' => 500, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); - } - - /** - * Tests chunking without embeddings - chunks should not have embedding vectors - */ - public function test_embedding_disabled(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping embedding_disabled: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['chunking' => ['max_chars' => 500, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, false, null, null); - } - - /** - * Tests embedding generation with fast preset model - */ - public function test_embedding_fast_preset(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping embedding_fast_preset: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('embeddings'); - - $config = Helpers::buildConfig(['chunking' => ['embedding' => ['model' => ['name' => 'fast', 'type' => 'preset'], 'normalize' => true], 'max_chars' => 500, 'max_overlap' => 50]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php deleted file mode 100644 index ac68313acb5..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/HtmlTest.php +++ /dev/null @@ -1,57 +0,0 @@ -markTestSkipped('Skipping html_complex_layout: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/html']); - Helpers::assertMinContentLength($result, 1000); - } - - /** - * HTML table converted to markdown should retain structure. - */ - public function test_html_simple_table(): void - { - $documentPath = Helpers::resolveDocument('html/simple_table.html'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping html_simple_table: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/html']); - Helpers::assertMinContentLength($result, 100); - Helpers::assertContentContainsAll($result, ['Product', 'Category', 'Price', 'Stock', 'Laptop', 'Electronics', 'Sample Data Table']); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php deleted file mode 100644 index 31c8adc0d53..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/ImageTest.php +++ /dev/null @@ -1,224 +0,0 @@ -markTestSkipped('Skipping image_bmp_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/bmp']); - Helpers::assertContentNotEmpty($result); - } - - /** - * GIF image extraction via OCR. - */ - public function test_image_gif_basic(): void - { - $documentPath = Helpers::resolveDocument('images_extra/ocr_image.gif'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_gif_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/gif']); - Helpers::assertContentNotEmpty($result); - } - - /** - * JPEG 2000 image extraction via OCR. - */ - public function test_image_jp2_basic(): void - { - $documentPath = Helpers::resolveDocument('images_extra/ocr_image.jp2'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_jp2_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/jp2', 'image/jpeg2000']); - Helpers::assertContentNotEmpty($result); - } - - /** - * JPEG image to validate metadata extraction without OCR. - */ - public function test_image_metadata_only(): void - { - $documentPath = Helpers::resolveDocument('images/example.jpg'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_metadata_only: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['ocr' => null]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/jpeg']); - Helpers::assertMaxContentLength($result, 200); - } - - /** - * PBM (portable bitmap) image extraction via OCR. - */ - public function test_image_pbm_basic(): void - { - $documentPath = Helpers::resolveDocument('images_extra/ocr_image.pbm'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_pbm_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/x-portable-bitmap', 'image/x-pbm']); - Helpers::assertContentNotEmpty($result); - } - - /** - * PGM (portable graymap) image extraction via OCR. - */ - public function test_image_pgm_basic(): void - { - $documentPath = Helpers::resolveDocument('images_extra/ocr_image.pgm'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_pgm_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/x-portable-graymap', 'image/x-pgm']); - Helpers::assertContentNotEmpty($result); - } - - /** - * PPM (portable pixmap) image extraction via OCR. - */ - public function test_image_ppm_basic(): void - { - $documentPath = Helpers::resolveDocument('images_extra/ocr_image.ppm'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_ppm_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/x-portable-pixmap', 'image/x-ppm']); - Helpers::assertContentNotEmpty($result); - } - - /** - * SVG image extraction. - */ - public function test_image_svg_basic(): void - { - $documentPath = Helpers::resolveDocument('xml/simple_svg.svg'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_svg_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/svg+xml']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * TIFF image extraction via OCR. - */ - public function test_image_tiff_basic(): void - { - $documentPath = Helpers::resolveDocument('images_extra/ocr_image.tif'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_tiff_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/tiff']); - Helpers::assertContentNotEmpty($result); - } - - /** - * WebP image extraction via OCR. - */ - public function test_image_webp_basic(): void - { - $documentPath = Helpers::resolveDocument('images_extra/ocr_image.webp'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping image_webp_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/webp']); - Helpers::assertContentNotEmpty($result); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/KeywordsTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/KeywordsTest.php deleted file mode 100644 index 23f45154c88..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/KeywordsTest.php +++ /dev/null @@ -1,62 +0,0 @@ -markTestSkipped('Skipping keywords_rake: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('keywords-rake'); - - $config = Helpers::buildConfig(['keywords' => ['algorithm' => 'rake', 'max_keywords' => 10]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertKeywords($result, true, 1, 10); - } - - /** - * Tests keyword extraction using YAKE algorithm - */ - public function test_keywords_yake(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping keywords_yake: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('keywords-yake'); - - $config = Helpers::buildConfig(['keywords' => ['algorithm' => 'yake', 'max_keywords' => 10]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertKeywords($result, true, 1, 10); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php deleted file mode 100644 index d221a5e41ac..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/OcrTest.php +++ /dev/null @@ -1,396 +0,0 @@ -markTestSkipped('Skipping ocr_image_hello_world: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertContentContainsAny($result, ['hello', 'world']); - } - - /** - * Image with no text to ensure OCR handles empty results gracefully. - */ - public function test_ocr_image_no_text(): void - { - $documentPath = Helpers::resolveDocument('images/flower_no_text.jpg'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_image_no_text: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/jpeg']); - Helpers::assertMaxContentLength($result, 300); - } - - /** - * PaddleOCR with minimum confidence threshold filtering. - */ - public function test_ocr_paddle_confidence_filter(): void - { - $documentPath = Helpers::resolveDocument('images/ocr_image.jpg'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_confidence_filter: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en', 'paddle_ocr_config' => ['min_confidence' => 80.0]]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/jpeg']); - Helpers::assertMinContentLength($result, 1); - } - - /** - * Tests PaddleOCR with element hierarchy building enabled - */ - public function test_ocr_paddle_element_hierarchy(): void - { - $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_element_hierarchy: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'element_config' => ['build_hierarchy' => true, 'include_elements' => true], 'language' => 'en']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertOcrElements($result, true, true, true, null); - } - - /** - * Tests PaddleOCR with word-level element extraction - */ - public function test_ocr_paddle_element_levels(): void - { - $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_element_levels: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'element_config' => ['include_elements' => true, 'min_level' => 'word'], 'language' => 'en']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertOcrElements($result, true, true, null, 1); - } - - /** - * Chinese OCR with PaddleOCR - its core strength. - */ - public function test_ocr_paddle_image_chinese(): void - { - $documentPath = Helpers::resolveDocument('images/chi_sim_image.jpeg'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_image_chinese: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'ch']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/jpeg']); - Helpers::assertMinContentLength($result, 1); - } - - /** - * Simple English image OCR with PaddleOCR backend. - */ - public function test_ocr_paddle_image_english(): void - { - $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_image_english: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertContentContainsAny($result, ['hello', 'Hello', 'world', 'World']); - } - - /** - * PaddleOCR with markdown output format. - */ - public function test_ocr_paddle_markdown(): void - { - $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_markdown: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en', 'paddle_ocr_config' => ['output_format' => 'markdown']]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertContentContainsAny($result, ['hello', 'Hello', 'world', 'World']); - } - - /** - * Scanned PDF requires PaddleOCR to extract text. - */ - public function test_ocr_paddle_pdf_scanned(): void - { - $documentPath = Helpers::resolveDocument('pdf/ocr_test.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_pdf_scanned: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 20); - Helpers::assertContentContainsAny($result, ['Docling', 'Markdown', 'JSON']); - } - - /** - * PaddleOCR with structured output preserving all metadata. - */ - public function test_ocr_paddle_structured(): void - { - $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_structured: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'element_config' => ['include_elements' => true], 'language' => 'en']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertOcrElements($result, true, true, true, null); - } - - /** - * Table detection and extraction with PaddleOCR. - */ - public function test_ocr_paddle_table_detection(): void - { - $documentPath = Helpers::resolveDocument('images/simple_table.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_paddle_table_detection: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - Helpers::skipIfFeatureUnavailable('paddle-ocr'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'paddle-ocr', 'language' => 'en', 'paddle_ocr_config' => ['enable_table_detection' => true]]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertTableCount($result, 1, null); - } - - /** - * Image-only German PDF requiring OCR to extract text. - */ - public function test_ocr_pdf_image_only_german(): void - { - $documentPath = Helpers::resolveDocument('pdf/image_only_german_pdf.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_pdf_image_only_german: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'deu']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 20); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - - /** - * Rotated page PDF requiring OCR to verify orientation handling. - */ - public function test_ocr_pdf_rotated_90(): void - { - $documentPath = Helpers::resolveDocument('pdf/ocr_test_rotated_90.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_pdf_rotated_90: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Scanned PDF requires OCR to extract text. - */ - public function test_ocr_pdf_tesseract(): void - { - $documentPath = Helpers::resolveDocument('pdf/ocr_test.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_pdf_tesseract: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'eng']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 20); - Helpers::assertContentContainsAny($result, ['Docling', 'Markdown', 'JSON']); - } - - /** - * Tests Tesseract OCR with element-level structured output including geometry - */ - public function test_ocr_tesseract_elements(): void - { - $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_tesseract_elements: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'element_config' => ['include_elements' => true], 'language' => 'eng']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertOcrElements($result, true, true, true, null); - } - - /** - * Tests Tesseract OCR element output with min_count assertion - */ - public function test_ocr_tesseract_elements_min_count(): void - { - $documentPath = Helpers::resolveDocument('images/test_hello_world.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_tesseract_elements_min_count: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'element_config' => ['include_elements' => true, 'min_level' => 'line'], 'language' => 'eng']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertOcrElements($result, true, null, null, 1); - } - - /** - * Tests Tesseract OCR with German language configuration - */ - public function test_ocr_tesseract_language_german(): void - { - $documentPath = Helpers::resolveDocument('pdf/image_only_german_pdf.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping ocr_tesseract_language_german: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('tesseract'); - - $config = Helpers::buildConfig(['force_ocr' => true, 'ocr' => ['backend' => 'tesseract', 'language' => 'deu']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 20); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php deleted file mode 100644 index f463e15bfe7..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/OfficeTest.php +++ /dev/null @@ -1,1007 +0,0 @@ -markTestSkipped('Skipping office_bibtex_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-bibtex', 'text/x-bibtex']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * CommonMark (.commonmark) text extraction. - */ - public function test_office_commonmark_basic(): void - { - $documentPath = Helpers::resolveDocument('markdown/sample.commonmark'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_commonmark_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/markdown', 'text/plain', 'text/x-commonmark']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * dBASE (.dbf) table extraction as markdown. - */ - public function test_office_dbf_basic(): void - { - $documentPath = Helpers::resolveDocument('dbf/stations.dbf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_dbf_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('office'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-dbf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['|']); - } - - /** - * Djot markup text extraction. - */ - public function test_office_djot_basic(): void - { - $documentPath = Helpers::resolveDocument('markdown/tables.djot'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_djot_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/x-djot', 'text/djot']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Legacy .doc document extraction via native OLE/CFB parsing. - */ - public function test_office_doc_legacy(): void - { - $documentPath = Helpers::resolveDocument('doc/unit_test_lists.doc'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_doc_legacy: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('office'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/msword']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * DocBook XML document extraction. - */ - public function test_office_docbook_basic(): void - { - $documentPath = Helpers::resolveDocument('docbook/docbook-reader.docbook'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_docbook_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/docbook+xml', 'text/docbook']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * DOCX document extraction baseline. - */ - public function test_office_docx_basic(): void - { - $documentPath = Helpers::resolveDocument('docx/sample_document.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_docx_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * DOCX file containing equations to validate math extraction. - */ - public function test_office_docx_equations(): void - { - $documentPath = Helpers::resolveDocument('docx/equations.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_docx_equations: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * Simple DOCX document to verify baseline extraction. - */ - public function test_office_docx_fake(): void - { - $documentPath = Helpers::resolveDocument('docx/fake.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_docx_fake: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * DOCX document heavy on formatting for style preservation. - */ - public function test_office_docx_formatting(): void - { - $documentPath = Helpers::resolveDocument('docx/unit_test_formatting.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_docx_formatting: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * DOCX document with complex headers. - */ - public function test_office_docx_headers(): void - { - $documentPath = Helpers::resolveDocument('docx/unit_test_headers.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_docx_headers: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * DOCX document emphasizing list formatting. - */ - public function test_office_docx_lists(): void - { - $documentPath = Helpers::resolveDocument('docx/unit_test_lists.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_docx_lists: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * DOCX document containing tables for table-aware extraction. - */ - public function test_office_docx_tables(): void - { - $documentPath = Helpers::resolveDocument('docx/docx_tables.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_docx_tables: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertContentContainsAll($result, ['Simple uniform table', 'Nested Table', 'merged cells', 'Header Col']); - Helpers::assertTableCount($result, 1, null); - } - - /** - * EPUB book extraction with text content. - */ - public function test_office_epub_basic(): void - { - $documentPath = Helpers::resolveDocument('epub/features.epub'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_epub_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/epub+zip']); - Helpers::assertMinContentLength($result, 50); - } - - /** - * FictionBook (FB2) document extraction baseline. - */ - public function test_office_fb2_basic(): void - { - $documentPath = Helpers::resolveDocument('fictionbook/basic.fb2'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_fb2_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-fictionbook+xml']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * FictionBook (.fb2) text extraction. - */ - public function test_office_fictionbook_basic(): void - { - $documentPath = Helpers::resolveDocument('fictionbook/basic.fb2'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_fictionbook_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-fictionbook+xml', 'application/x-fictionbook']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Hangul Word Processor (.hwp) text extraction. - */ - public function test_office_hwp_basic(): void - { - $documentPath = Helpers::resolveDocument('hwp/converted_output.hwp'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_hwp_basic: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('office'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-hwp']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Hangul Word Processor (.hwp) styled document extraction. - */ - public function test_office_hwp_styled(): void - { - $documentPath = Helpers::resolveDocument('hwp/styled_document.hwp'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_hwp_styled: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('hwp'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-hwp']); - } - - /** - * JATS scientific article extraction. - */ - public function test_office_jats_basic(): void - { - $documentPath = Helpers::resolveDocument('jats/sample_article.jats'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_jats_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-jats+xml', 'text/jats']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Jupyter notebook extraction. - */ - public function test_office_jupyter_basic(): void - { - $documentPath = Helpers::resolveDocument('jupyter/rank.ipynb'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_jupyter_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-ipynb+json']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Keynote document extraction baseline. - */ - public function test_office_keynote_basic(): void - { - $documentPath = Helpers::resolveDocument('iwork/test.key'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_keynote_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-iwork-keynote-sffkey']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * LaTeX document text extraction. - */ - public function test_office_latex_basic(): void - { - $documentPath = Helpers::resolveDocument('latex/basic_sections.tex'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_latex_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-latex', 'text/x-latex']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * Markdown document extraction baseline. - */ - public function test_office_markdown_basic(): void - { - $documentPath = Helpers::resolveDocument('markdown/comprehensive.md'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_markdown_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/markdown']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * MDX document extraction with JSX stripping and frontmatter. - */ - public function test_office_mdx_basic(): void - { - $documentPath = Helpers::resolveDocument('markdown/sample.mdx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_mdx_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/mdx', 'text/x-mdx']); - Helpers::assertMinContentLength($result, 50); - } - - /** - * Real-world MDX extraction from mdx-js/mdx getting-started.mdx with imports, exports, JSX components, code blocks, and reference links. - */ - public function test_office_mdx_getting_started(): void - { - $documentPath = Helpers::resolveDocument('markdown/mdx_getting_started.mdx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_mdx_getting_started: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/mdx', 'text/x-mdx']); - Helpers::assertMinContentLength($result, 2000); - } - - /** - * Real-world MDX extraction from mdx-js/mdx troubleshooting-mdx.mdx with error documentation, code blocks, and JSX comments. - */ - public function test_office_mdx_troubleshooting(): void - { - $documentPath = Helpers::resolveDocument('markdown/mdx_troubleshooting.mdx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_mdx_troubleshooting: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/mdx', 'text/x-mdx']); - Helpers::assertMinContentLength($result, 2000); - } - - /** - * Real-world MDX extraction from mdx-js/mdx using-mdx.mdx with complex JSX examples, component passing, and MDX provider patterns. - */ - public function test_office_mdx_using_mdx(): void - { - $documentPath = Helpers::resolveDocument('markdown/mdx_using_mdx.mdx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_mdx_using_mdx: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/mdx', 'text/x-mdx']); - Helpers::assertMinContentLength($result, 2000); - } - - /** - * Numbers document extraction baseline. - */ - public function test_office_numbers_basic(): void - { - $documentPath = Helpers::resolveDocument('iwork/test.numbers'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_numbers_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-iwork-numbers-sffnumbers']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Basic ODS spreadsheet extraction. - */ - public function test_office_ods_basic(): void - { - $documentPath = Helpers::resolveDocument('data_formats/test_01.ods'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_ods_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.spreadsheet']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * ODT document with bold formatting. - */ - public function test_office_odt_bold(): void - { - $documentPath = Helpers::resolveDocument('odt/bold.odt'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_odt_bold: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.text']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * ODT document containing unordered lists with nesting. - */ - public function test_office_odt_list(): void - { - $documentPath = Helpers::resolveDocument('odt/unorderedList.odt'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_odt_list: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.text']); - Helpers::assertMinContentLength($result, 30); - Helpers::assertContentContainsAny($result, ['list item', 'New level', 'Pushed us']); - } - - /** - * Basic ODT document with paragraphs and headings. - */ - public function test_office_odt_simple(): void - { - $documentPath = Helpers::resolveDocument('odt/simple.odt'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_odt_simple: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.text']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertContentContainsAny($result, ['favorite things', 'Parrots', 'Analysis']); - } - - /** - * ODT document with a table structure. - */ - public function test_office_odt_table(): void - { - $documentPath = Helpers::resolveDocument('odt/table.odt'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_odt_table: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.oasis.opendocument.text']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertTableCount($result, 1, null); - } - - /** - * OPML outline document extraction. - */ - public function test_office_opml_basic(): void - { - $documentPath = Helpers::resolveDocument('opml/outline.opml'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_opml_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/xml+opml', 'text/x-opml', 'application/x-opml+xml']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Org-mode document text extraction. - */ - public function test_office_org_basic(): void - { - $documentPath = Helpers::resolveDocument('org/comprehensive.org'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_org_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/x-org', 'text/org']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * Pages document extraction baseline. - */ - public function test_office_pages_basic(): void - { - $documentPath = Helpers::resolveDocument('iwork/test.pages'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_pages_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-iwork-pages-sffpages']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * PPSX (PowerPoint Show) files should extract slides content identical to PPTX. GitHub Issue #321 Bug 2. - */ - public function test_office_ppsx_slideshow(): void - { - $documentPath = Helpers::resolveDocument('pptx/sample.ppsx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_ppsx_slideshow: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.slideshow']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Legacy PowerPoint .ppt extraction via native OLE/CFB parsing. - */ - public function test_office_ppt_legacy(): void - { - $documentPath = Helpers::resolveDocument('ppt/simple.ppt'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_ppt_legacy: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('office'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.ms-powerpoint']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * PowerPoint macro-enabled presentation (.pptm) extraction. - */ - public function test_office_pptm_basic(): void - { - $documentPath = Helpers::resolveDocument('pptx/powerpoint_with_image.pptm'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_pptm_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.ms-powerpoint.presentation.macroEnabled.12', 'application/vnd.openxmlformats-officedocument.presentationml.presentation']); - Helpers::assertContentNotEmpty($result); - } - - /** - * PPTX deck should extract slides content. - */ - public function test_office_pptx_basic(): void - { - $documentPath = Helpers::resolveDocument('pptx/simple.pptx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_pptx_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.presentation']); - Helpers::assertMinContentLength($result, 50); - } - - /** - * PPTX presentation containing images to ensure metadata extraction. - */ - public function test_office_pptx_images(): void - { - $documentPath = Helpers::resolveDocument('pptx/powerpoint_with_image.pptx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_pptx_images: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.presentation']); - Helpers::assertMinContentLength($result, 15); - } - - /** - * Pitch deck PPTX used to validate large slide extraction. - */ - public function test_office_pptx_pitch_deck(): void - { - $documentPath = Helpers::resolveDocument('pptx/pitch_deck_presentation.pptx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_pptx_pitch_deck: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.presentationml.presentation']); - Helpers::assertMinContentLength($result, 100); - } - - /** - * reStructuredText document extraction. - */ - public function test_office_rst_basic(): void - { - $documentPath = Helpers::resolveDocument('rst/restructured_text.rst'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_rst_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/x-rst', 'text/prs.fallenstein.rst']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * RTF document text extraction. - */ - public function test_office_rtf_basic(): void - { - $documentPath = Helpers::resolveDocument('rtf/extraction_test.rtf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_rtf_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/rtf', 'text/rtf']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Typst document text extraction. - */ - public function test_office_typst_basic(): void - { - $documentPath = Helpers::resolveDocument('typst/headings.typ'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_typst_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-typst', 'text/x-typst']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Legacy XLS spreadsheet to ensure backward compatibility. - */ - public function test_office_xls_legacy(): void - { - $documentPath = Helpers::resolveDocument('xls/test_excel.xls'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_xls_legacy: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.ms-excel']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Excel binary workbook (.xlsb) extraction. - */ - public function test_office_xlsb_basic(): void - { - $documentPath = Helpers::resolveDocument('xlsx/test_xlsb.xlsb'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_xlsb_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.ms-excel.sheet.binary.macroEnabled.12', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); - Helpers::assertContentNotEmpty($result); - } - - /** - * Excel macro-enabled workbook (.xlsm) extraction. - */ - public function test_office_xlsm_basic(): void - { - $documentPath = Helpers::resolveDocument('xlsx/test_01.xlsm'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_xlsm_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.ms-excel.sheet.macroEnabled.12', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); - Helpers::assertContentNotEmpty($result); - } - - /** - * XLSX spreadsheet should produce metadata and table content. - */ - public function test_office_xlsx_basic(): void - { - $documentPath = Helpers::resolveDocument('xlsx/stanley_cups.xlsx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_xlsx_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); - Helpers::assertMinContentLength($result, 100); - Helpers::assertContentContainsAll($result, ['Team', 'Location', 'Stanley Cups']); - Helpers::assertTableCount($result, 1, null); - Helpers::assertMetadataExpectation($result, 'sheet_count', ['gte' => 2]); - Helpers::assertMetadataExpectation($result, 'sheet_names', ['contains' => ['Stanley Cups']]); - } - - /** - * XLSX workbook with multiple sheets. - */ - public function test_office_xlsx_multi_sheet(): void - { - $documentPath = Helpers::resolveDocument('xlsx/excel_multi_sheet.xlsx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_xlsx_multi_sheet: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); - Helpers::assertMinContentLength($result, 20); - Helpers::assertMetadataExpectation($result, 'sheet_count', ['gte' => 2]); - } - - /** - * Simple XLSX spreadsheet shipped alongside office integration tests. - */ - public function test_office_xlsx_office_example(): void - { - $documentPath = Helpers::resolveDocument('xlsx/test_01.xlsx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping office_xlsx_office_example: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); - Helpers::assertMinContentLength($result, 10); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php deleted file mode 100644 index f03039a2915..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/PdfTest.php +++ /dev/null @@ -1,388 +0,0 @@ -markTestSkipped('Skipping pdf_annotations: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['pdf_options' => ['extract_annotations' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertAnnotations($result, true, 1); - } - - /** - * Assembly language technical manual with large body of text. - */ - public function test_pdf_assembly_technical(): void - { - $documentPath = Helpers::resolveDocument('pdf/assembly_language_for_beginners_al4_b_en.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_assembly_technical: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 5000); - Helpers::assertContentContainsAny($result, ['assembly', 'register', 'instruction']); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - - /** - * Bayesian data analysis textbook PDF with large content volume. - */ - public function test_pdf_bayesian_data_analysis(): void - { - $documentPath = Helpers::resolveDocument('pdf/bayesian_data_analysis_third_edition_13th_feb_2020.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_bayesian_data_analysis: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10000); - Helpers::assertContentContainsAny($result, ['Bayesian', 'probability', 'distribution']); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - - /** - * Tests bounding box extraction on PDF tables and images - */ - public function test_pdf_bounding_boxes(): void - { - $documentPath = Helpers::resolveDocument('pdf/tiny.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_bounding_boxes: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('pdf'); - - $config = Helpers::buildConfig(['images' => ['extract_images' => true]]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertTableCount($result, 1, null); - Helpers::assertTableBoundingBoxes($result, true); - } - - /** - * PDF containing code snippets and formulas should retain substantial content. - */ - public function test_pdf_code_and_formula(): void - { - $documentPath = Helpers::resolveDocument('pdf/code_and_formula.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_code_and_formula: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 100); - } - - /** - * Deep learning textbook PDF to ensure long-form extraction quality. - */ - public function test_pdf_deep_learning(): void - { - $documentPath = Helpers::resolveDocument('pdf/fundamentals_of_deep_learning_2014.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_deep_learning: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 1000); - Helpers::assertContentContainsAny($result, ['neural', 'network', 'deep learning']); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - - /** - * PDF with embedded images should extract text and tables when present. - */ - public function test_pdf_embedded_images(): void - { - $documentPath = Helpers::resolveDocument('pdf/embedded_images_tables.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_embedded_images: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertTableCount($result, 0, null); - } - - /** - * Google Docs exported PDF to verify conversion fidelity. - */ - public function test_pdf_google_doc(): void - { - $documentPath = Helpers::resolveDocument('pdf/google_doc_document.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_google_doc: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - - /** - * Large machine learning textbook PDF to stress extraction length. - */ - public function test_pdf_large_ciml(): void - { - $documentPath = Helpers::resolveDocument('pdf/a_course_in_machine_learning_ciml_v0_9_all.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_large_ciml: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10000); - Helpers::assertContentContainsAny($result, ['machine learning', 'algorithm', 'training']); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - - /** - * PDF extraction with layout detection enabled should produce content from a document with mixed structure. - */ - public function test_pdf_layout_detection(): void - { - $documentPath = Helpers::resolveDocument('pdf/docling.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_layout_detection: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('layout-detection'); - - $config = Helpers::buildConfig(['layout' => ['preset' => 'accurate', 'table_model' => 'tatr'], 'output_format' => 'markdown']); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 100); - Helpers::assertContentNotEmpty($result); - } - - /** - * German technical PDF to ensure non-ASCII content extraction. - */ - public function test_pdf_non_english_german(): void - { - $documentPath = Helpers::resolveDocument('pdf/5_level_paging_and_5_level_ept_intel_revision_1_1_may_2017.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_non_english_german: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 100); - Helpers::assertContentContainsAny($result, ['Intel', 'paging']); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - - /** - * Copy-protected PDF should extract content (pdfium handles copy-protection transparently). - */ - public function test_pdf_password_protected(): void - { - $documentPath = Helpers::resolveDocument('pdf/copy_protected.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_password_protected: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertContentContainsAny($result, ['LayoutParser', 'document image analysis', 'deep learning']); - } - - /** - * Right-to-left language PDF to verify RTL extraction. - */ - public function test_pdf_right_to_left(): void - { - $documentPath = Helpers::resolveDocument('pdf/right_to_left_01.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_right_to_left: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - - /** - * Simple text-heavy PDF should extract content without OCR or tables. - */ - public function test_pdf_simple_text(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_simple_text: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'To Whom it May Concern', 'Mallori']); - } - - /** - * Large PDF with extensive tables to stress table extraction. - */ - public function test_pdf_tables_large(): void - { - $documentPath = Helpers::resolveDocument('pdf/large.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_tables_large: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 500); - } - - /** - * Medium-sized PDF with multiple tables. - */ - public function test_pdf_tables_medium(): void - { - $documentPath = Helpers::resolveDocument('pdf/medium.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_tables_medium: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 100); - } - - /** - * Small PDF containing tables to validate table extraction. - */ - public function test_pdf_tables_small(): void - { - $documentPath = Helpers::resolveDocument('pdf/tiny.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_tables_small: missing document at ' . $documentPath); - } - - Helpers::skipIfFeatureUnavailable('ocr'); - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertContentContainsAll($result, ['Table 1', 'Selected Numbers', 'Celsius', 'Fahrenheit', 'Water Freezing Point', 'Water Boiling Point']); - Helpers::assertTableCount($result, 1, null); - } - - /** - * Technical statistical learning PDF requiring substantial extraction. - */ - public function test_pdf_technical_stat_learning(): void - { - $documentPath = Helpers::resolveDocument('pdf/an_introduction_to_statistical_learning_with_applications_in_r_islr_sixth_printing.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping pdf_technical_stat_learning: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10000); - Helpers::assertContentContainsAny($result, ['statistical', 'regression', 'learning']); - Helpers::assertMetadataExpectation($result, 'format_type', ['eq' => 'pdf']); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/PluginApisTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/PluginApisTest.php deleted file mode 100644 index ad5abb73b20..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/PluginApisTest.php +++ /dev/null @@ -1,208 +0,0 @@ -assertNotNull($config); - - $this->assertNotNull($config->chunking); - $this->assertEquals(50, $config->chunking->maxChars); - chdir($oldCwd); - unlink($configPath); - rmdir($subdir); - rmdir($tmpDir); - } - - /** - * Load configuration from a TOML file - */ - public function test_config_from_file(): void - { - $tmpDir = sys_get_temp_dir(); - $configPath = $tmpDir . '/' . 'test_config.toml'; - file_put_contents($configPath, "[chunking]\nmax_chars = 100\nmax_overlap = 20\n\n[language_detection]\nenabled = false\n"); - - $config = ExtractionConfig::fromFile($configPath); - - $this->assertNotNull($config->chunking); - $this->assertEquals(100, $config->chunking->maxChars); - $this->assertEquals(20, $config->chunking->maxOverlap); - $this->assertNotNull($config->languageDetection); - $this->assertEquals(false, $config->languageDetection->enabled); - unlink($configPath); - } - - /** - * Clear all document extractors and verify list is empty - */ - public function test_extractors_clear(): void - { - Kreuzberg::clearDocumentExtractors(); - $result = Kreuzberg::listDocumentExtractors(); - $this->assertEmpty($result); - } - - /** - * List all registered document extractors - */ - public function test_extractors_list(): void - { - $result = Kreuzberg::listDocumentExtractors(); - $this->assertIsArray($result); - foreach ($result as $item) { - $this->assertIsString($item); - } - } - - /** - * Unregister nonexistent document extractor gracefully - */ - public function test_extractors_unregister(): void - { - Kreuzberg::unregisterDocumentExtractor('nonexistent-extractor-xyz'); - $this->assertTrue(true); // Should not throw - } - - /** - * Detect MIME type from file bytes - */ - public function test_mime_detect_bytes(): void - { - $testBytes = '%PDF-1.4\\n'; - $result = Kreuzberg::detectMimeType($testBytes); - - $this->assertStringContainsStringIgnoringCase('pdf', $result); - } - - /** - * Detect MIME type from file path - */ - public function test_mime_detect_path(): void - { - $tmpDir = sys_get_temp_dir(); - $testFile = $tmpDir . '/' . 'test.txt'; - file_put_contents($testFile, 'Hello, world!'); - - $result = Kreuzberg::detectMimeTypeFromPath($testFile); - - $this->assertStringContainsStringIgnoringCase('text', $result); - unlink($testFile); - } - - /** - * Get file extensions for a MIME type - */ - public function test_mime_get_extensions(): void - { - $result = Kreuzberg::getExtensionsForMime('application/pdf'); - $this->assertIsArray($result); - $this->assertContains('pdf', $result); - } - - /** - * Clear all OCR backends and verify list is empty - */ - public function test_ocr_backends_clear(): void - { - Kreuzberg::clearOcrBackends(); - $result = Kreuzberg::listOcrBackends(); - $this->assertEmpty($result); - } - - /** - * List all registered OCR backends - */ - public function test_ocr_backends_list(): void - { - $result = Kreuzberg::listOcrBackends(); - $this->assertIsArray($result); - foreach ($result as $item) { - $this->assertIsString($item); - } - } - - /** - * Unregister nonexistent OCR backend gracefully - */ - public function test_ocr_backends_unregister(): void - { - Kreuzberg::unregisterOcrBackend('nonexistent-backend-xyz'); - $this->assertTrue(true); // Should not throw - } - - /** - * Clear all post-processors and verify list is empty - */ - public function test_post_processors_clear(): void - { - Kreuzberg::clearPostProcessors(); - $this->assertTrue(true); // Should not throw - } - - /** - * List all registered post-processors - */ - public function test_post_processors_list(): void - { - $result = Kreuzberg::listPostProcessors(); - $this->assertIsArray($result); - foreach ($result as $item) { - $this->assertIsString($item); - } - } - - /** - * Clear all validators and verify list is empty - */ - public function test_validators_clear(): void - { - Kreuzberg::clearValidators(); - $result = Kreuzberg::listValidators(); - $this->assertEmpty($result); - } - - /** - * List all registered validators - */ - public function test_validators_list(): void - { - $result = Kreuzberg::listValidators(); - $this->assertIsArray($result); - foreach ($result as $item) { - $this->assertIsString($item); - } - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php deleted file mode 100644 index 36cb31ed39a..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/RenderTest.php +++ /dev/null @@ -1,60 +0,0 @@ -markTestSkipped('Missing document: ' . $documentPath); - } - $pngData = render_pdf_page($documentPath, 0, 72); - if (is_array($pngData)) { $pngData = pack('C*', ...$pngData); } - Helpers::assertIsPng($pngData); - Helpers::assertMinByteLength($pngData, 50); - } - - public function test_render_iterator(): void - { - $documentPath = Helpers::resolveDocument('pdf/tiny.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Missing document: ' . $documentPath); - } - $pages = []; - foreach (render_pdf_pages_iter($documentPath, 150) as $pngData) { - if (is_array($pngData)) { $pngData = pack('C*', ...$pngData); } - Helpers::assertIsPng($pngData); - $pages[] = $pngData; - } - $this->assertGreaterThanOrEqual(1, count($pages), - sprintf('Expected at least 1 pages, got %d', count($pages))); - } - - public function test_render_single_page(): void - { - $documentPath = Helpers::resolveDocument('pdf/tiny.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Missing document: ' . $documentPath); - } - $pngData = render_pdf_page($documentPath, 0, 150); - if (is_array($pngData)) { $pngData = pack('C*', ...$pngData); } - Helpers::assertIsPng($pngData); - Helpers::assertMinByteLength($pngData, 100); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php deleted file mode 100644 index bf90831adcf..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/SmokeTest.php +++ /dev/null @@ -1,177 +0,0 @@ -markTestSkipped('Skipping smoke_cache_namespace: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['cache_namespace' => 'test_tenant', 'cache_ttl_secs' => 3600, 'use_cache' => true]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/plain']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * Smoke test: DOCX with formatted text - */ - public function test_smoke_docx_basic(): void - { - $documentPath = Helpers::resolveDocument('docx/fake.docx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping smoke_docx_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.wordprocessingml.document']); - Helpers::assertMinContentLength($result, 20); - Helpers::assertContentContainsAny($result, ['Lorem', 'ipsum', 'document', 'text']); - } - - /** - * Smoke test: HTML table extraction - */ - public function test_smoke_html_basic(): void - { - $documentPath = Helpers::resolveDocument('html/simple_table.html'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping smoke_html_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/html']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['Sample Data Table', 'Laptop', 'Electronics', 'Product']); - } - - /** - * Smoke test: PNG image (without OCR, metadata only) - */ - public function test_smoke_image_png(): void - { - $documentPath = Helpers::resolveDocument('images/sample.png'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping smoke_image_png: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['image/png']); - Helpers::assertMetadataExpectation($result, 'format', ['eq' => 'PNG']); - } - - /** - * Smoke test: JSON file extraction - */ - public function test_smoke_json_basic(): void - { - $documentPath = Helpers::resolveDocument('json/simple.json'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping smoke_json_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/json']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * Smoke test: PDF with simple text extraction - */ - public function test_smoke_pdf_basic(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping smoke_pdf_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 50); - Helpers::assertContentContainsAny($result, ['May 5, 2023', 'To Whom it May Concern']); - } - - /** - * Smoke test: Plain text file - */ - public function test_smoke_txt_basic(): void - { - $documentPath = Helpers::resolveDocument('text/report.txt'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping smoke_txt_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/plain']); - Helpers::assertMinContentLength($result, 5); - } - - /** - * Smoke test: XLSX with basic spreadsheet data including tables - */ - public function test_smoke_xlsx_basic(): void - { - $documentPath = Helpers::resolveDocument('xlsx/stanley_cups.xlsx'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping smoke_xlsx_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']); - Helpers::assertMinContentLength($result, 100); - Helpers::assertContentContainsAll($result, ['Team', 'Location', 'Stanley Cups', 'Blues', 'Flyers', 'Maple Leafs', 'STL', 'PHI', 'TOR']); - Helpers::assertTableCount($result, 1, null); - Helpers::assertMetadataExpectation($result, 'sheet_count', ['gte' => 2]); - Helpers::assertMetadataExpectation($result, 'sheet_names', ['contains' => ['Stanley Cups']]); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/StructuredTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/StructuredTest.php deleted file mode 100644 index 2af675deef0..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/StructuredTest.php +++ /dev/null @@ -1,209 +0,0 @@ -markTestSkipped('Skipping structured_csv_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/csv']); - Helpers::assertMinContentLength($result, 20); - } - - /** - * EndNote ENW citation format extraction. - */ - public function test_structured_enw_basic(): void - { - $documentPath = Helpers::resolveDocument('data_formats/sample.enw'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_enw_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-endnote-refer', 'application/x-endnote+xml', 'text/plain']); - } - - /** - * Structured JSON extraction should stream and preserve content. - */ - public function test_structured_json_basic(): void - { - $documentPath = Helpers::resolveDocument('json/sample_document.json'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_json_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/json']); - Helpers::assertMinContentLength($result, 20); - Helpers::assertContentContainsAny($result, ['Sample Document', 'Test Author']); - } - - /** - * Simple JSON document to verify structured extraction. - */ - public function test_structured_json_simple(): void - { - $documentPath = Helpers::resolveDocument('json/simple.json'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_json_simple: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/json']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentContainsAny($result, ['{', 'name']); - } - - /** - * PubMed NBIB citation format extraction. - */ - public function test_structured_nbib_basic(): void - { - $documentPath = Helpers::resolveDocument('data_formats/sample.nbib'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_nbib_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/nbib', 'application/x-pubmed', 'text/plain']); - Helpers::assertContentNotEmpty($result); - } - - /** - * RIS citation format extraction. - */ - public function test_structured_ris_basic(): void - { - $documentPath = Helpers::resolveDocument('data_formats/sample.ris'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_ris_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-research-info-systems', 'text/plain']); - Helpers::assertContentNotEmpty($result); - } - - /** - * TOML configuration file extraction. - */ - public function test_structured_toml_basic(): void - { - $documentPath = Helpers::resolveDocument('data_formats/cargo.toml'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_toml_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/toml', 'text/toml']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * TSV (tab-separated values) data file extraction. - */ - public function test_structured_tsv_basic(): void - { - $documentPath = Helpers::resolveDocument('data_formats/employees.tsv'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_tsv_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['text/tab-separated-values', 'text/plain']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * YAML file text extraction. - */ - public function test_structured_yaml_basic(): void - { - $documentPath = Helpers::resolveDocument('yaml/simple.yaml'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_yaml_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/yaml', 'text/yaml', 'text/x-yaml', 'application/x-yaml']); - Helpers::assertMinContentLength($result, 10); - } - - /** - * Simple YAML document to validate structured extraction. - */ - public function test_structured_yaml_simple(): void - { - $documentPath = Helpers::resolveDocument('yaml/simple.yaml'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping structured_yaml_simple: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/x-yaml']); - Helpers::assertMinContentLength($result, 10); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/Token_reductionTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/Token_reductionTest.php deleted file mode 100644 index 71c8b8319a0..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/Token_reductionTest.php +++ /dev/null @@ -1,102 +0,0 @@ -markTestSkipped('Skipping token_reduction_aggressive: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['token_reduction' => ['mode' => 'aggressive']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertMaxContentLength($result, 150); - Helpers::assertContentNotEmpty($result); - } - - /** - * Tests basic token reduction on PDF document - */ - public function test_token_reduction_basic(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping token_reduction_basic: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['token_reduction' => ['mode' => 'moderate']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertMaxContentLength($result, 200); - Helpers::assertContentNotEmpty($result); - } - - /** - * Tests light token reduction mode preserves most content - */ - public function test_token_reduction_light(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping token_reduction_light: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['token_reduction' => ['mode' => 'light']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 10); - Helpers::assertContentNotEmpty($result); - } - - /** - * Tests token reduction combined with chunking - */ - public function test_token_reduction_with_chunking(): void - { - $documentPath = Helpers::resolveDocument('pdf/fake_memo.pdf'); - if (!file_exists($documentPath)) { - $this->markTestSkipped('Skipping token_reduction_with_chunking: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(['chunking' => ['max_chars' => 500, 'max_overlap' => 50], 'token_reduction' => ['mode' => 'moderate']]); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/pdf']); - Helpers::assertMinContentLength($result, 5); - Helpers::assertMaxContentLength($result, 200); - Helpers::assertChunks($result, 1, null, true, null, null, null); - Helpers::assertContentNotEmpty($result); - } - -} diff --git a/e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php b/e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php deleted file mode 100644 index 7d2d1ba77ad..00000000000 --- a/e2e/e2e/php/tests/E2EPhp/Tests/XmlTest.php +++ /dev/null @@ -1,37 +0,0 @@ -markTestSkipped('Skipping xml_plant_catalog: missing document at ' . $documentPath); - } - - $config = Helpers::buildConfig(null); - - $kreuzberg = new Kreuzberg($config); - $result = $kreuzberg->extractFile($documentPath); - - Helpers::assertExpectedMime($result, ['application/xml']); - Helpers::assertMinContentLength($result, 100); - } - -} diff --git a/e2e/e2e/php/tests/Helpers.php b/e2e/e2e/php/tests/Helpers.php deleted file mode 100644 index 449cacbb87a..00000000000 --- a/e2e/e2e/php/tests/Helpers.php +++ /dev/null @@ -1,831 +0,0 @@ -mimeType, $token)) { - $matches = true; - break; - } - } - - Assert::assertTrue( - $matches, - sprintf( - "Expected MIME '%s' to match one of %s", - $result->mimeType, - json_encode($expected) - ) - ); - } - - public static function assertMinContentLength(ExtractionResult $result, int $minimum): void - { - Assert::assertGreaterThanOrEqual( - $minimum, - strlen($result->content), - sprintf("Expected content length >= %d, got %d", $minimum, strlen($result->content)) - ); - } - - public static function assertMaxContentLength(ExtractionResult $result, int $maximum): void - { - Assert::assertLessThanOrEqual( - $maximum, - strlen($result->content), - sprintf("Expected content length <= %d, got %d", $maximum, strlen($result->content)) - ); - } - - public static function assertContentContainsAny(ExtractionResult $result, array $snippets): void - { - if (empty($snippets)) { - return; - } - - $lowered = strtolower($result->content); - $found = false; - foreach ($snippets as $snippet) { - if (str_contains($lowered, strtolower($snippet))) { - $found = true; - break; - } - } - - Assert::assertTrue( - $found, - sprintf( - "Expected content to contain any of %s. Preview: %s", - json_encode($snippets), - json_encode(substr($result->content, 0, 160)) - ) - ); - } - - public static function assertContentContainsAll(ExtractionResult $result, array $snippets): void - { - if (empty($snippets)) { - return; - } - - $lowered = strtolower($result->content); - $missing = []; - foreach ($snippets as $snippet) { - if (!str_contains($lowered, strtolower($snippet))) { - $missing[] = $snippet; - } - } - - Assert::assertEmpty( - $missing, - sprintf( - "Expected content to contain all snippets %s. Missing %s", - json_encode($snippets), - json_encode($missing) - ) - ); - } - - public static function assertTableCount(ExtractionResult $result, ?int $minimum, ?int $maximum): void - { - $count = count($result->tables ?? []); - - if ($minimum !== null) { - Assert::assertGreaterThanOrEqual( - $minimum, - $count, - sprintf("Expected at least %d tables, found %d", $minimum, $count) - ); - } - - if ($maximum !== null) { - Assert::assertLessThanOrEqual( - $maximum, - $count, - sprintf("Expected at most %d tables, found %d", $maximum, $count) - ); - } - } - - public static function assertDetectedLanguages( - ExtractionResult $result, - array $expected, - ?float $minConfidence - ): void { - if (empty($expected)) { - return; - } - - Assert::assertNotNull($result->detectedLanguages, "Expected detected languages but field is null"); - - $missing = []; - foreach ($expected as $lang) { - if (!in_array($lang, $result->detectedLanguages, true)) { - $missing[] = $lang; - } - } - - Assert::assertEmpty( - $missing, - sprintf("Expected languages %s, missing %s", json_encode($expected), json_encode($missing)) - ); - - $metaArr = self::metadataToArray($result->metadata); - if ($minConfidence !== null && isset($metaArr['confidence'])) { - $confidence = $metaArr['confidence']; - Assert::assertGreaterThanOrEqual( - $minConfidence, - $confidence, - sprintf("Expected confidence >= %f, got %f", $minConfidence, $confidence) - ); - } - } - - public static function assertChunks( - $result, - ?int $minCount = null, - ?int $maxCount = null, - ?bool $eachHasContent = null, - ?bool $eachHasEmbedding = null, - ?bool $eachHasHeadingContext = null, - ?bool $eachHasChunkType = null, - ?bool $contentStartsWithHeading = null - ): void { - $chunks = $result->chunks ?? null; - if ($chunks === null) { - throw new \Exception("Expected chunks but field is null"); - } - - $count = count($chunks); - if ($minCount !== null && $count < $minCount) { - throw new \Exception("Expected at least $minCount chunks, found $count"); - } - if ($maxCount !== null && $count > $maxCount) { - throw new \Exception("Expected at most $maxCount chunks, found $count"); - } - - foreach ($chunks as $i => $chunk) { - if ($eachHasContent && empty($chunk->content)) { - throw new \Exception("Chunk $i has no content"); - } - if ($eachHasEmbedding && (empty($chunk->embedding))) { - throw new \Exception("Chunk $i has no embedding"); - } - if ($eachHasHeadingContext !== null) { - $hc = $chunk->metadata->headingContext ?? null; - if ($eachHasHeadingContext && $hc === null) { - throw new \Exception("Chunk $i has no headingContext"); - } - if (!$eachHasHeadingContext && $hc !== null) { - throw new \Exception("Chunk $i should have no headingContext"); - } - } - if ($eachHasChunkType === true) { - $type = $chunk->chunkType ?? $chunk->chunk_type ?? null; - if ($type === null || $type === "unknown") { - throw new \Exception("Chunk $i has no specific chunkType, got " . var_export($type, true)); - } - } - if ($contentStartsWithHeading && (empty($chunk->content) || $chunk->content[0] !== '#')) { - throw new \Exception("Chunk $i content does not start with #"); - } - } - } - - public static function assertImages( - ExtractionResult $result, - ?int $minCount, - ?int $maxCount, - ?array $formatsInclude - ): void { - $images = $result->images ?? []; - $count = count($images); - - if ($minCount !== null) { - Assert::assertGreaterThanOrEqual( - $minCount, - $count, - sprintf("Expected at least %d images, found %d", $minCount, $count) - ); - } - - if ($maxCount !== null) { - Assert::assertLessThanOrEqual( - $maxCount, - $count, - sprintf("Expected at most %d images, found %d", $maxCount, $count) - ); - } - - if ($formatsInclude !== null && !empty($formatsInclude)) { - $foundFormats = []; - foreach ($images as $image) { - if (isset($image->format)) { - $foundFormats[] = strtolower($image->format); - } - } - - foreach ($formatsInclude as $format) { - Assert::assertContains( - strtolower($format), - $foundFormats, - sprintf("Expected image format '%s' not found in %s", $format, json_encode($foundFormats)) - ); - } - } - } - - public static function assertPages( - ExtractionResult $result, - ?int $minCount, - ?int $exactCount - ): void { - $pages = $result->pages ?? []; - $count = count($pages); - - if ($exactCount !== null) { - Assert::assertEquals( - $exactCount, - $count, - sprintf("Expected exactly %d pages, found %d", $exactCount, $count) - ); - } - - if ($minCount !== null) { - Assert::assertGreaterThanOrEqual( - $minCount, - $count, - sprintf("Expected at least %d pages, found %d", $minCount, $count) - ); - } - - foreach ($pages as $page) { - if (property_exists($page, 'isBlank')) { - Assert::assertTrue( - $page->isBlank === null || is_bool($page->isBlank), - 'isBlank should be null or bool' - ); - } - } - } - - public static function assertElements( - ExtractionResult $result, - ?int $minCount, - ?array $typesInclude - ): void { - $elements = $result->elements ?? []; - $count = count($elements); - - if ($minCount !== null) { - Assert::assertGreaterThanOrEqual( - $minCount, - $count, - sprintf("Expected at least %d elements, found %d", $minCount, $count) - ); - } - - if ($typesInclude !== null && !empty($typesInclude)) { - $foundTypes = []; - foreach ($elements as $element) { - if (isset($element->elementType)) { - $foundTypes[] = strtolower($element->elementType); - } - } - - foreach ($typesInclude as $type) { - Assert::assertContains( - strtolower($type), - $foundTypes, - sprintf("Expected element type '%s' not found in %s", $type, json_encode($foundTypes)) - ); - } - } - } - - public static function assertMetadataExpectation( - ExtractionResult $result, - string $path, - array $expectation - ): void { - // Convert Metadata object to array for lookup - $metadataArray = self::metadataToArray($result->metadata); - $value = self::lookupMetadataPath($metadataArray, $path); - - Assert::assertNotNull( - $value, - sprintf("Metadata path '%s' missing in %s", $path, json_encode($metadataArray)) - ); - - if (isset($expectation['eq'])) { - Assert::assertTrue( - self::valuesEqual($value, $expectation['eq']), - sprintf( - "Expected metadata '%s' == %s, got %s", - $path, - json_encode($expectation['eq']), - json_encode($value) - ) - ); - } - - if (isset($expectation['gte'])) { - Assert::assertGreaterThanOrEqual( - (float)$expectation['gte'], - (float)$value, - sprintf("Expected metadata '%s' >= %s, got %s", $path, $expectation['gte'], $value) - ); - } - - if (isset($expectation['lte'])) { - Assert::assertLessThanOrEqual( - (float)$expectation['lte'], - (float)$value, - sprintf("Expected metadata '%s' <= %s, got %s", $path, $expectation['lte'], $value) - ); - } - - if (isset($expectation['contains'])) { - $contains = $expectation['contains']; - if (is_string($value) && is_string($contains)) { - Assert::assertStringContainsString( - $contains, - $value, - sprintf("Expected metadata '%s' string to contain %s", $path, json_encode($contains)) - ); - } elseif (is_array($value) && is_string($contains)) { - Assert::assertContains( - $contains, - $value, - sprintf("Expected metadata '%s' to contain %s", $path, json_encode($contains)) - ); - } elseif (is_array($value) && is_array($contains)) { - $missing = array_diff($contains, $value); - Assert::assertEmpty( - $missing, - sprintf( - "Expected metadata '%s' to contain %s, missing %s", - $path, - json_encode($contains), - json_encode($missing) - ) - ); - } else { - Assert::fail(sprintf("Unsupported contains expectation for metadata '%s'", $path)); - } - } - } - - private static function metadataToArray($metadata): array - { - if (is_array($metadata)) { - return $metadata; - } - - // Use to_array() if available (extension Metadata object) - if (method_exists($metadata, 'to_array')) { - return $metadata->to_array(); - } - - // Fallback: Convert Metadata object to array using snake_case properties - $result = []; - $fields = [ - 'language', 'subject', 'format_type', 'title', 'authors', - 'keywords', 'created_at', 'modified_at', 'created_by', - 'modified_by', 'page_count', 'sheet_count', 'format', - ]; - foreach ($fields as $field) { - if (isset($metadata->$field)) { - $result[$field] = $metadata->$field; - } - } - - // Include custom/additional fields - if (method_exists($metadata, 'get_additional')) { - foreach ($metadata->get_additional() as $key => $value) { - $result[$key] = $value; - } - } elseif (isset($metadata->custom) && is_array($metadata->custom)) { - foreach ($metadata->custom as $key => $value) { - $result[$key] = $value; - } - } - - return $result; - } - - private static function lookupMetadataPath(array $metadata, string $path) - { - $current = $metadata; - $segments = explode('.', $path); - - foreach ($segments as $segment) { - if (!is_array($current) || !isset($current[$segment])) { - // Try format metadata fallback - if (isset($metadata['format']) && is_array($metadata['format'])) { - $current = $metadata['format']; - foreach ($segments as $seg) { - if (!is_array($current) || !isset($current[$seg])) { - return null; - } - $current = $current[$seg]; - } - return $current; - } - return null; - } - $current = $current[$segment]; - } - - return $current; - } - - private static function valuesEqual($lhs, $rhs): bool - { - if (is_string($lhs) && is_string($rhs)) { - return $lhs === $rhs; - } - if (is_numeric($lhs) && is_numeric($rhs)) { - return (float)$lhs === (float)$rhs; - } - if (is_bool($lhs) && is_bool($rhs)) { - return $lhs === $rhs; - } - return $lhs == $rhs; - } - - public static function assertDocument( - ExtractionResult $result, - bool $hasDocument, - ?int $minNodeCount = null, - ?array $nodeTypesInclude = null, - ?bool $hasGroups = null - ): void { - $document = $result->document ?? null; - if ($hasDocument) { - Assert::assertNotNull($document, 'Expected document but got null'); - $nodes = is_array($document) ? $document : ($document->nodes ?? []); - Assert::assertNotNull($nodes, 'Expected document.nodes but got null'); - if ($minNodeCount !== null) { - Assert::assertGreaterThanOrEqual( - $minNodeCount, - count($nodes), - sprintf('Expected at least %d nodes, found %d', $minNodeCount, count($nodes)) - ); - } - if ($nodeTypesInclude !== null && !empty($nodeTypesInclude)) { - $foundTypes = []; - foreach ($nodes as $node) { - $content = is_object($node) ? ($node->content ?? null) : ($node['content'] ?? null); - if ($content !== null) { - $nodeType = is_object($content) ? ($content->node_type ?? $content->nodeType ?? null) : ($content['node_type'] ?? null); - if ($nodeType !== null) { - $foundTypes[] = $nodeType; - } - } - } - foreach ($nodeTypesInclude as $type) { - Assert::assertContains( - $type, - $foundTypes, - sprintf("Expected node type '%s' not found in %s", $type, json_encode($foundTypes)) - ); - } - } - if ($hasGroups !== null) { - $hasGroupNodes = false; - foreach ($nodes as $node) { - $content = is_object($node) ? ($node->content ?? null) : ($node['content'] ?? null); - if ($content !== null) { - $nodeType = is_object($content) ? ($content->node_type ?? $content->nodeType ?? null) : ($content['node_type'] ?? null); - if ($nodeType === 'group') { - $hasGroupNodes = true; - break; - } - } - } - Assert::assertEquals($hasGroups, $hasGroupNodes); - } - } else { - Assert::assertNull($document, 'Expected document to be null'); - } - } - - public static function assertOcrElements( - ExtractionResult $result, - ?bool $hasElements = null, - ?bool $elementsHaveGeometry = null, - ?bool $elementsHaveConfidence = null, - ?int $minCount = null - ): void { - $ocrElements = $result->ocrElements ?? null; - if ($hasElements) { - Assert::assertNotNull($ocrElements, 'Expected ocr_elements but got null'); - Assert::assertIsArray($ocrElements); - Assert::assertNotEmpty($ocrElements, 'Expected ocr_elements to be non-empty'); - } - if (is_array($ocrElements)) { - if ($minCount !== null) { - Assert::assertGreaterThanOrEqual( - $minCount, - count($ocrElements), - sprintf('Expected at least %d ocr_elements, found %d', $minCount, count($ocrElements)) - ); - } - } - } - - public static function assertKeywords( - ExtractionResult $result, - ?bool $hasKeywords = null, - ?int $minCount = null, - ?int $maxCount = null - ): void { - if ($hasKeywords === true) { - Assert::assertNotNull($result->extractedKeywords, 'Expected keywords but got null'); - Assert::assertNotEmpty($result->extractedKeywords ?? [], 'Expected keywords to be non-empty'); - } elseif ($hasKeywords === false) { - $keywords = $result->extractedKeywords ?? []; - Assert::assertTrue( - $keywords === null || count($keywords) === 0, - 'Expected keywords to be null or empty' - ); - } - - $keywords = $result->extractedKeywords ?? []; - $count = count($keywords); - - if ($minCount !== null) { - Assert::assertGreaterThanOrEqual( - $minCount, - $count, - sprintf("Expected at least %d keywords, found %d", $minCount, $count) - ); - } - - if ($maxCount !== null) { - Assert::assertLessThanOrEqual( - $maxCount, - $count, - sprintf("Expected at most %d keywords, found %d", $maxCount, $count) - ); - } - } - - public static function assertContentNotEmpty(ExtractionResult $result): void - { - Assert::assertNotEmpty( - $result->content ?? '', - "Expected content to be non-empty" - ); - } - - public static function assertTableBoundingBoxes(ExtractionResult $result, bool $expected): void - { - if ($expected) { - $tables = $result->tables ?? []; - Assert::assertNotEmpty($tables, 'Expected tables with bounding boxes but no tables found'); - foreach ($tables as $table) { - $bb = $table->boundingBox ?? $table->bounding_box ?? null; - Assert::assertNotNull($bb, 'Expected table to have bounding_box but it was null'); - } - } - } - - public static function assertTableContentContainsAny(ExtractionResult $result, array $snippets): void - { - $tables = $result->tables ?? []; - Assert::assertNotEmpty($tables, 'Expected tables but none found'); - - $allCells = []; - foreach ($tables as $table) { - $cells = $table->cells ?? []; - foreach ($cells as $row) { - foreach ($row as $cell) { - $allCells[] = strtolower((string)$cell); - } - } - } - - $found = false; - foreach ($snippets as $snippet) { - foreach ($allCells as $cell) { - if (str_contains($cell, strtolower($snippet))) { - $found = true; - break 2; - } - } - } - - Assert::assertTrue( - $found, - sprintf('No table cell contains any of %s', json_encode($snippets)) - ); - } - - public static function assertImageBoundingBoxes(ExtractionResult $result, bool $expected): void - { - if ($expected) { - $images = $result->images ?? []; - Assert::assertNotEmpty($images, 'Expected images with bounding boxes but no images found'); - foreach ($images as $image) { - $bb = $image->boundingBox ?? $image->bounding_box ?? null; - Assert::assertNotNull($bb, 'Expected image to have bounding_box but it was null'); - } - } - } - - public static function assertQualityScore( - ExtractionResult $result, - ?bool $hasScore = null, - ?float $minScore = null, - ?float $maxScore = null - ): void { - $score = $result->qualityScore ?? $result->quality_score ?? null; - - if ($hasScore === true) { - Assert::assertNotNull($score, 'Expected quality_score to be present'); - } - - if ($hasScore === false) { - Assert::assertNull($score, 'Expected quality_score to be absent'); - } - - if ($minScore !== null) { - Assert::assertNotNull($score, 'quality_score required for min_score assertion'); - Assert::assertGreaterThanOrEqual( - $minScore, - (float)$score, - sprintf('quality_score %f < %f', (float)$score, $minScore) - ); - } - - if ($maxScore !== null) { - Assert::assertNotNull($score, 'quality_score required for max_score assertion'); - Assert::assertLessThanOrEqual( - $maxScore, - (float)$score, - sprintf('quality_score %f > %f', (float)$score, $maxScore) - ); - } - } - - public static function assertProcessingWarnings( - ExtractionResult $result, - ?int $maxCount = null, - ?bool $isEmpty = null - ): void { - $warnings = $result->processingWarnings ?? $result->processing_warnings ?? []; - - if ($maxCount !== null) { - Assert::assertLessThanOrEqual( - $maxCount, - count($warnings), - sprintf('processing_warnings count %d > %d', count($warnings), $maxCount) - ); - } - - if ($isEmpty === true) { - Assert::assertCount( - 0, - $warnings, - sprintf('Expected empty processing_warnings, got %d', count($warnings)) - ); - } - } - - public static function assertDjotContent( - ExtractionResult $result, - ?bool $hasContent = null, - ?int $minBlocks = null - ): void { - $djot = $result->djotContent ?? $result->djot_content ?? null; - - if ($hasContent === true) { - Assert::assertNotNull($djot, 'Expected djot_content to be present'); - } - - if ($hasContent === false) { - Assert::assertNull($djot, 'Expected djot_content to be absent'); - } - - if ($minBlocks !== null) { - Assert::assertNotNull($djot, 'djot_content required for min_blocks assertion'); - $blocks = is_object($djot) ? ($djot->blocks ?? []) : ($djot['blocks'] ?? []); - Assert::assertGreaterThanOrEqual( - $minBlocks, - count($blocks), - sprintf('djot_content blocks %d < %d', count($blocks), $minBlocks) - ); - } - } - - public static function assertAnnotations( - ExtractionResult $result, - bool $hasAnnotations = false, - ?int $minCount = null - ): void { - $annotations = $result->annotations ?? null; - - if ($hasAnnotations) { - Assert::assertNotNull($annotations, 'Expected annotations to be present'); - Assert::assertIsArray($annotations); - Assert::assertNotEmpty($annotations, 'Expected annotations to be non-empty'); - } - - if ($annotations !== null && is_array($annotations) && $minCount !== null) { - Assert::assertGreaterThanOrEqual( - $minCount, - count($annotations), - sprintf('Expected at least %d annotations, got %d', $minCount, count($annotations)) - ); - } - } - - public static function skipIfFeatureUnavailable(string $feature): void - { - $envVar = 'KREUZBERG_' . strtoupper(str_replace('-', '_', $feature)) . '_AVAILABLE'; - $flag = getenv($envVar); - if ($flag === false || $flag === '' || $flag === '0' || strtolower($flag) === 'false') { - Assert::markTestSkipped( - sprintf('Feature "%s" not available (set %s=1)', $feature, $envVar) - ); - } - } - - public static function assertIsPng(string $data): void - { - Assert::assertGreaterThanOrEqual(4, strlen($data), - sprintf('Data too short for PNG: %d bytes', strlen($data))); - Assert::assertSame("\x89PNG", substr($data, 0, 4), 'Missing PNG magic bytes'); - } - - public static function assertMinByteLength(string $data, int $minLength): void - { - Assert::assertGreaterThanOrEqual($minLength, strlen($data), - sprintf('Expected at least %d bytes, got %d', $minLength, strlen($data))); - } - - } -} diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/ContractTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/ContractTest.java index 2584fcae2b9..82a409e26df 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/ContractTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/ContractTest.java @@ -553,7 +553,7 @@ public void configChunking() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null, null); }); } @@ -572,7 +572,7 @@ public void configChunkingHeadingContext() throws Exception { true, result -> { E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, true, null); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, true, null, null); }); } @@ -592,7 +592,7 @@ public void configChunkingMarkdown() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null, null); }); } @@ -611,7 +611,7 @@ public void configChunkingNoHeadings() throws Exception { true, result -> { E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, false, null); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, false, null, null); }); } @@ -630,7 +630,7 @@ public void configChunkingPrependHeadingContext() throws Exception { true, result -> { E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, true, true); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, true, null, true); }); } @@ -648,7 +648,7 @@ public void configChunkingSmall() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, null, null, null); }); } @@ -667,7 +667,7 @@ public void configChunkingText() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null, null); }); } @@ -686,7 +686,7 @@ public void configChunkingTokenizer() throws Exception { true, result -> { E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 2, null, true, null, null, null, null); }); } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java b/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java index 2751d9651d0..400bcd75d52 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java @@ -343,6 +343,7 @@ public static void assertChunks( Boolean eachHasContent, Boolean eachHasEmbedding, Boolean eachHasHeadingContext, + Boolean eachHasChunkType, Boolean contentStartsWithHeading) { var chunks = result.getChunks(); int count = chunks != null ? chunks.size() : 0; @@ -381,6 +382,13 @@ public static void assertChunks( "Expected each chunk to have no heading_context"); } } + if (chunks != null && eachHasChunkType != null && eachHasChunkType) { + for (var chunk : chunks) { + String type = chunk.getChunkType(); + assertTrue(type != null && !"unknown".equals(type), + "Expected each chunk to have a specific chunk_type"); + } + } if (chunks != null && contentStartsWithHeading != null && contentStartsWithHeading) { String headingPrefix = String.valueOf((char) 35); for (var chunk : chunks) { diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/EmbeddingsTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/EmbeddingsTest.java index 60efee5b620..4f3d535ee0e 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/EmbeddingsTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/EmbeddingsTest.java @@ -69,7 +69,7 @@ public void embeddingAsync() throws Exception { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null, null); } @Test @@ -94,7 +94,7 @@ public void embeddingBalancedPreset() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null, null); }); } @@ -111,7 +111,7 @@ public void embeddingDisabled() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, false, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, false, null, null, null); }); } @@ -137,7 +137,7 @@ public void embeddingFastPreset() throws Exception { result -> { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 10); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, true, null, null, null); }); } } diff --git a/e2e/java/src/test/java/com/kreuzberg/e2e/TokenReductionTest.java b/e2e/java/src/test/java/com/kreuzberg/e2e/TokenReductionTest.java index 6b347b41493..b00176ad3d9 100644 --- a/e2e/java/src/test/java/com/kreuzberg/e2e/TokenReductionTest.java +++ b/e2e/java/src/test/java/com/kreuzberg/e2e/TokenReductionTest.java @@ -87,7 +87,7 @@ public void tokenReductionWithChunking() throws Exception { E2EHelpers.Assertions.assertExpectedMime(result, Arrays.asList("application/pdf")); E2EHelpers.Assertions.assertMinContentLength(result, 5); E2EHelpers.Assertions.assertMaxContentLength(result, 200); - E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null); + E2EHelpers.Assertions.assertChunks(result, 1, null, true, null, null, null, null); E2EHelpers.Assertions.assertContentNotEmpty(result); }); } diff --git a/e2e/php/tests/ContractTest.php b/e2e/php/tests/ContractTest.php index 52ed3e84568..2d311d5321a 100644 --- a/e2e/php/tests/ContractTest.php +++ b/e2e/php/tests/ContractTest.php @@ -334,7 +334,7 @@ public function test_config_chunking(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertChunks($result, 1, null, true, null, null, null, null); } /** @@ -355,7 +355,7 @@ public function test_config_chunking_heading_context(): void $result = $kreuzberg->extractFile($documentPath); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, true, null); + Helpers::assertChunks($result, 2, null, true, null, true, null, null); } /** @@ -377,7 +377,7 @@ public function test_config_chunking_markdown(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertChunks($result, 1, null, true, null, null, null, null); } /** @@ -398,7 +398,7 @@ public function test_config_chunking_no_headings(): void $result = $kreuzberg->extractFile($documentPath); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, false, null); + Helpers::assertChunks($result, 2, null, true, null, false, null, null); } /** @@ -419,7 +419,7 @@ public function test_config_chunking_prepend_heading_context(): void $result = $kreuzberg->extractFile($documentPath); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, true, true); + Helpers::assertChunks($result, 2, null, true, null, true, null, true); } /** @@ -441,7 +441,7 @@ public function test_config_chunking_small(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, null, null); + Helpers::assertChunks($result, 2, null, true, null, null, null, null); } /** @@ -461,7 +461,7 @@ public function test_config_chunking_text(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertChunks($result, 1, null, true, null, null, null, null); } /** @@ -482,7 +482,7 @@ public function test_config_chunking_tokenizer(): void $result = $kreuzberg->extractFile($documentPath); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 2, null, true, null, null, null); + Helpers::assertChunks($result, 2, null, true, null, null, null, null); } /** diff --git a/e2e/php/tests/EmbeddingsTest.php b/e2e/php/tests/EmbeddingsTest.php index f906aa9abbe..1baf60227fb 100644 --- a/e2e/php/tests/EmbeddingsTest.php +++ b/e2e/php/tests/EmbeddingsTest.php @@ -35,7 +35,7 @@ public function test_embedding_async(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); + Helpers::assertChunks($result, 1, null, true, true, null, null, null); } /** @@ -57,7 +57,7 @@ public function test_embedding_balanced_preset(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); + Helpers::assertChunks($result, 1, null, true, true, null, null, null); } /** @@ -77,7 +77,7 @@ public function test_embedding_disabled(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, false, null, null); + Helpers::assertChunks($result, 1, null, true, false, null, null, null); } /** @@ -99,7 +99,7 @@ public function test_embedding_fast_preset(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 10); - Helpers::assertChunks($result, 1, null, true, true, null, null); + Helpers::assertChunks($result, 1, null, true, true, null, null, null); } } diff --git a/e2e/php/tests/Helpers.php b/e2e/php/tests/Helpers.php index 88caaf899d3..17bf9d44f92 100644 --- a/e2e/php/tests/Helpers.php +++ b/e2e/php/tests/Helpers.php @@ -209,6 +209,7 @@ public static function assertChunks( ?bool $eachHasContent, ?bool $eachHasEmbedding, ?bool $eachHasHeadingContext = null, + ?bool $eachHasChunkType = null, ?bool $contentStartsWithHeading = null ): void { $chunks = $result->chunks ?? []; @@ -264,6 +265,13 @@ public static function assertChunks( ); } } + if ($eachHasChunkType === true) { + foreach ($chunks as $i => $chunk) { + $type = $chunk->chunk_type ?? null; + Assert::assertNotNull($type, sprintf("Chunk %d should have chunk_type", $i)); + Assert::assertNotSame('unknown', $type, sprintf("Chunk %d should have specific chunk_type, got 'unknown'", $i)); + } + } if ($contentStartsWithHeading === true) { foreach ($chunks as $i => $chunk) { if (empty($chunk->metadata->heading_context ?? null)) { diff --git a/e2e/php/tests/Token_reductionTest.php b/e2e/php/tests/Token_reductionTest.php index e09b33f23bb..c89cce2d86c 100644 --- a/e2e/php/tests/Token_reductionTest.php +++ b/e2e/php/tests/Token_reductionTest.php @@ -95,7 +95,7 @@ public function test_token_reduction_with_chunking(): void Helpers::assertExpectedMime($result, ['application/pdf']); Helpers::assertMinContentLength($result, 5); Helpers::assertMaxContentLength($result, 200); - Helpers::assertChunks($result, 1, null, true, null, null, null); + Helpers::assertChunks($result, 1, null, true, null, null, null, null); Helpers::assertContentNotEmpty($result); } diff --git a/e2e/r/tests/testthat/helper-kreuzberg.R b/e2e/r/tests/testthat/helper-kreuzberg.R index 41924f0b9a0..c0fe48bc82e 100644 --- a/e2e/r/tests/testthat/helper-kreuzberg.R +++ b/e2e/r/tests/testthat/helper-kreuzberg.R @@ -216,6 +216,7 @@ assert_metadata_expectation <- function(result, path, expectation) { assert_chunks <- function(result, min_count = NULL, max_count = NULL, each_has_content = NULL, each_has_embedding = NULL, each_has_heading_context = NULL, + each_has_chunk_type = NULL, content_starts_with_heading = NULL) { chunks <- if (is.null(result$chunks)) list() else result$chunks if (!is.null(min_count)) testthat::expect_gte(length(chunks), min_count) @@ -232,6 +233,14 @@ assert_chunks <- function(result, min_count = NULL, max_count = NULL, if (isFALSE(each_has_heading_context)) { for (chunk in chunks) testthat::expect_true(is.null(chunk$metadata$heading_context)) } + if (isTRUE(each_has_chunk_type)) { + for (chunk in chunks) { + chunk_type <- chunk[["chunk_type"]] + if (is.null(chunk_type) || chunk_type == "unknown") { + testthat::fail(paste0("Expected specific chunk_type, got: ", chunk_type)) + } + } + } if (isTRUE(content_starts_with_heading)) { heading_char <- rawToChar(as.raw(35)) for (chunk in chunks) { diff --git a/e2e/ruby/spec/helpers.rb b/e2e/ruby/spec/helpers.rb index c72edf57123..29c6baf629b 100644 --- a/e2e/ruby/spec/helpers.rb +++ b/e2e/ruby/spec/helpers.rb @@ -240,7 +240,7 @@ def self.assert_metadata_expectation(result, path, expectation) end end - def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, content_starts_with_heading: nil) + def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, each_has_chunk_type: nil, content_starts_with_heading: nil) chunks = Array(result.chunks) expect(chunks.length).to be >= min_count if min_count expect(chunks.length).to be <= max_count if max_count @@ -251,6 +251,12 @@ def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: elsif each_has_heading_context == false chunks.each { |chunk| expect(chunk.metadata&.heading_context).to be_nil } end + if each_has_chunk_type == true + chunks.each { |chunk| + expect(chunk.chunk_type).not_to be_nil + expect(chunk.chunk_type).not_to eq("unknown") + } + end return unless content_starts_with_heading == true chunks.each do |chunk| diff --git a/e2e/typescript/tests/contract.spec.ts b/e2e/typescript/tests/contract.spec.ts index d710be46a31..dfe7166d856 100644 --- a/e2e/typescript/tests/contract.spec.ts +++ b/e2e/typescript/tests/contract.spec.ts @@ -453,7 +453,7 @@ describe("contract fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, null, null, null); + chunkAssertions.assertChunks(result, 1, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -480,7 +480,7 @@ describe("contract fixtures", () => { return; } assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, true, null); + chunkAssertions.assertChunks(result, 2, null, true, null, true, null, null); }, TEST_TIMEOUT_MS, ); @@ -508,7 +508,7 @@ describe("contract fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, null, null, null); + chunkAssertions.assertChunks(result, 1, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -535,7 +535,7 @@ describe("contract fixtures", () => { return; } assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, false, null); + chunkAssertions.assertChunks(result, 2, null, true, null, false, null, null); }, TEST_TIMEOUT_MS, ); @@ -564,7 +564,7 @@ describe("contract fixtures", () => { return; } assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, true, true); + chunkAssertions.assertChunks(result, 2, null, true, null, true, null, true); }, TEST_TIMEOUT_MS, ); @@ -592,7 +592,7 @@ describe("contract fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, null, null); + chunkAssertions.assertChunks(result, 2, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -620,7 +620,7 @@ describe("contract fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, null, null, null); + chunkAssertions.assertChunks(result, 1, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -657,7 +657,7 @@ describe("contract fixtures", () => { return; } assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 2, null, true, null, null, null); + chunkAssertions.assertChunks(result, 2, null, true, null, null, null, null); }, TEST_TIMEOUT_MS, ); diff --git a/e2e/typescript/tests/embeddings.spec.ts b/e2e/typescript/tests/embeddings.spec.ts index e87cc6e4bab..0b2177cfb8d 100644 --- a/e2e/typescript/tests/embeddings.spec.ts +++ b/e2e/typescript/tests/embeddings.spec.ts @@ -46,7 +46,7 @@ describe("embeddings fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, true, null, null); + chunkAssertions.assertChunks(result, 1, null, true, true, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -87,7 +87,7 @@ describe("embeddings fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, true, null, null); + chunkAssertions.assertChunks(result, 1, null, true, true, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -115,7 +115,7 @@ describe("embeddings fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, false, null, null); + chunkAssertions.assertChunks(result, 1, null, true, false, null, null, null); }, TEST_TIMEOUT_MS, ); @@ -154,7 +154,7 @@ describe("embeddings fixtures", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - chunkAssertions.assertChunks(result, 1, null, true, true, null, null); + chunkAssertions.assertChunks(result, 1, null, true, true, null, null, null); }, TEST_TIMEOUT_MS, ); diff --git a/e2e/typescript/tests/helpers.ts b/e2e/typescript/tests/helpers.ts index 0e86b0c191e..fae2c145c4a 100644 --- a/e2e/typescript/tests/helpers.ts +++ b/e2e/typescript/tests/helpers.ts @@ -783,6 +783,7 @@ export const chunkAssertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -816,6 +817,13 @@ export const chunkAssertions = { expect(((chunk as PlainRecord).metadata as PlainRecord)?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const meta = (chunk as PlainRecord).metadata as PlainRecord | undefined; diff --git a/e2e/typescript/tests/token-reduction.spec.ts b/e2e/typescript/tests/token-reduction.spec.ts index 826eb54f251..510d6caddd5 100644 --- a/e2e/typescript/tests/token-reduction.spec.ts +++ b/e2e/typescript/tests/token-reduction.spec.ts @@ -125,7 +125,7 @@ describe("token_reduction fixtures", () => { assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 5); assertions.assertMaxContentLength(result, 200); - chunkAssertions.assertChunks(result, 1, null, true, null, null, null); + chunkAssertions.assertChunks(result, 1, null, true, null, null, null, null); assertions.assertContentNotEmpty(result); }, TEST_TIMEOUT_MS, diff --git a/e2e/wasm-deno/contract.test.ts b/e2e/wasm-deno/contract.test.ts index 53e93adfaf5..b69b1d29ec8 100644 --- a/e2e/wasm-deno/contract.test.ts +++ b/e2e/wasm-deno/contract.test.ts @@ -334,7 +334,7 @@ Deno.test("config_chunking", { permissions: { read: true, net: true } }, async ( } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); Deno.test("config_chunking_heading_context", { permissions: { read: true, net: true } }, async () => { @@ -354,7 +354,7 @@ Deno.test("config_chunking_heading_context", { permissions: { read: true, net: t return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, true, null); + assertions.assertChunks(result, 2, null, true, null, true, null, null); }); Deno.test("config_chunking_markdown", { permissions: { read: true, net: true } }, async () => { @@ -375,7 +375,7 @@ Deno.test("config_chunking_markdown", { permissions: { read: true, net: true } } } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); Deno.test("config_chunking_no_headings", { permissions: { read: true, net: true } }, async () => { @@ -395,7 +395,7 @@ Deno.test("config_chunking_no_headings", { permissions: { read: true, net: true return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, false, null); + assertions.assertChunks(result, 2, null, true, null, false, null, null); }); Deno.test("config_chunking_prepend_heading_context", { permissions: { read: true, net: true } }, async () => { @@ -417,7 +417,7 @@ Deno.test("config_chunking_prepend_heading_context", { permissions: { read: true return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, true, true); + assertions.assertChunks(result, 2, null, true, null, true, null, true); }); Deno.test("config_chunking_small", { permissions: { read: true, net: true } }, async () => { @@ -438,7 +438,7 @@ Deno.test("config_chunking_small", { permissions: { read: true, net: true } }, a } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, null, null); + assertions.assertChunks(result, 2, null, true, null, null, null, null); }); Deno.test("config_chunking_text", { permissions: { read: true, net: true } }, async () => { @@ -459,7 +459,7 @@ Deno.test("config_chunking_text", { permissions: { read: true, net: true } }, as } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); Deno.test("config_disable_ocr", { permissions: { read: true, net: true } }, async () => { diff --git a/e2e/wasm-deno/embeddings.test.ts b/e2e/wasm-deno/embeddings.test.ts index 8b753c914c3..e901c204ceb 100644 --- a/e2e/wasm-deno/embeddings.test.ts +++ b/e2e/wasm-deno/embeddings.test.ts @@ -28,6 +28,6 @@ Deno.test("embedding_disabled", { permissions: { read: true, net: true } }, asyn } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, false, null, null); + assertions.assertChunks(result, 1, null, true, false, null, null, null); }); diff --git a/e2e/wasm-deno/helpers.ts b/e2e/wasm-deno/helpers.ts index 16120b565a6..4525260becd 100644 --- a/e2e/wasm-deno/helpers.ts +++ b/e2e/wasm-deno/helpers.ts @@ -445,6 +445,7 @@ export const assertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -482,6 +483,13 @@ export const assertions = { ); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + assertExists(chunkType, "Chunk missing chunk_type"); + assertEquals(chunkType !== "unknown", true, "Chunk chunk_type should not be 'unknown'"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const meta = (chunk as PlainRecord).metadata as PlainRecord | undefined; diff --git a/e2e/wasm-deno/token_reduction.test.ts b/e2e/wasm-deno/token_reduction.test.ts index 55901f8e37c..3de61fd4f13 100644 --- a/e2e/wasm-deno/token_reduction.test.ts +++ b/e2e/wasm-deno/token_reduction.test.ts @@ -102,6 +102,6 @@ Deno.test("token_reduction_with_chunking", { permissions: { read: true, net: tru assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 5); assertions.assertMaxContentLength(result, 200); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); assertions.assertContentNotEmpty(result); }); diff --git a/e2e/wasm-workers/tests/contract.spec.ts b/e2e/wasm-workers/tests/contract.spec.ts index bab0f5c6481..dd7dd60a9fa 100644 --- a/e2e/wasm-workers/tests/contract.spec.ts +++ b/e2e/wasm-workers/tests/contract.spec.ts @@ -400,7 +400,7 @@ describe("contract", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); it("config_chunking_heading_context", async () => { @@ -424,7 +424,7 @@ describe("contract", () => { return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, true, null); + assertions.assertChunks(result, 2, null, true, null, true, null, null); }); it("config_chunking_markdown", async () => { @@ -449,7 +449,7 @@ describe("contract", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); it("config_chunking_no_headings", async () => { @@ -473,7 +473,7 @@ describe("contract", () => { return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, false, null); + assertions.assertChunks(result, 2, null, true, null, false, null, null); }); it("config_chunking_prepend_heading_context", async () => { @@ -499,7 +499,7 @@ describe("contract", () => { return; } assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, true, true); + assertions.assertChunks(result, 2, null, true, null, true, null, true); }); it("config_chunking_small", async () => { @@ -524,7 +524,7 @@ describe("contract", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 2, null, true, null, null, null); + assertions.assertChunks(result, 2, null, true, null, null, null, null); }); it("config_chunking_text", async () => { @@ -549,7 +549,7 @@ describe("contract", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); }); it("config_disable_ocr", async () => { diff --git a/e2e/wasm-workers/tests/embeddings.spec.ts b/e2e/wasm-workers/tests/embeddings.spec.ts index 50c2ad3bf5c..a5ba2cd49c6 100644 --- a/e2e/wasm-workers/tests/embeddings.spec.ts +++ b/e2e/wasm-workers/tests/embeddings.spec.ts @@ -31,7 +31,7 @@ describe("embeddings", () => { } assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, false, null, null); + assertions.assertChunks(result, 1, null, true, false, null, null, null); }); }); diff --git a/e2e/wasm-workers/tests/helpers.ts b/e2e/wasm-workers/tests/helpers.ts index 6af8502e30e..e3d9d710791 100644 --- a/e2e/wasm-workers/tests/helpers.ts +++ b/e2e/wasm-workers/tests/helpers.ts @@ -396,6 +396,7 @@ export const assertions = { eachHasContent: boolean | null, eachHasEmbedding: boolean | null, eachHasHeadingContext: boolean | null, + eachHasChunkType: boolean | null, contentStartsWithHeading: boolean | null, ): void { const chunks = Array.isArray(result.chunks) ? result.chunks : []; @@ -425,6 +426,13 @@ export const assertions = { expect(chunk.metadata?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { if (chunk.metadata?.headingContext == null) continue; diff --git a/e2e/wasm-workers/tests/token_reduction.spec.ts b/e2e/wasm-workers/tests/token_reduction.spec.ts index f2d271fea9f..d8ae8136689 100644 --- a/e2e/wasm-workers/tests/token_reduction.spec.ts +++ b/e2e/wasm-workers/tests/token_reduction.spec.ts @@ -112,7 +112,7 @@ describe("token_reduction", () => { assertions.assertExpectedMime(result, ["application/pdf"]); assertions.assertMinContentLength(result, 5); assertions.assertMaxContentLength(result, 200); - assertions.assertChunks(result, 1, null, true, null, null, null); + assertions.assertChunks(result, 1, null, true, null, null, null, null); assertions.assertContentNotEmpty(result); }); }); diff --git a/fixtures/schema.json b/fixtures/schema.json index 5e23d2168f5..aeaabc3c196 100644 --- a/fixtures/schema.json +++ b/fixtures/schema.json @@ -181,6 +181,14 @@ "each_has_heading_context": { "type": "boolean", "description": "Assert each chunk has heading_context populated (markdown chunker only)." + }, + "each_has_chunk_type": { + "type": "boolean", + "description": "Assert each chunk has a specific (non-Unknown) chunk_type classification." + }, + "content_starts_with_heading": { + "type": "boolean", + "description": "Assert each chunk's content starts with a Markdown heading (#)." } }, "additionalProperties": false diff --git a/packages/csharp/Kreuzberg/Models.cs b/packages/csharp/Kreuzberg/Models.cs index 268b1215c80..a74a3d49322 100644 --- a/packages/csharp/Kreuzberg/Models.cs +++ b/packages/csharp/Kreuzberg/Models.cs @@ -794,6 +794,12 @@ public sealed class Chunk /// [JsonPropertyName("metadata")] public ChunkMetadata Metadata { get; set; } = new(); + + ///

+ /// Semantic type classification of this chunk. + /// + [JsonPropertyName("chunk_type")] + public string ChunkType { get; set; } = "unknown"; } /// diff --git a/packages/elixir/lib/kreuzberg/chunk.ex b/packages/elixir/lib/kreuzberg/chunk.ex index 994072c73dc..4d2f241bf3d 100644 --- a/packages/elixir/lib/kreuzberg/chunk.ex +++ b/packages/elixir/lib/kreuzberg/chunk.ex @@ -9,15 +9,17 @@ defmodule Kreuzberg.Chunk do * `:content` - The text content of this chunk * `:embedding` - Vector embedding (list of floats) for semantic search * `:metadata` - ChunkMetadata struct with position and token info + * `:chunk_type` - Semantic type classification of this chunk (e.g. "heading", "unknown") """ @type t :: %__MODULE__{ content: String.t(), embedding: list(float()) | nil, - metadata: Kreuzberg.ChunkMetadata.t() + metadata: Kreuzberg.ChunkMetadata.t(), + chunk_type: String.t() } - defstruct content: "", embedding: nil, metadata: %Kreuzberg.ChunkMetadata{} + defstruct content: "", embedding: nil, metadata: %Kreuzberg.ChunkMetadata{}, chunk_type: "unknown" @doc """ Creates a new Chunk struct. @@ -25,14 +27,15 @@ defmodule Kreuzberg.Chunk do ## Parameters * `content` - The text content of the chunk - * `opts` - Optional keyword list with `:embedding` and `:metadata` + * `opts` - Optional keyword list with `:embedding`, `:metadata`, and `:chunk_type` """ @spec new(String.t(), keyword()) :: t() def new(content, opts \\ []) when is_binary(content) do %__MODULE__{ content: content, embedding: Keyword.get(opts, :embedding), - metadata: Keyword.get(opts, :metadata, %Kreuzberg.ChunkMetadata{}) + metadata: Keyword.get(opts, :metadata, %Kreuzberg.ChunkMetadata{}), + chunk_type: Keyword.get(opts, :chunk_type, "unknown") } end @@ -42,7 +45,7 @@ defmodule Kreuzberg.Chunk do ## Examples iex> Kreuzberg.Chunk.from_map(%{"content" => "chunk text", "embedding" => [0.1, 0.2]}) - %Kreuzberg.Chunk{content: "chunk text", embedding: [0.1, 0.2]} + %Kreuzberg.Chunk{content: "chunk text", embedding: [0.1, 0.2], metadata: %Kreuzberg.ChunkMetadata{}, chunk_type: "unknown"} """ @spec from_map(map()) :: t() def from_map(data) when is_map(data) do @@ -56,7 +59,8 @@ defmodule Kreuzberg.Chunk do %__MODULE__{ content: data["content"] || "", embedding: data["embedding"], - metadata: metadata + metadata: metadata, + chunk_type: data["chunk_type"] || "unknown" } end @@ -68,7 +72,8 @@ defmodule Kreuzberg.Chunk do %{ "content" => chunk.content, "embedding" => chunk.embedding, - "metadata" => Kreuzberg.ChunkMetadata.to_map(chunk.metadata) + "metadata" => Kreuzberg.ChunkMetadata.to_map(chunk.metadata), + "chunk_type" => chunk.chunk_type } end end diff --git a/packages/go/v4/types.go b/packages/go/v4/types.go index 18393f098d4..0841f90edb6 100644 --- a/packages/go/v4/types.go +++ b/packages/go/v4/types.go @@ -190,6 +190,7 @@ type Chunk struct { Content string `json:"content"` Embedding []float32 `json:"embedding,omitempty"` Metadata ChunkMetadata `json:"metadata"` + ChunkType string `json:"chunk_type,omitempty"` } // ChunkMetadata provides positional information for a chunk. diff --git a/packages/java/src/main/java/dev/kreuzberg/Chunk.java b/packages/java/src/main/java/dev/kreuzberg/Chunk.java index 13321b88279..f2340644d5f 100644 --- a/packages/java/src/main/java/dev/kreuzberg/Chunk.java +++ b/packages/java/src/main/java/dev/kreuzberg/Chunk.java @@ -16,10 +16,11 @@ public final class Chunk { private final String content; private final List embedding; private final ChunkMetadata metadata; + private final String chunkType; @JsonCreator public Chunk(@JsonProperty("content") String content, @JsonProperty("embedding") List embedding, - @JsonProperty("metadata") ChunkMetadata metadata) { + @JsonProperty("metadata") ChunkMetadata metadata, @JsonProperty("chunk_type") String chunkType) { this.content = Objects.requireNonNull(content, "content must not be null"); this.metadata = Objects.requireNonNull(metadata, "metadata must not be null"); if (embedding != null) { @@ -27,6 +28,7 @@ public Chunk(@JsonProperty("content") String content, @JsonProperty("embedding") } else { this.embedding = null; } + this.chunkType = chunkType != null ? chunkType : "unknown"; } public String getContent() { @@ -41,6 +43,10 @@ public ChunkMetadata getMetadata() { return metadata; } + public String getChunkType() { + return chunkType; + } + @Override public boolean equals(Object obj) { if (this == obj) { @@ -51,17 +57,17 @@ public boolean equals(Object obj) { } Chunk other = (Chunk) obj; return Objects.equals(content, other.content) && Objects.equals(embedding, other.embedding) - && Objects.equals(metadata, other.metadata); + && Objects.equals(metadata, other.metadata) && Objects.equals(chunkType, other.chunkType); } @Override public int hashCode() { - return Objects.hash(content, embedding, metadata); + return Objects.hash(content, embedding, metadata, chunkType); } @Override public String toString() { return "Chunk{" + "contentLength=" + content.length() + ", embeddingDimensions=" - + (embedding == null ? 0 : embedding.size()) + ", metadata=" + metadata + '}'; + + (embedding == null ? 0 : embedding.size()) + ", metadata=" + metadata + ", chunkType=" + chunkType + '}'; } } diff --git a/packages/typescript/core/src/types/results.ts b/packages/typescript/core/src/types/results.ts index c696d8325ed..cd7e0072b35 100644 --- a/packages/typescript/core/src/types/results.ts +++ b/packages/typescript/core/src/types/results.ts @@ -48,10 +48,26 @@ export interface ChunkMetadata { headingContext?: HeadingContext | null; } +export type ChunkType = + | "heading" + | "party_list" + | "definitions" + | "operative_clause" + | "signature_block" + | "schedule" + | "table_like" + | "formula" + | "code_block" + | "image" + | "org_chart" + | "diagram" + | "unknown"; + export interface Chunk { content: string; embedding?: number[] | null; metadata: ChunkMetadata; + chunkType: ChunkType; } export interface ExtractedImage { diff --git a/tools/e2e-generator/e2e/go/helpers_test.go b/tools/e2e-generator/e2e/go/helpers_test.go index e00eafcc94c..dcbde55239a 100644 --- a/tools/e2e-generator/e2e/go/helpers_test.go +++ b/tools/e2e-generator/e2e/go/helpers_test.go @@ -241,7 +241,7 @@ func boolPtr(value bool) *bool { return &value } -func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext *bool) { +func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, eachHasContent, eachHasEmbedding, eachHasHeadingContext, eachHasChunkType, contentStartsWithHeading *bool) { t.Helper() count := len(result.Chunks) if minCount != nil && count < *minCount { @@ -278,6 +278,20 @@ func assertChunks(t *testing.T, result *kreuzberg.ExtractionResult, minCount, ma } } } + if eachHasChunkType != nil && *eachHasChunkType { + for i, chunk := range result.Chunks { + if chunk.ChunkType == "" || chunk.ChunkType == "unknown" { + t.Fatalf("chunk %d has no specific chunk_type, got %q", i, chunk.ChunkType) + } + } + } + if contentStartsWithHeading != nil && *contentStartsWithHeading { + for i, chunk := range result.Chunks { + if !strings.HasPrefix(chunk.Content, "#") { + t.Fatalf("chunk %d content does not start with '#'", i) + } + } + } } func assertImages(t *testing.T, result *kreuzberg.ExtractionResult, minCount, maxCount *int, formatsInclude []string) { diff --git a/tools/e2e-generator/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java b/tools/e2e-generator/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java index b3bbc49e243..bb584aa5386 100644 --- a/tools/e2e-generator/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java +++ b/tools/e2e-generator/e2e/java/src/test/java/com/kreuzberg/e2e/E2EHelpers.java @@ -337,7 +337,9 @@ public static void assertChunks( Integer maxCount, Boolean eachHasContent, Boolean eachHasEmbedding, - Boolean eachHasHeadingContext) { + Boolean eachHasHeadingContext, + Boolean eachHasChunkType, + Boolean contentStartsWithHeading) { var chunks = result.getChunks(); int count = chunks != null ? chunks.size() : 0; if (minCount != null) { @@ -375,6 +377,22 @@ public static void assertChunks( "Expected each chunk to have no heading_context"); } } + if (chunks != null && eachHasChunkType != null && eachHasChunkType) { + for (var chunk : chunks) { + String type = chunk.getChunkType(); + assertTrue(type != null && !"unknown".equals(type), + "Expected each chunk to have a specific chunk_type"); + } + } + if (chunks != null && contentStartsWithHeading != null && contentStartsWithHeading) { + String headingPrefix = String.valueOf((char) 35); + for (var chunk : chunks) { + String content = chunk.getContent(); + assertTrue( + content != null && content.startsWith(headingPrefix), + "Expected each chunk content to start with a heading"); + } + } } public static void assertImages( diff --git a/tools/e2e-generator/e2e/php/tests/Helpers.php b/tools/e2e-generator/e2e/php/tests/Helpers.php index 245c73886b0..49b433e68ac 100644 --- a/tools/e2e-generator/e2e/php/tests/Helpers.php +++ b/tools/e2e-generator/e2e/php/tests/Helpers.php @@ -208,7 +208,9 @@ public static function assertChunks( ?int $maxCount, ?bool $eachHasContent, ?bool $eachHasEmbedding, - ?bool $eachHasHeadingContext = null + ?bool $eachHasHeadingContext = null, + ?bool $eachHasChunkType = null, + ?bool $contentStartsWithHeading = null ): void { $chunks = $result->chunks ?? []; $count = count($chunks); @@ -263,6 +265,22 @@ public static function assertChunks( ); } } + if ($eachHasChunkType === true) { + foreach ($chunks as $i => $chunk) { + $type = $chunk->chunk_type ?? null; + Assert::assertNotNull($type, sprintf("Chunk %d should have chunk_type", $i)); + Assert::assertNotSame('unknown', $type, sprintf("Chunk %d should have specific chunk_type, got 'unknown'", $i)); + } + } + if ($contentStartsWithHeading === true) { + foreach ($chunks as $i => $chunk) { + Assert::assertStringStartsWith( + '#', + $chunk->content ?? '', + sprintf("Chunk %d content should start with a heading (#)", $i) + ); + } + } } public static function assertImages( diff --git a/tools/e2e-generator/e2e/python/tests/helpers.py b/tools/e2e-generator/e2e/python/tests/helpers.py index bdc46ca3ecb..0ab7a0f8403 100644 --- a/tools/e2e-generator/e2e/python/tests/helpers.py +++ b/tools/e2e-generator/e2e/python/tests/helpers.py @@ -288,6 +288,13 @@ def _get_heading_context(chunk: Any) -> Any: return metadata.get("heading_context") if isinstance(metadata, dict) else getattr(metadata, "heading_context", None) +def _assert_chunks_chunk_type(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + chunk_type = getattr(chunk, "chunk_type", None) + if chunk_type is None or str(chunk_type) == "unknown": + pytest.fail(f"Chunk {i} has no specific chunk_type, got {chunk_type}") + + def assert_chunks( result: Any, min_count: int | None = None, @@ -295,6 +302,8 @@ def assert_chunks( each_has_content: bool | None = None, each_has_embedding: bool | None = None, each_has_heading_context: bool | None = None, + each_has_chunk_type: bool | None = None, + content_starts_with_heading: bool | None = None, ) -> None: chunks = getattr(result, "chunks", None) if chunks is None: @@ -319,6 +328,13 @@ def assert_chunks( pytest.fail(f"Chunk {i} has no heading_context") if not each_has_heading_context and hc is not None: pytest.fail(f"Chunk {i} should have no heading_context") + if each_has_chunk_type: + _assert_chunks_chunk_type(chunks) + if content_starts_with_heading: + for i, chunk in enumerate(chunks): + content = getattr(chunk, "content", None) + if not isinstance(content, str) or content[0:1] != chr(35): + pytest.fail(f"Chunk {i} content does not start with a heading") def assert_images( diff --git a/tools/e2e-generator/e2e/r/tests/testthat/helper-kreuzberg.R b/tools/e2e-generator/e2e/r/tests/testthat/helper-kreuzberg.R index 8dd336080fc..2000f975e87 100644 --- a/tools/e2e-generator/e2e/r/tests/testthat/helper-kreuzberg.R +++ b/tools/e2e-generator/e2e/r/tests/testthat/helper-kreuzberg.R @@ -236,7 +236,9 @@ assert_metadata_expectation <- function(result, path, expectation) { assert_chunks <- function(result, min_count = NULL, max_count = NULL, each_has_content = NULL, each_has_embedding = NULL, - each_has_heading_context = NULL) { + each_has_heading_context = NULL, + each_has_chunk_type = NULL, + content_starts_with_heading = NULL) { chunks <- if (is.null(result$chunks)) list() else result$chunks if (!is.null(min_count)) testthat::expect_gte(length(chunks), min_count) if (!is.null(max_count)) testthat::expect_lte(length(chunks), max_count) @@ -252,6 +254,18 @@ assert_chunks <- function(result, min_count = NULL, max_count = NULL, if (isFALSE(each_has_heading_context)) { for (chunk in chunks) testthat::expect_true(is.null(chunk$metadata$heading_context)) } + if (isTRUE(each_has_chunk_type)) { + for (chunk in chunks) { + chunk_type <- chunk[["chunk_type"]] + if (is.null(chunk_type) || chunk_type == "unknown") { + testthat::fail(paste0("Expected specific chunk_type, got: ", chunk_type)) + } + } + } + if (isTRUE(content_starts_with_heading)) { + heading_char <- rawToChar(as.raw(35)) + for (chunk in chunks) testthat::expect_true(startsWith(chunk$content %||% "", heading_char)) + } } assert_images <- function(result, min_count = NULL, max_count = NULL, formats_include = NULL) { diff --git a/tools/e2e-generator/e2e/ruby/spec/helpers.rb b/tools/e2e-generator/e2e/ruby/spec/helpers.rb index d21bf81edef..266301ca9de 100644 --- a/tools/e2e-generator/e2e/ruby/spec/helpers.rb +++ b/tools/e2e-generator/e2e/ruby/spec/helpers.rb @@ -230,7 +230,7 @@ def self.assert_metadata_expectation(result, path, expectation) end end - def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil) + def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: nil, each_has_embedding: nil, each_has_heading_context: nil, each_has_chunk_type: nil, content_starts_with_heading: nil) chunks = Array(result.chunks) expect(chunks.length).to be >= min_count if min_count expect(chunks.length).to be <= max_count if max_count @@ -241,6 +241,15 @@ def self.assert_chunks(result, min_count: nil, max_count: nil, each_has_content: elsif each_has_heading_context == false chunks.each { |chunk| expect(chunk.metadata&.heading_context).to be_nil } end + if each_has_chunk_type == true + chunks.each { |chunk| + expect(chunk.chunk_type).not_to be_nil + expect(chunk.chunk_type).not_to eq("unknown") + } + end + return unless content_starts_with_heading == true + + chunks.each { |chunk| expect(chunk.content).to start_with('#') } end def self.assert_images(result, min_count: nil, max_count: nil, formats_include: nil) diff --git a/tools/e2e-generator/e2e/typescript/tests/helpers.ts b/tools/e2e-generator/e2e/typescript/tests/helpers.ts index d5a29e83619..cb131978a12 100644 --- a/tools/e2e-generator/e2e/typescript/tests/helpers.ts +++ b/tools/e2e-generator/e2e/typescript/tests/helpers.ts @@ -726,6 +726,7 @@ export const chunkAssertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -759,6 +760,13 @@ export const chunkAssertions = { expect(((chunk as PlainRecord).metadata as PlainRecord)?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const headingContext = ((chunk as PlainRecord).metadata as PlainRecord | undefined)?.headingContext; diff --git a/tools/e2e-generator/e2e/wasm-deno/helpers.ts b/tools/e2e-generator/e2e/wasm-deno/helpers.ts index d78edc59fbc..3c56e93849d 100644 --- a/tools/e2e-generator/e2e/wasm-deno/helpers.ts +++ b/tools/e2e-generator/e2e/wasm-deno/helpers.ts @@ -402,6 +402,8 @@ export const assertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, + contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; assertExists(chunks, "Expected chunks to be defined"); @@ -438,6 +440,23 @@ export const assertions = { ); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + assertExists(chunkType, "Chunk missing chunk_type"); + assertEquals(chunkType !== "unknown", true, "Chunk chunk_type should not be 'unknown'"); + } + } + if (contentStartsWithHeading === true) { + for (const chunk of chunks) { + const content = (chunk as PlainRecord).content; + assertEquals( + typeof content === "string" && (content as string).charCodeAt(0) === 35, + true, + "Chunk content should start with heading", + ); + } + } }, assertImages( diff --git a/tools/e2e-generator/e2e/wasm-workers/tests/helpers.ts b/tools/e2e-generator/e2e/wasm-workers/tests/helpers.ts index f51c836a572..9685a2a042a 100644 --- a/tools/e2e-generator/e2e/wasm-workers/tests/helpers.ts +++ b/tools/e2e-generator/e2e/wasm-workers/tests/helpers.ts @@ -353,6 +353,8 @@ export const assertions = { eachHasContent: boolean | null, eachHasEmbedding: boolean | null, eachHasHeadingContext: boolean | null, + eachHasChunkType: boolean | null, + contentStartsWithHeading: boolean | null, ): void { const chunks = Array.isArray(result.chunks) ? result.chunks : []; if (typeof minCount === "number") { @@ -381,6 +383,18 @@ export const assertions = { expect(chunk.metadata?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } + if (contentStartsWithHeading === true) { + for (const chunk of chunks) { + expect(typeof chunk.content === "string" && chunk.content.charCodeAt(0) === 35).toBe(true); + } + } }, assertImages( diff --git a/tools/e2e-generator/src/python.rs b/tools/e2e-generator/src/python.rs index de2f0e300d4..5fd53d44db8 100644 --- a/tools/e2e-generator/src/python.rs +++ b/tools/e2e-generator/src/python.rs @@ -1077,6 +1077,12 @@ fn render_assertions(assertions: &Assertions) -> String { if has_heading_context { "True" } else { "False" } )); } + if let Some(has_chunk_type) = chunks.each_has_chunk_type { + args.push(format!( + "each_has_chunk_type={}", + if has_chunk_type { "True" } else { "False" } + )); + } if let Some(starts_with_heading) = chunks.content_starts_with_heading { args.push(format!( "content_starts_with_heading={}", diff --git a/tools/e2e-generator/src/r/assertions.rs b/tools/e2e-generator/src/r/assertions.rs index 90e22e17cd8..c36b8963ad4 100644 --- a/tools/e2e-generator/src/r/assertions.rs +++ b/tools/e2e-generator/src/r/assertions.rs @@ -114,6 +114,12 @@ pub fn render_assertions(assertions: &Assertions) -> String { if has_heading_context { "TRUE" } else { "FALSE" } )); } + if let Some(has_chunk_type) = chunks.each_has_chunk_type { + args.push(format!( + "each_has_chunk_type = {}", + if has_chunk_type { "TRUE" } else { "FALSE" } + )); + } if let Some(starts_with_heading) = chunks.content_starts_with_heading { args.push(format!( "content_starts_with_heading = {}", diff --git a/tools/e2e-generator/src/r/helpers.rs b/tools/e2e-generator/src/r/helpers.rs index 6b3d7f58e9e..4919472bf24 100644 --- a/tools/e2e-generator/src/r/helpers.rs +++ b/tools/e2e-generator/src/r/helpers.rs @@ -216,6 +216,7 @@ assert_metadata_expectation <- function(result, path, expectation) { assert_chunks <- function(result, min_count = NULL, max_count = NULL, each_has_content = NULL, each_has_embedding = NULL, each_has_heading_context = NULL, + each_has_chunk_type = NULL, content_starts_with_heading = NULL) { chunks <- if (is.null(result$chunks)) list() else result$chunks if (!is.null(min_count)) testthat::expect_gte(length(chunks), min_count) @@ -232,6 +233,14 @@ assert_chunks <- function(result, min_count = NULL, max_count = NULL, if (isFALSE(each_has_heading_context)) { for (chunk in chunks) testthat::expect_true(is.null(chunk$metadata$heading_context)) } + if (isTRUE(each_has_chunk_type)) { + for (chunk in chunks) { + chunk_type <- chunk[["chunk_type"]] + if (is.null(chunk_type) || chunk_type == "unknown") { + testthat::fail(paste0("Expected specific chunk_type, got: ", chunk_type)) + } + } + } if (isTRUE(content_starts_with_heading)) { heading_char <- rawToChar(as.raw(35)) for (chunk in chunks) { diff --git a/tools/e2e-generator/src/wasm_deno.rs b/tools/e2e-generator/src/wasm_deno.rs index 71965dbeb25..8d140294d85 100644 --- a/tools/e2e-generator/src/wasm_deno.rs +++ b/tools/e2e-generator/src/wasm_deno.rs @@ -442,6 +442,7 @@ export const assertions = { eachHasContent?: boolean | null, eachHasEmbedding?: boolean | null, eachHasHeadingContext?: boolean | null, + eachHasChunkType?: boolean | null, contentStartsWithHeading?: boolean | null, ): void { const chunks = (result as unknown as PlainRecord).chunks as unknown[] | undefined; @@ -475,6 +476,13 @@ export const assertions = { assertEquals(((chunk as PlainRecord).metadata as PlainRecord)?.headingContext ?? null, null, "Chunk should have no heading_context"); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + assertExists(chunkType, "Chunk missing chunk_type"); + assertEquals(chunkType !== "unknown", true, "Chunk chunk_type should not be 'unknown'"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { const meta = (chunk as PlainRecord).metadata as PlainRecord | undefined; @@ -1158,12 +1166,16 @@ fn render_assertions(assertions: &Assertions, _requirements: &[String]) -> Strin .each_has_heading_context .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| v.to_string()) + .unwrap_or_else(|| "null".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); buffer.push_str(&format!( - " assertions.assertChunks(result, {min}, {max}, {has_content}, {has_embedding}, {has_heading_context}, {content_starts_with_heading});\n" + " assertions.assertChunks(result, {min}, {max}, {has_content}, {has_embedding}, {has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" )); } diff --git a/tools/e2e-generator/src/wasm_workers.rs b/tools/e2e-generator/src/wasm_workers.rs index 20e513ca863..e57ce0b1c5d 100644 --- a/tools/e2e-generator/src/wasm_workers.rs +++ b/tools/e2e-generator/src/wasm_workers.rs @@ -411,6 +411,7 @@ export const assertions = { eachHasContent: boolean | null, eachHasEmbedding: boolean | null, eachHasHeadingContext: boolean | null, + eachHasChunkType: boolean | null, contentStartsWithHeading: boolean | null, ): void { const chunks = Array.isArray(result.chunks) ? result.chunks : []; @@ -440,6 +441,13 @@ export const assertions = { expect(chunk.metadata?.headingContext ?? null).toBeNull(); } } + if (eachHasChunkType === true) { + for (const chunk of chunks) { + const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + expect(chunkType).toBeDefined(); + expect(chunkType).not.toBe("unknown"); + } + } if (contentStartsWithHeading === true) { for (const chunk of chunks) { if (chunk.metadata?.headingContext == null) continue; @@ -1097,12 +1105,16 @@ fn render_assertions(assertions: &Assertions) -> String { .each_has_heading_context .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); + let each_has_chunk_type = chunks + .each_has_chunk_type + .map(|v| v.to_string()) + .unwrap_or_else(|| "null".into()); let content_starts_with_heading = chunks .content_starts_with_heading .map(|v| v.to_string()) .unwrap_or_else(|| "null".into()); buffer.push_str(&format!( - " assertions.assertChunks(result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {content_starts_with_heading});\n" + " assertions.assertChunks(result, {min_count}, {max_count}, {each_has_content}, {each_has_embedding}, {each_has_heading_context}, {each_has_chunk_type}, {content_starts_with_heading});\n" )); } From f66e7eef5ffce862ff6c159a096b08c1ad43b7e9 Mon Sep 17 00:00:00 2001 From: kh3rld Date: Thu, 2 Apr 2026 03:06:06 -0400 Subject: [PATCH 11/12] fix(ci): resolve chunk_type CI failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add missing chunk_type field in benchmark and manifest Chunk literals - Fix TypeScript double-cast for Chunk → PlainRecord - Reformat Elixir defstruct to pass mix format --check-formatted - Refactor assert_chunks helpers in Python e2e snapshot (complexity lint) - Fix trailing whitespace and missing EOF newlines in generated e2e files --- .../benches/result_view_benchmark.rs | 1 + e2e/elixir/test/e2e/plugin_apis_test.exs | 1 - e2e/ruby/spec/plugin_apis_spec.rb | 2 +- e2e/typescript/tests/archive.spec.ts | 1 - e2e/typescript/tests/email.spec.ts | 1 - e2e/typescript/tests/html.spec.ts | 1 - e2e/typescript/tests/image.spec.ts | 1 - e2e/typescript/tests/keywords.spec.ts | 1 - e2e/typescript/tests/office.spec.ts | 1 - e2e/typescript/tests/plugin-apis.spec.ts | 3 +- e2e/typescript/tests/structured.spec.ts | 1 - e2e/typescript/tests/xml.spec.ts | 1 - e2e/wasm-deno/archive.test.ts | 1 - e2e/wasm-deno/email.test.ts | 1 - e2e/wasm-deno/embeddings.test.ts | 1 - e2e/wasm-deno/html.test.ts | 1 - e2e/wasm-deno/image.test.ts | 1 - e2e/wasm-deno/office.test.ts | 1 - e2e/wasm-deno/plugin-apis.test.ts | 1 - e2e/wasm-deno/structured.test.ts | 1 - e2e/wasm-deno/xml.test.ts | 1 - e2e/wasm-workers/tests/helpers.ts | 2 +- e2e/wasm-workers/tests/plugin-apis.spec.ts | 1 - packages/elixir/lib/kreuzberg/chunk.ex | 5 +- .../e2e-generator/e2e/python/tests/helpers.py | 90 +- .../e2e/python/tests/test_archive.py | 73 ++ .../e2e/python/tests/test_code.py | 104 ++ .../e2e/python/tests/test_contract.py | 1145 +++++++++++++++++ .../e2e/python/tests/test_email.py | 103 ++ .../e2e/python/tests/test_embeddings.py | 118 ++ .../e2e/python/tests/test_html.py | 46 + .../e2e/python/tests/test_image.py | 163 +++ .../e2e/python/tests/test_keywords.py | 45 + .../e2e/python/tests/test_ocr.py | 332 +++++ .../e2e/python/tests/test_office.py | 811 ++++++++++++ .../e2e/python/tests/test_parity.py | 158 +++ .../e2e/python/tests/test_pdf.py | 316 +++++ .../e2e/python/tests/test_plugin_apis.py | 162 +++ .../e2e/python/tests/test_render.py | 53 + .../e2e/python/tests/test_smoke.py | 142 ++ .../e2e/python/tests/test_structured.py | 164 +++ .../e2e/python/tests/test_token_reduction.py | 83 ++ .../e2e/python/tests/test_xml.py | 28 + tools/e2e-generator/src/manifest.rs | 1 + 44 files changed, 4127 insertions(+), 42 deletions(-) create mode 100644 tools/e2e-generator/e2e/python/tests/test_archive.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_code.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_contract.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_email.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_embeddings.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_html.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_image.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_keywords.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_ocr.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_office.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_parity.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_pdf.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_plugin_apis.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_render.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_smoke.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_structured.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_token_reduction.py create mode 100644 tools/e2e-generator/e2e/python/tests/test_xml.py diff --git a/crates/kreuzberg-ffi/benches/result_view_benchmark.rs b/crates/kreuzberg-ffi/benches/result_view_benchmark.rs index 00f30c57e3f..bb2fbda4ade 100644 --- a/crates/kreuzberg-ffi/benches/result_view_benchmark.rs +++ b/crates/kreuzberg-ffi/benches/result_view_benchmark.rs @@ -44,6 +44,7 @@ fn create_test_result(content_size: usize, chunk_count: usize) -> ExtractionResu }; Chunk { content: content[start..end].to_string(), + chunk_type: Default::default(), embedding: None, metadata: ChunkMetadata { byte_start: start, diff --git a/e2e/elixir/test/e2e/plugin_apis_test.exs b/e2e/elixir/test/e2e/plugin_apis_test.exs index 5dc8d37cbf4..774a4e5ca1a 100644 --- a/e2e/elixir/test/e2e/plugin_apis_test.exs +++ b/e2e/elixir/test/e2e/plugin_apis_test.exs @@ -174,4 +174,3 @@ defmodule E2E.ValidatorManagementTest do end end - diff --git a/e2e/ruby/spec/plugin_apis_spec.rb b/e2e/ruby/spec/plugin_apis_spec.rb index 27290d25251..92759aeb3ef 100644 --- a/e2e/ruby/spec/plugin_apis_spec.rb +++ b/e2e/ruby/spec/plugin_apis_spec.rb @@ -42,7 +42,7 @@ [chunking] max_chars = 100 max_overlap = 20 - + [language_detection] enabled = false TOML diff --git a/e2e/typescript/tests/archive.spec.ts b/e2e/typescript/tests/archive.spec.ts index 468c4348abe..d2684c40a4e 100644 --- a/e2e/typescript/tests/archive.spec.ts +++ b/e2e/typescript/tests/archive.spec.ts @@ -105,4 +105,3 @@ describe("archive fixtures", () => { }, TEST_TIMEOUT_MS); }); - diff --git a/e2e/typescript/tests/email.spec.ts b/e2e/typescript/tests/email.spec.ts index b5f88cdbbe5..448a9ae1db2 100644 --- a/e2e/typescript/tests/email.spec.ts +++ b/e2e/typescript/tests/email.spec.ts @@ -151,4 +151,3 @@ describe("email fixtures", () => { }, TEST_TIMEOUT_MS); }); - diff --git a/e2e/typescript/tests/html.spec.ts b/e2e/typescript/tests/html.spec.ts index 06a1071b7bc..862a47bce02 100644 --- a/e2e/typescript/tests/html.spec.ts +++ b/e2e/typescript/tests/html.spec.ts @@ -60,4 +60,3 @@ describe("html fixtures", () => { }, TEST_TIMEOUT_MS); }); - diff --git a/e2e/typescript/tests/image.spec.ts b/e2e/typescript/tests/image.spec.ts index adb786e37b3..944bb505902 100644 --- a/e2e/typescript/tests/image.spec.ts +++ b/e2e/typescript/tests/image.spec.ts @@ -243,4 +243,3 @@ describe("image fixtures", () => { }, TEST_TIMEOUT_MS); }); - diff --git a/e2e/typescript/tests/keywords.spec.ts b/e2e/typescript/tests/keywords.spec.ts index c14947e5527..eb170615191 100644 --- a/e2e/typescript/tests/keywords.spec.ts +++ b/e2e/typescript/tests/keywords.spec.ts @@ -61,4 +61,3 @@ describe("keywords fixtures", () => { }, TEST_TIMEOUT_MS); }); - diff --git a/e2e/typescript/tests/office.spec.ts b/e2e/typescript/tests/office.spec.ts index 88919ea8cb1..6d93eeec5e0 100644 --- a/e2e/typescript/tests/office.spec.ts +++ b/e2e/typescript/tests/office.spec.ts @@ -1205,4 +1205,3 @@ describe("office fixtures", () => { }, TEST_TIMEOUT_MS); }); - diff --git a/e2e/typescript/tests/plugin-apis.spec.ts b/e2e/typescript/tests/plugin-apis.spec.ts index 5e064b00e06..80692ff82ee 100644 --- a/e2e/typescript/tests/plugin-apis.spec.ts +++ b/e2e/typescript/tests/plugin-apis.spec.ts @@ -26,7 +26,7 @@ describe("Configuration", () => { process.chdir(subDir); const config = kreuzberg.ExtractionConfig.discover(); - + expect(config.chunking).toBeDefined(); expect(config.chunking?.maxChars).toBe(50); } finally { @@ -142,4 +142,3 @@ describe("Validator Management", () => { }); }); - diff --git a/e2e/typescript/tests/structured.spec.ts b/e2e/typescript/tests/structured.spec.ts index df47ec7942a..d556af729c8 100644 --- a/e2e/typescript/tests/structured.spec.ts +++ b/e2e/typescript/tests/structured.spec.ts @@ -244,4 +244,3 @@ describe("structured fixtures", () => { }, TEST_TIMEOUT_MS); }); - diff --git a/e2e/typescript/tests/xml.spec.ts b/e2e/typescript/tests/xml.spec.ts index f28b0d9c1bb..9ddea2b2048 100644 --- a/e2e/typescript/tests/xml.spec.ts +++ b/e2e/typescript/tests/xml.spec.ts @@ -36,4 +36,3 @@ describe("xml fixtures", () => { }, TEST_TIMEOUT_MS); }); - diff --git a/e2e/wasm-deno/archive.test.ts b/e2e/wasm-deno/archive.test.ts index 6acde9adf94..de53656c643 100644 --- a/e2e/wasm-deno/archive.test.ts +++ b/e2e/wasm-deno/archive.test.ts @@ -89,4 +89,3 @@ Deno.test("archive_zip_basic", { permissions: { read: true, net: true } }, async assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); assertions.assertMinContentLength(result, 10); }); - diff --git a/e2e/wasm-deno/email.test.ts b/e2e/wasm-deno/email.test.ts index 1cfaa97f8f1..c9525e71524 100644 --- a/e2e/wasm-deno/email.test.ts +++ b/e2e/wasm-deno/email.test.ts @@ -129,4 +129,3 @@ Deno.test("email_sample_eml", { permissions: { read: true, net: true } }, async assertions.assertExpectedMime(result, ["message/rfc822"]); assertions.assertMinContentLength(result, 20); }); - diff --git a/e2e/wasm-deno/embeddings.test.ts b/e2e/wasm-deno/embeddings.test.ts index e901c204ceb..79512239fde 100644 --- a/e2e/wasm-deno/embeddings.test.ts +++ b/e2e/wasm-deno/embeddings.test.ts @@ -30,4 +30,3 @@ Deno.test("embedding_disabled", { permissions: { read: true, net: true } }, asyn assertions.assertMinContentLength(result, 10); assertions.assertChunks(result, 1, null, true, false, null, null, null); }); - diff --git a/e2e/wasm-deno/html.test.ts b/e2e/wasm-deno/html.test.ts index 6d66937ccc9..34903a32d5d 100644 --- a/e2e/wasm-deno/html.test.ts +++ b/e2e/wasm-deno/html.test.ts @@ -30,4 +30,3 @@ Deno.test("html_simple_table", { permissions: { read: true, net: true } }, async assertions.assertMinContentLength(result, 100); assertions.assertContentContainsAll(result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]); }); - diff --git a/e2e/wasm-deno/image.test.ts b/e2e/wasm-deno/image.test.ts index 273dc28f702..7051ca70538 100644 --- a/e2e/wasm-deno/image.test.ts +++ b/e2e/wasm-deno/image.test.ts @@ -209,4 +209,3 @@ Deno.test("image_webp_basic", { permissions: { read: true, net: true } }, async assertions.assertExpectedMime(result, ["image/webp"]); assertions.assertContentNotEmpty(result); }); - diff --git a/e2e/wasm-deno/office.test.ts b/e2e/wasm-deno/office.test.ts index 527d25e12c2..b8236b94eda 100644 --- a/e2e/wasm-deno/office.test.ts +++ b/e2e/wasm-deno/office.test.ts @@ -1028,4 +1028,3 @@ Deno.test("office_xlsx_office_example", { permissions: { read: true, net: true } assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); assertions.assertMinContentLength(result, 10); }); - diff --git a/e2e/wasm-deno/plugin-apis.test.ts b/e2e/wasm-deno/plugin-apis.test.ts index c81191cda5a..c8d6356a819 100644 --- a/e2e/wasm-deno/plugin-apis.test.ts +++ b/e2e/wasm-deno/plugin-apis.test.ts @@ -85,4 +85,3 @@ Deno.test("List all registered validators", () => { assertEquals(Array.isArray(result), true); assertEquals(result.every((item: unknown) => typeof item === "string"), true); }); - diff --git a/e2e/wasm-deno/structured.test.ts b/e2e/wasm-deno/structured.test.ts index 97348138eef..c76e43b93c7 100644 --- a/e2e/wasm-deno/structured.test.ts +++ b/e2e/wasm-deno/structured.test.ts @@ -210,4 +210,3 @@ Deno.test("structured_yaml_simple", { permissions: { read: true, net: true } }, assertions.assertExpectedMime(result, ["application/x-yaml"]); assertions.assertMinContentLength(result, 10); }); - diff --git a/e2e/wasm-deno/xml.test.ts b/e2e/wasm-deno/xml.test.ts index c75f1acc26c..ae8a5ce299c 100644 --- a/e2e/wasm-deno/xml.test.ts +++ b/e2e/wasm-deno/xml.test.ts @@ -29,4 +29,3 @@ Deno.test("xml_plant_catalog", { permissions: { read: true, net: true } }, async assertions.assertExpectedMime(result, ["application/xml"]); assertions.assertMinContentLength(result, 100); }); - diff --git a/e2e/wasm-workers/tests/helpers.ts b/e2e/wasm-workers/tests/helpers.ts index e3d9d710791..2a07c055ccc 100644 --- a/e2e/wasm-workers/tests/helpers.ts +++ b/e2e/wasm-workers/tests/helpers.ts @@ -428,7 +428,7 @@ export const assertions = { } if (eachHasChunkType === true) { for (const chunk of chunks) { - const chunkType = (chunk as PlainRecord).chunkType ?? (chunk as PlainRecord).chunk_type; + const chunkType = (chunk as unknown as PlainRecord).chunkType ?? (chunk as unknown as PlainRecord).chunk_type; expect(chunkType).toBeDefined(); expect(chunkType).not.toBe("unknown"); } diff --git a/e2e/wasm-workers/tests/plugin-apis.spec.ts b/e2e/wasm-workers/tests/plugin-apis.spec.ts index 7d027ff3321..85f8b337955 100644 --- a/e2e/wasm-workers/tests/plugin-apis.spec.ts +++ b/e2e/wasm-workers/tests/plugin-apis.spec.ts @@ -80,4 +80,3 @@ describe("Validator Management", () => { }); }); - diff --git a/packages/elixir/lib/kreuzberg/chunk.ex b/packages/elixir/lib/kreuzberg/chunk.ex index 4d2f241bf3d..6fe33bcfd87 100644 --- a/packages/elixir/lib/kreuzberg/chunk.ex +++ b/packages/elixir/lib/kreuzberg/chunk.ex @@ -19,7 +19,10 @@ defmodule Kreuzberg.Chunk do chunk_type: String.t() } - defstruct content: "", embedding: nil, metadata: %Kreuzberg.ChunkMetadata{}, chunk_type: "unknown" + defstruct content: "", + embedding: nil, + metadata: %Kreuzberg.ChunkMetadata{}, + chunk_type: "unknown" @doc """ Creates a new Chunk struct. diff --git a/tools/e2e-generator/e2e/python/tests/helpers.py b/tools/e2e-generator/e2e/python/tests/helpers.py index 0ab7a0f8403..24d26f1c66a 100644 --- a/tools/e2e-generator/e2e/python/tests/helpers.py +++ b/tools/e2e-generator/e2e/python/tests/helpers.py @@ -30,6 +30,8 @@ PostProcessorConfig, ResultFormat, TokenReductionConfig, + TreeSitterConfig, + TreeSitterProcessConfig, ) _WORKSPACE_ROOT = Path(__file__).resolve().parent.parent.parent.parent @@ -104,6 +106,12 @@ def _build_config_objects(config: dict[str, Any], kwargs: dict[str, Any]) -> Non kwargs["acceleration"] = AccelerationConfig(**accel_data) if (email_data := config.get("email")) is not None and isinstance(email_data, dict): kwargs["email"] = EmailConfig(**email_data) + if (tree_sitter_data := config.get("tree_sitter")) is not None: + ts = dict(tree_sitter_data) + process_data = ts.pop("process", None) + if process_data is not None: + ts["process"] = TreeSitterProcessConfig(**process_data) + kwargs["tree_sitter"] = TreeSitterConfig(**ts) def build_config(config: dict[str, Any] | None) -> ExtractionConfig: @@ -114,10 +122,20 @@ def build_config(config: dict[str, Any] | None) -> ExtractionConfig: kwargs: dict[str, Any] = {} - for key in ("use_cache", "enable_quality_processing", "force_ocr", "include_document_structure"): + for key in ( + "use_cache", + "enable_quality_processing", + "force_ocr", + "disable_ocr", + "force_ocr_pages", + "include_document_structure", + ): if key in config: kwargs[key] = config[key] + if (extraction_timeout_secs := config.get("extraction_timeout_secs")) is not None: + kwargs["extraction_timeout_secs"] = int(extraction_timeout_secs) + _build_config_objects(config, kwargs) if (output_format := config.get("output_format")) is not None: @@ -137,10 +155,19 @@ def build_file_config(config: dict[str, Any] | None) -> FileExtractionConfig: kwargs: dict[str, Any] = {} - for key in ("enable_quality_processing", "force_ocr", "include_document_structure"): + for key in ( + "enable_quality_processing", + "force_ocr", + "disable_ocr", + "force_ocr_pages", + "include_document_structure", + ): if key in config: kwargs[key] = config[key] + if (timeout_secs := config.get("timeout_secs")) is not None: + kwargs["timeout_secs"] = int(timeout_secs) + _build_config_objects(config, kwargs) if (output_format := config.get("output_format")) is not None: @@ -288,6 +315,27 @@ def _get_heading_context(chunk: Any) -> Any: return metadata.get("heading_context") if isinstance(metadata, dict) else getattr(metadata, "heading_context", None) +def _assert_chunks_content(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + if not getattr(chunk, "content", None): + pytest.fail(f"Chunk {i} has no content") + + +def _assert_chunks_embedding(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + if not getattr(chunk, "embedding", None): + pytest.fail(f"Chunk {i} has no embedding") + + +def _assert_chunks_heading_context(chunks: Any, expected: bool) -> None: + for i, chunk in enumerate(chunks): + hc = _get_heading_context(chunk) + if expected and hc is None: + pytest.fail(f"Chunk {i} has no heading_context") + if not expected and hc is not None: + pytest.fail(f"Chunk {i} should have no heading_context") + + def _assert_chunks_chunk_type(chunks: Any) -> None: for i, chunk in enumerate(chunks): chunk_type = getattr(chunk, "chunk_type", None) @@ -295,6 +343,13 @@ def _assert_chunks_chunk_type(chunks: Any) -> None: pytest.fail(f"Chunk {i} has no specific chunk_type, got {chunk_type}") +def _assert_chunks_heading_prefix(chunks: Any) -> None: + for i, chunk in enumerate(chunks): + content = getattr(chunk, "content", None) + if not isinstance(content, str) or content[0:1] != chr(35): + pytest.fail(f"Chunk {i} content does not start with a heading") + + def assert_chunks( result: Any, min_count: int | None = None, @@ -314,27 +369,15 @@ def assert_chunks( if max_count is not None and count > max_count: pytest.fail(f"Expected at most {max_count} chunks, found {count}") if each_has_content: - for i, chunk in enumerate(chunks): - if not getattr(chunk, "content", None): - pytest.fail(f"Chunk {i} has no content") + _assert_chunks_content(chunks) if each_has_embedding: - for i, chunk in enumerate(chunks): - if not getattr(chunk, "embedding", None): - pytest.fail(f"Chunk {i} has no embedding") + _assert_chunks_embedding(chunks) if each_has_heading_context is not None: - for i, chunk in enumerate(chunks): - hc = _get_heading_context(chunk) - if each_has_heading_context and hc is None: - pytest.fail(f"Chunk {i} has no heading_context") - if not each_has_heading_context and hc is not None: - pytest.fail(f"Chunk {i} should have no heading_context") + _assert_chunks_heading_context(chunks, each_has_heading_context) if each_has_chunk_type: _assert_chunks_chunk_type(chunks) if content_starts_with_heading: - for i, chunk in enumerate(chunks): - content = getattr(chunk, "content", None) - if not isinstance(content, str) or content[0:1] != chr(35): - pytest.fail(f"Chunk {i} content does not start with a heading") + _assert_chunks_heading_prefix(chunks) def assert_images( @@ -638,3 +681,14 @@ def assert_annotations(result: Any, has_annotations: bool = False, min_count: in and len(annotations) < min_count ): pytest.fail(f"Expected at least {min_count} annotations, found {len(annotations)}") + + +def assert_is_png(data: bytes) -> None: + """Assert data starts with PNG magic bytes.""" + assert len(data) >= 4, f"Data too short for PNG: {len(data)} bytes" + assert data[:4] == b"\x89PNG", f"Missing PNG magic bytes, got: {data[:4]!r}" + + +def assert_min_byte_length(data: bytes, min_length: int) -> None: + """Assert data is at least min_length bytes.""" + assert len(data) >= min_length, f"Expected at least {min_length} bytes, got {len(data)}" diff --git a/tools/e2e-generator/e2e/python/tests/test_archive.py b/tools/e2e-generator/e2e/python/tests/test_archive.py new file mode 100644 index 00000000000..bba1e42eea0 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_archive.py @@ -0,0 +1,73 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for archive fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_archive_gz_basic() -> None: + """Gzip compressed file extraction.""" + + document_path = helpers.resolve_document("archives/book_war_and_peace_1p.txt.gz") + if not document_path.exists(): + pytest.skip(f"Skipping archive_gz_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/gzip", "application/x-gzip"]) + helpers.assert_min_content_length(result, 10) + + +def test_archive_sevenz_basic() -> None: + """7-Zip archive extraction.""" + + document_path = helpers.resolve_document("archives/documents.7z") + if not document_path.exists(): + pytest.skip(f"Skipping archive_sevenz_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-7z-compressed"]) + helpers.assert_min_content_length(result, 10) + + +def test_archive_tar_basic() -> None: + """TAR archive extraction.""" + + document_path = helpers.resolve_document("archives/documents.tar") + if not document_path.exists(): + pytest.skip(f"Skipping archive_tar_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-tar", "application/tar"]) + helpers.assert_min_content_length(result, 10) + + +def test_archive_zip_basic() -> None: + """ZIP archive extraction.""" + + document_path = helpers.resolve_document("archives/documents.zip") + if not document_path.exists(): + pytest.skip(f"Skipping archive_zip_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/zip", "application/x-zip-compressed"]) + helpers.assert_min_content_length(result, 10) diff --git a/tools/e2e-generator/e2e/python/tests/test_code.py b/tools/e2e-generator/e2e/python/tests/test_code.py new file mode 100644 index 00000000000..63ed1e4f5cc --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_code.py @@ -0,0 +1,104 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for code fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + detect_mime_type_from_path, + extract_bytes_sync, + extract_file_sync, +) + +from . import helpers + + +def test_code_javascript_basic() -> None: + """Smoke test: JavaScript source code file extraction""" + + document_path = helpers.resolve_document("code/app.js") + if not document_path.exists(): + pytest.skip(f"Skipping code_javascript_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["class Application", "export function", "import"]) + + +def test_code_python_basic() -> None: + """Smoke test: Python source code file extraction""" + + document_path = helpers.resolve_document("code/hello.py") + if not document_path.exists(): + pytest.skip(f"Skipping code_python_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["def greet", "class Greeter", "import os"]) + + +def test_code_rust_basic() -> None: + """Smoke test: Rust source code file extraction""" + + document_path = helpers.resolve_document("code/main.rs") + if not document_path.exists(): + pytest.skip(f"Skipping code_rust_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["struct Store", "fn main", "HashMap"]) + + +def test_code_shebang_detection() -> None: + """Test language detection from shebang line via bytes input""" + + document_path = helpers.resolve_document("code/script.sh") + if not document_path.exists(): + pytest.skip(f"Skipping code_shebang_detection: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + result = extract_bytes_sync(file_bytes, mime_type, config=config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["build", "clean"]) + + +def test_code_tree_sitter_config() -> None: + """Test tree-sitter config with structure, comments, and docstrings enabled""" + + document_path = helpers.resolve_document("code/hello.py") + if not document_path.exists(): + pytest.skip(f"Skipping code_tree_sitter_config: missing document at {document_path}") + + config = helpers.build_config( + { + "tree_sitter": { + "process": {"structure": True, "imports": True, "exports": True, "comments": True, "docstrings": True} + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_all(result, ["def greet", "class Greeter"]) diff --git a/tools/e2e-generator/e2e/python/tests/test_contract.py b/tools/e2e-generator/e2e/python/tests/test_contract.py new file mode 100644 index 00000000000..76a8b16aebf --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_contract.py @@ -0,0 +1,1145 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for contract fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + batch_extract_bytes, + batch_extract_bytes_sync, + batch_extract_files, + batch_extract_files_sync, + detect_mime_type_from_path, + extract_bytes, + extract_bytes_sync, + extract_file, + extract_file_sync, +) + +from . import helpers + + +@pytest.mark.asyncio +async def test_api_batch_bytes_async() -> None: + """Tests async batch bytes extraction API (batch_extract_bytes)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_bytes_async: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + results = await batch_extract_bytes([file_bytes], [mime_type], config=config) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_api_batch_bytes_sync() -> None: + """Tests sync batch bytes extraction API (batch_extract_bytes_sync)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_bytes_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + results = batch_extract_bytes_sync([file_bytes], [mime_type], config=config) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +@pytest.mark.asyncio +async def test_api_batch_bytes_with_configs_async() -> None: + """Tests async batch bytes extraction with per-file configs (batch_extract_bytes with file_configs parameter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_bytes_with_configs_async: missing document at {document_path}") + + config = helpers.build_config(None) + + file_configs = [helpers.build_file_config({"output_format": "markdown"})] + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + results = await batch_extract_bytes([file_bytes], [mime_type], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_api_batch_bytes_with_configs_sync() -> None: + """Tests sync batch bytes extraction with per-file configs (batch_extract_bytes_sync with file_configs parameter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_bytes_with_configs_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + file_configs = [helpers.build_file_config({"output_format": "markdown"})] + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + results = batch_extract_bytes_sync([file_bytes], [mime_type], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +@pytest.mark.asyncio +async def test_api_batch_file_async() -> None: + """Tests async batch file extraction API (batch_extract_file)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_async: missing document at {document_path}") + + config = helpers.build_config(None) + + results = await batch_extract_files([document_path], config=config) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_api_batch_file_sync() -> None: + """Tests sync batch file extraction API (batch_extract_file_sync)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + results = batch_extract_files_sync([document_path], config=config) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +@pytest.mark.asyncio +async def test_api_batch_file_with_configs_async() -> None: + """Tests async batch file extraction with per-file configs (batch_extract_files with file_configs parameter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_with_configs_async: missing document at {document_path}") + + config = helpers.build_config(None) + + file_configs = [helpers.build_file_config({"output_format": "markdown"})] + + results = await batch_extract_files([document_path], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_api_batch_file_with_configs_sync() -> None: + """Tests sync batch file extraction with per-file configs (batch_extract_files_sync with file_configs parameter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_with_configs_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + file_configs = [helpers.build_file_config({"output_format": "markdown"})] + + results = batch_extract_files_sync([document_path], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_api_batch_file_with_timeout_sync() -> None: + """Tests sync batch file extraction with per-file timeout config override""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_batch_file_with_timeout_sync: missing document at {document_path}") + + config = helpers.build_config({"extraction_timeout_secs": 300}) + + file_configs = [helpers.build_file_config({"timeout_secs": 600})] + + results = batch_extract_files_sync([document_path], config=config, file_configs=file_configs) + result = results[0] + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + + +@pytest.mark.asyncio +async def test_api_extract_bytes_async() -> None: + """Tests async bytes extraction API (extract_bytes)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_extract_bytes_async: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + result = await extract_bytes(file_bytes, mime_type, config=config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_api_extract_bytes_sync() -> None: + """Tests sync bytes extraction API (extract_bytes_sync)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_extract_bytes_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + result = extract_bytes_sync(file_bytes, mime_type, config=config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +@pytest.mark.asyncio +async def test_api_extract_file_async() -> None: + """Tests async file extraction API (extract_file)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_extract_file_async: missing document at {document_path}") + + config = helpers.build_config(None) + + result = await extract_file(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_api_extract_file_sync() -> None: + """Tests sync file extraction API (extract_file_sync)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping api_extract_file_sync: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["May 5, 2023", "Mallori"]) + + +def test_config_acceleration_cpu_provider() -> None: + """Tests explicit CPU acceleration provider configuration""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_acceleration_cpu_provider: missing document at {document_path}") + + config = helpers.build_config({"acceleration": {"provider": "cpu", "device_id": 0}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["May 5, 2023", "To Whom it May Concern"]) + + +def test_config_chunking() -> None: + """Tests chunking configuration with chunk assertions""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"max_chars": 500, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True) + + +def test_config_chunking_heading_context() -> None: + """Tests markdown chunker populates heading context on chunks""" + + document_path = helpers.resolve_document("markdown/extraction_test.md") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_heading_context: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"chunker_type": "markdown", "max_chars": 300, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=2, each_has_content=True, each_has_heading_context=True) + + +def test_config_chunking_markdown() -> None: + """Tests markdown-aware chunker type""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_markdown: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"chunker_type": "markdown", "max_chars": 500, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True) + + +def test_config_chunking_no_headings() -> None: + """Tests markdown chunker on text with no headings produces null heading_context""" + + document_path = helpers.resolve_document("text/book_war_and_peace_1p.txt") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_no_headings: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"chunker_type": "markdown", "max_chars": 300, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=2, each_has_content=True, each_has_heading_context=False) + + +def test_config_chunking_prepend_heading_context() -> None: + """Tests markdown chunker prepends heading hierarchy to chunk content""" + + document_path = helpers.resolve_document("markdown/extraction_test.md") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_prepend_heading_context: missing document at {document_path}") + + config = helpers.build_config( + {"chunking": {"chunker_type": "markdown", "max_chars": 300, "max_overlap": 50, "prepend_heading_context": True}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks( + result, min_count=2, each_has_content=True, each_has_heading_context=True, content_starts_with_heading=True + ) + + +def test_config_chunking_small() -> None: + """Tests chunking with very small chunk size produces more chunks""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_small: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"max_chars": 100, "max_overlap": 20}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=2, each_has_content=True) + + +def test_config_chunking_text() -> None: + """Tests text chunker type (generic whitespace/punctuation splitter)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_text: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"chunker_type": "text", "max_chars": 500, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True) + + +def test_config_chunking_tokenizer() -> None: + """Tests token-based chunk sizing with HuggingFace tokenizer""" + + document_path = helpers.resolve_document("markdown/comprehensive.md") + if not document_path.exists(): + pytest.skip(f"Skipping config_chunking_tokenizer: missing document at {document_path}") + + config = helpers.build_config( + {"chunking": {"max_chars": 200, "max_overlap": 40, "sizing": {"type": "tokenizer", "model": "Xenova/gpt-4o"}}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=2, each_has_content=True) + + +def test_config_disable_ocr() -> None: + """Tests disable_ocr configuration option - OCR is skipped for images""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping config_disable_ocr: missing document at {document_path}") + + config = helpers.build_config({"disable_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_max_content_length(result, 5) + + +def test_config_djot_content() -> None: + """Tests djot output format converts content to djot markup""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_djot_content: missing document at {document_path}") + + config = helpers.build_config({"output_format": "djot"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "djot") + + +def test_config_document_structure() -> None: + """Tests include_document_structure config produces document tree""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure: missing document at {document_path}") + + config = helpers.build_config({"include_document_structure": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_document(result, has_document=True, min_node_count=1, node_types_include=["paragraph"]) + + +def test_config_document_structure_disabled() -> None: + """Tests document field is null when include_document_structure is false""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure_disabled: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_document(result, has_document=False) + + +def test_config_document_structure_groups() -> None: + """Tests document structure extraction with group node assertion on DOCX with headings""" + + document_path = helpers.resolve_document("docx/unit_test_headers.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure_groups: missing document at {document_path}") + + config = helpers.build_config({"include_document_structure": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_document(result, has_document=True, has_groups=True) + + +def test_config_document_structure_headings() -> None: + """Tests document structure extraction with heading nodes on a DOCX""" + + document_path = helpers.resolve_document("docx/unit_test_headers.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure_headings: missing document at {document_path}") + + config = helpers.build_config({"include_document_structure": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_document(result, has_document=True, min_node_count=1, node_types_include=["heading", "paragraph"]) + + +def test_config_document_structure_with_headings() -> None: + """Tests document structure with DOCX heading-driven nesting""" + + document_path = helpers.resolve_document("docx/fake.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_document_structure_with_headings: missing document at {document_path}") + + config = helpers.build_config({"include_document_structure": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_document(result, has_document=True, min_node_count=1) + + +def test_config_element_types() -> None: + """Tests element-based result format with element type assertions on DOCX""" + + document_path = helpers.resolve_document("docx/unit_test_headers.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_element_types: missing document at {document_path}") + + config = helpers.build_config({"result_format": "element_based"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_elements(result, min_count=1, types_include=["narrative_text"]) + helpers.assert_result_format(result, "element_based") + + +def test_config_email_msg_fallback_codepage() -> None: + """Tests MSG extraction with custom fallback codepage for Cyrillic""" + + document_path = helpers.resolve_document("email/fake_email.msg") + if not document_path.exists(): + pytest.skip(f"Skipping config_email_msg_fallback_codepage: missing document at {document_path}") + + config = helpers.build_config({"email": {"msg_fallback_codepage": 1251}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-outlook"]) + helpers.assert_min_content_length(result, 10) + + +def test_config_extraction_timeout() -> None: + """Tests that extraction_timeout_secs config field is accepted and does not affect fast extractions""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_extraction_timeout: missing document at {document_path}") + + config = helpers.build_config({"extraction_timeout_secs": 300}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + + +def test_config_force_ocr() -> None: + """Tests force_ocr configuration option""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_force_ocr: missing document at {document_path}") + + config = helpers.build_config({"force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + + +def test_config_force_ocr_pages() -> None: + """Tests that force_ocr_pages config field is accepted for selective page OCR""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_force_ocr_pages: missing document at {document_path}") + + config = helpers.build_config({"force_ocr_pages": [1], "ocr": {"backend": "tesseract", "language": "eng"}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 1) + + +def test_config_html_options() -> None: + """Tests extraction with HTML conversion options configured""" + + document_path = helpers.resolve_document("html/complex_table.html") + if not document_path.exists(): + pytest.skip(f"Skipping config_html_options: missing document at {document_path}") + + config = helpers.build_config({"html_options": {"extractMetadata": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/html"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_not_empty(result) + + +def test_config_images() -> None: + """Tests image extraction configuration with image assertions""" + + document_path = helpers.resolve_document("pdf/embedded_images_tables.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_images: missing document at {document_path}") + + config = helpers.build_config({"images": {"extract_images": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_images(result, min_count=1) + + +def test_config_images_with_formats() -> None: + """Tests image extraction on PPTX containing embedded images""" + + document_path = helpers.resolve_document("pptx/powerpoint_with_image.pptx") + if not document_path.exists(): + pytest.skip(f"Skipping config_images_with_formats: missing document at {document_path}") + + config = helpers.build_config({"images": {"extract_images": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) + helpers.assert_images(result, min_count=1) + + +def test_config_keywords() -> None: + """Tests keyword extraction via YAKE algorithm""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_keywords: missing document at {document_path}") + + config = helpers.build_config({"keywords": {"algorithm": "yake", "max_keywords": 10}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_keywords(result, has_keywords=True, min_count=1) + + +def test_config_language_detection() -> None: + """Tests language detection configuration""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_language_detection: missing document at {document_path}") + + config = helpers.build_config({"language_detection": {"enabled": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_detected_languages(result, ["eng"], 0.5) + + +def test_config_language_detection_multi() -> None: + """Tests language detection with detect_multiple and min_confidence options""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_language_detection_multi: missing document at {document_path}") + + config = helpers.build_config( + {"language_detection": {"enabled": True, "detect_multiple": True, "min_confidence": 0.3}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_detected_languages(result, ["eng"], None) + + +def test_config_language_multi() -> None: + """Tests multi-language detection config""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_language_multi: missing document at {document_path}") + + config = helpers.build_config({"language_detection": {"enabled": True, "detect_multiple": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_detected_languages(result, ["eng"], None) + + +def test_config_pages() -> None: + """Tests page extraction and page marker configuration""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pages: missing document at {document_path}") + + config = helpers.build_config({"pages": {"extract_pages": True, "insert_page_markers": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["PAGE"]) + + +def test_config_pages_exact_count() -> None: + """Tests page extraction with exact page count assertion on multi-page PDF""" + + document_path = helpers.resolve_document("pdf/multi_page.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pages_exact_count: missing document at {document_path}") + + config = helpers.build_config({"pages": {"extract_pages": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_pages(result, exact_count=5) + + +def test_config_pages_extract() -> None: + """Tests page extraction config producing per-page content array""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pages_extract: missing document at {document_path}") + + config = helpers.build_config({"pages": {"extract_pages": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_pages(result, min_count=1) + + +def test_config_pages_markers() -> None: + """Tests page marker insertion in extracted content""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pages_markers: missing document at {document_path}") + + config = helpers.build_config({"pages": {"insert_page_markers": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["PAGE"]) + + +def test_config_pdf_annotations_count() -> None: + """Tests PDF annotation extraction with min_count assertion""" + + document_path = helpers.resolve_document("vendored/pdfplumber/pdf/annotations.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pdf_annotations_count: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "aarch64" and _platform.system() == "Linux": + pytest.skip("Skipping config_pdf_annotations_count: not supported on this platform") + + config = helpers.build_config({"pdf_options": {"extract_annotations": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_annotations(result, has_annotations=True, min_count=3) + + +def test_config_pdf_hierarchy() -> None: + """Tests PDF hierarchy extraction config with block-level structure""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pdf_hierarchy: missing document at {document_path}") + + config = helpers.build_config( + {"pdf_options": {"hierarchy": {"enabled": True, "include_bbox": True}}, "pages": {"extract_pages": True}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + + +def test_config_pdf_margins() -> None: + """Tests PDF margin exclusion configuration""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_pdf_margins: missing document at {document_path}") + + config = helpers.build_config({"pdf_options": {"top_margin_fraction": 0.1, "bottom_margin_fraction": 0.1}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + + +def test_config_postprocessor() -> None: + """Tests postprocessor config is accepted and extraction succeeds""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_postprocessor: missing document at {document_path}") + + config = helpers.build_config({"postprocessor": {"enabled": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_not_empty(result) + + +def test_config_processing_warnings_empty() -> None: + """Tests that a clean PDF extraction produces no processing warnings""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_processing_warnings_empty: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_processing_warnings(result, is_empty=True) + + +def test_config_quality_disabled() -> None: + """Tests extraction with quality processing explicitly disabled""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_quality_disabled: missing document at {document_path}") + + config = helpers.build_config({"enable_quality_processing": False}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_not_empty(result) + + +def test_config_quality_enabled() -> None: + """Tests quality scoring produces a score value in [0.0, 1.0]""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_quality_enabled: missing document at {document_path}") + + config = helpers.build_config({"enable_quality_processing": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_quality_score(result, has_score=True, min_score=0, max_score=1) + + +def test_config_quality_score_range() -> None: + """Tests quality scoring produces a score with minimum bound assertion""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_quality_score_range: missing document at {document_path}") + + config = helpers.build_config({"enable_quality_processing": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_quality_score(result, has_score=True, min_score=0.1) + + +def test_config_security_limits() -> None: + """Tests archive extraction with custom security limits""" + + document_path = helpers.resolve_document("archives/documents.zip") + if not document_path.exists(): + pytest.skip(f"Skipping config_security_limits: missing document at {document_path}") + + config = helpers.build_config( + {"security_limits": {"max_archive_size": 104857600, "max_compression_ratio": 50, "max_files_in_archive": 100}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/zip", "application/x-zip-compressed"]) + helpers.assert_min_content_length(result, 10) + + +def test_config_structured_output() -> None: + """Tests structured (JSON) output format config""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_structured_output: missing document at {document_path}") + + config = helpers.build_config({"output_format": "structured"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "structured") + + +def test_config_tables_content() -> None: + """Tests table extraction with content_contains_any assertion on DOCX with tables""" + + document_path = helpers.resolve_document("docx/docx_tables.docx") + if not document_path.exists(): + pytest.skip(f"Skipping config_tables_content: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_table_count(result, 1, None) + + +def test_config_tree_sitter() -> None: + """Tests tree-sitter configuration round-trip""" + + document_path = helpers.resolve_document("code/hello.py") + if not document_path.exists(): + pytest.skip(f"Skipping config_tree_sitter: missing document at {document_path}") + + config = helpers.build_config( + { + "tree_sitter": { + "languages": ["python", "rust"], + "groups": ["web"], + "process": { + "structure": True, + "imports": True, + "exports": True, + "comments": False, + "docstrings": False, + "symbols": False, + "diagnostics": False, + }, + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 5) + + +def test_config_tree_sitter_process() -> None: + """Tests tree-sitter process config with all options enabled""" + + document_path = helpers.resolve_document("code/hello.py") + if not document_path.exists(): + pytest.skip(f"Skipping config_tree_sitter_process: missing document at {document_path}") + + config = helpers.build_config( + { + "tree_sitter": { + "process": { + "structure": True, + "imports": True, + "exports": True, + "comments": True, + "docstrings": True, + "symbols": True, + "diagnostics": True, + "chunk_max_size": 2000, + } + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-source-code"]) + helpers.assert_min_content_length(result, 5) + + +def test_config_use_cache_false() -> None: + """Tests use_cache=false configuration option""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping config_use_cache_false: missing document at {document_path}") + + config = helpers.build_config({"use_cache": False}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + + +def test_output_format_bytes_markdown() -> None: + """Tests markdown output format via bytes extraction API""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_bytes_markdown: missing document at {document_path}") + + config = helpers.build_config({"output_format": "markdown"}) + + file_bytes = document_path.read_bytes() + mime_type = detect_mime_type_from_path(str(document_path)) + + result = extract_bytes_sync(file_bytes, mime_type, config=config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_output_format_djot() -> None: + """Tests Djot output format""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_djot: missing document at {document_path}") + + config = helpers.build_config({"output_format": "djot"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "djot") + + +def test_output_format_html() -> None: + """Tests HTML output format""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_html: missing document at {document_path}") + + config = helpers.build_config({"output_format": "html"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "html") + + +def test_output_format_markdown() -> None: + """Tests Markdown output format""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_markdown: missing document at {document_path}") + + config = helpers.build_config({"output_format": "markdown"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "markdown") + + +def test_output_format_plain() -> None: + """Tests Plain output format""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping output_format_plain: missing document at {document_path}") + + config = helpers.build_config({"output_format": "plain"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_output_format(result, "plain") + + +def test_result_format_element_based() -> None: + """Tests ElementBased result format with element assertions""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping result_format_element_based: missing document at {document_path}") + + config = helpers.build_config({"result_format": "element_based"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_elements(result, min_count=1) + helpers.assert_result_format(result, "element_based") + + +def test_result_format_unified() -> None: + """Tests Unified result format (default)""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping result_format_unified: missing document at {document_path}") + + config = helpers.build_config({"result_format": "unified"}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_result_format(result, "unified") diff --git a/tools/e2e-generator/e2e/python/tests/test_email.py b/tools/e2e-generator/e2e/python/tests/test_email.py new file mode 100644 index 00000000000..53dc11347f9 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_email.py @@ -0,0 +1,103 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for email fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_email_eml_html_body() -> None: + """EML with HTML body content.""" + + document_path = helpers.resolve_document("email/html_only.eml") + if not document_path.exists(): + pytest.skip(f"Skipping email_eml_html_body: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["message/rfc822"]) + helpers.assert_min_content_length(result, 10) + + +def test_email_eml_multipart() -> None: + """EML with multipart MIME content.""" + + document_path = helpers.resolve_document("email/html_email_multipart.eml") + if not document_path.exists(): + pytest.skip(f"Skipping email_eml_multipart: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["message/rfc822"]) + helpers.assert_min_content_length(result, 10) + + +def test_email_eml_utf16() -> None: + """UTF-16 encoded EML file with BOM.""" + + document_path = helpers.resolve_document("vendored/unstructured/eml/fake-email-utf-16.eml") + if not document_path.exists(): + pytest.skip(f"Skipping email_eml_utf16: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["message/rfc822"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["Test Email", "Roses are red"]) + + +def test_email_msg_basic() -> None: + """Outlook MSG file extraction.""" + + document_path = helpers.resolve_document("email/fake_email.msg") + if not document_path.exists(): + pytest.skip(f"Skipping email_msg_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-outlook"]) + helpers.assert_min_content_length(result, 10) + + +def test_email_pst_empty() -> None: + """Empty Outlook PST archive with no messages.""" + + document_path = helpers.resolve_document("email/empty.pst") + if not document_path.exists(): + pytest.skip(f"Skipping email_pst_empty: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-outlook-pst"]) + + +def test_email_sample_eml() -> None: + """Sample EML email file to verify email parsing.""" + + document_path = helpers.resolve_document("email/sample_email.eml") + if not document_path.exists(): + pytest.skip(f"Skipping email_sample_eml: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["message/rfc822"]) + helpers.assert_min_content_length(result, 20) diff --git a/tools/e2e-generator/e2e/python/tests/test_embeddings.py b/tools/e2e-generator/e2e/python/tests/test_embeddings.py new file mode 100644 index 00000000000..6e3ad344a9b --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_embeddings.py @@ -0,0 +1,118 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for embeddings fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file, + extract_file_sync, +) + +from . import helpers + + +@pytest.mark.asyncio +async def test_embedding_async() -> None: + """Tests embedding generation via async extraction path""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping embedding_async: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "AMD64" and _platform.system() == "Windows": + pytest.skip("Skipping embedding_async: not supported on this platform") + + config = helpers.build_config( + { + "chunking": { + "max_chars": 500, + "max_overlap": 50, + "embedding": {"model": {"type": "preset", "name": "balanced"}, "normalize": True}, + } + } + ) + + result = await extract_file(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True, each_has_embedding=True) + + +def test_embedding_balanced_preset() -> None: + """Tests embedding generation with balanced preset model via chunking""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping embedding_balanced_preset: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "AMD64" and _platform.system() == "Windows": + pytest.skip("Skipping embedding_balanced_preset: not supported on this platform") + + config = helpers.build_config( + { + "chunking": { + "max_chars": 500, + "max_overlap": 50, + "embedding": {"model": {"type": "preset", "name": "balanced"}, "normalize": True}, + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True, each_has_embedding=True) + + +def test_embedding_disabled() -> None: + """Tests chunking without embeddings - chunks should not have embedding vectors""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping embedding_disabled: missing document at {document_path}") + + config = helpers.build_config({"chunking": {"max_chars": 500, "max_overlap": 50}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True, each_has_embedding=False) + + +def test_embedding_fast_preset() -> None: + """Tests embedding generation with fast preset model""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping embedding_fast_preset: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "AMD64" and _platform.system() == "Windows": + pytest.skip("Skipping embedding_fast_preset: not supported on this platform") + + config = helpers.build_config( + { + "chunking": { + "max_chars": 500, + "max_overlap": 50, + "embedding": {"model": {"type": "preset", "name": "fast"}, "normalize": True}, + } + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_chunks(result, min_count=1, each_has_content=True, each_has_embedding=True) diff --git a/tools/e2e-generator/e2e/python/tests/test_html.py b/tools/e2e-generator/e2e/python/tests/test_html.py new file mode 100644 index 00000000000..264832f14c6 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_html.py @@ -0,0 +1,46 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for html fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_html_complex_layout() -> None: + """Large Wikipedia HTML page to validate complex conversion.""" + + document_path = helpers.resolve_document("html/taylor_swift.html") + if not document_path.exists(): + pytest.skip(f"Skipping html_complex_layout: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/html"]) + helpers.assert_min_content_length(result, 1000) + + +def test_html_simple_table() -> None: + """HTML table converted to markdown should retain structure.""" + + document_path = helpers.resolve_document("html/simple_table.html") + if not document_path.exists(): + pytest.skip(f"Skipping html_simple_table: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/html"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_contains_all( + result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"] + ) diff --git a/tools/e2e-generator/e2e/python/tests/test_image.py b/tools/e2e-generator/e2e/python/tests/test_image.py new file mode 100644 index 00000000000..32e4b367d3d --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_image.py @@ -0,0 +1,163 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for image fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_image_bmp_basic() -> None: + """BMP image extraction via OCR.""" + + document_path = helpers.resolve_document("images/bmp_24.bmp") + if not document_path.exists(): + pytest.skip(f"Skipping image_bmp_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/bmp"]) + helpers.assert_content_not_empty(result) + + +def test_image_gif_basic() -> None: + """GIF image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.gif") + if not document_path.exists(): + pytest.skip(f"Skipping image_gif_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/gif"]) + helpers.assert_content_not_empty(result) + + +def test_image_jp2_basic() -> None: + """JPEG 2000 image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.jp2") + if not document_path.exists(): + pytest.skip(f"Skipping image_jp2_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jp2", "image/jpeg2000"]) + helpers.assert_content_not_empty(result) + + +def test_image_metadata_only() -> None: + """JPEG image to validate metadata extraction without OCR.""" + + document_path = helpers.resolve_document("images/example.jpg") + if not document_path.exists(): + pytest.skip(f"Skipping image_metadata_only: missing document at {document_path}") + + config = helpers.build_config({"ocr": None}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jpeg"]) + helpers.assert_max_content_length(result, 200) + + +def test_image_pbm_basic() -> None: + """PBM (portable bitmap) image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.pbm") + if not document_path.exists(): + pytest.skip(f"Skipping image_pbm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/x-portable-bitmap", "image/x-pbm"]) + helpers.assert_content_not_empty(result) + + +def test_image_pgm_basic() -> None: + """PGM (portable graymap) image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.pgm") + if not document_path.exists(): + pytest.skip(f"Skipping image_pgm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/x-portable-graymap", "image/x-pgm"]) + helpers.assert_content_not_empty(result) + + +def test_image_ppm_basic() -> None: + """PPM (portable pixmap) image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.ppm") + if not document_path.exists(): + pytest.skip(f"Skipping image_ppm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/x-portable-pixmap", "image/x-ppm"]) + helpers.assert_content_not_empty(result) + + +def test_image_svg_basic() -> None: + """SVG image extraction.""" + + document_path = helpers.resolve_document("xml/simple_svg.svg") + if not document_path.exists(): + pytest.skip(f"Skipping image_svg_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/svg+xml"]) + helpers.assert_min_content_length(result, 5) + + +def test_image_tiff_basic() -> None: + """TIFF image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.tif") + if not document_path.exists(): + pytest.skip(f"Skipping image_tiff_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/tiff"]) + helpers.assert_content_not_empty(result) + + +def test_image_webp_basic() -> None: + """WebP image extraction via OCR.""" + + document_path = helpers.resolve_document("images_extra/ocr_image.webp") + if not document_path.exists(): + pytest.skip(f"Skipping image_webp_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/webp"]) + helpers.assert_content_not_empty(result) diff --git a/tools/e2e-generator/e2e/python/tests/test_keywords.py b/tools/e2e-generator/e2e/python/tests/test_keywords.py new file mode 100644 index 00000000000..5a3949a87d9 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_keywords.py @@ -0,0 +1,45 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for keywords fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_keywords_rake() -> None: + """Tests keyword extraction using RAKE algorithm""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping keywords_rake: missing document at {document_path}") + + config = helpers.build_config({"keywords": {"algorithm": "rake", "max_keywords": 10}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_keywords(result, has_keywords=True, min_count=1, max_count=10) + + +def test_keywords_yake() -> None: + """Tests keyword extraction using YAKE algorithm""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping keywords_yake: missing document at {document_path}") + + config = helpers.build_config({"keywords": {"algorithm": "yake", "max_keywords": 10}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_keywords(result, has_keywords=True, min_count=1, max_count=10) diff --git a/tools/e2e-generator/e2e/python/tests/test_ocr.py b/tools/e2e-generator/e2e/python/tests/test_ocr.py new file mode 100644 index 00000000000..50551498698 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_ocr.py @@ -0,0 +1,332 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for ocr fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_ocr_image_hello_world() -> None: + """PNG image with visible English text for OCR validation.""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_image_hello_world: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "eng"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_content_contains_any(result, ["hello", "world"]) + + +def test_ocr_image_no_text() -> None: + """Image with no text to ensure OCR handles empty results gracefully.""" + + document_path = helpers.resolve_document("images/flower_no_text.jpg") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_image_no_text: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "eng"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jpeg"]) + helpers.assert_max_content_length(result, 300) + + +def test_ocr_paddle_confidence_filter() -> None: + """PaddleOCR with minimum confidence threshold filtering.""" + + document_path = helpers.resolve_document("images/ocr_image.jpg") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_confidence_filter: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "paddle-ocr", "language": "en", "paddle_ocr_config": {"min_confidence": 80.0}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jpeg"]) + helpers.assert_min_content_length(result, 1) + + +def test_ocr_paddle_element_hierarchy() -> None: + """Tests PaddleOCR with element hierarchy building enabled""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_element_hierarchy: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": { + "backend": "paddle-ocr", + "language": "en", + "element_config": {"include_elements": True, "build_hierarchy": True}, + }, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, elements_have_geometry=True, elements_have_confidence=True) + + +def test_ocr_paddle_element_levels() -> None: + """Tests PaddleOCR with word-level element extraction""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_element_levels: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": { + "backend": "paddle-ocr", + "language": "en", + "element_config": {"include_elements": True, "min_level": "word"}, + }, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, elements_have_geometry=True, min_count=1) + + +def test_ocr_paddle_image_chinese() -> None: + """Chinese OCR with PaddleOCR - its core strength.""" + + document_path = helpers.resolve_document("images/chi_sim_image.jpeg") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_image_chinese: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "paddle-ocr", "language": "ch"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/jpeg"]) + helpers.assert_min_content_length(result, 1) + + +def test_ocr_paddle_image_english() -> None: + """Simple English image OCR with PaddleOCR backend.""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_image_english: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "paddle-ocr", "language": "en"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_content_contains_any(result, ["hello", "Hello", "world", "World"]) + + +def test_ocr_paddle_markdown() -> None: + """PaddleOCR with markdown output format.""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_markdown: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "paddle-ocr", "language": "en", "paddle_ocr_config": {"output_format": "markdown"}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_content_contains_any(result, ["hello", "Hello", "world", "World"]) + + +def test_ocr_paddle_pdf_scanned() -> None: + """Scanned PDF requires PaddleOCR to extract text.""" + + document_path = helpers.resolve_document("pdf/ocr_test.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_pdf_scanned: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "paddle-ocr", "language": "en"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_content_contains_any(result, ["Docling", "Markdown", "JSON"]) + + +def test_ocr_paddle_structured() -> None: + """PaddleOCR with structured output preserving all metadata.""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_structured: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "paddle-ocr", "language": "en", "element_config": {"include_elements": True}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, elements_have_geometry=True, elements_have_confidence=True) + + +def test_ocr_paddle_table_detection() -> None: + """Table detection and extraction with PaddleOCR.""" + + document_path = helpers.resolve_document("images/simple_table.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_paddle_table_detection: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "paddle-ocr", "language": "en", "paddle_ocr_config": {"enable_table_detection": True}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_table_count(result, 1, None) + + +def test_ocr_pdf_image_only_german() -> None: + """Image-only German PDF requiring OCR to extract text.""" + + document_path = helpers.resolve_document("pdf/image_only_german_pdf.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_pdf_image_only_german: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "deu"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_ocr_pdf_rotated_90() -> None: + """Rotated page PDF requiring OCR to verify orientation handling.""" + + document_path = helpers.resolve_document("pdf/ocr_test_rotated_90.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_pdf_rotated_90: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "eng"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + + +def test_ocr_pdf_tesseract() -> None: + """Scanned PDF requires OCR to extract text.""" + + document_path = helpers.resolve_document("pdf/ocr_test.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_pdf_tesseract: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "eng"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_content_contains_any(result, ["Docling", "Markdown", "JSON"]) + + +def test_ocr_tesseract_elements() -> None: + """Tests Tesseract OCR with element-level structured output including geometry""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_tesseract_elements: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": {"backend": "tesseract", "language": "eng", "element_config": {"include_elements": True}}, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, elements_have_geometry=True, elements_have_confidence=True) + + +def test_ocr_tesseract_elements_min_count() -> None: + """Tests Tesseract OCR element output with min_count assertion""" + + document_path = helpers.resolve_document("images/test_hello_world.png") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_tesseract_elements_min_count: missing document at {document_path}") + + config = helpers.build_config( + { + "ocr": { + "backend": "tesseract", + "language": "eng", + "element_config": {"include_elements": True, "min_level": "line"}, + }, + "force_ocr": True, + } + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_ocr_elements(result, has_elements=True, min_count=1) + + +def test_ocr_tesseract_language_german() -> None: + """Tests Tesseract OCR with German language configuration""" + + document_path = helpers.resolve_document("pdf/image_only_german_pdf.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping ocr_tesseract_language_german: missing document at {document_path}") + + config = helpers.build_config({"ocr": {"backend": "tesseract", "language": "deu"}, "force_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 20) diff --git a/tools/e2e-generator/e2e/python/tests/test_office.py b/tools/e2e-generator/e2e/python/tests/test_office.py new file mode 100644 index 00000000000..72239fb4306 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_office.py @@ -0,0 +1,811 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for office fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_office_bibtex_basic() -> None: + """BibTeX bibliography extraction.""" + + document_path = helpers.resolve_document("bibtex/comprehensive.bib") + if not document_path.exists(): + pytest.skip(f"Skipping office_bibtex_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-bibtex", "text/x-bibtex"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_commonmark_basic() -> None: + """CommonMark (.commonmark) text extraction.""" + + document_path = helpers.resolve_document("markdown/sample.commonmark") + if not document_path.exists(): + pytest.skip(f"Skipping office_commonmark_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/markdown", "text/plain", "text/x-commonmark"]) + helpers.assert_min_content_length(result, 5) + + +def test_office_dbf_basic() -> None: + """dBASE (.dbf) table extraction as markdown.""" + + document_path = helpers.resolve_document("dbf/stations.dbf") + if not document_path.exists(): + pytest.skip(f"Skipping office_dbf_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-dbf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["|"]) + + +def test_office_djot_basic() -> None: + """Djot markup text extraction.""" + + document_path = helpers.resolve_document("markdown/tables.djot") + if not document_path.exists(): + pytest.skip(f"Skipping office_djot_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-djot", "text/djot"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_doc_legacy() -> None: + """Legacy .doc document extraction via native OLE/CFB parsing.""" + + document_path = helpers.resolve_document("doc/unit_test_lists.doc") + if not document_path.exists(): + pytest.skip(f"Skipping office_doc_legacy: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/msword"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docbook_basic() -> None: + """DocBook XML document extraction.""" + + document_path = helpers.resolve_document("docbook/docbook-reader.docbook") + if not document_path.exists(): + pytest.skip(f"Skipping office_docbook_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/docbook+xml", "text/docbook"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_docx_basic() -> None: + """DOCX document extraction baseline.""" + + document_path = helpers.resolve_document("docx/sample_document.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_docx_equations() -> None: + """DOCX file containing equations to validate math extraction.""" + + document_path = helpers.resolve_document("docx/equations.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_equations: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_fake() -> None: + """Simple DOCX document to verify baseline extraction.""" + + document_path = helpers.resolve_document("docx/fake.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_fake: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_formatting() -> None: + """DOCX document heavy on formatting for style preservation.""" + + document_path = helpers.resolve_document("docx/unit_test_formatting.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_formatting: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_headers() -> None: + """DOCX document with complex headers.""" + + document_path = helpers.resolve_document("docx/unit_test_headers.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_headers: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_lists() -> None: + """DOCX document emphasizing list formatting.""" + + document_path = helpers.resolve_document("docx/unit_test_lists.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_lists: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_docx_tables() -> None: + """DOCX document containing tables for table-aware extraction.""" + + document_path = helpers.resolve_document("docx/docx_tables.docx") + if not document_path.exists(): + pytest.skip(f"Skipping office_docx_tables: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_all(result, ["Simple uniform table", "Nested Table", "merged cells", "Header Col"]) + helpers.assert_table_count(result, 1, None) + + +def test_office_epub_basic() -> None: + """EPUB book extraction with text content.""" + + document_path = helpers.resolve_document("epub/features.epub") + if not document_path.exists(): + pytest.skip(f"Skipping office_epub_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/epub+zip"]) + helpers.assert_min_content_length(result, 50) + + +def test_office_fb2_basic() -> None: + """FictionBook (FB2) document extraction baseline.""" + + document_path = helpers.resolve_document("fictionbook/basic.fb2") + if not document_path.exists(): + pytest.skip(f"Skipping office_fb2_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-fictionbook+xml"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_fictionbook_basic() -> None: + """FictionBook (.fb2) text extraction.""" + + document_path = helpers.resolve_document("fictionbook/basic.fb2") + if not document_path.exists(): + pytest.skip(f"Skipping office_fictionbook_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_hwp_basic() -> None: + """Hangul Word Processor (.hwp) text extraction.""" + + document_path = helpers.resolve_document("hwp/converted_output.hwp") + if not document_path.exists(): + pytest.skip(f"Skipping office_hwp_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-hwp"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_hwp_styled() -> None: + """Hangul Word Processor (.hwp) styled document extraction.""" + + document_path = helpers.resolve_document("hwp/styled_document.hwp") + if not document_path.exists(): + pytest.skip(f"Skipping office_hwp_styled: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "aarch64" and _platform.system() == "Linux": + pytest.skip("Skipping office_hwp_styled: not supported on this platform") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-hwp"]) + + +def test_office_jats_basic() -> None: + """JATS scientific article extraction.""" + + document_path = helpers.resolve_document("jats/sample_article.jats") + if not document_path.exists(): + pytest.skip(f"Skipping office_jats_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-jats+xml", "text/jats"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_jupyter_basic() -> None: + """Jupyter notebook extraction.""" + + document_path = helpers.resolve_document("jupyter/rank.ipynb") + if not document_path.exists(): + pytest.skip(f"Skipping office_jupyter_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-ipynb+json"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_keynote_basic() -> None: + """Keynote document extraction baseline.""" + + document_path = helpers.resolve_document("iwork/test.key") + if not document_path.exists(): + pytest.skip(f"Skipping office_keynote_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-iwork-keynote-sffkey"]) + helpers.assert_min_content_length(result, 5) + + +def test_office_latex_basic() -> None: + """LaTeX document text extraction.""" + + document_path = helpers.resolve_document("latex/basic_sections.tex") + if not document_path.exists(): + pytest.skip(f"Skipping office_latex_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-latex", "text/x-latex"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_markdown_basic() -> None: + """Markdown document extraction baseline.""" + + document_path = helpers.resolve_document("markdown/comprehensive.md") + if not document_path.exists(): + pytest.skip(f"Skipping office_markdown_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/markdown"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_mdx_basic() -> None: + """MDX document extraction with JSX stripping and frontmatter.""" + + document_path = helpers.resolve_document("markdown/sample.mdx") + if not document_path.exists(): + pytest.skip(f"Skipping office_mdx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/mdx", "text/x-mdx"]) + helpers.assert_min_content_length(result, 50) + + +def test_office_mdx_getting_started() -> None: + """Real-world MDX extraction from mdx-js/mdx getting-started.mdx with imports, exports, JSX components, code blocks, and reference links.""" + + document_path = helpers.resolve_document("markdown/mdx_getting_started.mdx") + if not document_path.exists(): + pytest.skip(f"Skipping office_mdx_getting_started: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/mdx", "text/x-mdx"]) + helpers.assert_min_content_length(result, 2000) + + +def test_office_mdx_troubleshooting() -> None: + """Real-world MDX extraction from mdx-js/mdx troubleshooting-mdx.mdx with error documentation, code blocks, and JSX comments.""" + + document_path = helpers.resolve_document("markdown/mdx_troubleshooting.mdx") + if not document_path.exists(): + pytest.skip(f"Skipping office_mdx_troubleshooting: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/mdx", "text/x-mdx"]) + helpers.assert_min_content_length(result, 2000) + + +def test_office_mdx_using_mdx() -> None: + """Real-world MDX extraction from mdx-js/mdx using-mdx.mdx with complex JSX examples, component passing, and MDX provider patterns.""" + + document_path = helpers.resolve_document("markdown/mdx_using_mdx.mdx") + if not document_path.exists(): + pytest.skip(f"Skipping office_mdx_using_mdx: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/mdx", "text/x-mdx"]) + helpers.assert_min_content_length(result, 2000) + + +def test_office_numbers_basic() -> None: + """Numbers document extraction baseline.""" + + document_path = helpers.resolve_document("iwork/test.numbers") + if not document_path.exists(): + pytest.skip(f"Skipping office_numbers_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-iwork-numbers-sffnumbers"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_ods_basic() -> None: + """Basic ODS spreadsheet extraction.""" + + document_path = helpers.resolve_document("data_formats/test_01.ods") + if not document_path.exists(): + pytest.skip(f"Skipping office_ods_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.spreadsheet"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_odt_bold() -> None: + """ODT document with bold formatting.""" + + document_path = helpers.resolve_document("odt/bold.odt") + if not document_path.exists(): + pytest.skip(f"Skipping office_odt_bold: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.text"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_odt_list() -> None: + """ODT document containing unordered lists with nesting.""" + + document_path = helpers.resolve_document("odt/unorderedList.odt") + if not document_path.exists(): + pytest.skip(f"Skipping office_odt_list: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.text"]) + helpers.assert_min_content_length(result, 30) + helpers.assert_content_contains_any(result, ["list item", "New level", "Pushed us"]) + + +def test_office_odt_simple() -> None: + """Basic ODT document with paragraphs and headings.""" + + document_path = helpers.resolve_document("odt/simple.odt") + if not document_path.exists(): + pytest.skip(f"Skipping office_odt_simple: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.text"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["favorite things", "Parrots", "Analysis"]) + + +def test_office_odt_table() -> None: + """ODT document with a table structure.""" + + document_path = helpers.resolve_document("odt/table.odt") + if not document_path.exists(): + pytest.skip(f"Skipping office_odt_table: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.oasis.opendocument.text"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_table_count(result, 1, None) + + +def test_office_opml_basic() -> None: + """OPML outline document extraction.""" + + document_path = helpers.resolve_document("opml/outline.opml") + if not document_path.exists(): + pytest.skip(f"Skipping office_opml_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_org_basic() -> None: + """Org-mode document text extraction.""" + + document_path = helpers.resolve_document("org/comprehensive.org") + if not document_path.exists(): + pytest.skip(f"Skipping office_org_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-org", "text/org"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_pages_basic() -> None: + """Pages document extraction baseline.""" + + document_path = helpers.resolve_document("iwork/test.pages") + if not document_path.exists(): + pytest.skip(f"Skipping office_pages_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-iwork-pages-sffpages"]) + helpers.assert_min_content_length(result, 5) + + +def test_office_ppsx_slideshow() -> None: + """PPSX (PowerPoint Show) files should extract slides content identical to PPTX. GitHub Issue #321 Bug 2.""" + + document_path = helpers.resolve_document("pptx/sample.ppsx") + if not document_path.exists(): + pytest.skip(f"Skipping office_ppsx_slideshow: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_ppt_legacy() -> None: + """Legacy PowerPoint .ppt extraction via native OLE/CFB parsing.""" + + document_path = helpers.resolve_document("ppt/simple.ppt") + if not document_path.exists(): + pytest.skip(f"Skipping office_ppt_legacy: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-powerpoint"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_pptm_basic() -> None: + """PowerPoint macro-enabled presentation (.pptm) extraction.""" + + document_path = helpers.resolve_document("pptx/powerpoint_with_image.pptm") + if not document_path.exists(): + pytest.skip(f"Skipping office_pptm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime( + result, + [ + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ], + ) + helpers.assert_content_not_empty(result) + + +def test_office_pptx_basic() -> None: + """PPTX deck should extract slides content.""" + + document_path = helpers.resolve_document("pptx/simple.pptx") + if not document_path.exists(): + pytest.skip(f"Skipping office_pptx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) + helpers.assert_min_content_length(result, 50) + + +def test_office_pptx_images() -> None: + """PPTX presentation containing images to ensure metadata extraction.""" + + document_path = helpers.resolve_document("pptx/powerpoint_with_image.pptx") + if not document_path.exists(): + pytest.skip(f"Skipping office_pptx_images: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) + helpers.assert_min_content_length(result, 15) + + +def test_office_pptx_pitch_deck() -> None: + """Pitch deck PPTX used to validate large slide extraction.""" + + document_path = helpers.resolve_document("pptx/pitch_deck_presentation.pptx") + if not document_path.exists(): + pytest.skip(f"Skipping office_pptx_pitch_deck: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]) + helpers.assert_min_content_length(result, 100) + + +def test_office_rst_basic() -> None: + """reStructuredText document extraction.""" + + document_path = helpers.resolve_document("rst/restructured_text.rst") + if not document_path.exists(): + pytest.skip(f"Skipping office_rst_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/x-rst", "text/prs.fallenstein.rst"]) + helpers.assert_min_content_length(result, 20) + + +def test_office_rtf_basic() -> None: + """RTF document text extraction.""" + + document_path = helpers.resolve_document("rtf/extraction_test.rtf") + if not document_path.exists(): + pytest.skip(f"Skipping office_rtf_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/rtf", "text/rtf"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_typst_basic() -> None: + """Typst document text extraction.""" + + document_path = helpers.resolve_document("typst/headings.typ") + if not document_path.exists(): + pytest.skip(f"Skipping office_typst_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-typst", "text/x-typst"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_xls_legacy() -> None: + """Legacy XLS spreadsheet to ensure backward compatibility.""" + + document_path = helpers.resolve_document("xls/test_excel.xls") + if not document_path.exists(): + pytest.skip(f"Skipping office_xls_legacy: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.ms-excel"]) + helpers.assert_min_content_length(result, 10) + + +def test_office_xlsb_basic() -> None: + """Excel binary workbook (.xlsb) extraction.""" + + document_path = helpers.resolve_document("xlsx/test_xlsb.xlsb") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsb_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime( + result, + [ + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ], + ) + helpers.assert_content_not_empty(result) + + +def test_office_xlsm_basic() -> None: + """Excel macro-enabled workbook (.xlsm) extraction.""" + + document_path = helpers.resolve_document("xlsx/test_01.xlsm") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsm_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime( + result, + [ + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ], + ) + helpers.assert_content_not_empty(result) + + +def test_office_xlsx_basic() -> None: + """XLSX spreadsheet should produce metadata and table content.""" + + document_path = helpers.resolve_document("xlsx/stanley_cups.xlsx") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_contains_all(result, ["Team", "Location", "Stanley Cups"]) + helpers.assert_table_count(result, 1, None) + helpers.assert_metadata_expectation(result, "sheet_count", {"gte": 2}) + helpers.assert_metadata_expectation(result, "sheet_names", {"contains": ["Stanley Cups"]}) + + +def test_office_xlsx_multi_sheet() -> None: + """XLSX workbook with multiple sheets.""" + + document_path = helpers.resolve_document("xlsx/excel_multi_sheet.xlsx") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsx_multi_sheet: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_metadata_expectation(result, "sheet_count", {"gte": 2}) + + +def test_office_xlsx_office_example() -> None: + """Simple XLSX spreadsheet shipped alongside office integration tests.""" + + document_path = helpers.resolve_document("xlsx/test_01.xlsx") + if not document_path.exists(): + pytest.skip(f"Skipping office_xlsx_office_example: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) + helpers.assert_min_content_length(result, 10) diff --git a/tools/e2e-generator/e2e/python/tests/test_parity.py b/tools/e2e-generator/e2e/python/tests/test_parity.py new file mode 100644 index 00000000000..1699413b55f --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_parity.py @@ -0,0 +1,158 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python + +from __future__ import annotations + +from kreuzberg import ExtractionConfig, extract_file_sync + +from . import helpers + + +def test_extraction_result_field_parity(): + """Verify ExtractionResult has all expected fields.""" + result = extract_file_sync(helpers.resolve_document("pdf/fake_memo.pdf")) + + required_fields = [ + "content", + "metadata", + "mime_type", + "tables", + ] + for field in required_fields: + assert hasattr(result, field), f"ExtractionResult missing required field: {field}" + + all_fields = [ + "annotations", + "children", + "chunks", + "content", + "detected_languages", + "djot_content", + "document", + "elements", + "extracted_keywords", + "images", + "metadata", + "mime_type", + "ocr_elements", + "pages", + "processing_warnings", + "quality_score", + "tables", + "uris", + ] + for field in all_fields: + assert hasattr(result, field), f"ExtractionResult missing field: {field}" + + +def test_extraction_config_field_parity(): + """Verify ExtractionConfig accepts all expected fields.""" + config = ExtractionConfig() + + all_fields = [ + "acceleration", + "cache_namespace", + "cache_ttl_secs", + "chunking", + "concurrency", + "disable_ocr", + "email", + "enable_quality_processing", + "extraction_timeout_secs", + "force_ocr", + "force_ocr_pages", + "html_options", + "images", + "include_document_structure", + "keywords", + "language_detection", + "layout", + "max_archive_depth", + "max_concurrent_extractions", + "ocr", + "output_format", + "pages", + "pdf_options", + "postprocessor", + "result_format", + "security_limits", + "token_reduction", + "tree_sitter", + "use_cache", + ] + for field in all_fields: + assert hasattr(config, field), f"ExtractionConfig missing field: {field}" + + +def test_archive_entry_field_parity(): + """Verify ArchiveEntry has all expected fields in the manifest.""" + expected_fields = [ + "mime_type", + "path", + "result", + ] + assert len(expected_fields) > 0, "ArchiveEntry should have at least one field" + + +def test_bounding_box_field_parity(): + """Verify BoundingBox has all expected fields in the manifest.""" + expected_fields = [ + "x0", + "x1", + "y0", + "y1", + ] + assert len(expected_fields) > 0, "BoundingBox should have at least one field" + + +def test_keyword_field_parity(): + """Verify Keyword has all expected fields in the manifest.""" + expected_fields = [ + "algorithm", + "positions", + "score", + "text", + ] + assert len(expected_fields) > 0, "Keyword should have at least one field" + + +def test_pdf_annotation_field_parity(): + """Verify PdfAnnotation has all expected fields in the manifest.""" + expected_fields = [ + "annotation_type", + "bounding_box", + "content", + "page_number", + ] + assert len(expected_fields) > 0, "PdfAnnotation should have at least one field" + + +def test_processing_warning_field_parity(): + """Verify ProcessingWarning has all expected fields in the manifest.""" + expected_fields = [ + "message", + "source", + ] + assert len(expected_fields) > 0, "ProcessingWarning should have at least one field" + + +def test_table_field_parity(): + """Verify Table has all expected fields in the manifest.""" + expected_fields = [ + "bounding_box", + "cells", + "markdown", + "page_number", + ] + assert len(expected_fields) > 0, "Table should have at least one field" + + +def test_uri_field_parity(): + """Verify Uri has all expected fields in the manifest.""" + expected_fields = [ + "kind", + "label", + "page", + "url", + ] + assert len(expected_fields) > 0, "Uri should have at least one field" diff --git a/tools/e2e-generator/e2e/python/tests/test_pdf.py b/tools/e2e-generator/e2e/python/tests/test_pdf.py new file mode 100644 index 00000000000..bb88f8ca96e --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_pdf.py @@ -0,0 +1,316 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for pdf fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_pdf_annotations() -> None: + """PDF with annotations should extract annotation data when enabled.""" + + document_path = helpers.resolve_document("pdf/test_article.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_annotations: missing document at {document_path}") + + import platform as _platform + + if _platform.machine() == "aarch64" and _platform.system() == "Linux": + pytest.skip("Skipping pdf_annotations: not supported on this platform") + + config = helpers.build_config({"pdf_options": {"extract_annotations": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_annotations(result, has_annotations=True, min_count=1) + + +def test_pdf_assembly_technical() -> None: + """Assembly language technical manual with large body of text.""" + + document_path = helpers.resolve_document("pdf/assembly_language_for_beginners_al4_b_en.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_assembly_technical: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5000) + helpers.assert_content_contains_any(result, ["assembly", "register", "instruction"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_bayesian_data_analysis() -> None: + """Bayesian data analysis textbook PDF with large content volume.""" + + document_path = helpers.resolve_document("pdf/bayesian_data_analysis_third_edition_13th_feb_2020.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_bayesian_data_analysis: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10000) + helpers.assert_content_contains_any(result, ["Bayesian", "probability", "distribution"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_bounding_boxes() -> None: + """Tests bounding box extraction on PDF tables and images""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_bounding_boxes: missing document at {document_path}") + + config = helpers.build_config({"images": {"extract_images": True}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_table_count(result, 1, None) + helpers.assert_table_bounding_boxes(result, True) + + +def test_pdf_code_and_formula() -> None: + """PDF containing code snippets and formulas should retain substantial content.""" + + document_path = helpers.resolve_document("pdf/code_and_formula.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_code_and_formula: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 100) + + +def test_pdf_deep_learning() -> None: + """Deep learning textbook PDF to ensure long-form extraction quality.""" + + document_path = helpers.resolve_document("pdf/fundamentals_of_deep_learning_2014.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_deep_learning: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 1000) + helpers.assert_content_contains_any(result, ["neural", "network", "deep learning"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_embedded_images() -> None: + """PDF with embedded images should extract text and tables when present.""" + + document_path = helpers.resolve_document("pdf/embedded_images_tables.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_embedded_images: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_table_count(result, 0, None) + + +def test_pdf_google_doc() -> None: + """Google Docs exported PDF to verify conversion fidelity.""" + + document_path = helpers.resolve_document("pdf/google_doc_document.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_google_doc: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_large_ciml() -> None: + """Large machine learning textbook PDF to stress extraction length.""" + + document_path = helpers.resolve_document("pdf/a_course_in_machine_learning_ciml_v0_9_all.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_large_ciml: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10000) + helpers.assert_content_contains_any(result, ["machine learning", "algorithm", "training"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_layout_detection() -> None: + """PDF extraction with layout detection enabled should produce content from a document with mixed structure.""" + + document_path = helpers.resolve_document("pdf/docling.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_layout_detection: missing document at {document_path}") + + config = helpers.build_config( + {"layout": {"preset": "accurate", "table_model": "tatr"}, "output_format": "markdown"} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_not_empty(result) + + +def test_pdf_non_english_german() -> None: + """German technical PDF to ensure non-ASCII content extraction.""" + + document_path = helpers.resolve_document("pdf/5_level_paging_and_5_level_ept_intel_revision_1_1_may_2017.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_non_english_german: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_contains_any(result, ["Intel", "paging"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_password_protected() -> None: + """Copy-protected PDF should extract content (pdfium handles copy-protection transparently).""" + + document_path = helpers.resolve_document("pdf/copy_protected.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_password_protected: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["LayoutParser", "document image analysis", "deep learning"]) + + +def test_pdf_right_to_left() -> None: + """Right-to-left language PDF to verify RTL extraction.""" + + document_path = helpers.resolve_document("pdf/right_to_left_01.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_right_to_left: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) + + +def test_pdf_simple_text() -> None: + """Simple text-heavy PDF should extract content without OCR or tables.""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_simple_text: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["May 5, 2023", "To Whom it May Concern", "Mallori"]) + + +def test_pdf_tables_large() -> None: + """Large PDF with extensive tables to stress table extraction.""" + + document_path = helpers.resolve_document("pdf/large.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_tables_large: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 500) + + +def test_pdf_tables_medium() -> None: + """Medium-sized PDF with multiple tables.""" + + document_path = helpers.resolve_document("pdf/medium.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_tables_medium: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 100) + + +def test_pdf_tables_small() -> None: + """Small PDF containing tables to validate table extraction.""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping pdf_tables_small: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_all( + result, ["Table 1", "Selected Numbers", "Celsius", "Fahrenheit", "Water Freezing Point", "Water Boiling Point"] + ) + helpers.assert_table_count(result, 1, None) + + +def test_pdf_technical_stat_learning() -> None: + """Technical statistical learning PDF requiring substantial extraction.""" + + document_path = helpers.resolve_document( + "pdf/an_introduction_to_statistical_learning_with_applications_in_r_islr_sixth_printing.pdf" + ) + if not document_path.exists(): + pytest.skip(f"Skipping pdf_technical_stat_learning: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10000) + helpers.assert_content_contains_any(result, ["statistical", "regression", "learning"]) + helpers.assert_metadata_expectation(result, "format_type", {"eq": "pdf"}) diff --git a/tools/e2e-generator/e2e/python/tests/test_plugin_apis.py b/tools/e2e-generator/e2e/python/tests/test_plugin_apis.py new file mode 100644 index 00000000000..54a4011bbcb --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_plugin_apis.py @@ -0,0 +1,162 @@ +# Auto-generated from fixtures/plugin_api/ - DO NOT EDIT +""" +E2E tests for plugin/config/utility APIs. + +Generated from plugin API fixtures. +To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import kreuzberg +from kreuzberg import ExtractionConfig + +if TYPE_CHECKING: + from pathlib import Path + + +# Configuration Tests + + +def test_config_discover(tmp_path: Path, monkeypatch) -> None: + """Discover configuration from current or parent directories""" + config_path = tmp_path / "kreuzberg.toml" + config_path.write_text("""[chunking] +max_chars = 50 +""") + + subdir = tmp_path / "subdir" + subdir.mkdir() + monkeypatch.chdir(subdir) + + config = ExtractionConfig.discover() + assert config is not None + + assert config.chunking is not None + assert config.chunking.max_chars == 50 + + +def test_config_from_file(tmp_path: Path) -> None: + """Load configuration from a TOML file""" + config_path = tmp_path / "test_config.toml" + config_path.write_text("""[chunking] +max_chars = 100 +max_overlap = 20 + +[language_detection] +enabled = false +""") + + config = ExtractionConfig.from_file(str(config_path)) + + assert config.chunking is not None + assert config.chunking.max_chars == 100 + assert config.chunking.max_overlap == 20 + assert config.language_detection is not None + assert config.language_detection.enabled is False + + +# Document Extractor Management Tests + + +def test_extractors_clear() -> None: + """Clear all document extractors and verify list is empty""" + kreuzberg.clear_document_extractors() + result = kreuzberg.list_document_extractors() + assert len(result) == 0 + + +def test_extractors_list() -> None: + """List all registered document extractors""" + result = kreuzberg.list_document_extractors() + assert isinstance(result, list) + assert all(isinstance(item, str) for item in result) + + +def test_extractors_unregister() -> None: + """Unregister nonexistent document extractor gracefully""" + kreuzberg.unregister_document_extractor("nonexistent-extractor-xyz") + + +# Mime Utilities Tests + + +def test_mime_detect_bytes() -> None: + """Detect MIME type from file bytes""" + test_bytes = b"%PDF-1.4\n" + result = kreuzberg.detect_mime_type(test_bytes) + + assert "pdf" in result.lower() + + +def test_mime_detect_path(tmp_path: Path) -> None: + """Detect MIME type from file path""" + test_file = tmp_path / "test.txt" + test_file.write_text("Hello, world!") + + result = kreuzberg.detect_mime_type_from_path(str(test_file)) + + assert "text" in result.lower() + + +def test_mime_get_extensions() -> None: + """Get file extensions for a MIME type""" + result = kreuzberg.get_extensions_for_mime("application/pdf") + assert isinstance(result, list) + assert "pdf" in result + + +# Ocr Backend Management Tests + + +def test_ocr_backends_clear() -> None: + """Clear all OCR backends and verify list is empty""" + kreuzberg.clear_ocr_backends() + result = kreuzberg.list_ocr_backends() + assert len(result) == 0 + + +def test_ocr_backends_list() -> None: + """List all registered OCR backends""" + result = kreuzberg.list_ocr_backends() + assert isinstance(result, list) + assert all(isinstance(item, str) for item in result) + + +def test_ocr_backends_unregister() -> None: + """Unregister nonexistent OCR backend gracefully""" + kreuzberg.unregister_ocr_backend("nonexistent-backend-xyz") + + +# Post Processor Management Tests + + +def test_post_processors_clear() -> None: + """Clear all post-processors and verify list is empty""" + kreuzberg.clear_post_processors() + + +def test_post_processors_list() -> None: + """List all registered post-processors""" + result = kreuzberg.list_post_processors() + assert isinstance(result, list) + assert all(isinstance(item, str) for item in result) + + +# Validator Management Tests + + +def test_validators_clear() -> None: + """Clear all validators and verify list is empty""" + kreuzberg.clear_validators() + result = kreuzberg.list_validators() + assert len(result) == 0 + + +def test_validators_list() -> None: + """List all registered validators""" + result = kreuzberg.list_validators() + assert isinstance(result, list) + assert all(isinstance(item, str) for item in result) diff --git a/tools/e2e-generator/e2e/python/tests/test_render.py b/tools/e2e-generator/e2e/python/tests/test_render.py new file mode 100644 index 00000000000..517cff43756 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_render.py @@ -0,0 +1,53 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for render fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import PdfPageIterator, render_pdf_page + +from . import helpers + + +def test_render_custom_dpi() -> None: + """Render PDF page with custom DPI""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping render_custom_dpi: missing document at {document_path}") + + png_data = render_pdf_page(document_path, 0, dpi=72) + + helpers.assert_is_png(png_data) + helpers.assert_min_byte_length(png_data, 50) + + +def test_render_iterator() -> None: + """Iterate all PDF pages as PNG images""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping render_iterator: missing document at {document_path}") + + pages = [] + with PdfPageIterator(str(document_path), dpi=150) as page_iter: + for _page_index, png_data in page_iter: + helpers.assert_is_png(png_data) + pages.append(png_data) + + assert len(pages) >= 1, f"Expected at least 1 pages, got {len(pages)}" + + +def test_render_single_page() -> None: + """Render a single PDF page to PNG""" + + document_path = helpers.resolve_document("pdf/tiny.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping render_single_page: missing document at {document_path}") + + png_data = render_pdf_page(document_path, 0, dpi=150) + + helpers.assert_is_png(png_data) + helpers.assert_min_byte_length(png_data, 100) diff --git a/tools/e2e-generator/e2e/python/tests/test_smoke.py b/tools/e2e-generator/e2e/python/tests/test_smoke.py new file mode 100644 index 00000000000..be976dd45d1 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_smoke.py @@ -0,0 +1,142 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for smoke fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_smoke_cache_namespace() -> None: + """Smoke test: Extraction with cache namespace and TTL configuration""" + + document_path = helpers.resolve_document("text/report.txt") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_cache_namespace: missing document at {document_path}") + + config = helpers.build_config({"cache_namespace": "test_tenant", "cache_ttl_secs": 3600, "use_cache": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/plain"]) + helpers.assert_min_content_length(result, 5) + + +def test_smoke_docx_basic() -> None: + """Smoke test: DOCX with formatted text""" + + document_path = helpers.resolve_document("docx/fake.docx") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_docx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_content_contains_any(result, ["Lorem", "ipsum", "document", "text"]) + + +def test_smoke_html_basic() -> None: + """Smoke test: HTML table extraction""" + + document_path = helpers.resolve_document("html/simple_table.html") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_html_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/html"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["Sample Data Table", "Laptop", "Electronics", "Product"]) + + +def test_smoke_image_png() -> None: + """Smoke test: PNG image (without OCR, metadata only)""" + + document_path = helpers.resolve_document("images/sample.png") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_image_png: missing document at {document_path}") + + config = helpers.build_config({"disable_ocr": True}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["image/png"]) + helpers.assert_metadata_expectation(result, "format", {"eq": "PNG"}) + + +def test_smoke_json_basic() -> None: + """Smoke test: JSON file extraction""" + + document_path = helpers.resolve_document("json/simple.json") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_json_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/json"]) + helpers.assert_min_content_length(result, 5) + + +def test_smoke_pdf_basic() -> None: + """Smoke test: PDF with simple text extraction""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_pdf_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 50) + helpers.assert_content_contains_any(result, ["May 5, 2023", "To Whom it May Concern"]) + + +def test_smoke_txt_basic() -> None: + """Smoke test: Plain text file""" + + document_path = helpers.resolve_document("text/report.txt") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_txt_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/plain"]) + helpers.assert_min_content_length(result, 5) + + +def test_smoke_xlsx_basic() -> None: + """Smoke test: XLSX with basic spreadsheet data including tables""" + + document_path = helpers.resolve_document("xlsx/stanley_cups.xlsx") + if not document_path.exists(): + pytest.skip(f"Skipping smoke_xlsx_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]) + helpers.assert_min_content_length(result, 100) + helpers.assert_content_contains_all( + result, ["Team", "Location", "Stanley Cups", "Blues", "Flyers", "Maple Leafs", "STL", "PHI", "TOR"] + ) + helpers.assert_table_count(result, 1, None) + helpers.assert_metadata_expectation(result, "sheet_count", {"gte": 2}) + helpers.assert_metadata_expectation(result, "sheet_names", {"contains": ["Stanley Cups"]}) diff --git a/tools/e2e-generator/e2e/python/tests/test_structured.py b/tools/e2e-generator/e2e/python/tests/test_structured.py new file mode 100644 index 00000000000..4cbf2907372 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_structured.py @@ -0,0 +1,164 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for structured fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_structured_csv_basic() -> None: + """CSV data file extraction.""" + + document_path = helpers.resolve_document("csv/stanley_cups.csv") + if not document_path.exists(): + pytest.skip(f"Skipping structured_csv_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/csv"]) + helpers.assert_min_content_length(result, 20) + + +def test_structured_enw_basic() -> None: + """EndNote ENW citation format extraction.""" + + document_path = helpers.resolve_document("data_formats/sample.enw") + if not document_path.exists(): + pytest.skip(f"Skipping structured_enw_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]) + + +def test_structured_json_basic() -> None: + """Structured JSON extraction should stream and preserve content.""" + + document_path = helpers.resolve_document("json/sample_document.json") + if not document_path.exists(): + pytest.skip(f"Skipping structured_json_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/json"]) + helpers.assert_min_content_length(result, 20) + helpers.assert_content_contains_any(result, ["Sample Document", "Test Author"]) + + +def test_structured_json_simple() -> None: + """Simple JSON document to verify structured extraction.""" + + document_path = helpers.resolve_document("json/simple.json") + if not document_path.exists(): + pytest.skip(f"Skipping structured_json_simple: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/json"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_contains_any(result, ["{", "name"]) + + +def test_structured_nbib_basic() -> None: + """PubMed NBIB citation format extraction.""" + + document_path = helpers.resolve_document("data_formats/sample.nbib") + if not document_path.exists(): + pytest.skip(f"Skipping structured_nbib_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/nbib", "application/x-pubmed", "text/plain"]) + helpers.assert_content_not_empty(result) + + +def test_structured_ris_basic() -> None: + """RIS citation format extraction.""" + + document_path = helpers.resolve_document("data_formats/sample.ris") + if not document_path.exists(): + pytest.skip(f"Skipping structured_ris_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-research-info-systems", "text/plain"]) + helpers.assert_content_not_empty(result) + + +def test_structured_toml_basic() -> None: + """TOML configuration file extraction.""" + + document_path = helpers.resolve_document("data_formats/cargo.toml") + if not document_path.exists(): + pytest.skip(f"Skipping structured_toml_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/toml", "text/toml"]) + helpers.assert_min_content_length(result, 10) + + +def test_structured_tsv_basic() -> None: + """TSV (tab-separated values) data file extraction.""" + + document_path = helpers.resolve_document("data_formats/employees.tsv") + if not document_path.exists(): + pytest.skip(f"Skipping structured_tsv_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["text/tab-separated-values", "text/plain"]) + helpers.assert_min_content_length(result, 10) + + +def test_structured_yaml_basic() -> None: + """YAML file text extraction.""" + + document_path = helpers.resolve_document("yaml/simple.yaml") + if not document_path.exists(): + pytest.skip(f"Skipping structured_yaml_basic: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]) + helpers.assert_min_content_length(result, 10) + + +def test_structured_yaml_simple() -> None: + """Simple YAML document to validate structured extraction.""" + + document_path = helpers.resolve_document("yaml/simple.yaml") + if not document_path.exists(): + pytest.skip(f"Skipping structured_yaml_simple: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/x-yaml"]) + helpers.assert_min_content_length(result, 10) diff --git a/tools/e2e-generator/e2e/python/tests/test_token_reduction.py b/tools/e2e-generator/e2e/python/tests/test_token_reduction.py new file mode 100644 index 00000000000..73e3ed5168c --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_token_reduction.py @@ -0,0 +1,83 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for token_reduction fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_token_reduction_aggressive() -> None: + """Tests aggressive token reduction mode significantly reduces content""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping token_reduction_aggressive: missing document at {document_path}") + + config = helpers.build_config({"token_reduction": {"mode": "aggressive"}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_max_content_length(result, 150) + helpers.assert_content_not_empty(result) + + +def test_token_reduction_basic() -> None: + """Tests basic token reduction on PDF document""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping token_reduction_basic: missing document at {document_path}") + + config = helpers.build_config({"token_reduction": {"mode": "moderate"}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_max_content_length(result, 200) + helpers.assert_content_not_empty(result) + + +def test_token_reduction_light() -> None: + """Tests light token reduction mode preserves most content""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping token_reduction_light: missing document at {document_path}") + + config = helpers.build_config({"token_reduction": {"mode": "light"}}) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 10) + helpers.assert_content_not_empty(result) + + +def test_token_reduction_with_chunking() -> None: + """Tests token reduction combined with chunking""" + + document_path = helpers.resolve_document("pdf/fake_memo.pdf") + if not document_path.exists(): + pytest.skip(f"Skipping token_reduction_with_chunking: missing document at {document_path}") + + config = helpers.build_config( + {"token_reduction": {"mode": "moderate"}, "chunking": {"max_chars": 500, "max_overlap": 50}} + ) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/pdf"]) + helpers.assert_min_content_length(result, 5) + helpers.assert_max_content_length(result, 200) + helpers.assert_chunks(result, min_count=1, each_has_content=True) + helpers.assert_content_not_empty(result) diff --git a/tools/e2e-generator/e2e/python/tests/test_xml.py b/tools/e2e-generator/e2e/python/tests/test_xml.py new file mode 100644 index 00000000000..6b08fd44216 --- /dev/null +++ b/tools/e2e-generator/e2e/python/tests/test_xml.py @@ -0,0 +1,28 @@ +# Code generated by kreuzberg-e2e-generator. DO NOT EDIT. +# To regenerate: cargo run -p kreuzberg-e2e-generator -- generate --lang python +# +# Tests for xml fixtures. +from __future__ import annotations + +import pytest + +from kreuzberg import ( + extract_file_sync, +) + +from . import helpers + + +def test_xml_plant_catalog() -> None: + """XML plant catalog to validate streaming XML extraction.""" + + document_path = helpers.resolve_document("xml/plant_catalog.xml") + if not document_path.exists(): + pytest.skip(f"Skipping xml_plant_catalog: missing document at {document_path}") + + config = helpers.build_config(None) + + result = extract_file_sync(document_path, None, config) + + helpers.assert_expected_mime(result, ["application/xml"]) + helpers.assert_min_content_length(result, 100) diff --git a/tools/e2e-generator/src/manifest.rs b/tools/e2e-generator/src/manifest.rs index d4febe3f607..bedc2e03126 100644 --- a/tools/e2e-generator/src/manifest.rs +++ b/tools/e2e-generator/src/manifest.rs @@ -256,6 +256,7 @@ fn sample_extraction_result() -> Value { let chunk = kreuzberg::types::extraction::Chunk { content: "chunk text".to_string(), + chunk_type: Default::default(), embedding: Some(vec![0.1, 0.2]), metadata: kreuzberg::types::extraction::ChunkMetadata { byte_start: 0, From 9cf7f848d2bd4e21c9addd488bd9ba71a6d9d5dc Mon Sep 17 00:00:00 2001 From: kh3rld Date: Thu, 2 Apr 2026 04:15:45 -0400 Subject: [PATCH 12/12] style: apply biome formatting to TypeScript e2e files --- e2e/typescript/tests/archive.spec.ts | 193 +- e2e/typescript/tests/email.spec.ts | 289 ++- e2e/typescript/tests/helpers.ts | 5 +- e2e/typescript/tests/html.spec.ts | 107 +- e2e/typescript/tests/image.spec.ts | 481 ++-- e2e/typescript/tests/keywords.spec.ts | 101 +- e2e/typescript/tests/office.spec.ts | 2530 +++++++++++--------- e2e/typescript/tests/plugin-apis.spec.ts | 223 +- e2e/typescript/tests/render.spec.ts | 57 +- e2e/typescript/tests/structured.spec.ts | 483 ++-- e2e/typescript/tests/xml.spec.ts | 49 +- e2e/wasm-deno/archive.test.ts | 146 +- e2e/wasm-deno/email.test.ts | 214 +- e2e/wasm-deno/embeddings.test.ts | 46 +- e2e/wasm-deno/html.test.ts | 54 +- e2e/wasm-deno/image.test.ts | 350 +-- e2e/wasm-deno/office.test.ts | 1820 +++++++------- e2e/wasm-deno/plugin-apis.test.ts | 68 +- e2e/wasm-deno/structured.test.ts | 352 +-- e2e/wasm-deno/xml.test.ts | 44 +- e2e/wasm-workers/tests/archive.spec.ts | 177 +- e2e/wasm-workers/tests/email.spec.ts | 265 +- e2e/wasm-workers/tests/embeddings.spec.ts | 47 +- e2e/wasm-workers/tests/html.spec.ts | 99 +- e2e/wasm-workers/tests/image.spec.ts | 479 ++-- e2e/wasm-workers/tests/plugin-apis.spec.ts | 102 +- e2e/wasm-workers/tests/structured.spec.ts | 481 ++-- e2e/wasm-workers/tests/xml.spec.ts | 45 +- 28 files changed, 4928 insertions(+), 4379 deletions(-) diff --git a/e2e/typescript/tests/archive.spec.ts b/e2e/typescript/tests/archive.spec.ts index d2684c40a4e..e8aee483ea5 100644 --- a/e2e/typescript/tests/archive.spec.ts +++ b/e2e/typescript/tests/archive.spec.ts @@ -12,96 +12,111 @@ import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("archive fixtures", () => { - it("archive_gz_basic", () => { - const documentPath = resolveDocument("archives/book_war_and_peace_1p.txt.gz"); - if (!existsSync(documentPath)) { - console.warn("Skipping archive_gz_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "archive_gz_basic", + () => { + const documentPath = resolveDocument("archives/book_war_and_peace_1p.txt.gz"); + if (!existsSync(documentPath)) { + console.warn("Skipping archive_gz_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("archive_sevenz_basic", () => { - const documentPath = resolveDocument("archives/documents.7z"); - if (!existsSync(documentPath)) { - console.warn("Skipping archive_sevenz_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "archive_sevenz_basic", + () => { + const documentPath = resolveDocument("archives/documents.7z"); + if (!existsSync(documentPath)) { + console.warn("Skipping archive_sevenz_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("archive_tar_basic", () => { - const documentPath = resolveDocument("archives/documents.tar"); - if (!existsSync(documentPath)) { - console.warn("Skipping archive_tar_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); - - it("archive_zip_basic", () => { - const documentPath = resolveDocument("archives/documents.zip"); - if (!existsSync(documentPath)) { - console.warn("Skipping archive_zip_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "archive_tar_basic", + () => { + const documentPath = resolveDocument("archives/documents.tar"); + if (!existsSync(documentPath)) { + console.warn("Skipping archive_tar_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); + it( + "archive_zip_basic", + () => { + const documentPath = resolveDocument("archives/documents.zip"); + if (!existsSync(documentPath)) { + console.warn("Skipping archive_zip_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/e2e/typescript/tests/email.spec.ts b/e2e/typescript/tests/email.spec.ts index 448a9ae1db2..0858980f436 100644 --- a/e2e/typescript/tests/email.spec.ts +++ b/e2e/typescript/tests/email.spec.ts @@ -12,142 +12,165 @@ import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("email fixtures", () => { - it("email_eml_html_body", () => { - const documentPath = resolveDocument("email/html_only.eml"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_eml_html_body: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "email_eml_html_body", + () => { + const documentPath = resolveDocument("email/html_only.eml"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_eml_html_body: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("email_eml_multipart", () => { - const documentPath = resolveDocument("email/html_email_multipart.eml"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_eml_multipart: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "email_eml_multipart", + () => { + const documentPath = resolveDocument("email/html_email_multipart.eml"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_eml_multipart: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("email_eml_utf16", () => { - const documentPath = resolveDocument("vendored/unstructured/eml/fake-email-utf-16.eml"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_eml_utf16: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); - }, TEST_TIMEOUT_MS); + it( + "email_eml_utf16", + () => { + const documentPath = resolveDocument("vendored/unstructured/eml/fake-email-utf-16.eml"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_eml_utf16: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); + }, + TEST_TIMEOUT_MS, + ); - it("email_msg_basic", () => { - const documentPath = resolveDocument("email/fake_email.msg"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_msg_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "email_msg_basic", + () => { + const documentPath = resolveDocument("email/fake_email.msg"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_msg_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("email_pst_empty", () => { - const documentPath = resolveDocument("email/empty.pst"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_pst_empty: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); - }, TEST_TIMEOUT_MS); - - it("email_sample_eml", () => { - const documentPath = resolveDocument("email/sample_email.eml"); - if (!existsSync(documentPath)) { - console.warn("Skipping email_sample_eml: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "email_pst_empty", + () => { + const documentPath = resolveDocument("email/empty.pst"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_pst_empty: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); + }, + TEST_TIMEOUT_MS, + ); + it( + "email_sample_eml", + () => { + const documentPath = resolveDocument("email/sample_email.eml"); + if (!existsSync(documentPath)) { + console.warn("Skipping email_sample_eml: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/e2e/typescript/tests/helpers.ts b/e2e/typescript/tests/helpers.ts index fae2c145c4a..05bb3585196 100644 --- a/e2e/typescript/tests/helpers.ts +++ b/e2e/typescript/tests/helpers.ts @@ -415,9 +415,8 @@ export function buildConfig(raw: unknown): ExtractionConfig { if (isPlainRecord(source.email)) { const email = source.email as PlainRecord; - target.email = { - ...(typeof email.msg_fallback_codepage === "number" ? { msgFallbackCodepage: email.msg_fallback_codepage } : {}), - }; + target.email = + typeof email.msg_fallback_codepage === "number" ? { msgFallbackCodepage: email.msg_fallback_codepage } : {}; } if (isPlainRecord(source.tree_sitter)) { diff --git a/e2e/typescript/tests/html.spec.ts b/e2e/typescript/tests/html.spec.ts index 862a47bce02..7f7b177f00f 100644 --- a/e2e/typescript/tests/html.spec.ts +++ b/e2e/typescript/tests/html.spec.ts @@ -12,51 +12,66 @@ import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("html fixtures", () => { - it("html_complex_layout", () => { - const documentPath = resolveDocument("html/taylor_swift.html"); - if (!existsSync(documentPath)) { - console.warn("Skipping html_complex_layout: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "html_complex_layout", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 1000); - }, TEST_TIMEOUT_MS); - - it("html_simple_table", () => { - const documentPath = resolveDocument("html/simple_table.html"); - if (!existsSync(documentPath)) { - console.warn("Skipping html_simple_table: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]); - }, TEST_TIMEOUT_MS); + it( + "html_complex_layout", + () => { + const documentPath = resolveDocument("html/taylor_swift.html"); + if (!existsSync(documentPath)) { + console.warn("Skipping html_complex_layout: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "html_complex_layout", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 1000); + }, + TEST_TIMEOUT_MS, + ); + it( + "html_simple_table", + () => { + const documentPath = resolveDocument("html/simple_table.html"); + if (!existsSync(documentPath)) { + console.warn("Skipping html_simple_table: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, [ + "Product", + "Category", + "Price", + "Stock", + "Laptop", + "Electronics", + "Sample Data Table", + ]); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/e2e/typescript/tests/image.spec.ts b/e2e/typescript/tests/image.spec.ts index 944bb505902..d7ea275a8b5 100644 --- a/e2e/typescript/tests/image.spec.ts +++ b/e2e/typescript/tests/image.spec.ts @@ -12,234 +12,273 @@ import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("image fixtures", () => { - it("image_bmp_basic", () => { - const documentPath = resolveDocument("images/bmp_24.bmp"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_bmp_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/bmp"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "image_bmp_basic", + () => { + const documentPath = resolveDocument("images/bmp_24.bmp"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_bmp_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/bmp"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("image_gif_basic", () => { - const documentPath = resolveDocument("images_extra/ocr_image.gif"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_gif_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/gif"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "image_gif_basic", + () => { + const documentPath = resolveDocument("images_extra/ocr_image.gif"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_gif_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/gif"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("image_jp2_basic", () => { - const documentPath = resolveDocument("images_extra/ocr_image.jp2"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_jp2_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "image_jp2_basic", + () => { + const documentPath = resolveDocument("images_extra/ocr_image.jp2"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_jp2_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("image_metadata_only", () => { - const documentPath = resolveDocument("images/example.jpg"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_metadata_only: missing document at", documentPath); - return; - } - const config = buildConfig({"ocr":null}); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jpeg"]); - assertions.assertMaxContentLength(result, 200); - }, TEST_TIMEOUT_MS); + it( + "image_metadata_only", + () => { + const documentPath = resolveDocument("images/example.jpg"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_metadata_only: missing document at", documentPath); + return; + } + const config = buildConfig({ ocr: null }); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jpeg"]); + assertions.assertMaxContentLength(result, 200); + }, + TEST_TIMEOUT_MS, + ); - it("image_pbm_basic", () => { - const documentPath = resolveDocument("images_extra/ocr_image.pbm"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_pbm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "image_pbm_basic", + () => { + const documentPath = resolveDocument("images_extra/ocr_image.pbm"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_pbm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("image_pgm_basic", () => { - const documentPath = resolveDocument("images_extra/ocr_image.pgm"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_pgm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "image_pgm_basic", + () => { + const documentPath = resolveDocument("images_extra/ocr_image.pgm"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_pgm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("image_ppm_basic", () => { - const documentPath = resolveDocument("images_extra/ocr_image.ppm"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_ppm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "image_ppm_basic", + () => { + const documentPath = resolveDocument("images_extra/ocr_image.ppm"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_ppm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("image_svg_basic", () => { - const documentPath = resolveDocument("xml/simple_svg.svg"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_svg_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/svg+xml"]); - assertions.assertMinContentLength(result, 5); - }, TEST_TIMEOUT_MS); + it( + "image_svg_basic", + () => { + const documentPath = resolveDocument("xml/simple_svg.svg"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_svg_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/svg+xml"]); + assertions.assertMinContentLength(result, 5); + }, + TEST_TIMEOUT_MS, + ); - it("image_tiff_basic", () => { - const documentPath = resolveDocument("images_extra/ocr_image.tif"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_tiff_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/tiff"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); - - it("image_webp_basic", () => { - const documentPath = resolveDocument("images_extra/ocr_image.webp"); - if (!existsSync(documentPath)) { - console.warn("Skipping image_webp_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/webp"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "image_tiff_basic", + () => { + const documentPath = resolveDocument("images_extra/ocr_image.tif"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_tiff_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/tiff"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); + it( + "image_webp_basic", + () => { + const documentPath = resolveDocument("images_extra/ocr_image.webp"); + if (!existsSync(documentPath)) { + console.warn("Skipping image_webp_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/webp"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/e2e/typescript/tests/keywords.spec.ts b/e2e/typescript/tests/keywords.spec.ts index eb170615191..6797169a9fc 100644 --- a/e2e/typescript/tests/keywords.spec.ts +++ b/e2e/typescript/tests/keywords.spec.ts @@ -12,52 +12,59 @@ import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("keywords fixtures", () => { - it("keywords_rake", () => { - const documentPath = resolveDocument("pdf/fake_memo.pdf"); - if (!existsSync(documentPath)) { - console.warn("Skipping keywords_rake: missing document at", documentPath); - return; - } - const config = buildConfig({"keywords":{"algorithm":"rake","max_keywords":10}}); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "keywords_rake", ["keywords-rake"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/pdf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertKeywords(result, true, 1, 10); - }, TEST_TIMEOUT_MS); - - it("keywords_yake", () => { - const documentPath = resolveDocument("pdf/fake_memo.pdf"); - if (!existsSync(documentPath)) { - console.warn("Skipping keywords_yake: missing document at", documentPath); - return; - } - const config = buildConfig({"keywords":{"algorithm":"yake","max_keywords":10}}); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "keywords_yake", ["keywords-yake"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/pdf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertKeywords(result, true, 1, 10); - }, TEST_TIMEOUT_MS); + it( + "keywords_rake", + () => { + const documentPath = resolveDocument("pdf/fake_memo.pdf"); + if (!existsSync(documentPath)) { + console.warn("Skipping keywords_rake: missing document at", documentPath); + return; + } + const config = buildConfig({ keywords: { algorithm: "rake", max_keywords: 10 } }); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "keywords_rake", ["keywords-rake"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/pdf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertKeywords(result, true, 1, 10); + }, + TEST_TIMEOUT_MS, + ); + it( + "keywords_yake", + () => { + const documentPath = resolveDocument("pdf/fake_memo.pdf"); + if (!existsSync(documentPath)) { + console.warn("Skipping keywords_yake: missing document at", documentPath); + return; + } + const config = buildConfig({ keywords: { algorithm: "yake", max_keywords: 10 } }); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "keywords_yake", ["keywords-yake"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/pdf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertKeywords(result, true, 1, 10); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/e2e/typescript/tests/office.spec.ts b/e2e/typescript/tests/office.spec.ts index 6d93eeec5e0..a15c3d307ac 100644 --- a/e2e/typescript/tests/office.spec.ts +++ b/e2e/typescript/tests/office.spec.ts @@ -12,1196 +12,1442 @@ import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("office fixtures", () => { - it("office_bibtex_basic", () => { - const documentPath = resolveDocument("bibtex/comprehensive.bib"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_bibtex_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_bibtex_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-bibtex", "text/x-bibtex"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_bibtex_basic", + () => { + const documentPath = resolveDocument("bibtex/comprehensive.bib"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_bibtex_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_bibtex_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-bibtex", "text/x-bibtex"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_commonmark_basic", () => { - const documentPath = resolveDocument("markdown/sample.commonmark"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_commonmark_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_commonmark_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/markdown", "text/plain", "text/x-commonmark"]); - assertions.assertMinContentLength(result, 5); - }, TEST_TIMEOUT_MS); + it( + "office_commonmark_basic", + () => { + const documentPath = resolveDocument("markdown/sample.commonmark"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_commonmark_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_commonmark_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/markdown", "text/plain", "text/x-commonmark"]); + assertions.assertMinContentLength(result, 5); + }, + TEST_TIMEOUT_MS, + ); - it("office_dbf_basic", () => { - const documentPath = resolveDocument("dbf/stations.dbf"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_dbf_basic: missing document at", documentPath); - console.warn("Notes: Requires the office feature."); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_dbf_basic", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-dbf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["|"]); - }, TEST_TIMEOUT_MS); + it( + "office_dbf_basic", + () => { + const documentPath = resolveDocument("dbf/stations.dbf"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_dbf_basic: missing document at", documentPath); + console.warn("Notes: Requires the office feature."); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_dbf_basic", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-dbf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["|"]); + }, + TEST_TIMEOUT_MS, + ); - it("office_djot_basic", () => { - const documentPath = resolveDocument("markdown/tables.djot"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_djot_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_djot_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-djot", "text/djot"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_djot_basic", + () => { + const documentPath = resolveDocument("markdown/tables.djot"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_djot_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_djot_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-djot", "text/djot"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_doc_legacy", () => { - const documentPath = resolveDocument("doc/unit_test_lists.doc"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_doc_legacy: missing document at", documentPath); - console.warn("Notes: Requires the office feature."); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_doc_legacy", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/msword"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_doc_legacy", + () => { + const documentPath = resolveDocument("doc/unit_test_lists.doc"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_doc_legacy: missing document at", documentPath); + console.warn("Notes: Requires the office feature."); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_doc_legacy", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/msword"]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_docbook_basic", () => { - const documentPath = resolveDocument("docbook/docbook-reader.docbook"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docbook_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docbook_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/docbook+xml", "text/docbook"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_docbook_basic", + () => { + const documentPath = resolveDocument("docbook/docbook-reader.docbook"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docbook_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docbook_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/docbook+xml", "text/docbook"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_docx_basic", () => { - const documentPath = resolveDocument("docx/sample_document.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_docx_basic", + () => { + const documentPath = resolveDocument("docx/sample_document.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_docx_equations", () => { - const documentPath = resolveDocument("docx/equations.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_equations: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_equations", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_docx_equations", + () => { + const documentPath = resolveDocument("docx/equations.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_equations: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_equations", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_docx_fake", () => { - const documentPath = resolveDocument("docx/fake.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_fake: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_fake", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_docx_fake", + () => { + const documentPath = resolveDocument("docx/fake.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_fake: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_fake", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_docx_formatting", () => { - const documentPath = resolveDocument("docx/unit_test_formatting.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_formatting: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_formatting", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_docx_formatting", + () => { + const documentPath = resolveDocument("docx/unit_test_formatting.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_formatting: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_formatting", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_docx_headers", () => { - const documentPath = resolveDocument("docx/unit_test_headers.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_headers: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_headers", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_docx_headers", + () => { + const documentPath = resolveDocument("docx/unit_test_headers.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_headers: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_headers", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_docx_lists", () => { - const documentPath = resolveDocument("docx/unit_test_lists.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_lists: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_lists", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_docx_lists", + () => { + const documentPath = resolveDocument("docx/unit_test_lists.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_lists: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_lists", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_docx_tables", () => { - const documentPath = resolveDocument("docx/docx_tables.docx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_docx_tables: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_tables", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAll(result, ["Simple uniform table", "Nested Table", "merged cells", "Header Col"]); - assertions.assertTableCount(result, 1, null); - }, TEST_TIMEOUT_MS); + it( + "office_docx_tables", + () => { + const documentPath = resolveDocument("docx/docx_tables.docx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_docx_tables: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_tables", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAll(result, [ + "Simple uniform table", + "Nested Table", + "merged cells", + "Header Col", + ]); + assertions.assertTableCount(result, 1, null); + }, + TEST_TIMEOUT_MS, + ); - it("office_epub_basic", () => { - const documentPath = resolveDocument("epub/features.epub"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_epub_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_epub_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/epub+zip"]); - assertions.assertMinContentLength(result, 50); - }, TEST_TIMEOUT_MS); + it( + "office_epub_basic", + () => { + const documentPath = resolveDocument("epub/features.epub"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_epub_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_epub_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/epub+zip"]); + assertions.assertMinContentLength(result, 50); + }, + TEST_TIMEOUT_MS, + ); - it("office_fb2_basic", () => { - const documentPath = resolveDocument("fictionbook/basic.fb2"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_fb2_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_fb2_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-fictionbook+xml"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_fb2_basic", + () => { + const documentPath = resolveDocument("fictionbook/basic.fb2"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_fb2_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_fb2_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-fictionbook+xml"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_fictionbook_basic", () => { - const documentPath = resolveDocument("fictionbook/basic.fb2"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_fictionbook_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_fictionbook_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_fictionbook_basic", + () => { + const documentPath = resolveDocument("fictionbook/basic.fb2"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_fictionbook_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_fictionbook_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_hwp_basic", () => { - const documentPath = resolveDocument("hwp/converted_output.hwp"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_hwp_basic: missing document at", documentPath); - console.warn("Notes: Requires the office feature."); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_hwp_basic", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-hwp"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_hwp_basic", + () => { + const documentPath = resolveDocument("hwp/converted_output.hwp"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_hwp_basic: missing document at", documentPath); + console.warn("Notes: Requires the office feature."); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_hwp_basic", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-hwp"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_hwp_styled", () => { - const documentPath = resolveDocument("hwp/styled_document.hwp"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_hwp_styled: missing document at", documentPath); - console.warn("Notes: HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux."); - return; - } - if ((process.arch === "arm64" && process.platform === "linux")) { - console.warn("Skipping office_hwp_styled: not supported on this platform"); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_hwp_styled", ["hwp"], "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-hwp"]); - }, TEST_TIMEOUT_MS); + it( + "office_hwp_styled", + () => { + const documentPath = resolveDocument("hwp/styled_document.hwp"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_hwp_styled: missing document at", documentPath); + console.warn( + "Notes: HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", + ); + return; + } + if (process.arch === "arm64" && process.platform === "linux") { + console.warn("Skipping office_hwp_styled: not supported on this platform"); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if ( + shouldSkipFixture( + error, + "office_hwp_styled", + ["hwp"], + "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", + ) + ) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-hwp"]); + }, + TEST_TIMEOUT_MS, + ); - it("office_jats_basic", () => { - const documentPath = resolveDocument("jats/sample_article.jats"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_jats_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_jats_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-jats+xml", "text/jats"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_jats_basic", + () => { + const documentPath = resolveDocument("jats/sample_article.jats"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_jats_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_jats_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-jats+xml", "text/jats"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_jupyter_basic", () => { - const documentPath = resolveDocument("jupyter/rank.ipynb"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_jupyter_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_jupyter_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-ipynb+json"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_jupyter_basic", + () => { + const documentPath = resolveDocument("jupyter/rank.ipynb"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_jupyter_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_jupyter_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-ipynb+json"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_keynote_basic", () => { - const documentPath = resolveDocument("iwork/test.key"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_keynote_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_keynote_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-keynote-sffkey"]); - assertions.assertMinContentLength(result, 5); - }, TEST_TIMEOUT_MS); + it( + "office_keynote_basic", + () => { + const documentPath = resolveDocument("iwork/test.key"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_keynote_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_keynote_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-keynote-sffkey"]); + assertions.assertMinContentLength(result, 5); + }, + TEST_TIMEOUT_MS, + ); - it("office_latex_basic", () => { - const documentPath = resolveDocument("latex/basic_sections.tex"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_latex_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_latex_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-latex", "text/x-latex"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_latex_basic", + () => { + const documentPath = resolveDocument("latex/basic_sections.tex"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_latex_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_latex_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-latex", "text/x-latex"]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_markdown_basic", () => { - const documentPath = resolveDocument("markdown/comprehensive.md"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_markdown_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_markdown_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/markdown"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_markdown_basic", + () => { + const documentPath = resolveDocument("markdown/comprehensive.md"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_markdown_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_markdown_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/markdown"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_mdx_basic", () => { - const documentPath = resolveDocument("markdown/sample.mdx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_mdx_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 50); - }, TEST_TIMEOUT_MS); + it( + "office_mdx_basic", + () => { + const documentPath = resolveDocument("markdown/sample.mdx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_mdx_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 50); + }, + TEST_TIMEOUT_MS, + ); - it("office_mdx_getting_started", () => { - const documentPath = resolveDocument("markdown/mdx_getting_started.mdx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_mdx_getting_started: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_getting_started", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); - }, TEST_TIMEOUT_MS); + it( + "office_mdx_getting_started", + () => { + const documentPath = resolveDocument("markdown/mdx_getting_started.mdx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_mdx_getting_started: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_getting_started", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); + }, + TEST_TIMEOUT_MS, + ); - it("office_mdx_troubleshooting", () => { - const documentPath = resolveDocument("markdown/mdx_troubleshooting.mdx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_mdx_troubleshooting: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_troubleshooting", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); - }, TEST_TIMEOUT_MS); + it( + "office_mdx_troubleshooting", + () => { + const documentPath = resolveDocument("markdown/mdx_troubleshooting.mdx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_mdx_troubleshooting: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_troubleshooting", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); + }, + TEST_TIMEOUT_MS, + ); - it("office_mdx_using_mdx", () => { - const documentPath = resolveDocument("markdown/mdx_using_mdx.mdx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_mdx_using_mdx: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_using_mdx", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); - }, TEST_TIMEOUT_MS); + it( + "office_mdx_using_mdx", + () => { + const documentPath = resolveDocument("markdown/mdx_using_mdx.mdx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_mdx_using_mdx: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_using_mdx", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); + }, + TEST_TIMEOUT_MS, + ); - it("office_numbers_basic", () => { - const documentPath = resolveDocument("iwork/test.numbers"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_numbers_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_numbers_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-numbers-sffnumbers"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_numbers_basic", + () => { + const documentPath = resolveDocument("iwork/test.numbers"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_numbers_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_numbers_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-numbers-sffnumbers"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_ods_basic", () => { - const documentPath = resolveDocument("data_formats/test_01.ods"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_ods_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_ods_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.spreadsheet"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_ods_basic", + () => { + const documentPath = resolveDocument("data_formats/test_01.ods"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_ods_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_ods_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.spreadsheet"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_odt_bold", () => { - const documentPath = resolveDocument("odt/bold.odt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_odt_bold: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_bold", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_odt_bold", + () => { + const documentPath = resolveDocument("odt/bold.odt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_odt_bold: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_bold", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_odt_list", () => { - const documentPath = resolveDocument("odt/unorderedList.odt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_odt_list: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_list", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 30); - assertions.assertContentContainsAny(result, ["list item", "New level", "Pushed us"]); - }, TEST_TIMEOUT_MS); + it( + "office_odt_list", + () => { + const documentPath = resolveDocument("odt/unorderedList.odt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_odt_list: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_list", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 30); + assertions.assertContentContainsAny(result, ["list item", "New level", "Pushed us"]); + }, + TEST_TIMEOUT_MS, + ); - it("office_odt_simple", () => { - const documentPath = resolveDocument("odt/simple.odt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_odt_simple: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["favorite things", "Parrots", "Analysis"]); - }, TEST_TIMEOUT_MS); + it( + "office_odt_simple", + () => { + const documentPath = resolveDocument("odt/simple.odt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_odt_simple: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["favorite things", "Parrots", "Analysis"]); + }, + TEST_TIMEOUT_MS, + ); - it("office_odt_table", () => { - const documentPath = resolveDocument("odt/table.odt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_odt_table: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 10); - assertions.assertTableCount(result, 1, null); - }, TEST_TIMEOUT_MS); + it( + "office_odt_table", + () => { + const documentPath = resolveDocument("odt/table.odt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_odt_table: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 10); + assertions.assertTableCount(result, 1, null); + }, + TEST_TIMEOUT_MS, + ); - it("office_opml_basic", () => { - const documentPath = resolveDocument("opml/outline.opml"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_opml_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_opml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_opml_basic", + () => { + const documentPath = resolveDocument("opml/outline.opml"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_opml_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_opml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_org_basic", () => { - const documentPath = resolveDocument("org/comprehensive.org"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_org_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_org_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-org", "text/org"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_org_basic", + () => { + const documentPath = resolveDocument("org/comprehensive.org"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_org_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_org_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-org", "text/org"]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_pages_basic", () => { - const documentPath = resolveDocument("iwork/test.pages"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pages_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pages_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-pages-sffpages"]); - assertions.assertMinContentLength(result, 5); - }, TEST_TIMEOUT_MS); + it( + "office_pages_basic", + () => { + const documentPath = resolveDocument("iwork/test.pages"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pages_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pages_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-pages-sffpages"]); + assertions.assertMinContentLength(result, 5); + }, + TEST_TIMEOUT_MS, + ); - it("office_ppsx_slideshow", () => { - const documentPath = resolveDocument("pptx/sample.ppsx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_ppsx_slideshow: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_ppsx_slideshow", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_ppsx_slideshow", + () => { + const documentPath = resolveDocument("pptx/sample.ppsx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_ppsx_slideshow: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_ppsx_slideshow", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_ppt_legacy", () => { - const documentPath = resolveDocument("ppt/simple.ppt"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_ppt_legacy: missing document at", documentPath); - console.warn("Notes: Requires the office feature."); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_ppt_legacy", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_ppt_legacy", + () => { + const documentPath = resolveDocument("ppt/simple.ppt"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_ppt_legacy: missing document at", documentPath); + console.warn("Notes: Requires the office feature."); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_ppt_legacy", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_pptm_basic", () => { - const documentPath = resolveDocument("pptx/powerpoint_with_image.pptm"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pptm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptm_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "office_pptm_basic", + () => { + const documentPath = resolveDocument("pptx/powerpoint_with_image.pptm"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pptm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptm_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("office_pptx_basic", () => { - const documentPath = resolveDocument("pptx/simple.pptx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pptx_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 50); - }, TEST_TIMEOUT_MS); + it( + "office_pptx_basic", + () => { + const documentPath = resolveDocument("pptx/simple.pptx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pptx_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ]); + assertions.assertMinContentLength(result, 50); + }, + TEST_TIMEOUT_MS, + ); - it("office_pptx_images", () => { - const documentPath = resolveDocument("pptx/powerpoint_with_image.pptx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pptx_images: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_images", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 15); - }, TEST_TIMEOUT_MS); + it( + "office_pptx_images", + () => { + const documentPath = resolveDocument("pptx/powerpoint_with_image.pptx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pptx_images: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_images", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ]); + assertions.assertMinContentLength(result, 15); + }, + TEST_TIMEOUT_MS, + ); - it("office_pptx_pitch_deck", () => { - const documentPath = resolveDocument("pptx/pitch_deck_presentation.pptx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_pptx_pitch_deck: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_pitch_deck", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 100); - }, TEST_TIMEOUT_MS); + it( + "office_pptx_pitch_deck", + () => { + const documentPath = resolveDocument("pptx/pitch_deck_presentation.pptx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_pptx_pitch_deck: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_pitch_deck", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ]); + assertions.assertMinContentLength(result, 100); + }, + TEST_TIMEOUT_MS, + ); - it("office_rst_basic", () => { - const documentPath = resolveDocument("rst/restructured_text.rst"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_rst_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_rst_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-rst", "text/prs.fallenstein.rst"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "office_rst_basic", + () => { + const documentPath = resolveDocument("rst/restructured_text.rst"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_rst_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_rst_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-rst", "text/prs.fallenstein.rst"]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("office_rtf_basic", () => { - const documentPath = resolveDocument("rtf/extraction_test.rtf"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_rtf_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_rtf_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/rtf", "text/rtf"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_rtf_basic", + () => { + const documentPath = resolveDocument("rtf/extraction_test.rtf"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_rtf_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_rtf_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/rtf", "text/rtf"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_typst_basic", () => { - const documentPath = resolveDocument("typst/headings.typ"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_typst_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_typst_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-typst", "text/x-typst"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_typst_basic", + () => { + const documentPath = resolveDocument("typst/headings.typ"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_typst_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_typst_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-typst", "text/x-typst"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_xls_legacy", () => { - const documentPath = resolveDocument("xls/test_excel.xls"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xls_legacy: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xls_legacy", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-excel"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_xls_legacy", + () => { + const documentPath = resolveDocument("xls/test_excel.xls"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xls_legacy: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xls_legacy", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-excel"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("office_xlsb_basic", () => { - const documentPath = resolveDocument("xlsx/test_xlsb.xlsb"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsb_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsb_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "office_xlsb_basic", + () => { + const documentPath = resolveDocument("xlsx/test_xlsb.xlsb"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsb_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsb_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("office_xlsm_basic", () => { - const documentPath = resolveDocument("xlsx/test_01.xlsm"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsm_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsm_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "office_xlsm_basic", + () => { + const documentPath = resolveDocument("xlsx/test_01.xlsm"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsm_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsm_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("office_xlsx_basic", () => { - const documentPath = resolveDocument("xlsx/stanley_cups.xlsx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsx_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, ["Team", "Location", "Stanley Cups"]); - assertions.assertTableCount(result, 1, null); - assertions.assertMetadataExpectation(result, "sheetCount", {"gte":2}); - assertions.assertMetadataExpectation(result, "sheetNames", {"contains":["Stanley Cups"]}); - }, TEST_TIMEOUT_MS); + it( + "office_xlsx_basic", + () => { + const documentPath = resolveDocument("xlsx/stanley_cups.xlsx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsx_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, ["Team", "Location", "Stanley Cups"]); + assertions.assertTableCount(result, 1, null); + assertions.assertMetadataExpectation(result, "sheetCount", { gte: 2 }); + assertions.assertMetadataExpectation(result, "sheetNames", { contains: ["Stanley Cups"] }); + }, + TEST_TIMEOUT_MS, + ); - it("office_xlsx_multi_sheet", () => { - const documentPath = resolveDocument("xlsx/excel_multi_sheet.xlsx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsx_multi_sheet: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_multi_sheet", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 20); - assertions.assertMetadataExpectation(result, "sheetCount", {"gte":2}); - }, TEST_TIMEOUT_MS); - - it("office_xlsx_office_example", () => { - const documentPath = resolveDocument("xlsx/test_01.xlsx"); - if (!existsSync(documentPath)) { - console.warn("Skipping office_xlsx_office_example: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_office_example", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "office_xlsx_multi_sheet", + () => { + const documentPath = resolveDocument("xlsx/excel_multi_sheet.xlsx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsx_multi_sheet: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_multi_sheet", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 20); + assertions.assertMetadataExpectation(result, "sheetCount", { gte: 2 }); + }, + TEST_TIMEOUT_MS, + ); + it( + "office_xlsx_office_example", + () => { + const documentPath = resolveDocument("xlsx/test_01.xlsx"); + if (!existsSync(documentPath)) { + console.warn("Skipping office_xlsx_office_example: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_office_example", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/e2e/typescript/tests/plugin-apis.spec.ts b/e2e/typescript/tests/plugin-apis.spec.ts index 80692ff82ee..86552eb9c7f 100644 --- a/e2e/typescript/tests/plugin-apis.spec.ts +++ b/e2e/typescript/tests/plugin-apis.spec.ts @@ -13,132 +13,129 @@ import { describe, expect, it } from "vitest"; import * as kreuzberg from "@kreuzberg/node"; describe("Configuration", () => { - it("Discover configuration from current or parent directories", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); - const configPath = path.join(tmpDir, "kreuzberg.toml"); - fs.writeFileSync(configPath, "[chunking]\nmax_chars = 50\n"); - - const subDir = path.join(tmpDir, "subdir"); - fs.mkdirSync(subDir); - - const originalCwd = process.cwd(); - try { - process.chdir(subDir); - - const config = kreuzberg.ExtractionConfig.discover(); - - expect(config.chunking).toBeDefined(); - expect(config.chunking?.maxChars).toBe(50); - } finally { - process.chdir(originalCwd); - } - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); - - it("Load configuration from a TOML file", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); - const configPath = path.join(tmpDir, "test_config.toml"); - fs.writeFileSync(configPath, "[chunking]\nmax_chars = 100\nmax_overlap = 20\n\n[language_detection]\nenabled = false\n"); - - const config = kreuzberg.ExtractionConfig.fromFile(configPath); - - expect(config.chunking).toBeDefined(); - expect(config.chunking?.maxChars).toBe(100); - expect(config.chunking?.maxOverlap).toBe(20); - expect(config.languageDetection).toBeDefined(); - expect(config.languageDetection?.enabled).toBe(false); - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); - + it("Discover configuration from current or parent directories", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); + const configPath = path.join(tmpDir, "kreuzberg.toml"); + fs.writeFileSync(configPath, "[chunking]\nmax_chars = 50\n"); + + const subDir = path.join(tmpDir, "subdir"); + fs.mkdirSync(subDir); + + const originalCwd = process.cwd(); + try { + process.chdir(subDir); + + const config = kreuzberg.ExtractionConfig.discover(); + + expect(config.chunking).toBeDefined(); + expect(config.chunking?.maxChars).toBe(50); + } finally { + process.chdir(originalCwd); + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("Load configuration from a TOML file", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); + const configPath = path.join(tmpDir, "test_config.toml"); + fs.writeFileSync( + configPath, + "[chunking]\nmax_chars = 100\nmax_overlap = 20\n\n[language_detection]\nenabled = false\n", + ); + + const config = kreuzberg.ExtractionConfig.fromFile(configPath); + + expect(config.chunking).toBeDefined(); + expect(config.chunking?.maxChars).toBe(100); + expect(config.chunking?.maxOverlap).toBe(20); + expect(config.languageDetection).toBeDefined(); + expect(config.languageDetection?.enabled).toBe(false); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); }); describe("Document Extractor Management", () => { - it("Clear all document extractors and verify list is empty", () => { - kreuzberg.clearDocumentExtractors(); - const result = kreuzberg.listDocumentExtractors(); - expect(result).toHaveLength(0); - }); - - it("List all registered document extractors", () => { - const result = kreuzberg.listDocumentExtractors(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item) => typeof item === "string")).toBe(true); - }); - - it("Unregister nonexistent document extractor gracefully", () => { - expect(() => kreuzberg.unregisterDocumentExtractor("nonexistent-extractor-xyz")).not.toThrow(); - }); - + it("Clear all document extractors and verify list is empty", () => { + kreuzberg.clearDocumentExtractors(); + const result = kreuzberg.listDocumentExtractors(); + expect(result).toHaveLength(0); + }); + + it("List all registered document extractors", () => { + const result = kreuzberg.listDocumentExtractors(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item) => typeof item === "string")).toBe(true); + }); + + it("Unregister nonexistent document extractor gracefully", () => { + expect(() => kreuzberg.unregisterDocumentExtractor("nonexistent-extractor-xyz")).not.toThrow(); + }); }); describe("Mime Utilities", () => { - it("Detect MIME type from file bytes", () => { - const testData = Buffer.from("%PDF-1.4\\n"); - const result = kreuzberg.detectMimeType(testData); - expect(result.toLowerCase()).toContain("pdf"); - }); - - it("Detect MIME type from file path", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); - const filePath = path.join(tmpDir, "test.txt"); - fs.writeFileSync(filePath, "Hello, world!"); - - const result = kreuzberg.detectMimeTypeFromPath(filePath); - expect(result.toLowerCase()).toContain("text"); - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); - - it("Get file extensions for a MIME type", () => { - const result = kreuzberg.getExtensionsForMime("application/pdf"); - expect(Array.isArray(result)).toBe(true); - expect(result).toContain("pdf"); - }); - + it("Detect MIME type from file bytes", () => { + const testData = Buffer.from("%PDF-1.4\\n"); + const result = kreuzberg.detectMimeType(testData); + expect(result.toLowerCase()).toContain("pdf"); + }); + + it("Detect MIME type from file path", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kreuzberg-test-")); + const filePath = path.join(tmpDir, "test.txt"); + fs.writeFileSync(filePath, "Hello, world!"); + + const result = kreuzberg.detectMimeTypeFromPath(filePath); + expect(result.toLowerCase()).toContain("text"); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("Get file extensions for a MIME type", () => { + const result = kreuzberg.getExtensionsForMime("application/pdf"); + expect(Array.isArray(result)).toBe(true); + expect(result).toContain("pdf"); + }); }); describe("Ocr Backend Management", () => { - it("Clear all OCR backends and verify list is empty", () => { - kreuzberg.clearOcrBackends(); - const result = kreuzberg.listOcrBackends(); - expect(result).toHaveLength(0); - }); - - it("List all registered OCR backends", () => { - const result = kreuzberg.listOcrBackends(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item) => typeof item === "string")).toBe(true); - }); - - it("Unregister nonexistent OCR backend gracefully", () => { - expect(() => kreuzberg.unregisterOcrBackend("nonexistent-backend-xyz")).not.toThrow(); - }); - + it("Clear all OCR backends and verify list is empty", () => { + kreuzberg.clearOcrBackends(); + const result = kreuzberg.listOcrBackends(); + expect(result).toHaveLength(0); + }); + + it("List all registered OCR backends", () => { + const result = kreuzberg.listOcrBackends(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item) => typeof item === "string")).toBe(true); + }); + + it("Unregister nonexistent OCR backend gracefully", () => { + expect(() => kreuzberg.unregisterOcrBackend("nonexistent-backend-xyz")).not.toThrow(); + }); }); describe("Post Processor Management", () => { - it("Clear all post-processors and verify list is empty", () => { - kreuzberg.clearPostProcessors(); - }); - - it("List all registered post-processors", () => { - const result = kreuzberg.listPostProcessors(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item) => typeof item === "string")).toBe(true); - }); - + it("Clear all post-processors and verify list is empty", () => { + kreuzberg.clearPostProcessors(); + }); + + it("List all registered post-processors", () => { + const result = kreuzberg.listPostProcessors(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item) => typeof item === "string")).toBe(true); + }); }); describe("Validator Management", () => { - it("Clear all validators and verify list is empty", () => { - kreuzberg.clearValidators(); - const result = kreuzberg.listValidators(); - expect(result).toHaveLength(0); - }); - - it("List all registered validators", () => { - const result = kreuzberg.listValidators(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item) => typeof item === "string")).toBe(true); - }); - + it("Clear all validators and verify list is empty", () => { + kreuzberg.clearValidators(); + const result = kreuzberg.listValidators(); + expect(result).toHaveLength(0); + }); + + it("List all registered validators", () => { + const result = kreuzberg.listValidators(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item) => typeof item === "string")).toBe(true); + }); }); diff --git a/e2e/typescript/tests/render.spec.ts b/e2e/typescript/tests/render.spec.ts index 62f0d1aeb0d..4d8f42a3731 100644 --- a/e2e/typescript/tests/render.spec.ts +++ b/e2e/typescript/tests/render.spec.ts @@ -9,35 +9,34 @@ import { renderPdfPageSync, PdfPageIterator } from "@kreuzberg/node"; import { assertions, resolveDocument } from "./helpers.js"; describe("render", () => { - it("render_custom_dpi", () => { - const documentPath = resolveDocument("pdf/tiny.pdf"); - if (!existsSync(documentPath)) return; - const pngData = renderPdfPageSync(documentPath, 0, { dpi: 72 }); - assertions.assertIsPng(pngData); - assertions.assertMinByteLength(pngData, 50); - }); + it("render_custom_dpi", () => { + const documentPath = resolveDocument("pdf/tiny.pdf"); + if (!existsSync(documentPath)) return; + const pngData = renderPdfPageSync(documentPath, 0, { dpi: 72 }); + assertions.assertIsPng(pngData); + assertions.assertMinByteLength(pngData, 50); + }); - it("render_iterator", () => { - const documentPath = resolveDocument("pdf/tiny.pdf"); - if (!existsSync(documentPath)) return; - const pages: Buffer[] = []; - const iter = new PdfPageIterator(documentPath, { dpi: 150 }); - let result = iter.next(); - while (result !== null) { - assertions.assertIsPng(result.data); - pages.push(result.data); - result = iter.next(); - } - iter.close(); - expect(pages.length).toBeGreaterThanOrEqual(1); - }); - - it("render_single_page", () => { - const documentPath = resolveDocument("pdf/tiny.pdf"); - if (!existsSync(documentPath)) return; - const pngData = renderPdfPageSync(documentPath, 0, { dpi: 150 }); - assertions.assertIsPng(pngData); - assertions.assertMinByteLength(pngData, 100); - }); + it("render_iterator", () => { + const documentPath = resolveDocument("pdf/tiny.pdf"); + if (!existsSync(documentPath)) return; + const pages: Buffer[] = []; + const iter = new PdfPageIterator(documentPath, { dpi: 150 }); + let result = iter.next(); + while (result !== null) { + assertions.assertIsPng(result.data); + pages.push(result.data); + result = iter.next(); + } + iter.close(); + expect(pages.length).toBeGreaterThanOrEqual(1); + }); + it("render_single_page", () => { + const documentPath = resolveDocument("pdf/tiny.pdf"); + if (!existsSync(documentPath)) return; + const pngData = renderPdfPageSync(documentPath, 0, { dpi: 150 }); + assertions.assertIsPng(pngData); + assertions.assertMinByteLength(pngData, 100); + }); }); diff --git a/e2e/typescript/tests/structured.spec.ts b/e2e/typescript/tests/structured.spec.ts index d556af729c8..e954274567f 100644 --- a/e2e/typescript/tests/structured.spec.ts +++ b/e2e/typescript/tests/structured.spec.ts @@ -12,235 +12,274 @@ import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("structured fixtures", () => { - it("structured_csv_basic", () => { - const documentPath = resolveDocument("csv/stanley_cups.csv"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_csv_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/csv"]); - assertions.assertMinContentLength(result, 20); - }, TEST_TIMEOUT_MS); + it( + "structured_csv_basic", + () => { + const documentPath = resolveDocument("csv/stanley_cups.csv"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_csv_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/csv"]); + assertions.assertMinContentLength(result, 20); + }, + TEST_TIMEOUT_MS, + ); - it("structured_enw_basic", () => { - const documentPath = resolveDocument("data_formats/sample.enw"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_enw_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); - }, TEST_TIMEOUT_MS); + it( + "structured_enw_basic", + () => { + const documentPath = resolveDocument("data_formats/sample.enw"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_enw_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); + }, + TEST_TIMEOUT_MS, + ); - it("structured_json_basic", () => { - const documentPath = resolveDocument("json/sample_document.json"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_json_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 20); - assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); - }, TEST_TIMEOUT_MS); + it( + "structured_json_basic", + () => { + const documentPath = resolveDocument("json/sample_document.json"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_json_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 20); + assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); + }, + TEST_TIMEOUT_MS, + ); - it("structured_json_simple", () => { - const documentPath = resolveDocument("json/simple.json"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_json_simple: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["{", "name"]); - }, TEST_TIMEOUT_MS); + it( + "structured_json_simple", + () => { + const documentPath = resolveDocument("json/simple.json"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_json_simple: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["{", "name"]); + }, + TEST_TIMEOUT_MS, + ); - it("structured_nbib_basic", () => { - const documentPath = resolveDocument("data_formats/sample.nbib"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_nbib_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "structured_nbib_basic", + () => { + const documentPath = resolveDocument("data_formats/sample.nbib"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_nbib_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("structured_ris_basic", () => { - const documentPath = resolveDocument("data_formats/sample.ris"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_ris_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); - assertions.assertContentNotEmpty(result); - }, TEST_TIMEOUT_MS); + it( + "structured_ris_basic", + () => { + const documentPath = resolveDocument("data_formats/sample.ris"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_ris_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); + assertions.assertContentNotEmpty(result); + }, + TEST_TIMEOUT_MS, + ); - it("structured_toml_basic", () => { - const documentPath = resolveDocument("data_formats/cargo.toml"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_toml_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "structured_toml_basic", + () => { + const documentPath = resolveDocument("data_formats/cargo.toml"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_toml_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("structured_tsv_basic", () => { - const documentPath = resolveDocument("data_formats/employees.tsv"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_tsv_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "structured_tsv_basic", + () => { + const documentPath = resolveDocument("data_formats/employees.tsv"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_tsv_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); - it("structured_yaml_basic", () => { - const documentPath = resolveDocument("yaml/simple.yaml"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_yaml_basic: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); - - it("structured_yaml_simple", () => { - const documentPath = resolveDocument("yaml/simple.yaml"); - if (!existsSync(documentPath)) { - console.warn("Skipping structured_yaml_simple: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-yaml"]); - assertions.assertMinContentLength(result, 10); - }, TEST_TIMEOUT_MS); + it( + "structured_yaml_basic", + () => { + const documentPath = resolveDocument("yaml/simple.yaml"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_yaml_basic: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); + it( + "structured_yaml_simple", + () => { + const documentPath = resolveDocument("yaml/simple.yaml"); + if (!existsSync(documentPath)) { + console.warn("Skipping structured_yaml_simple: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-yaml"]); + assertions.assertMinContentLength(result, 10); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/e2e/typescript/tests/xml.spec.ts b/e2e/typescript/tests/xml.spec.ts index 9ddea2b2048..1a3191978a7 100644 --- a/e2e/typescript/tests/xml.spec.ts +++ b/e2e/typescript/tests/xml.spec.ts @@ -12,27 +12,30 @@ import type { ExtractionResult } from "@kreuzberg/node"; const TEST_TIMEOUT_MS = 60_000; describe("xml fixtures", () => { - it("xml_plant_catalog", () => { - const documentPath = resolveDocument("xml/plant_catalog.xml"); - if (!existsSync(documentPath)) { - console.warn("Skipping xml_plant_catalog: missing document at", documentPath); - return; - } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = extractFileSync(documentPath, null, config); - } catch (error) { - if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml"]); - assertions.assertMinContentLength(result, 100); - }, TEST_TIMEOUT_MS); - + it( + "xml_plant_catalog", + () => { + const documentPath = resolveDocument("xml/plant_catalog.xml"); + if (!existsSync(documentPath)) { + console.warn("Skipping xml_plant_catalog: missing document at", documentPath); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = extractFileSync(documentPath, null, config); + } catch (error) { + if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml"]); + assertions.assertMinContentLength(result, 100); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/e2e/wasm-deno/archive.test.ts b/e2e/wasm-deno/archive.test.ts index de53656c643..eee27332263 100644 --- a/e2e/wasm-deno/archive.test.ts +++ b/e2e/wasm-deno/archive.test.ts @@ -3,7 +3,15 @@ // Tests for archive fixtures. Run with: deno test --allow-read -import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; +import { + assertions, + buildConfig, + enableOcr, + extractBytes, + initWasm, + resolveDocument, + shouldSkipFixture, +} from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time @@ -11,81 +19,81 @@ await initWasm(); await enableOcr(); Deno.test("archive_gz_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("archives/book_war_and_peace_1p.txt.gz"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/gzip", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("archives/book_war_and_peace_1p.txt.gz"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/gzip", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("archive_sevenz_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("archives/documents.7z"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-7z-compressed", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("archives/documents.7z"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-7z-compressed", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("archive_tar_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("archives/documents.tar"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-tar", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("archives/documents.tar"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-tar", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("archive_zip_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("archives/documents.zip"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/zip", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("archives/documents.zip"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/zip", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); + assertions.assertMinContentLength(result, 10); }); diff --git a/e2e/wasm-deno/email.test.ts b/e2e/wasm-deno/email.test.ts index c9525e71524..b1c903543a9 100644 --- a/e2e/wasm-deno/email.test.ts +++ b/e2e/wasm-deno/email.test.ts @@ -3,7 +3,15 @@ // Tests for email fixtures. Run with: deno test --allow-read -import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; +import { + assertions, + buildConfig, + enableOcr, + extractBytes, + initWasm, + resolveDocument, + shouldSkipFixture, +} from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time @@ -11,121 +19,121 @@ await initWasm(); await enableOcr(); Deno.test("email_eml_html_body", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/html_only.eml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/html_only.eml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("email_eml_multipart", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/html_email_multipart.eml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/html_email_multipart.eml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("email_eml_utf16", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("vendored/unstructured/eml/fake-email-utf-16.eml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("vendored/unstructured/eml/fake-email-utf-16.eml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); }); Deno.test("email_msg_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/fake_email.msg"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-outlook", config); - } catch (error) { - if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/fake_email.msg"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-outlook", config); + } catch (error) { + if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("email_pst_empty", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/empty.pst"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-outlook-pst", config); - } catch (error) { - if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/empty.pst"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-outlook-pst", config); + } catch (error) { + if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); }); Deno.test("email_sample_eml", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("email/sample_email.eml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("email/sample_email.eml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 20); }); diff --git a/e2e/wasm-deno/embeddings.test.ts b/e2e/wasm-deno/embeddings.test.ts index 79512239fde..abf97a996d4 100644 --- a/e2e/wasm-deno/embeddings.test.ts +++ b/e2e/wasm-deno/embeddings.test.ts @@ -3,7 +3,15 @@ // Tests for embeddings fixtures. Run with: deno test --allow-read -import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; +import { + assertions, + buildConfig, + enableOcr, + extractBytes, + initWasm, + resolveDocument, + shouldSkipFixture, +} from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time @@ -11,22 +19,22 @@ await initWasm(); await enableOcr(); Deno.test("embedding_disabled", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig({"chunking":{"max_chars":500,"max_overlap":50}}); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pdf/fake_memo.pdf"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/octet-stream", config); - } catch (error) { - if (shouldSkipFixture(error, "embedding_disabled", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/pdf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, false, null, null, null); + const config = buildConfig({ chunking: { max_chars: 500, max_overlap: 50 } }); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pdf/fake_memo.pdf"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/octet-stream", config); + } catch (error) { + if (shouldSkipFixture(error, "embedding_disabled", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/pdf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertChunks(result, 1, null, true, false, null, null, null); }); diff --git a/e2e/wasm-deno/html.test.ts b/e2e/wasm-deno/html.test.ts index 34903a32d5d..d633dc5b6ae 100644 --- a/e2e/wasm-deno/html.test.ts +++ b/e2e/wasm-deno/html.test.ts @@ -3,7 +3,15 @@ // Tests for html fixtures. Run with: deno test --allow-read -import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; +import { + assertions, + buildConfig, + enableOcr, + extractBytes, + initWasm, + resolveDocument, + shouldSkipFixture, +} from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time @@ -11,22 +19,30 @@ await initWasm(); await enableOcr(); Deno.test("html_simple_table", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("html/simple_table.html"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/html", config); - } catch (error) { - if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("html/simple_table.html"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/html", config); + } catch (error) { + if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, [ + "Product", + "Category", + "Price", + "Stock", + "Laptop", + "Electronics", + "Sample Data Table", + ]); }); diff --git a/e2e/wasm-deno/image.test.ts b/e2e/wasm-deno/image.test.ts index 7051ca70538..c7f94d2bbba 100644 --- a/e2e/wasm-deno/image.test.ts +++ b/e2e/wasm-deno/image.test.ts @@ -3,7 +3,15 @@ // Tests for image fixtures. Run with: deno test --allow-read -import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; +import { + assertions, + buildConfig, + enableOcr, + extractBytes, + initWasm, + resolveDocument, + shouldSkipFixture, +} from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time @@ -11,201 +19,201 @@ await initWasm(); await enableOcr(); Deno.test("image_bmp_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images/bmp_24.bmp"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/bmp", config); - } catch (error) { - if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/bmp"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images/bmp_24.bmp"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/bmp", config); + } catch (error) { + if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/bmp"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_gif_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.gif"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/gif", config); - } catch (error) { - if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/gif"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.gif"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/gif", config); + } catch (error) { + if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/gif"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_jp2_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.jp2"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/jp2", config); - } catch (error) { - if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.jp2"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/jp2", config); + } catch (error) { + if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_metadata_only", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig({"ocr":null}); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images/example.jpg"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/jpeg", config); - } catch (error) { - if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jpeg"]); - assertions.assertMaxContentLength(result, 200); + const config = buildConfig({ ocr: null }); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images/example.jpg"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/jpeg", config); + } catch (error) { + if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jpeg"]); + assertions.assertMaxContentLength(result, 200); }); Deno.test("image_pbm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.pbm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/x-portable-bitmap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.pbm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/x-portable-bitmap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_pgm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.pgm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/x-portable-graymap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.pgm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/x-portable-graymap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_ppm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.ppm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/x-portable-pixmap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.ppm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/x-portable-pixmap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_svg_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xml/simple_svg.svg"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/svg+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/svg+xml"]); - assertions.assertMinContentLength(result, 5); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xml/simple_svg.svg"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/svg+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/svg+xml"]); + assertions.assertMinContentLength(result, 5); }); Deno.test("image_tiff_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.tif"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/tiff", config); - } catch (error) { - if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/tiff"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.tif"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/tiff", config); + } catch (error) { + if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/tiff"]); + assertions.assertContentNotEmpty(result); }); Deno.test("image_webp_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("images_extra/ocr_image.webp"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "image/webp", config); - } catch (error) { - if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/webp"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("images_extra/ocr_image.webp"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "image/webp", config); + } catch (error) { + if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/webp"]); + assertions.assertContentNotEmpty(result); }); diff --git a/e2e/wasm-deno/office.test.ts b/e2e/wasm-deno/office.test.ts index b8236b94eda..c73c36f91d9 100644 --- a/e2e/wasm-deno/office.test.ts +++ b/e2e/wasm-deno/office.test.ts @@ -3,7 +3,15 @@ // Tests for office fixtures. Run with: deno test --allow-read -import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; +import { + assertions, + buildConfig, + enableOcr, + extractBytes, + initWasm, + resolveDocument, + shouldSkipFixture, +} from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time @@ -11,1020 +19,1092 @@ await initWasm(); await enableOcr(); Deno.test("office_bibtex_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("bibtex/comprehensive.bib"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-bibtex", config); - } catch (error) { - if (shouldSkipFixture(error, "office_bibtex_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-bibtex", "text/x-bibtex"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("bibtex/comprehensive.bib"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-bibtex", config); + } catch (error) { + if (shouldSkipFixture(error, "office_bibtex_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-bibtex", "text/x-bibtex"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_commonmark_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/sample.commonmark"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/markdown", config); - } catch (error) { - if (shouldSkipFixture(error, "office_commonmark_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/markdown", "text/plain", "text/x-commonmark"]); - assertions.assertMinContentLength(result, 5); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/sample.commonmark"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/markdown", config); + } catch (error) { + if (shouldSkipFixture(error, "office_commonmark_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/markdown", "text/plain", "text/x-commonmark"]); + assertions.assertMinContentLength(result, 5); }); Deno.test("office_dbf_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("dbf/stations.dbf"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-dbf", config); - } catch (error) { - if (shouldSkipFixture(error, "office_dbf_basic", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-dbf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["|"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("dbf/stations.dbf"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-dbf", config); + } catch (error) { + if (shouldSkipFixture(error, "office_dbf_basic", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-dbf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["|"]); }); Deno.test("office_djot_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/tables.djot"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/x-djot", config); - } catch (error) { - if (shouldSkipFixture(error, "office_djot_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-djot", "text/djot"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/tables.djot"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/x-djot", config); + } catch (error) { + if (shouldSkipFixture(error, "office_djot_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-djot", "text/djot"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_doc_legacy", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("doc/unit_test_lists.doc"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/msword", config); - } catch (error) { - if (shouldSkipFixture(error, "office_doc_legacy", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/msword"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("doc/unit_test_lists.doc"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/msword", config); + } catch (error) { + if (shouldSkipFixture(error, "office_doc_legacy", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/msword"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docbook_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docbook/docbook-reader.docbook"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/docbook+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docbook_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/docbook+xml", "text/docbook"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docbook/docbook-reader.docbook"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/docbook+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_docbook_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/docbook+xml", "text/docbook"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_docx_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/sample_document.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/sample_document.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_docx_equations", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/equations.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_equations", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/equations.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_equations", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_fake", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/fake.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_fake", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/fake.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_fake", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_formatting", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/unit_test_formatting.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_formatting", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/unit_test_formatting.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_formatting", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_headers", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/unit_test_headers.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_headers", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/unit_test_headers.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_headers", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_lists", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/unit_test_lists.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_lists", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/unit_test_lists.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_lists", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_docx_tables", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("docx/docx_tables.docx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config); - } catch (error) { - if (shouldSkipFixture(error, "office_docx_tables", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAll(result, ["Simple uniform table", "Nested Table", "merged cells", "Header Col"]); - // Table and bounding box assertions require OCR feature for PDF table extraction - if (result.tables.length > 0) { - assertions.assertTableCount(result, 1, null); - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("docx/docx_tables.docx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_docx_tables", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAll(result, ["Simple uniform table", "Nested Table", "merged cells", "Header Col"]); + // Table and bounding box assertions require OCR feature for PDF table extraction + if (result.tables.length > 0) { + assertions.assertTableCount(result, 1, null); + } }); Deno.test("office_epub_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("epub/features.epub"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/epub+zip", config); - } catch (error) { - if (shouldSkipFixture(error, "office_epub_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/epub+zip"]); - assertions.assertMinContentLength(result, 50); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("epub/features.epub"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/epub+zip", config); + } catch (error) { + if (shouldSkipFixture(error, "office_epub_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/epub+zip"]); + assertions.assertMinContentLength(result, 50); }); Deno.test("office_fb2_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("fictionbook/basic.fb2"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-fictionbook+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_fb2_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-fictionbook+xml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("fictionbook/basic.fb2"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-fictionbook+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_fb2_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-fictionbook+xml"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_fictionbook_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("fictionbook/basic.fb2"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-fictionbook+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_fictionbook_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("fictionbook/basic.fb2"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-fictionbook+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_fictionbook_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-fictionbook+xml", "application/x-fictionbook"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_hwp_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("hwp/converted_output.hwp"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-hwp", config); - } catch (error) { - if (shouldSkipFixture(error, "office_hwp_basic", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-hwp"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("hwp/converted_output.hwp"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-hwp", config); + } catch (error) { + if (shouldSkipFixture(error, "office_hwp_basic", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-hwp"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_hwp_styled", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("hwp/styled_document.hwp"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-hwp", config); - } catch (error) { - if (shouldSkipFixture(error, "office_hwp_styled", ["hwp"], "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-hwp"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("hwp/styled_document.hwp"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-hwp", config); + } catch (error) { + if ( + shouldSkipFixture( + error, + "office_hwp_styled", + ["hwp"], + "HWP styled doc yields no extractable plain text with current parser. Extraction returns empty content on ARM Linux.", + ) + ) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-hwp"]); }); Deno.test("office_jats_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("jats/sample_article.jats"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-jats+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_jats_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-jats+xml", "text/jats"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("jats/sample_article.jats"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-jats+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_jats_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-jats+xml", "text/jats"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_keynote_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("iwork/test.key"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-iwork-keynote-sffkey", config); - } catch (error) { - if (shouldSkipFixture(error, "office_keynote_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-keynote-sffkey"]); - assertions.assertMinContentLength(result, 5); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("iwork/test.key"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-iwork-keynote-sffkey", config); + } catch (error) { + if (shouldSkipFixture(error, "office_keynote_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-keynote-sffkey"]); + assertions.assertMinContentLength(result, 5); }); Deno.test("office_latex_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("latex/basic_sections.tex"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-latex", config); - } catch (error) { - if (shouldSkipFixture(error, "office_latex_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-latex", "text/x-latex"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("latex/basic_sections.tex"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-latex", config); + } catch (error) { + if (shouldSkipFixture(error, "office_latex_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-latex", "text/x-latex"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_markdown_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/comprehensive.md"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/markdown", config); - } catch (error) { - if (shouldSkipFixture(error, "office_markdown_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/markdown"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/comprehensive.md"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/markdown", config); + } catch (error) { + if (shouldSkipFixture(error, "office_markdown_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/markdown"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_mdx_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/sample.mdx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/mdx", config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 50); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/sample.mdx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/mdx", config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 50); }); Deno.test("office_mdx_getting_started", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/mdx_getting_started.mdx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/mdx", config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_getting_started", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/mdx_getting_started.mdx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/mdx", config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_getting_started", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); }); Deno.test("office_mdx_troubleshooting", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/mdx_troubleshooting.mdx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/mdx", config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_troubleshooting", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/mdx_troubleshooting.mdx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/mdx", config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_troubleshooting", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); }); Deno.test("office_mdx_using_mdx", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("markdown/mdx_using_mdx.mdx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/mdx", config); - } catch (error) { - if (shouldSkipFixture(error, "office_mdx_using_mdx", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); - assertions.assertMinContentLength(result, 2000); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("markdown/mdx_using_mdx.mdx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/mdx", config); + } catch (error) { + if (shouldSkipFixture(error, "office_mdx_using_mdx", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/mdx", "text/x-mdx"]); + assertions.assertMinContentLength(result, 2000); }); Deno.test("office_numbers_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("iwork/test.numbers"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-iwork-numbers-sffnumbers", config); - } catch (error) { - if (shouldSkipFixture(error, "office_numbers_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-numbers-sffnumbers"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("iwork/test.numbers"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-iwork-numbers-sffnumbers", config); + } catch (error) { + if (shouldSkipFixture(error, "office_numbers_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-numbers-sffnumbers"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_ods_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/test_01.ods"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.spreadsheet", config); - } catch (error) { - if (shouldSkipFixture(error, "office_ods_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.spreadsheet"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/test_01.ods"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.spreadsheet", config); + } catch (error) { + if (shouldSkipFixture(error, "office_ods_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.spreadsheet"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_odt_bold", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("odt/bold.odt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_bold", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("odt/bold.odt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_bold", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_odt_list", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("odt/unorderedList.odt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_list", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 30); - assertions.assertContentContainsAny(result, ["list item", "New level", "Pushed us"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("odt/unorderedList.odt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_list", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 30); + assertions.assertContentContainsAny(result, ["list item", "New level", "Pushed us"]); }); Deno.test("office_odt_simple", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("odt/simple.odt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["favorite things", "Parrots", "Analysis"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("odt/simple.odt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["favorite things", "Parrots", "Analysis"]); }); Deno.test("office_odt_table", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("odt/table.odt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); - } catch (error) { - if (shouldSkipFixture(error, "office_odt_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); - assertions.assertMinContentLength(result, 10); - // Table and bounding box assertions require OCR feature for PDF table extraction - if (result.tables.length > 0) { - assertions.assertTableCount(result, 1, null); - } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("odt/table.odt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.oasis.opendocument.text", config); + } catch (error) { + if (shouldSkipFixture(error, "office_odt_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.oasis.opendocument.text"]); + assertions.assertMinContentLength(result, 10); + // Table and bounding box assertions require OCR feature for PDF table extraction + if (result.tables.length > 0) { + assertions.assertTableCount(result, 1, null); + } }); Deno.test("office_opml_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("opml/outline.opml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/xml+opml", config); - } catch (error) { - if (shouldSkipFixture(error, "office_opml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("opml/outline.opml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/xml+opml", config); + } catch (error) { + if (shouldSkipFixture(error, "office_opml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml+opml", "text/x-opml", "application/x-opml+xml"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_org_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("org/comprehensive.org"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/x-org", config); - } catch (error) { - if (shouldSkipFixture(error, "office_org_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-org", "text/org"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("org/comprehensive.org"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/x-org", config); + } catch (error) { + if (shouldSkipFixture(error, "office_org_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-org", "text/org"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_pages_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("iwork/test.pages"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-iwork-pages-sffpages", config); - } catch (error) { - if (shouldSkipFixture(error, "office_pages_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-iwork-pages-sffpages"]); - assertions.assertMinContentLength(result, 5); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("iwork/test.pages"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-iwork-pages-sffpages", config); + } catch (error) { + if (shouldSkipFixture(error, "office_pages_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-iwork-pages-sffpages"]); + assertions.assertMinContentLength(result, 5); }); Deno.test("office_ppsx_slideshow", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/sample.ppsx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.presentationml.slideshow", config); - } catch (error) { - if (shouldSkipFixture(error, "office_ppsx_slideshow", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/sample.ppsx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_ppsx_slideshow", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_ppt_legacy", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("ppt/simple.ppt"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-powerpoint", config); - } catch (error) { - if (shouldSkipFixture(error, "office_ppt_legacy", ["office"], "Requires the office feature.")) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("ppt/simple.ppt"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-powerpoint", config); + } catch (error) { + if (shouldSkipFixture(error, "office_ppt_legacy", ["office"], "Requires the office feature.")) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_pptm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/powerpoint_with_image.pptm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-powerpoint.presentation.macroEnabled.12", config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptm_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-powerpoint.presentation.macroEnabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/powerpoint_with_image.pptm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-powerpoint.presentation.macroEnabled.12", config); + } catch (error) { + if (shouldSkipFixture(error, "office_pptm_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ]); + assertions.assertContentNotEmpty(result); }); Deno.test("office_pptx_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/simple.pptx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.presentationml.presentation", config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 50); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/simple.pptx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 50); }); Deno.test("office_pptx_images", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/powerpoint_with_image.pptx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.presentationml.presentation", config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_images", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 15); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/powerpoint_with_image.pptx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_images", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 15); }); Deno.test("office_pptx_pitch_deck", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("pptx/pitch_deck_presentation.pptx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.presentationml.presentation", config); - } catch (error) { - if (shouldSkipFixture(error, "office_pptx_pitch_deck", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); - assertions.assertMinContentLength(result, 100); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("pptx/pitch_deck_presentation.pptx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_pptx_pitch_deck", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]); + assertions.assertMinContentLength(result, 100); }); Deno.test("office_rst_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("rst/restructured_text.rst"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/x-rst", config); - } catch (error) { - if (shouldSkipFixture(error, "office_rst_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/x-rst", "text/prs.fallenstein.rst"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("rst/restructured_text.rst"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/x-rst", config); + } catch (error) { + if (shouldSkipFixture(error, "office_rst_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/x-rst", "text/prs.fallenstein.rst"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("office_rtf_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("rtf/extraction_test.rtf"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/rtf", config); - } catch (error) { - if (shouldSkipFixture(error, "office_rtf_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/rtf", "text/rtf"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("rtf/extraction_test.rtf"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/rtf", config); + } catch (error) { + if (shouldSkipFixture(error, "office_rtf_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/rtf", "text/rtf"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_typst_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("typst/headings.typ"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-typst", config); - } catch (error) { - if (shouldSkipFixture(error, "office_typst_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-typst", "text/x-typst"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("typst/headings.typ"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-typst", config); + } catch (error) { + if (shouldSkipFixture(error, "office_typst_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-typst", "text/x-typst"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_xls_legacy", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xls/test_excel.xls"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-excel", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xls_legacy", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-excel"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xls/test_excel.xls"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-excel", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xls_legacy", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-excel"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("office_xlsb_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/test_xlsb.xlsb"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-excel.sheet.binary.macroEnabled.12", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsb_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-excel.sheet.binary.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/test_xlsb.xlsb"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-excel.sheet.binary.macroEnabled.12", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsb_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ]); + assertions.assertContentNotEmpty(result); }); Deno.test("office_xlsm_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/test_01.xlsm"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.ms-excel.sheet.macroEnabled.12", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsm_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-excel.sheet.macroEnabled.12", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/test_01.xlsm"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/vnd.ms-excel.sheet.macroEnabled.12", config); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsm_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, [ + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ]); + assertions.assertContentNotEmpty(result); }); Deno.test("office_xlsx_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/stanley_cups.xlsx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, ["Team", "Location", "Stanley Cups"]); - // Table and bounding box assertions require OCR feature for PDF table extraction - if (result.tables.length > 0) { - assertions.assertTableCount(result, 1, null); - } - assertions.assertMetadataExpectation(result, "sheetCount", {"gte":2}); - assertions.assertMetadataExpectation(result, "sheetNames", {"contains":["Stanley Cups"]}); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/stanley_cups.xlsx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, ["Team", "Location", "Stanley Cups"]); + // Table and bounding box assertions require OCR feature for PDF table extraction + if (result.tables.length > 0) { + assertions.assertTableCount(result, 1, null); + } + assertions.assertMetadataExpectation(result, "sheetCount", { gte: 2 }); + assertions.assertMetadataExpectation(result, "sheetNames", { contains: ["Stanley Cups"] }); }); Deno.test("office_xlsx_multi_sheet", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/excel_multi_sheet.xlsx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_multi_sheet", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 20); - assertions.assertMetadataExpectation(result, "sheetCount", {"gte":2}); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/excel_multi_sheet.xlsx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_multi_sheet", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 20); + assertions.assertMetadataExpectation(result, "sheetCount", { gte: 2 }); }); Deno.test("office_xlsx_office_example", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xlsx/test_01.xlsx"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", config); - } catch (error) { - if (shouldSkipFixture(error, "office_xlsx_office_example", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xlsx/test_01.xlsx"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes( + documentBytes, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + config, + ); + } catch (error) { + if (shouldSkipFixture(error, "office_xlsx_office_example", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]); + assertions.assertMinContentLength(result, 10); }); diff --git a/e2e/wasm-deno/plugin-apis.test.ts b/e2e/wasm-deno/plugin-apis.test.ts index c8d6356a819..6252a9a31fb 100644 --- a/e2e/wasm-deno/plugin-apis.test.ts +++ b/e2e/wasm-deno/plugin-apis.test.ts @@ -8,7 +8,18 @@ import { assertEquals } from "@std/assert"; // @deno-types="../../crates/kreuzberg-wasm/dist/index.d.ts" -import { clearOcrBackends, clearPostProcessors, clearValidators, detectMimeFromBytes, getExtensionsForMime, initWasm, listOcrBackends, listPostProcessors, listValidators, unregisterOcrBackend } from "npm:@kreuzberg/wasm@^4.0.0"; +import { + clearOcrBackends, + clearPostProcessors, + clearValidators, + detectMimeFromBytes, + getExtensionsForMime, + initWasm, + listOcrBackends, + listPostProcessors, + listValidators, + unregisterOcrBackend, +} from "npm:@kreuzberg/wasm@^4.0.0"; await initWasm(); @@ -29,59 +40,68 @@ Deno.test({ name: "Unregister nonexistent document extractor gracefully", ignore // Mime Utilities Deno.test("Detect MIME type from file bytes", () => { - const testData = new TextEncoder().encode("%PDF-1.4\\n"); - const result = detectMimeFromBytes(testData); - assertEquals(result.toLowerCase().includes("pdf"), true); + const testData = new TextEncoder().encode("%PDF-1.4\\n"); + const result = detectMimeFromBytes(testData); + assertEquals(result.toLowerCase().includes("pdf"), true); }); Deno.test({ name: "Detect MIME type from file path", ignore: true, fn() {} }); Deno.test("Get file extensions for a MIME type", () => { - const result = getExtensionsForMime("application/pdf"); - assertEquals(Array.isArray(result), true); - assertEquals(result.includes("pdf"), true); + const result = getExtensionsForMime("application/pdf"); + assertEquals(Array.isArray(result), true); + assertEquals(result.includes("pdf"), true); }); // Ocr Backend Management Deno.test("Clear all OCR backends and verify list is empty", () => { - clearOcrBackends(); - const result = listOcrBackends(); - assertEquals(result.length, 0); + clearOcrBackends(); + const result = listOcrBackends(); + assertEquals(result.length, 0); }); Deno.test("List all registered OCR backends", () => { - const result = listOcrBackends(); - assertEquals(Array.isArray(result), true); - assertEquals(result.every((item: unknown) => typeof item === "string"), true); + const result = listOcrBackends(); + assertEquals(Array.isArray(result), true); + assertEquals( + result.every((item: unknown) => typeof item === "string"), + true, + ); }); Deno.test("Unregister nonexistent OCR backend gracefully", () => { - unregisterOcrBackend("nonexistent-backend-xyz"); + unregisterOcrBackend("nonexistent-backend-xyz"); }); // Post Processor Management Deno.test("Clear all post-processors and verify list is empty", () => { - clearPostProcessors(); + clearPostProcessors(); }); Deno.test("List all registered post-processors", () => { - const result = listPostProcessors(); - assertEquals(Array.isArray(result), true); - assertEquals(result.every((item: unknown) => typeof item === "string"), true); + const result = listPostProcessors(); + assertEquals(Array.isArray(result), true); + assertEquals( + result.every((item: unknown) => typeof item === "string"), + true, + ); }); // Validator Management Deno.test("Clear all validators and verify list is empty", () => { - clearValidators(); - const result = listValidators(); - assertEquals(result.length, 0); + clearValidators(); + const result = listValidators(); + assertEquals(result.length, 0); }); Deno.test("List all registered validators", () => { - const result = listValidators(); - assertEquals(Array.isArray(result), true); - assertEquals(result.every((item: unknown) => typeof item === "string"), true); + const result = listValidators(); + assertEquals(Array.isArray(result), true); + assertEquals( + result.every((item: unknown) => typeof item === "string"), + true, + ); }); diff --git a/e2e/wasm-deno/structured.test.ts b/e2e/wasm-deno/structured.test.ts index c76e43b93c7..d8d45b451fb 100644 --- a/e2e/wasm-deno/structured.test.ts +++ b/e2e/wasm-deno/structured.test.ts @@ -3,7 +3,15 @@ // Tests for structured fixtures. Run with: deno test --allow-read -import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; +import { + assertions, + buildConfig, + enableOcr, + extractBytes, + initWasm, + resolveDocument, + shouldSkipFixture, +} from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time @@ -11,202 +19,202 @@ await initWasm(); await enableOcr(); Deno.test("structured_csv_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("csv/stanley_cups.csv"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/csv", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/csv"]); - assertions.assertMinContentLength(result, 20); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("csv/stanley_cups.csv"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/csv", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/csv"]); + assertions.assertMinContentLength(result, 20); }); Deno.test("structured_enw_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/sample.enw"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-endnote-refer", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/sample.enw"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-endnote-refer", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); }); Deno.test("structured_json_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("json/sample_document.json"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/json", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 20); - assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("json/sample_document.json"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/json", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 20); + assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); }); Deno.test("structured_json_simple", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("json/simple.json"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/json", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["{", "name"]); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("json/simple.json"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/json", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["{", "name"]); }); Deno.test("structured_nbib_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/sample.nbib"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/nbib", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/sample.nbib"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/nbib", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); + assertions.assertContentNotEmpty(result); }); Deno.test("structured_ris_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/sample.ris"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-research-info-systems", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); - assertions.assertContentNotEmpty(result); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/sample.ris"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-research-info-systems", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); + assertions.assertContentNotEmpty(result); }); Deno.test("structured_toml_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/cargo.toml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/toml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/cargo.toml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/toml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("structured_tsv_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("data_formats/employees.tsv"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "text/tab-separated-values", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("data_formats/employees.tsv"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "text/tab-separated-values", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("structured_yaml_basic", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("yaml/simple.yaml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/yaml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("yaml/simple.yaml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/yaml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); + assertions.assertMinContentLength(result, 10); }); Deno.test("structured_yaml_simple", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("yaml/simple.yaml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/x-yaml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-yaml"]); - assertions.assertMinContentLength(result, 10); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("yaml/simple.yaml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/x-yaml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-yaml"]); + assertions.assertMinContentLength(result, 10); }); diff --git a/e2e/wasm-deno/xml.test.ts b/e2e/wasm-deno/xml.test.ts index ae8a5ce299c..e38eb793059 100644 --- a/e2e/wasm-deno/xml.test.ts +++ b/e2e/wasm-deno/xml.test.ts @@ -3,7 +3,15 @@ // Tests for xml fixtures. Run with: deno test --allow-read -import { assertions, buildConfig, enableOcr, extractBytes, initWasm, resolveDocument, shouldSkipFixture } from "./helpers.ts"; +import { + assertions, + buildConfig, + enableOcr, + extractBytes, + initWasm, + resolveDocument, + shouldSkipFixture, +} from "./helpers.ts"; import type { ExtractionResult } from "./helpers.ts"; // Initialize WASM module and enable OCR once at module load time @@ -11,21 +19,21 @@ await initWasm(); await enableOcr(); Deno.test("xml_plant_catalog", { permissions: { read: true, net: true } }, async () => { - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - const documentBytes = await resolveDocument("xml/plant_catalog.xml"); - // Sync file extraction - WASM uses extractBytes with pre-read bytes - result = await extractBytes(documentBytes, "application/xml", config); - } catch (error) { - if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml"]); - assertions.assertMinContentLength(result, 100); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + const documentBytes = await resolveDocument("xml/plant_catalog.xml"); + // Sync file extraction - WASM uses extractBytes with pre-read bytes + result = await extractBytes(documentBytes, "application/xml", config); + } catch (error) { + if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml"]); + assertions.assertMinContentLength(result, 100); }); diff --git a/e2e/wasm-workers/tests/archive.spec.ts b/e2e/wasm-workers/tests/archive.spec.ts index 940e91ab256..9b96d9a8122 100644 --- a/e2e/wasm-workers/tests/archive.spec.ts +++ b/e2e/wasm-workers/tests/archive.spec.ts @@ -9,100 +9,99 @@ import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helper import type { ExtractionResult } from "@kreuzberg/wasm"; describe("archive", () => { - it("archive_gz_basic", async () => { - const documentBytes = getFixture("archives/book_war_and_peace_1p.txt.gz"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("archive_gz_basic", async () => { + const documentBytes = getFixture("archives/book_war_and_peace_1p.txt.gz"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/gzip", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); - assertions.assertMinContentLength(result, 10); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/gzip", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_gz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/gzip", "application/x-gzip"]); + assertions.assertMinContentLength(result, 10); + }); - it("archive_sevenz_basic", async () => { - const documentBytes = getFixture("archives/documents.7z"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("archive_sevenz_basic", async () => { + const documentBytes = getFixture("archives/documents.7z"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-7z-compressed", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); - assertions.assertMinContentLength(result, 10); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-7z-compressed", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_sevenz_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-7z-compressed"]); + assertions.assertMinContentLength(result, 10); + }); - it("archive_tar_basic", async () => { - const documentBytes = getFixture("archives/documents.tar"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("archive_tar_basic", async () => { + const documentBytes = getFixture("archives/documents.tar"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-tar", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); - assertions.assertMinContentLength(result, 10); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-tar", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_tar_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-tar", "application/tar"]); + assertions.assertMinContentLength(result, 10); + }); - it("archive_zip_basic", async () => { - const documentBytes = getFixture("archives/documents.zip"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/zip", config); - } catch (error) { - if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); - assertions.assertMinContentLength(result, 10); - }); + it("archive_zip_basic", async () => { + const documentBytes = getFixture("archives/documents.zip"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/zip", config); + } catch (error) { + if (shouldSkipFixture(error, "archive_zip_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/zip", "application/x-zip-compressed"]); + assertions.assertMinContentLength(result, 10); + }); }); diff --git a/e2e/wasm-workers/tests/email.spec.ts b/e2e/wasm-workers/tests/email.spec.ts index 45403772123..91523f05e91 100644 --- a/e2e/wasm-workers/tests/email.spec.ts +++ b/e2e/wasm-workers/tests/email.spec.ts @@ -9,148 +9,147 @@ import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helper import type { ExtractionResult } from "@kreuzberg/wasm"; describe("email", () => { - it("email_eml_html_body", async () => { - const documentBytes = getFixture("email/html_only.eml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("email_eml_html_body", async () => { + const documentBytes = getFixture("email/html_only.eml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_html_body", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); + }); - it("email_eml_multipart", async () => { - const documentBytes = getFixture("email/html_email_multipart.eml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("email_eml_multipart", async () => { + const documentBytes = getFixture("email/html_email_multipart.eml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 10); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_multipart", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 10); + }); - it("email_eml_utf16", async () => { - const documentBytes = getFixture("vendored/unstructured/eml/fake-email-utf-16.eml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("email_eml_utf16", async () => { + const documentBytes = getFixture("vendored/unstructured/eml/fake-email-utf-16.eml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 50); - assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_eml_utf16", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 50); + assertions.assertContentContainsAny(result, ["Test Email", "Roses are red"]); + }); - it("email_msg_basic", async () => { - const documentBytes = getFixture("email/fake_email.msg"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("email_msg_basic", async () => { + const documentBytes = getFixture("email/fake_email.msg"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/vnd.ms-outlook", config); - } catch (error) { - if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); - assertions.assertMinContentLength(result, 10); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/vnd.ms-outlook", config); + } catch (error) { + if (shouldSkipFixture(error, "email_msg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook"]); + assertions.assertMinContentLength(result, 10); + }); - it("email_pst_empty", async () => { - const documentBytes = getFixture("email/empty.pst"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("email_pst_empty", async () => { + const documentBytes = getFixture("email/empty.pst"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/vnd.ms-outlook-pst", config); - } catch (error) { - if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/vnd.ms-outlook-pst", config); + } catch (error) { + if (shouldSkipFixture(error, "email_pst_empty", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/vnd.ms-outlook-pst"]); + }); - it("email_sample_eml", async () => { - const documentBytes = getFixture("email/sample_email.eml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "message/rfc822", config); - } catch (error) { - if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["message/rfc822"]); - assertions.assertMinContentLength(result, 20); - }); + it("email_sample_eml", async () => { + const documentBytes = getFixture("email/sample_email.eml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "message/rfc822", config); + } catch (error) { + if (shouldSkipFixture(error, "email_sample_eml", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["message/rfc822"]); + assertions.assertMinContentLength(result, 20); + }); }); diff --git a/e2e/wasm-workers/tests/embeddings.spec.ts b/e2e/wasm-workers/tests/embeddings.spec.ts index a5ba2cd49c6..776d11ba60f 100644 --- a/e2e/wasm-workers/tests/embeddings.spec.ts +++ b/e2e/wasm-workers/tests/embeddings.spec.ts @@ -9,29 +9,28 @@ import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helper import type { ExtractionResult } from "@kreuzberg/wasm"; describe("embeddings", () => { - it("embedding_disabled", async () => { - const documentBytes = getFixture("pdf/fake_memo.pdf"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig({"chunking":{"max_chars":500,"max_overlap":50}}); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/octet-stream", config); - } catch (error) { - if (shouldSkipFixture(error, "embedding_disabled", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/pdf"]); - assertions.assertMinContentLength(result, 10); - assertions.assertChunks(result, 1, null, true, false, null, null, null); - }); + it("embedding_disabled", async () => { + const documentBytes = getFixture("pdf/fake_memo.pdf"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + const config = buildConfig({ chunking: { max_chars: 500, max_overlap: 50 } }); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/octet-stream", config); + } catch (error) { + if (shouldSkipFixture(error, "embedding_disabled", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/pdf"]); + assertions.assertMinContentLength(result, 10); + assertions.assertChunks(result, 1, null, true, false, null, null, null); + }); }); diff --git a/e2e/wasm-workers/tests/html.spec.ts b/e2e/wasm-workers/tests/html.spec.ts index cda5547695f..d67201ded95 100644 --- a/e2e/wasm-workers/tests/html.spec.ts +++ b/e2e/wasm-workers/tests/html.spec.ts @@ -9,53 +9,60 @@ import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helper import type { ExtractionResult } from "@kreuzberg/wasm"; describe("html", () => { - it("html_complex_layout", async () => { - const documentBytes = getFixture("html/taylor_swift.html"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } + it("html_complex_layout", async () => { + const documentBytes = getFixture("html/taylor_swift.html"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "text/html", config); - } catch (error) { - if (shouldSkipFixture(error, "html_complex_layout", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 1000); - }); + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "text/html", config); + } catch (error) { + if (shouldSkipFixture(error, "html_complex_layout", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 1000); + }); - it("html_simple_table", async () => { - const documentBytes = getFixture("html/simple_table.html"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "text/html", config); - } catch (error) { - if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/html"]); - assertions.assertMinContentLength(result, 100); - assertions.assertContentContainsAll(result, ["Product", "Category", "Price", "Stock", "Laptop", "Electronics", "Sample Data Table"]); - }); + it("html_simple_table", async () => { + const documentBytes = getFixture("html/simple_table.html"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "text/html", config); + } catch (error) { + if (shouldSkipFixture(error, "html_simple_table", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/html"]); + assertions.assertMinContentLength(result, 100); + assertions.assertContentContainsAll(result, [ + "Product", + "Category", + "Price", + "Stock", + "Laptop", + "Electronics", + "Sample Data Table", + ]); + }); }); diff --git a/e2e/wasm-workers/tests/image.spec.ts b/e2e/wasm-workers/tests/image.spec.ts index 87897ae7255..31e14599f1a 100644 --- a/e2e/wasm-workers/tests/image.spec.ts +++ b/e2e/wasm-workers/tests/image.spec.ts @@ -9,244 +9,243 @@ import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helper import type { ExtractionResult } from "@kreuzberg/wasm"; describe("image", () => { - it("image_bmp_basic", async () => { - const documentBytes = getFixture("images/bmp_24.bmp"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/bmp", config); - } catch (error) { - if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/bmp"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_gif_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.gif"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/gif", config); - } catch (error) { - if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/gif"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_jp2_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.jp2"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/jp2", config); - } catch (error) { - if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_metadata_only", async () => { - const documentBytes = getFixture("images/example.jpg"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig({"ocr":null}); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/jpeg", config); - } catch (error) { - if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/jpeg"]); - assertions.assertMaxContentLength(result, 200); - }); - - it("image_pbm_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.pbm"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/x-portable-bitmap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_pgm_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.pgm"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/x-portable-graymap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_ppm_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.ppm"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/x-portable-pixmap", config); - } catch (error) { - if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_svg_basic", async () => { - const documentBytes = getFixture("xml/simple_svg.svg"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/svg+xml", config); - } catch (error) { - if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/svg+xml"]); - assertions.assertMinContentLength(result, 5); - }); - - it("image_tiff_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.tif"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/tiff", config); - } catch (error) { - if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/tiff"]); - assertions.assertContentNotEmpty(result); - }); - - it("image_webp_basic", async () => { - const documentBytes = getFixture("images_extra/ocr_image.webp"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "image/webp", config); - } catch (error) { - if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["image/webp"]); - assertions.assertContentNotEmpty(result); - }); - + it("image_bmp_basic", async () => { + const documentBytes = getFixture("images/bmp_24.bmp"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/bmp", config); + } catch (error) { + if (shouldSkipFixture(error, "image_bmp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/bmp"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_gif_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.gif"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/gif", config); + } catch (error) { + if (shouldSkipFixture(error, "image_gif_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/gif"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_jp2_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.jp2"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/jp2", config); + } catch (error) { + if (shouldSkipFixture(error, "image_jp2_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jp2", "image/jpeg2000"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_metadata_only", async () => { + const documentBytes = getFixture("images/example.jpg"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig({ ocr: null }); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/jpeg", config); + } catch (error) { + if (shouldSkipFixture(error, "image_metadata_only", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/jpeg"]); + assertions.assertMaxContentLength(result, 200); + }); + + it("image_pbm_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.pbm"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/x-portable-bitmap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_pbm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-bitmap", "image/x-pbm"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_pgm_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.pgm"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/x-portable-graymap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_pgm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-graymap", "image/x-pgm"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_ppm_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.ppm"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/x-portable-pixmap", config); + } catch (error) { + if (shouldSkipFixture(error, "image_ppm_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/x-portable-pixmap", "image/x-ppm"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_svg_basic", async () => { + const documentBytes = getFixture("xml/simple_svg.svg"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/svg+xml", config); + } catch (error) { + if (shouldSkipFixture(error, "image_svg_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/svg+xml"]); + assertions.assertMinContentLength(result, 5); + }); + + it("image_tiff_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.tif"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/tiff", config); + } catch (error) { + if (shouldSkipFixture(error, "image_tiff_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/tiff"]); + assertions.assertContentNotEmpty(result); + }); + + it("image_webp_basic", async () => { + const documentBytes = getFixture("images_extra/ocr_image.webp"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "image/webp", config); + } catch (error) { + if (shouldSkipFixture(error, "image_webp_basic", ["tesseract"], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["image/webp"]); + assertions.assertContentNotEmpty(result); + }); }); diff --git a/e2e/wasm-workers/tests/plugin-apis.spec.ts b/e2e/wasm-workers/tests/plugin-apis.spec.ts index 85f8b337955..97162bad852 100644 --- a/e2e/wasm-workers/tests/plugin-apis.spec.ts +++ b/e2e/wasm-workers/tests/plugin-apis.spec.ts @@ -7,76 +7,78 @@ */ import { describe, it, expect } from "vitest"; -import { clearOcrBackends, clearPostProcessors, clearValidators, listOcrBackends, listPostProcessors, listValidators, unregisterOcrBackend } from "@kreuzberg/wasm"; +import { + clearOcrBackends, + clearPostProcessors, + clearValidators, + listOcrBackends, + listPostProcessors, + listValidators, + unregisterOcrBackend, +} from "@kreuzberg/wasm"; describe("Configuration", () => { - it.skip("Discover configuration from current or parent directories (not available in WASM)", () => {}); - - it.skip("Load configuration from a TOML file (not available in WASM)", () => {}); + it.skip("Discover configuration from current or parent directories (not available in WASM)", () => {}); + it.skip("Load configuration from a TOML file (not available in WASM)", () => {}); }); describe("Document Extractor Management", () => { - it.skip("Clear all document extractors and verify list is empty (not available in WASM)", () => {}); - - it.skip("List all registered document extractors (not available in WASM)", () => {}); + it.skip("Clear all document extractors and verify list is empty (not available in WASM)", () => {}); - it.skip("Unregister nonexistent document extractor gracefully (not available in WASM)", () => {}); + it.skip("List all registered document extractors (not available in WASM)", () => {}); + it.skip("Unregister nonexistent document extractor gracefully (not available in WASM)", () => {}); }); describe("Mime Utilities", () => { - it.skip("Detect MIME type from file bytes (not available in WASM)", () => {}); + it.skip("Detect MIME type from file bytes (not available in WASM)", () => {}); - it.skip("Detect MIME type from file path (not available in WASM)", () => {}); - - it.skip("Get file extensions for a MIME type (not available in WASM)", () => {}); + it.skip("Detect MIME type from file path (not available in WASM)", () => {}); + it.skip("Get file extensions for a MIME type (not available in WASM)", () => {}); }); describe("Ocr Backend Management", () => { - it("Clear all OCR backends and verify list is empty", () => { - clearOcrBackends(); - const result = listOcrBackends(); - expect(result).toHaveLength(0); - }); - - it("List all registered OCR backends", () => { - const result = listOcrBackends(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item: unknown) => typeof item === "string")).toBe(true); - }); - - it("Unregister nonexistent OCR backend gracefully", () => { - expect(() => unregisterOcrBackend("nonexistent-backend-xyz")).not.toThrow(); - }); - + it("Clear all OCR backends and verify list is empty", () => { + clearOcrBackends(); + const result = listOcrBackends(); + expect(result).toHaveLength(0); + }); + + it("List all registered OCR backends", () => { + const result = listOcrBackends(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item: unknown) => typeof item === "string")).toBe(true); + }); + + it("Unregister nonexistent OCR backend gracefully", () => { + expect(() => unregisterOcrBackend("nonexistent-backend-xyz")).not.toThrow(); + }); }); describe("Post Processor Management", () => { - it("Clear all post-processors and verify list is empty", () => { - clearPostProcessors(); - }); - - it("List all registered post-processors", () => { - const result = listPostProcessors(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item: unknown) => typeof item === "string")).toBe(true); - }); - + it("Clear all post-processors and verify list is empty", () => { + clearPostProcessors(); + }); + + it("List all registered post-processors", () => { + const result = listPostProcessors(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item: unknown) => typeof item === "string")).toBe(true); + }); }); describe("Validator Management", () => { - it("Clear all validators and verify list is empty", () => { - clearValidators(); - const result = listValidators(); - expect(result).toHaveLength(0); - }); - - it("List all registered validators", () => { - const result = listValidators(); - expect(Array.isArray(result)).toBe(true); - expect(result.every((item: unknown) => typeof item === "string")).toBe(true); - }); - + it("Clear all validators and verify list is empty", () => { + clearValidators(); + const result = listValidators(); + expect(result).toHaveLength(0); + }); + + it("List all registered validators", () => { + const result = listValidators(); + expect(Array.isArray(result)).toBe(true); + expect(result.every((item: unknown) => typeof item === "string")).toBe(true); + }); }); diff --git a/e2e/wasm-workers/tests/structured.spec.ts b/e2e/wasm-workers/tests/structured.spec.ts index 7cc23b43f1e..2e529cbb3a4 100644 --- a/e2e/wasm-workers/tests/structured.spec.ts +++ b/e2e/wasm-workers/tests/structured.spec.ts @@ -9,245 +9,244 @@ import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helper import type { ExtractionResult } from "@kreuzberg/wasm"; describe("structured", () => { - it("structured_csv_basic", async () => { - const documentBytes = getFixture("csv/stanley_cups.csv"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "text/csv", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/csv"]); - assertions.assertMinContentLength(result, 20); - }); - - it("structured_enw_basic", async () => { - const documentBytes = getFixture("data_formats/sample.enw"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-endnote-refer", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); - }); - - it("structured_json_basic", async () => { - const documentBytes = getFixture("json/sample_document.json"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/json", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 20); - assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); - }); - - it("structured_json_simple", async () => { - const documentBytes = getFixture("json/simple.json"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/json", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/json"]); - assertions.assertMinContentLength(result, 10); - assertions.assertContentContainsAny(result, ["{", "name"]); - }); - - it("structured_nbib_basic", async () => { - const documentBytes = getFixture("data_formats/sample.nbib"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/nbib", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); - assertions.assertContentNotEmpty(result); - }); - - it("structured_ris_basic", async () => { - const documentBytes = getFixture("data_formats/sample.ris"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-research-info-systems", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); - assertions.assertContentNotEmpty(result); - }); - - it("structured_toml_basic", async () => { - const documentBytes = getFixture("data_formats/cargo.toml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/toml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); - assertions.assertMinContentLength(result, 10); - }); - - it("structured_tsv_basic", async () => { - const documentBytes = getFixture("data_formats/employees.tsv"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "text/tab-separated-values", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); - assertions.assertMinContentLength(result, 10); - }); - - it("structured_yaml_basic", async () => { - const documentBytes = getFixture("yaml/simple.yaml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/yaml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); - assertions.assertMinContentLength(result, 10); - }); - - it("structured_yaml_simple", async () => { - const documentBytes = getFixture("yaml/simple.yaml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/x-yaml", config); - } catch (error) { - if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/x-yaml"]); - assertions.assertMinContentLength(result, 10); - }); - + it("structured_csv_basic", async () => { + const documentBytes = getFixture("csv/stanley_cups.csv"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "text/csv", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_csv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/csv"]); + assertions.assertMinContentLength(result, 20); + }); + + it("structured_enw_basic", async () => { + const documentBytes = getFixture("data_formats/sample.enw"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-endnote-refer", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_enw_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-endnote-refer", "application/x-endnote+xml", "text/plain"]); + }); + + it("structured_json_basic", async () => { + const documentBytes = getFixture("json/sample_document.json"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/json", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 20); + assertions.assertContentContainsAny(result, ["Sample Document", "Test Author"]); + }); + + it("structured_json_simple", async () => { + const documentBytes = getFixture("json/simple.json"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/json", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_json_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/json"]); + assertions.assertMinContentLength(result, 10); + assertions.assertContentContainsAny(result, ["{", "name"]); + }); + + it("structured_nbib_basic", async () => { + const documentBytes = getFixture("data_formats/sample.nbib"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/nbib", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_nbib_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/nbib", "application/x-pubmed", "text/plain"]); + assertions.assertContentNotEmpty(result); + }); + + it("structured_ris_basic", async () => { + const documentBytes = getFixture("data_formats/sample.ris"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-research-info-systems", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_ris_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-research-info-systems", "text/plain"]); + assertions.assertContentNotEmpty(result); + }); + + it("structured_toml_basic", async () => { + const documentBytes = getFixture("data_formats/cargo.toml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/toml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_toml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/toml", "text/toml"]); + assertions.assertMinContentLength(result, 10); + }); + + it("structured_tsv_basic", async () => { + const documentBytes = getFixture("data_formats/employees.tsv"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "text/tab-separated-values", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_tsv_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["text/tab-separated-values", "text/plain"]); + assertions.assertMinContentLength(result, 10); + }); + + it("structured_yaml_basic", async () => { + const documentBytes = getFixture("yaml/simple.yaml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/yaml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_basic", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml"]); + assertions.assertMinContentLength(result, 10); + }); + + it("structured_yaml_simple", async () => { + const documentBytes = getFixture("yaml/simple.yaml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/x-yaml", config); + } catch (error) { + if (shouldSkipFixture(error, "structured_yaml_simple", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/x-yaml"]); + assertions.assertMinContentLength(result, 10); + }); }); diff --git a/e2e/wasm-workers/tests/xml.spec.ts b/e2e/wasm-workers/tests/xml.spec.ts index 7222cf375d7..cdd7cb3de01 100644 --- a/e2e/wasm-workers/tests/xml.spec.ts +++ b/e2e/wasm-workers/tests/xml.spec.ts @@ -9,28 +9,27 @@ import { assertions, buildConfig, getFixture, shouldSkipFixture } from "./helper import type { ExtractionResult } from "@kreuzberg/wasm"; describe("xml", () => { - it("xml_plant_catalog", async () => { - const documentBytes = getFixture("xml/plant_catalog.xml"); - if (documentBytes === null) { - console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); - return; - } - - const config = buildConfig(undefined); - let result: ExtractionResult | null = null; - try { - result = await extractBytes(documentBytes, "application/xml", config); - } catch (error) { - if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { - return; - } - throw error; - } - if (result === null) { - return; - } - assertions.assertExpectedMime(result, ["application/xml"]); - assertions.assertMinContentLength(result, 100); - }); + it("xml_plant_catalog", async () => { + const documentBytes = getFixture("xml/plant_catalog.xml"); + if (documentBytes === null) { + console.warn("[SKIP] Test skipped: fixture not available in Cloudflare Workers environment"); + return; + } + const config = buildConfig(undefined); + let result: ExtractionResult | null = null; + try { + result = await extractBytes(documentBytes, "application/xml", config); + } catch (error) { + if (shouldSkipFixture(error, "xml_plant_catalog", [], undefined)) { + return; + } + throw error; + } + if (result === null) { + return; + } + assertions.assertExpectedMime(result, ["application/xml"]); + assertions.assertMinContentLength(result, 100); + }); });