From bb57f202b16471c549121998aa7b43b3e353b004 Mon Sep 17 00:00:00 2001 From: jonathanwxh-cell <258340642+jonathanwxh-cell@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:52:32 +0800 Subject: [PATCH 1/5] M8: redaction: clear redacted strings from /Info and XMP metadata (#2448) --- Cargo.lock | 1 + ROADMAP.md | 1 + crates/automation/tests/automation.rs | 28 ++++ crates/redact/Cargo.toml | 1 + crates/redact/README.md | 36 ++++- crates/redact/src/interp.rs | 32 +++- crates/redact/src/lib.rs | 198 ++++++++++++++++++++++++- crates/redact/src/tests.rs | 203 ++++++++++++++++++++++++++ parity/acrobat-features.toml | 6 +- xtask/src/layers.rs | 1 + 10 files changed, 501 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91ed3c4904..1b772b43fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4282,6 +4282,7 @@ dependencies = [ "pdfcraft-fonts", "pdfcraft-forms", "pdfcraft-model", + "pdfcraft-organize", "pdfcraft-testkit", "pdfcraft-xfa", "thiserror 2.0.21", diff --git a/ROADMAP.md b/ROADMAP.md index 66b9677569..5bfcf1cd7e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -157,6 +157,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md | 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 | ## Progress log +- **2026-10-15 (M8, #2448):** Applying redactions clears what the marks removed from document metadata too: `/Info` entries whose value repeats a removed text run (case-insensitive) go with their mirrored XMP property, and an XMP stream that still holds removed text where per-entry editing cannot reach — custom schemas, entity-escaped values, page-level packets, or a stream that cannot be decoded and read — is removed whole, like the hidden-information metadata category. Unrelated entries and the rest of an editable packet survive. Text inside images and runs under two characters leave no searchable string, so Remove hidden information (metadata) remains the certain path. `redact` now uses `organize` (registered as a sideways edge). Estimates unchanged. - **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. - **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged. - **2026-10-13 (community, M3, #1569 review):** Pixel-only Wayland wheel, wheel-tilt and continuous input use standalone moved events, so Ctrl zoom does not remain latched after the modifier is released (#991); vendored winit patch (13), with a policy test. Estimates unchanged. diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 1e662d8f38..1a27ece32f 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -3295,6 +3295,34 @@ fn redacting_through_tools() { assert!(matches!(a.call("redact_mark", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_)))); } +/// Issue #2448: applying redactions clears document information entries that repeat the redacted +/// text; unrelated entries stay, and the saved file keeps no trace of the secret. +#[test] +fn redacting_clears_matching_document_info_through_tools() { + let dir = workdir("redact-info"); + std::fs::write(dir.join("memo.txt"), "Project Nimbus clearance is TOPSECRETX today\nPublic line").unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "memo.txt" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_set_info", json!({ "doc": doc, "key": "Title", "value": "TOPSECRETX clearance memo" })); + ok(&mut a, "doc_set_info", json!({ "doc": doc, "key": "Author", "value": "Ada" })); + ok(&mut a, "redact_mark", json!({ "doc": doc, "find": "TOPSECRETX" })); + ok(&mut a, "redact_apply", json!({ "doc": doc })); + let info = ok(&mut a, "doc_info", json!({ "doc": doc })); + assert_eq!(info["title"].as_str(), None, "the title repeating redacted text is gone"); + assert_eq!(info["author"].as_str(), Some("Ada"), "unrelated entries stay"); + let text = page_text(&mut a, doc)[0].clone(); + assert!(!text.contains("TOPSECRETX") && text.contains("Public line"), "{text}"); + ok(&mut a, "doc_save", json!({ "doc": doc, "path": "memo.pdf" })); + let bytes = std::fs::read(dir.join("memo.pdf")).unwrap(); + assert!(!bytes.windows(10).any(|w| w == b"TOPSECRETX"), "the saved file has no trace of the secret"); + // A full write may pack /Info into a compressed object stream, so survival is checked by + // opening the saved file, not by scanning its bytes. + let saved = ok(&mut a, "doc_open", json!({ "path": "memo.pdf" }))["doc"].as_u64().unwrap(); + let info = ok(&mut a, "doc_info", json!({ "doc": saved })); + assert_eq!(info["author"].as_str(), Some("Ada"), "unrelated metadata survives in the saved file"); + assert_eq!(info["title"].as_str(), None, "the secret title is gone from the saved file"); +} + #[test] fn redacting_with_codes_through_tools() { let dir = workdir("redact-codes"); diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index 49593855e2..d1cba3643f 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -17,6 +17,7 @@ pdfcraft-forms = { workspace = true } pdfcraft-edit = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } +pdfcraft-organize = { workspace = true } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/redact/README.md b/crates/redact/README.md index 3b2d54e814..600c15b691 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -1,6 +1,6 @@ # pdfcraft-redact -Layer L4 (may use `edit`): redaction, architecture §11.2. +Layer L4 (may use `edit`, `organize`): redaction, architecture §11.2. ```rust // Marks are Redact annotations (pdfcraft-annot: Shape::Redact { quads, overlay }). @@ -27,6 +27,11 @@ clear_marks(&mut doc, None)?; // remove marks without app - **form XObjects**: rewritten recursively into new objects (pages sharing the original keep it); - **annotations** whose rectangle overlaps (with their pop-ups), and **form fields** with a widget under a mark; +- **document metadata** that repeats removed text: `/Info` entries whose value contains a removed + text run (case-insensitive) are cleared — through `pdfcraft_organize::set_info`, which also + removes the XMP property mirroring each entry (`dc:title`, `dc:creator`, `dc:description`, + `pdf:Keywords`, `xmp:CreatorTool`, `pdf:Producer`) while leaving the rest of the packet + byte-for-byte intact; - the marks themselves, replaced by boxes in their fill colour (and overlay text) drawn into the page as a `/PCMark /Redaction` stream. @@ -34,6 +39,35 @@ A verification pass re-reads each redacted page and fails the operation if any g image is still under an area. Content that can't be decoded makes the operation fail rather than leave it unredacted. +## Metadata redaction (#2448) + +What the marks removed must not survive in metadata either. The interpreter decodes every glyph +run it cuts out into *fragments*: maximal runs that join across `TJ` kerning numbers and split at +kept glyphs, at codes with no Unicode meaning, and at operator ends. After verification, each +`/Info` string entry whose value contains a fragment (lowercased, two characters or more) — or +is contained in one, for a value shorter than the run; the author of a whole redacted sentence +is removed text too — is cleared with its mirrored XMP property; then every catalog and page +`/Metadata` stream is checked in its raw bytes **and** in an entity-unescaped view of them (XMP +stores `AT&T` as `AT&T`), and value by value (each tag's text, each quoted attribute), +and removed whole when a fragment or part of one survives there — custom schemas cannot be +edited per-entry — or when the stream cannot be decoded and read as UTF-8 within 16 MiB. A +trailer `/Info` that is a direct dictionary (the one shape an entry clear cannot edit) is +removed whole when it still matches. All of this happens before the forced full save, so the +previous revision's bytes leave the file. + +The match errs towards clearing, in both directions: + +- **over-clearing**: a fully redacted common word is a fragment, and an unrelated entry or packet + merely containing it goes too; the wholesale fallback drops a whole stream for one + custom-schema hit; +- **under-matching**: text inside images never becomes a fragment, runs shorter than two + characters are ignored, fragments do not join across operators, and `/Metadata` on component + objects (images, fonts, form XObjects) is not scanned — the same scope as Remove hidden + information. + +Remove hidden information (metadata category, `doc_remove_hidden`) is therefore the certain +follow-up when a document's metadata must be provably free of the redacted text. + Before changing any page, `apply` prepares the redaction appearances with a shared 8 MiB budget for their uncompressed PDF content across the selected pages. It bounds repeated-text counts and working line buffers before allocating them, and measures and encodes each repeated diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index c76e119953..3db7a8db46 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -85,6 +85,10 @@ pub(crate) struct Scope<'a> { pub hidden_layers: Vec, /// Content removed from hidden layers (blocks and XObjects). pub layer_blocks: usize, + /// Redaction (apply mode): the text of each maximal run of removed glyphs, for the metadata + /// pass. A run joins across `TJ` kerning numbers and splits at kept glyphs, at codes with no + /// Unicode meaning, and at operator ends. + pub removed_fragments: Vec, fonts: HashMap, Rc>, used_names: Vec>, depth: usize, @@ -99,6 +103,7 @@ impl<'a> Scope<'a> { hidden_text: None, hidden_layers: Vec::new(), layer_blocks: 0, + removed_fragments: Vec::new(), fonts: HashMap::new(), used_names: Vec::new(), depth: 0, @@ -629,11 +634,22 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< /// Lay out a shown string (or `TJ` array) glyph by glyph, advancing `tm`. Returns the `TJ` /// items with the glyphs under a region replaced by displacements, and how many were removed. -fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { +fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { let f = &gs.font; let size = gs.size; let mut out: Vec = Vec::new(); let mut removed = 0; + // Redaction only: collect what is removed, for the metadata pass. Hidden-text sanitising + // (empty regions or a page box) collects nothing. + let collect = scope.mode == Mode::Apply && !scope.rects.is_empty() && scope.hidden_text.is_none(); + // The removed text of this one show() call: joined across `TJ` kerning numbers, split by + // kept glyphs and codes with no Unicode meaning. + let mut frag = String::new(); + let flush = |frag: &mut String, scope: &mut Scope<'_>| { + if !frag.is_empty() { + scope.removed_fragments.push(std::mem::take(frag)); + } + }; let push_num = |out: &mut Vec, v: f64| { if v == 0.0 { return; @@ -675,6 +691,16 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec) -> Option<[f64; 4]> { @@ -336,6 +347,8 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result = Vec::new(); let widget_owner: Vec<(ObjRef, String)> = pdfcraft_forms::fields(doc).into_iter().flat_map(|f| f.widgets.iter().map(|w| (w.obj, f.name.clone())).collect::>()).collect(); + // The text the marks removed (maximal runs), for the metadata pass below. + let mut fragments: Vec = Vec::new(); for &(pi, page_marks) in &groups { let page = pdfcraft_model::pages(doc).swap_remove(pi); @@ -346,7 +359,9 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result) -> Result 0 { return Err(RedactError::Residue(residue)); } + // 6. Metadata: what the marks removed must not survive in `/Info` or XMP either (#2448). + // Before the full-save flag, so the previous revision's metadata leaves the file too. + clear_marked_metadata(doc, &fragments, &mut report)?; // The previous revision still holds the removed content: the next save must rewrite. doc.require_full_save(); Ok(report) } +/// Decoding limit for a metadata stream (XMP packets are a few kB; padding rarely takes them +/// past 100 kB — the same cap `pdfcraft_organize` reads packets with). +const MAX_METADATA: usize = 16 << 20; + +/// `s` with XML entity references resolved: the five named ones and numeric character +/// references. XMP stores values escaped (`AT&T` as `AT&T`), so removed text has to be +/// looked for in an unescaped view of a packet too. Anything malformed stays as it stands. +fn xml_unescape(s: &str) -> Cow<'_, str> { + if !s.contains('&') { + return Cow::Borrowed(s); + } + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len()); + let mut i = 0; + while i < s.len() { + if bytes[i] == b'&' + && let Some(end) = s[i..].find(';').map(|e| i + e).filter(|e| *e - i <= 12) + { + let ent = &s[i + 1..end]; + let c = match ent { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + _ => ent + .strip_prefix("#x") + .and_then(|h| u32::from_str_radix(h, 16).ok()) + .or_else(|| ent.strip_prefix('#').and_then(|d| d.parse::().ok())) + .and_then(char::from_u32), + }; + if let Some(c) = c { + out.push(c); + i = end + 1; + continue; + } + } + match s[i..].chars().next() { + Some(c) => { + out.push(c); + i += c.len_utf8(); + } + None => break, + } + } + Cow::Owned(out) +} + +/// Does the XMP packet `text` hold removed text? Scanned whole, raw and entity-unescaped, for a +/// removed run; and value by value — each tag's text and each quoted attribute, unescaped — for +/// a value that is only part of one (a name inside a redacted sentence). Structure is read from +/// the raw text, where an escaped `<` is not a tag, before each value is unescaped. +fn packet_hit(text: &str, hit: &impl Fn(&str) -> bool) -> bool { + hit(text) || hit(&xml_unescape(text)) || xml_chunks(text).into_iter().any(|c| hit(&xml_unescape(c))) +} + +/// The places an XML packet can hold a value: the text between tags and the quoted attribute +/// values inside them, as they stand (the caller unescapes). Lenient: malformed markup just +/// yields fewer places. +fn xml_chunks(xmp: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut rest = xmp; + while let Some(open) = rest.find('<') { + if open > 0 { + out.push(&rest[..open]); + } + // A tag with no `>` ends the scan; the tail is then text. + let Some(rel) = rest[open..].find('>') else { break }; + let tag = &rest[open + 1..open + rel]; + rest = &rest[open + rel + 1..]; + let bytes = tag.as_bytes(); + let mut i = 0; + while i < tag.len() { + let q = bytes[i]; + if (q == b'"' || q == b'\'') && i > 0 && bytes[i - 1] == b'=' { + let value = &tag[i + 1..]; + let len = value.find(q as char).unwrap_or(value.len()); + out.push(&value[..len]); + i += len + 2; + } else { + i += 1; + } + } + } + if !rest.is_empty() { + out.push(rest); + } + out +} + +/// The metadata half of redaction (#2448): a string the marks took off the pages must not stay +/// in the document information dictionary or the XMP metadata. Runs of removed text two +/// characters or longer are matched case-insensitively (a single clipped glyph is no evidence a +/// secret was removed, and would clear unrelated entries that merely contain that character). +/// +/// Per entry first, so unrelated values survive: each `/Info` string entry whose value contains +/// a removed run — or is contained in one, for a value shorter than the run (the author of a +/// whole redacted sentence is removed text too) — goes through `pdfcraft_organize::set_info`, +/// which also removes the XMP property mirroring it and leaves the rest of the packet byte for +/// byte. Then, wholesale where per-entry editing cannot reach: every catalog and page +/// `/Metadata` stream is kept only when it decodes, reads as UTF-8, and holds no removed run or +/// part of one (raw, entity-unescaped, or value by value — custom schemas escape their values); +/// a trailer `/Info` that is a direct dictionary — the one shape an entry clear cannot edit — +/// is removed whole when it still matches. Matching errs towards clearing; text that leaves no +/// searchable run (image text, one surviving piece) needs `sanitize`'s metadata category to be +/// provably gone. +fn clear_marked_metadata(doc: &mut Document, fragments: &[String], report: &mut Report) -> Result<(), RedactError> { + let needles: Vec = fragments.iter().map(|f| f.trim().to_lowercase()).filter(|f| f.chars().count() >= 2).collect(); + if needles.is_empty() { + return Ok(()); + } + // A value goes when it holds a removed run, or when it is itself part of one. The second + // direction never matches a value under two characters: one shared character is no evidence + // a secret was removed. + let hit = |v: &str| { + let v = v.to_lowercase(); + let part = v.trim(); + needles.iter().any(|n| v.contains(n.as_str())) || (part.chars().count() >= 2 && needles.iter().any(|n| n.contains(part))) + }; + + // 1. Entry by entry: the value goes, and with it the XMP property that mirrors the entry. + let keys: Vec = doc + .trailer() + .get(b"Info") + .map(|o| doc.resolve(o)) + .and_then(|o| o.as_dict().map(|d| d.iter().filter_map(|(k, _)| String::from_utf8(k.clone()).ok()).collect())) + .unwrap_or_default(); + for key in keys { + if pdfcraft_organize::info(doc, &key).is_some_and(|v| hit(&v)) { + pdfcraft_organize::set_info(doc, &key, "").map_err(|e| pdfcraft_edit::EditError::Invalid(e.to_string()))?; + // A trailer `/Info` that is a direct dictionary is quietly left alone (only the + // wholesale step below can take it); count what really went. + if pdfcraft_organize::info(doc, &key).is_none() { + report.info_entries_cleared += 1; + } + } + } + + // 2. Wholesale: a stream that still holds removed text where per-entry editing cannot reach + // (custom schemas, page-level packets), or that cannot be inspected at all, goes whole. + let mut owners: Vec = doc.root().into_iter().collect(); + owners.extend(pdfcraft_model::pages(doc).iter().map(|p| p.obj)); + for owner in owners { + let Some(entry) = doc.get(owner).as_dict().and_then(|d| d.get(b"Metadata").cloned()) else { continue }; + let keeps = match &*doc.resolve(&entry) { + Object::Stream(s) => match s.decoded_within(MAX_METADATA) { + Ok(bytes) => match String::from_utf8(bytes) { + Ok(text) => !packet_hit(&text, &hit), + Err(_) => false, + }, + Err(_) => false, + }, + _ => false, + }; + if !keeps { + doc.update_dict(owner, |d| { + d.remove(b"Metadata"); + })?; + report.xmp_removed += 1; + } + } + + // 3. The dictionary one last time: a direct trailer dictionary cannot be edited per entry, + // and a value that survived step 1 takes the whole `/Info` with it (the sanitize shape). + if doc + .trailer() + .get(b"Info") + .map(|o| doc.resolve(o)) + .and_then(|o| o.as_dict().map(|d| d.iter().any(|(_, v)| doc.resolve(v).as_string().is_some_and(|s| hit(&s.to_text()))))) + .unwrap_or(false) + { + let n = doc.trailer().get(b"Info").map(|o| doc.resolve(o)).and_then(|o| o.as_dict().map(Dict::len)).unwrap_or(0); + doc.trailer_mut().remove(b"Info"); + report.info_entries_cleared += n; + } + Ok(()) +} + /// Remove redaction marks without applying them (`None` = all). pub fn clear_marks(doc: &mut Document, pages: Option<&[usize]>) -> Result { let all = marks(doc); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index b9a230fc41..aa15e8fc57 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -631,6 +631,209 @@ fn hidden_information_is_counted_and_removed() { assert!(shown, "the field's value stays visible as page content"); } +/// A one-page document (see `one_page`) plus `/Info` entries, a catalog XMP packet and a +/// page-level XMP packet, for the metadata-redaction tests (#2448). +fn meta_doc(content: &[u8], info: &[(&str, &str)], catalog_packet: Option<&str>, page_packet: Option<&str>) -> Document { + let mut doc = one_page(content, "", vec![]); + let metadata = |p: &str| { + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("Metadata")); + d.set(b"Subtype".to_vec(), Object::name("XML")); + Object::Stream(Stream::from_raw(d, p.as_bytes().to_vec())) + }; + if let Some(p) = catalog_packet { + let r = doc.add(metadata(p)); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| d.set(b"Metadata".to_vec(), Object::Ref(r))).unwrap(); + } + if let Some(p) = page_packet { + let r = doc.add(metadata(p)); + let page = pdfcraft_model::pages(&doc)[0].obj; + doc.update_dict(page, |d| d.set(b"Metadata".to_vec(), Object::Ref(r))).unwrap(); + } + if !info.is_empty() { + let mut d = Dict::new(); + for (k, v) in info { + d.set(k.as_bytes().to_vec(), Object::String(pdfcraft_cos::PdfString::text(v))); + } + let r = doc.add(Object::Dict(d)); + doc.trailer_mut().set(b"Info".to_vec(), Object::Ref(r)); + } + doc +} + +/// One `/Info` entry as text, straight from the trailer dictionary. +fn info_value(doc: &Document, key: &str) -> Option { + let info = doc.trailer().get(b"Info").map(|o| doc.resolve(o))?; + info.as_dict()?.get(key.as_bytes()).and_then(|v| doc.resolve(v).as_string().map(|s| s.to_text())) +} + +/// The catalog's XMP packet, decoded. +fn xmp_text(doc: &Document) -> Option { + let r = doc.get(doc.root()?).as_dict()?.reference(b"Metadata")?; + match &*doc.get(r) { + Object::Stream(s) => Some(String::from_utf8_lossy(&s.decoded().ok()?).into_owned()), + _ => None, + } +} + +/// Issue #2448: a string redacted off the page must not survive in `/Info` (here Title, while +/// Author survives) or in the XMP property mirroring it (here `dc:title`, while the rest of the +/// packet stays). The match is case-insensitive: the page shows TOPSECRET, the title doesn't. +#[test] +fn redacted_text_is_cleared_from_info_and_xmp_on_apply() { + let packet = "\ +TopSecret plan\ +Writer 1.0\ +"; + let clean = "\ +Writer\ +"; + let mut doc = + meta_doc(b"BT /F1 10 Tf 10 100 Td (TOPSECRET PLAN) Tj ET", &[("Title", "TopSecret plan"), ("Author", "Ada")], Some(packet), Some(clean)); + assert_eq!(under(&mut doc, 0, &[[10.0, 95.0, 55.0, 110.0]]), 9, "TOPSECRET is nine glyphs"); + mark(&mut doc, 0, &[[10.0, 95.0, 55.0, 110.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.info_entries_cleared, 1, "the Title entry and nothing else: {r:?}"); + let doc = reopen(&doc); + assert_eq!(info_value(&doc, "Title"), None, "the entry repeating removed text is gone"); + assert_eq!(info_value(&doc, "Author").as_deref(), Some("Ada"), "unrelated entries stay"); + let xmp = xmp_text(&doc).unwrap_or_default(); + assert!(!xmp.contains("dc:title"), "the mirrored property went with it: {xmp}"); + assert!(xmp.contains("Writer 1.0"), "the rest of the packet stays: {xmp}"); + let page = &pdfcraft_model::pages(&doc)[0]; + assert!(page.dict.get(b"Metadata").is_some(), "a clean page-level stream stays"); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).to_lowercase().contains("topsecret"), "no trace in the saved file"); +} + +/// Issue #2448, review follow-up: a value SHORTER than the removed run is still removed text — +/// the author of a whole redacted sentence — whether `/Info` or a custom XMP schema holds it. +#[test] +fn a_value_inside_a_longer_removed_run_is_cleared_too() { + let packet = "\ +James Bond\ +Writer 1.0\ +"; + let mut doc = + meta_doc(b"BT /F1 10 Tf 10 100 Td (JAMES BOND IS A SPY) Tj ET", &[("Author", "James Bond"), ("Subject", "Spy stories")], Some(packet), None); + assert_eq!(under(&mut doc, 0, &[[10.0, 95.0, 105.0, 110.0]]), 19, "the whole phrase is nineteen glyphs"); + mark(&mut doc, 0, &[[10.0, 95.0, 105.0, 110.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.info_entries_cleared, 1, "the Author entry: {r:?}"); + assert_eq!(r.xmp_removed, 1, "the packet holding the name in its custom schema: {r:?}"); + let doc = reopen(&doc); + assert_eq!(info_value(&doc, "Author"), None, "the shorter value is part of the removed text"); + assert_eq!(info_value(&doc, "Subject").as_deref(), Some("Spy stories"), "an unrelated value stays"); + let cat = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + assert!(!cat.contains(b"Metadata"), "the custom-schema value took the packet with it"); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).to_lowercase().contains("james bond"), "no trace in the saved file"); + let reread = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(info_value(&reread, "Subject").as_deref(), Some("Spy stories"), "unrelated metadata survives in the saved file"); +} + +/// Issue #2448: the fallback. A secret held where PdfCraft cannot edit per-entry — a custom XMP +/// schema, entity-escaped, and a page-level packet — takes the whole stream with it, the same +/// shape as removing hidden information's metadata category. +#[test] +fn an_xmp_stream_holding_redacted_text_is_removed_whole() { + let packet = "\ +<SECRET&PLAN>\ +Writer 1.0\ +"; + let page_packet = "\ +OTHERTEXT\ +"; + let mut doc = meta_doc(b"BT /F1 10 Tf 10 100 Td (MEMO OTHERTEXT HERE) Tj ET", &[], Some(packet), Some(page_packet)); + assert_eq!(under(&mut doc, 0, &[[35.0, 95.0, 100.0, 110.0]]), 13, " is thirteen glyphs"); + mark(&mut doc, 0, &[[35.0, 95.0, 100.0, 110.0], [105.0, 95.0, 150.0, 110.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.xmp_removed, 2, "the catalog and page streams: {r:?}"); + let doc = reopen(&doc); + let cat = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + assert!(!cat.contains(b"Metadata"), "the escaped secret took the catalog stream with it"); + let page = &pdfcraft_model::pages(&doc)[0]; + assert!(page.dict.get(b"Metadata").is_none(), "the page-level stream went too"); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + let text = String::from_utf8_lossy(&bytes).to_lowercase(); + assert!(!text.contains("secret&plan") && !text.contains("secret&plan"), "no escaped trace in the saved file"); + assert!(!text.contains("othertext"), "the page-level secret is gone too"); +} + +/// Issue #2448: a trailer `/Info` that is a direct dictionary is the one shape an entry-by-entry +/// clear cannot edit, so the final check removes it whole (the hidden-information shape); a save +/// then writes a fresh date-only `/Info`. +#[test] +fn a_direct_info_dictionary_holding_redacted_text_is_removed_whole() { + let mut doc = one_page(b"BT /F1 10 Tf 10 100 Td (TOPSECRET PLAN) Tj ET", "", vec![]); + let mut info = Dict::new(); + info.set(b"Title".to_vec(), Object::String(pdfcraft_cos::PdfString::text("TopSecret plan"))); + info.set(b"Author".to_vec(), Object::String(pdfcraft_cos::PdfString::text("Ada"))); + doc.trailer_mut().set(b"Info".to_vec(), Object::Dict(info)); + mark(&mut doc, 0, &[[10.0, 95.0, 55.0, 110.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.info_entries_cleared, 2, "the whole dictionary, both entries: {r:?}"); + assert!(doc.trailer().get(b"Info").is_none(), "the whole dictionary went"); + let doc = reopen(&doc); + let info = doc.trailer().get(b"Info").map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()); + if let Some(d) = &info { + assert!(!d.contains(b"Title") && !d.contains(b"Author"), "nothing of it came back: {d:?}"); + } + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).to_lowercase().contains("topsecret"), "no trace in the saved file"); +} + +/// Issue #2448: a non-ASCII title is stored as UTF-16BE (`PdfString::text`), and the removed +/// run's Euro sign is only known through the font's ToUnicode map; the match still has to find it. +#[test] +fn utf16_info_entries_holding_redacted_text_are_cleared() { + let packet = "\ +Writer 1.0\ +"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /Metadata 8 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 10 100 Td (TOPSECRET$PLAN) Tj ET"), + // The font of `one_page`, plus a ToUnicode map that reads code $ as U+20AC (€). + FONT.replace("95 0 R", "6 0 R").replace("/FontDescriptor", "/ToUnicode 7 0 R /FontDescriptor").into_bytes(), + widths(), + stream("", b"1 beginbfchar <24> <20AC> endbfchar"), + stream("/Type /Metadata /Subtype /XML", packet.as_bytes()), + ]); + let mut info = Dict::new(); + info.set(b"Title".to_vec(), Object::String(pdfcraft_cos::PdfString::text("Classified SECRET€ memo"))); + info.set(b"Author".to_vec(), Object::String(pdfcraft_cos::PdfString::text("Ada"))); + let r = doc.add(Object::Dict(info)); + doc.trailer_mut().set(b"Info".to_vec(), Object::Ref(r)); + assert_eq!(under(&mut doc, 0, &[[25.0, 95.0, 60.0, 110.0]]), 7, "SECRET$ is seven glyphs"); + mark(&mut doc, 0, &[[25.0, 95.0, 60.0, 110.0]], ""); + apply(&mut doc, None).unwrap(); + let doc = reopen(&doc); + assert_eq!(info_value(&doc, "Title"), None, "the UTF-16BE title repeating removed text is gone"); + assert_eq!(info_value(&doc, "Author").as_deref(), Some("Ada"), "unrelated entries stay"); + let needle: Vec = "SECRET€".encode_utf16().flat_map(u16::to_be_bytes).collect(); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!bytes.windows(needle.len()).any(|w| w == needle), "no UTF-16BE trace in the saved file"); + // A full write may pack /Info into a compressed object stream, so survival is checked by + // reading the file back, not by scanning its bytes. + let reread = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(info_value(&reread, "Author").as_deref(), Some("Ada"), "unrelated metadata survives"); +} + +/// Issue #2448: with nothing to apply, metadata is left alone. +#[test] +fn applying_with_no_marks_leaves_metadata_alone() { + let packet = "\ +TopSecret plan\ +"; + let mut doc = meta_doc(b"BT /F1 10 Tf 10 100 Td (PLAIN) Tj ET", &[("Title", "TopSecret plan")], Some(packet), None); + assert_eq!(apply(&mut doc, None), Err(RedactError::NothingToApply)); + assert_eq!(info_value(&doc, "Title").as_deref(), Some("TopSecret plan"), "the entry stays"); + assert!(xmp_text(&doc).is_some_and(|x| x.contains("TopSecret")), "the packet stays"); +} + /// Issue #1325: removing comments keeps the annotations that are not comments: the multimedia /// and print-production ones (Watermark, PrinterMark, TrapNet, Screen, Movie, RichMedia, 3D). #[test] diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 533013b547..24bf70340d 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5680,8 +5680,8 @@ milestone = "M8" status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] -evidence = ["crates/ui-egui/tests/ui_group/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::redaction_is_refused_when_the_page_still_shows_text_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources"] -notes = "confirmation; one undo step; next save is a full rewrite. Refused, with nothing changed, when the page still shows text under a mark afterwards: text whose font is set through an ExtGState or taken by a form from its caller, some predefined CMaps and rotated Type 3 fonts are not removed yet, so those documents cannot be redacted. Removed images and forms are unlinked from page-tree resources (own, inherited or shared), not yet from a form's own resources, annotation appearances or patterns" +evidence = ["crates/ui-egui/tests/ui_group/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::redaction_is_refused_when_the_page_still_shows_text_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources", "crates/redact/src/tests.rs::redacted_text_is_cleared_from_info_and_xmp_on_apply", "crates/redact/src/tests.rs::an_xmp_stream_holding_redacted_text_is_removed_whole", "crates/redact/src/tests.rs::a_value_inside_a_longer_removed_run_is_cleared_too", "crates/automation/tests/automation.rs::redacting_clears_matching_document_info_through_tools"] +notes = "confirmation; one undo step; next save is a full rewrite. Refused, with nothing changed, when the page still shows text under a mark afterwards: text whose font is set through an ExtGState or taken by a form from its caller, some predefined CMaps and rotated Type 3 fonts are not removed yet, so those documents cannot be redacted. Removed images and forms are unlinked from page-tree resources (own, inherited or shared), not yet from a form's own resources, annotation appearances or patterns. /Info entries repeating a removed text run — or forming part of one, for a value shorter than the run — are cleared with their mirrored XMP property, and an XMP stream still holding removed text where per-entry editing cannot reach (custom schemas, entity-escaped values, page-level packets) or that cannot be decoded and read is removed whole; text inside images, runs under two characters and component-level /Metadata are not matched, so remove hidden information (metadata) remains the certain path" [[feature]] id = "protect.redact-glyph-removal" @@ -5789,7 +5789,7 @@ status = "shipped" commands = ["protect.remove_hidden"] tools = ["doc_hidden_info", "doc_remove_hidden"] evidence = ["crates/redact/src/tests.rs::hidden_information_is_counted_and_removed", "crates/ui-egui/tests/ui_group/redact.rs::removing_hidden_information_and_sanitizing", "crates/automation/tests/automation.rs::removing_hidden_information_through_tools"] -notes = "metadata, attachments, comments, form fields (flattened), hidden text, hidden layers, bookmarks, links/actions/scripts, private data; overlapping objects and cropped content pending" +notes = "metadata, attachments, comments, form fields (flattened), hidden text, hidden layers, bookmarks, links/actions/scripts, private data; overlapping objects and cropped content pending; the certain metadata follow-up after apply-redactions, whose matcher cannot prove an entry clean (image text, short runs)" [[feature]] id = "protect.sanitize" diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index a0907cb1a6..d302bd62dd 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -97,6 +97,7 @@ pub const STANDALONE_DEPS: &[(&str, &[&str])] = &[("cos", &["filters", "crypt", /// Allowed same-layer edges (architecture §3 rule 1). `forms → js` is deliberately absent. pub const SIDEWAYS: &[(&str, &str)] = &[ ("redact", "edit"), + ("redact", "organize"), ("audit", "edit"), ("a11y", "preflight"), ("prepress", "preflight"), From 9248b8aa2368df115737f8732be307c694d56e3d Mon Sep 17 00:00:00 2001 From: jonathanwxh-cell <258340642+jonathanwxh-cell@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:54:20 +0800 Subject: [PATCH 2/5] fix(redact): join removed glyphs across operators by position (#2448) Many generators draw text one glyph per operator ((T) Tj 5 0 Td (O) Tj ...). The metadata pass collected each operator's removed text separately, so such text became one-character fragments, the two-character needle floor filtered them all, and a /Title repeating the removed text survived (#2598). The interpreter now carries the open run in the Scope and joins by device-space position: a removed glyph continues the run when it starts within 1.5 glyph heights of where the last one's advance - character and word spacing included, so tracking never reads as a word gap - landed; a repositioning gap past 0.15 heights becomes a real space. TJ kern numbers keep their old meaning of spacing inside one text run: the glyph after a number joins with no space and no split, so a kerned intra-word value still clears. Kept glyphs, codes with no Unicode meaning and page ends still split runs. Matching itself is unchanged: whole-run needles and both two-character floors, so unrelated entries like a /Subject (Spy stories) beside a redacted sentence still stay. --- crates/redact/README.md | 26 +++++--- crates/redact/src/interp.rs | 93 +++++++++++++++++++++++----- crates/redact/src/lib.rs | 1 + crates/redact/src/tests.rs | 118 ++++++++++++++++++++++++++++++++++++ 4 files changed, 215 insertions(+), 23 deletions(-) diff --git a/crates/redact/README.md b/crates/redact/README.md index 600c15b691..2866e00362 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -42,9 +42,14 @@ leave it unredacted. ## Metadata redaction (#2448) What the marks removed must not survive in metadata either. The interpreter decodes every glyph -run it cuts out into *fragments*: maximal runs that join across `TJ` kerning numbers and split at -kept glyphs, at codes with no Unicode meaning, and at operator ends. After verification, each -`/Info` string entry whose value contains a fragment (lowercased, two characters or more) — or +run it cuts out into *fragments*: maximal runs that join across `TJ` kerning numbers (spacing +inside one text run: no space, no split) **and operators** — text drawn one glyph per operator, +as many generators do, still collects as one run — whenever the next removed glyph starts within +1.5 glyph heights of where the last one's advance (character and word spacing included) landed +in device space; a repositioning gap past 0.15 heights becomes a space, and kept glyphs, codes +with no Unicode meaning, and page ends still split runs. +After verification, each `/Info` string entry whose value contains a fragment (lowercased, two +characters or more) — or is contained in one, for a value shorter than the run; the author of a whole redacted sentence is removed text too — is cleared with its mirrored XMP property; then every catalog and page `/Metadata` stream is checked in its raw bytes **and** in an entity-unescaped view of them (XMP @@ -58,12 +63,17 @@ previous revision's bytes leave the file. The match errs towards clearing, in both directions: - **over-clearing**: a fully redacted common word is a fragment, and an unrelated entry or packet - merely containing it goes too; the wholesale fallback drops a whole stream for one - custom-schema hit; + merely containing it goes too; glyphs removed under *different* marks also join when they land + within 1.5 em of each other in stream order, so two one-letter removals can form a two-character + needle; the wholesale fallback drops a whole stream for one custom-schema hit; - **under-matching**: text inside images never becomes a fragment, runs shorter than two - characters are ignored, fragments do not join across operators, and `/Metadata` on component - objects (images, fonts, form XObjects) is not scanned — the same scope as Remove hidden - information. + characters are ignored, fragments split at gaps wider than 1.5 em and carry narrower + repositioning gaps as spaces, so a value that spells the same text with or without one of those + spaces matches only the spelling the run collected — only glyphs shown in stream order at + adjacent positions join, so overprint-interleaved, character-batched or logical-order RTL + generators still leave one-character runs, and codes with no Unicode meaning still split + them — and `/Metadata` on component objects (images, fonts, form XObjects) is not scanned — the + same scope as Remove hidden information. Remove hidden information (metadata category, `doc_remove_hidden`) is therefore the certain follow-up when a document's metadata must be provably free of the redacted text. diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 3db7a8db46..36f03dbcb0 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -86,9 +86,16 @@ pub(crate) struct Scope<'a> { /// Content removed from hidden layers (blocks and XObjects). pub layer_blocks: usize, /// Redaction (apply mode): the text of each maximal run of removed glyphs, for the metadata - /// pass. A run joins across `TJ` kerning numbers and splits at kept glyphs, at codes with no - /// Unicode meaning, and at operator ends. + /// pass. A run joins across `TJ` kerning numbers (spacing inside one text run: no space, no + /// split) and across operators when the next removed glyph starts within 1.5 glyph heights + /// of where the last one's advance (spacing included) landed in device space — a + /// repositioning gap past 0.15 heights becomes a space — and splits at kept glyphs, at codes + /// with no Unicode meaning, and at page end. pub removed_fragments: Vec, + /// The run being collected: its text, and where its last removed glyph's advance ended in + /// device space with that glyph's height (`None` = no open run). + run: String, + run_end: Option<((f64, f64), f64)>, fonts: HashMap, Rc>, used_names: Vec>, depth: usize, @@ -104,6 +111,8 @@ impl<'a> Scope<'a> { hidden_layers: Vec::new(), layer_blocks: 0, removed_fragments: Vec::new(), + run: String::new(), + run_end: None, fonts: HashMap::new(), used_names: Vec::new(), depth: 0, @@ -163,6 +172,47 @@ impl<'a> Scope<'a> { ox > mx && oy > my }) } + + /// End the run of removed text being collected (nothing when it is empty). + pub(crate) fn flush_run(&mut self) { + self.run_end = None; + if !self.run.is_empty() { + self.removed_fragments.push(std::mem::take(&mut self.run)); + } + } + + /// A removed glyph: `kerned` says a `TJ` number displaced the text since the last removed + /// glyph — spacing inside one text run, which joins with no space and no split (a kern is not + /// a word gap). Otherwise the glyph continues the open run when it starts where the last + /// one's advance landed — device space, within 1.5 glyph heights, so text drawn one glyph per + /// operator still collects as one run — and a gap past 0.15 heights is a word gap (a real + /// space, or the joined text would not match the value it was drawn from). Degenerate + /// (zero-height) glyphs and non-finite positions never join. + fn removed_glyph(&mut self, text: &str, origin: (f64, f64), end: (f64, f64), height: f64, kerned: bool) { + if kerned && self.run_end.is_some() { + self.run.push_str(text); + self.run_end = Some((end, height)); + return; + } + if let Some(((lx, ly), lh)) = self.run_end + && lh > 0.0 + && height > 0.0 + { + let gap = (origin.0 - lx).hypot(origin.1 - ly); + let h = lh.max(height); + if gap.is_finite() && h.is_finite() && gap <= 1.5 * h { + if gap > 0.15 * h && !self.run.ends_with(char::is_whitespace) { + self.run.push(' '); + } + self.run.push_str(text); + self.run_end = Some((end, height)); + return; + } + } + self.flush_run(); + self.run.push_str(text); + self.run_end = Some((end, height)); + } } fn res_dict(doc: &Document, res: &Dict, key: &[u8]) -> Dict { @@ -352,6 +402,11 @@ struct Glyph { /// Advance in unscaled text space (before `Th`). advance: f64, hit: bool, + /// The glyph in device space: its origin, where its advance (spacing included) lands the + /// next glyph, and its height — for joining removed text into runs. + origin: (f64, f64), + end: (f64, f64), + height: f64, } /// Process a scope's content streams (state flows from one stream to the next). @@ -642,14 +697,6 @@ fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (V // Redaction only: collect what is removed, for the metadata pass. Hidden-text sanitising // (empty regions or a page box) collects nothing. let collect = scope.mode == Mode::Apply && !scope.rects.is_empty() && scope.hidden_text.is_none(); - // The removed text of this one show() call: joined across `TJ` kerning numbers, split by - // kept glyphs and codes with no Unicode meaning. - let mut frag = String::new(); - let flush = |frag: &mut String, scope: &mut Scope<'_>| { - if !frag.is_empty() { - scope.removed_fragments.push(std::mem::take(frag)); - } - }; let push_num = |out: &mut Vec, v: f64| { if v == 0.0 { return; @@ -662,6 +709,10 @@ fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (V } out.push(num(v)); }; + // A `TJ` number displaced the text since the last removed glyph: the glyph after it joins + // the run with no space and no split. Set by the number, spent by the next glyph drawn (so + // it never reaches the next operator). + let mut kerned = false; for item in items { match item { Object::String(s) => { @@ -675,6 +726,14 @@ fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (V let b = trm.bbox([0.0, f.descent, w0, f.ascent]); let origin = trm.apply(0.0, 0.0); let advance = w0 * size + spacing; + // Where this glyph's advance lands the next one (device space, spacing + // included, so character spacing is never read as a word gap), and the + // glyph's height there: `trm.apply(x, 0)` maps text space through + // `tm.apply(size·Th·x, rise)`, so `advance / size` is exactly the next + // origin while nothing repositions the text in between. + let top = trm.apply(0.0, 1.0); + let height = (top.0 - origin.0).hypot(top.1 - origin.1); + let end = if size != 0.0 { trm.apply(advance / size, 0.0) } else { origin }; let hit = match scope.hidden_text { // Invisible (Tr 3) and clip-only (Tr 7) text, or text wholly off the page. Some(page) => { @@ -683,7 +742,7 @@ fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (V } None => scope.glyph_hit(b, origin), }; - glyphs.push(Glyph { bytes: pos..pos + len, advance, hit }); + glyphs.push(Glyph { bytes: pos..pos + len, advance, hit, origin, end, height }); *tm = Matrix::translate(advance * gs.scale, 0.0).then(tm); pos += len; } @@ -696,9 +755,12 @@ fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (V if t.is_empty() { // A code the font gives no meaning: the run cannot be known to // continue through it, so each side matches on its own. - flush(&mut frag, scope); + scope.flush_run(); + kerned = false; } else { - frag.push_str(&t); + let k = kerned; + kerned = false; + scope.removed_glyph(&t, g.origin, g.end, g.height, k); } } if !run.is_empty() { @@ -710,8 +772,9 @@ fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (V } } else { if collect { - flush(&mut frag, scope); + scope.flush_run(); } + kerned = false; run.extend_from_slice(&bytes[g.bytes.clone()]); } } @@ -723,11 +786,11 @@ fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (V if let Some(v) = other.as_f64() { *tm = Matrix::translate(-v / 1000.0 * size * gs.scale, 0.0).then(tm); push_num(&mut out, v); + kerned = true; } } } } - flush(&mut frag, scope); (out, removed) } diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 2eb19a4958..fcb3ca6724 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -360,6 +360,7 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result Date: Fri, 16 Oct 2026 17:34:04 +0800 Subject: [PATCH 3/5] M5: control raster delivery in pen preview regressions A completed raster can legitimately clear the settling stroke before the integration test checks it. A temporary 300 ms receive delay reproduced the FreeBSD count-0-versus-1 assertion on parent 9248b8aa. Hold delivery while checking pending strokes, assert five extra frames, then release it and wait for the raster. Preserve the undo assertion and production defaults. Narrowly adapted from Guillem Hermida's pen-test changes in upstream PR https://github.com/storytold/pdfcraft/pull/2812 at 47e5398b6ba650dbc7008a23f5231b2767af41a0 (MIT OR Apache-2.0). Exclude its separate fullscreen and renderer changes. No redaction change. The temporary delay is not included. Validation: comments 45/45 with and without the diagnostic delay; native Clippy all targets -D warnings; formatting; layers 37; assets; wasm 31. Co-authored-by: Guillem Hermida --- ROADMAP.md | 2 ++ crates/ui-egui/src/canvas.rs | 7 ++++++- crates/ui-egui/tests/comments.rs | 10 ++++++++-- docs/roadmap.md | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 5bfcf1cd7e..2e4bd7fa38 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -157,6 +157,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md | 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 | ## Progress log + +- **2026-10-16 (M5, PR #2598 CI):** Pen preview regressions hold raster delivery while checking pending strokes, then wait for delivery by state. Adapted narrowly from #2812; no fullscreen or redaction behavior change. Estimates unchanged. - **2026-10-15 (M8, #2448):** Applying redactions clears what the marks removed from document metadata too: `/Info` entries whose value repeats a removed text run (case-insensitive) go with their mirrored XMP property, and an XMP stream that still holds removed text where per-entry editing cannot reach — custom schemas, entity-escaped values, page-level packets, or a stream that cannot be decoded and read — is removed whole, like the hidden-information metadata category. Unrelated entries and the rest of an editable packet survive. Text inside images and runs under two characters leave no searchable string, so Remove hidden information (metadata) remains the certain path. `redact` now uses `organize` (registered as a sideways edge). Estimates unchanged. - **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. - **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged. diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 989b731b7e..e1fa09190e 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -379,6 +379,8 @@ pub struct DocView { last_press_time: f64, last_press_pos: Option, last_queue: Vec, + /// Hold finished rasters in the pool so tests can inspect the view before delivery. + pub hold_rasters: bool, viewport_w: f32, viewport_h: f32, page_count: usize, @@ -654,6 +656,7 @@ impl DocView { last_press_time: 0.0, last_press_pos: None, last_queue: Vec::new(), + hold_rasters: false, viewport_w: 800.0, viewport_h: 600.0, page_count: info.pages.len(), @@ -1655,7 +1658,9 @@ impl DocView { pool.set_queue(queue.clone()); self.last_queue = queue; } - self.receive(ctx, pool); + if !self.hold_rasters { + self.receive(ctx, pool); + } if !self.last_queue.is_empty() { ctx.request_repaint_after(std::time::Duration::from_millis(30)); } diff --git a/crates/ui-egui/tests/comments.rs b/crates/ui-egui/tests/comments.rs index 914bc281b8..89b03879b9 100644 --- a/crates/ui-egui/tests/comments.rs +++ b/crates/ui-egui/tests/comments.rs @@ -7,6 +7,8 @@ use egui_kittest::kittest::Queryable; use pdfcraft_render::Annotation; use pdfcraft_ui_egui::{PdfCraftApp, QuickTool}; +mod support; + /// Two 300×200 pages of Helvetica text. const TEXT_FIXTURE: &[u8] = b"%PDF-1.7 1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj @@ -406,8 +408,9 @@ fn shapes_and_ink_are_drawn_by_dragging() { assert!((r[0] - 40.0).abs() < 3.0 && (r[1] - 40.0).abs() < 3.0 && (r[2] - 140.0).abs() < 3.0 && (r[3] - 100.0).abs() < 3.0, "{r:?}"); } -/// Press, drag and release a stroke, running only one frame after the release. +/// Hold raster delivery while releasing a stroke, regardless of renderer speed. fn stroke_without_settling(h: &mut Harness<'static, PdfCraftApp>, a: (f32, f32), b: (f32, f32)) { + h.state_mut().views[0].hold_rasters = true; let (from, to) = (at(h, a.0, a.1), at(h, b.0, b.1)); h.hover_at(from); h.run_steps(1); @@ -429,7 +432,10 @@ fn a_released_pen_stroke_stays_drawn_until_the_page_raster_has_it() { stroke_without_settling(&mut h, (40.0, 100.0), (140.0, 60.0)); assert_eq!(comments(&h).len(), 1, "the stroke is committed"); assert_eq!(h.state().views[0].comments.settling_len(), 1, "still drawn from its points"); - settle(&mut h); + h.run_steps(5); + assert_eq!(h.state().views[0].comments.settling_len(), 1, "drawn from its points for as long as its raster takes"); + h.state_mut().views[0].hold_rasters = false; + support::wait_until(&mut h, "the page raster with the stroke", |app| app.views[0].comments.settling_len() == 0); assert_eq!(h.state().views[0].comments.settling_len(), 0, "the raster draws it now"); } diff --git a/docs/roadmap.md b/docs/roadmap.md index 3148823ea5..8b5b8ffeb0 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -112,7 +112,7 @@ total matches the parity estimate. | M4 | Engine, history, save, organize | 100–180 | 95% | 8–15 | Done:
- command registry (menus, shortcuts and palette all use it);
- undo/redo; incremental, atomic and encrypted saves;
- autosave and crash recovery;
- organize, combine, extract, split and insert-from-file, with identical fonts and images stored once;
- CLI `edit/combine/extract/split`.
Done since: Organize opens directly into the page overview, with checkbox selection, an editable page-range selection (#1092), and per-page rotate/delete controls; Combine files in its own tab (sortable, resizable columns; folders; early warnings; unlocking protected files), bookmark editing, page labels (Number pages), CLI `run`, Set Page Boxes, Crop tool, Duplicate pages, Ctrl+A / Cmd+A to select all pages in Organize, Replace pages, bookmarks from tagged headings. Extract and Split preserve shared Type 3 glyph resource graphs, retaining conservative pruning limits. Missing: page transitions, recovery on the web M4: Insert-from-file preserves source bookmarks; integration tests retained with current main. | | M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). Renderer-creation errors return to the existing OpenGL startup fallback; a real-GPU regression covers errors and device reuse. The underlying x86 DX12 shader compiler failure remains unresolved; see `windows-renderer-startup.md`. With Windows' Animation effects turned off, the wheel scrolls and zooms without easing and egui's animations take no time (`motion.rs`; macOS, Linux and the browser aren't asked yet); Preferences ▸ Animation effects (Use system setting, On, Off) decides it on every platform. | | M4 | Engine, history, save, organize | 100–180 | 95% | 8–15 | Done:
- command registry (menus, shortcuts and palette all use it);
- undo/redo; incremental, atomic and encrypted saves;
- autosave and crash recovery;
- organize, combine, extract, split and insert-from-file, with identical fonts and images stored once;
- CLI `edit/combine/extract/split`.
Done since: Combine files in its own tab (sortable, resizable columns; folders; early warnings; unlocking protected files), bookmark editing, page labels (Number pages), CLI `run`, Set Page Boxes, Crop tool, Duplicate pages, Ctrl+A / Cmd+A to select all pages in Organize, Replace pages, bookmarks from tagged headings. Missing: page transitions, recovery on the web | -| M5 | Comments (all annotation types, XFDF) | 120–200 | 88% | 12–25 | Done: notes, highlight/underline/strikeout/squiggly, text boxes, ink, lines, arrows, rectangles, ovals, stamps (dynamic, Sign Here, Standard Business), with appearance streams; replies, status, checkmarks, locking, move/resize/restyle/delete; properties dialog; panel filter and sort; hide all; summaries (comments only); XFDF/FDF import and export (partial imports preserve existing reply and review-status links); flatten; Fill & Sign with adjustable text/date size; agent tools.
Done since: stamps placed on a rotated page read upright (#1398); polygons, connected lines, clouds, callouts, inserted text, colour and checkmark filters; image signature pointer previews, selection handles and live image movement/resizing with proportional corners and independent edge handles; Ctrl+Enter submission for highlight comment text and replies, with Shift+Enter newlines (reconciled with upstream main for PR #1457 on 2026-10-14). Missing: custom stamps, replace-text proposals, summary layouts with the page | +| M5 | Comments (all annotation types, XFDF) | 120–200 | 88% | 12–25 | Done: notes, highlight/underline/strikeout/squiggly, text boxes, ink, lines, arrows, rectangles, ovals, stamps (dynamic, Sign Here, Standard Business), with appearance streams; replies, status, checkmarks, locking, move/resize/restyle/delete; properties dialog; panel filter and sort; hide all; summaries (comments only); XFDF/FDF import and export (partial imports preserve existing reply and review-status links); flatten; Fill & Sign with adjustable text/date size; agent tools.
Done since: stamps placed on a rotated page read upright (#1398); polygons, connected lines, clouds, callouts, inserted text, colour and checkmark filters; image signature pointer previews, selection handles and live image movement/resizing with proportional corners and independent edge handles; Ctrl+Enter submission for highlight comment text and replies, with Shift+Enter newlines (reconciled with upstream main for PR #1457 on 2026-10-14). Pen preview regression timing controlled by raster delivery (PR #2598, adapted from #2812). Missing: custom stamps, replace-text proposals, summary layouts with the page | | M4 | Engine, history, save, organize | 100–180 | 95% | 8–15 | Done:
- command registry (menus, shortcuts and palette all use it);
- undo/redo; incremental, atomic and encrypted saves;
- autosave and crash recovery;
- organize, combine, extract, split and insert-from-file, with identical fonts and images stored once;
- CLI `edit/combine/extract/split`.
Done since: Combine files in its own tab (sortable, resizable columns; folders; early warnings; unlocking protected files), bookmark editing, page labels (Number pages), CLI `run`, Set Page Boxes, Crop tool, Duplicate pages, Ctrl+A / Cmd+A to select all pages in Organize, Replace pages, bookmarks from tagged headings. Missing: page transitions, recovery on the web M4: Insert-from-file preserves source bookmarks; integration tests retained with current main. | | M5 | Comments (all annotation types, XFDF) | 120–200 | 88% | 12–25 | Done: notes, highlight/underline/strikeout/squiggly, text boxes, ink, lines, arrows, rectangles, ovals, stamps (dynamic, Sign Here, Standard Business), with appearance streams; replies, status, checkmarks, locking, move/resize/restyle/delete; properties dialog; panel filter and sort; hide all; summaries (comments only); XFDF/FDF import and export (partial imports preserve existing reply and review-status links); flatten; Fill & Sign with adjustable text/date size; agent tools.
Done since: stamps placed on a rotated page read upright (#1398); polygons, connected lines, clouds, callouts, inserted text, colour and checkmark filters; image signature pointer previews, selection handles and live image movement/resizing with proportional corners and independent edge handles. Missing: custom stamps, replace-text proposals, summary layouts with the page | | M5 | Comments (all annotation types, XFDF) | 120–200 | 88% | 12–25 | Done: notes, highlight/underline/strikeout/squiggly, text boxes, ink, lines, arrows, rectangles, ovals, stamps (dynamic, Sign Here, Standard Business), with appearance streams; replies, status, checkmarks, locking, move/resize/restyle/delete; properties dialog; panel filter and sort; hide all; summaries (comments only); XFDF/FDF import and export (partial imports preserve existing reply and review-status links); flatten; Fill & Sign with adjustable text/date size; agent tools.
Done since: polygons, connected lines, clouds, callouts, inserted text, colour and checkmark filters; image signature pointer previews, selection handles and live image movement/resizing with proportional corners and independent edge handles. A refused restyle or resize of a text box another program drew leaves that comment unchanged (#1327). Make Current Properties Default on an arrow updates arrows, not plain lines (#1302). Flattening keeps a visible comment it cannot draw (#1317). Missing: custom stamps, replace-text proposals, summary layouts with the page | From 192c7457b5e7fd24af307c24de77f20a11d75b57 Mon Sep 17 00:00:00 2001 From: jonathanwxh-cell Date: Fri, 16 Oct 2026 19:54:35 +0800 Subject: [PATCH 4/5] M8: compare redacted metadata across TJ gaps and XML values A TJ word kern could collect JamesBond while metadata held James Bond. Record optional forward-gap spaces for comparison, preserving unspaced intra-word kerning and literal word boundaries without enumerating variants. Normalize internal whitespace in values and removed fragments. Parse XMP values with the already-locked roxmltree dependency rather than searching markup names; malformed XML and unsupported DTDs take the whole-packet fallback. Preserve complete-fragment/value matching, Spy stories, and existing -200/-300 kerning regressions. Partial/reordered names, Unicode normalization, short-value over-clearing, and resource-limit policy remain documented and unchanged. Review feedback: https://github.com/storytold/pdfcraft/pull/2598#issuecomment-6213443550 Validation: three expected-correctness groups failed before implementation; 67 redaction tests pass (one existing manual probe ignored); headless metadata redaction automation test passes; strict Clippy, formatting, layers, assets, parity and wasm (31 crates) pass. Synthetic full-save/reopen regressions cover mixed/rotated kerns, whitespace, XML names/namespaces, entities and CDATA. --- Cargo.lock | 1 + ROADMAP.md | 2 + crates/redact/Cargo.toml | 2 + crates/redact/README.md | 31 ++++-- crates/redact/src/interp.rs | 23 ++++- crates/redact/src/lib.rs | 108 +++------------------ crates/redact/src/metadata.rs | 73 ++++++++++++++ crates/redact/src/metadata_tests.rs | 143 ++++++++++++++++++++++++++++ crates/redact/src/tests.rs | 3 + docs/roadmap.md | 2 +- parity/acrobat-features.toml | 4 +- 11 files changed, 284 insertions(+), 108 deletions(-) create mode 100644 crates/redact/src/metadata.rs create mode 100644 crates/redact/src/metadata_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 1b772b43fd..8b229257e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4285,6 +4285,7 @@ dependencies = [ "pdfcraft-organize", "pdfcraft-testkit", "pdfcraft-xfa", + "roxmltree", "thiserror 2.0.21", ] diff --git a/ROADMAP.md b/ROADMAP.md index 2e4bd7fa38..1b5b87ca30 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -158,6 +158,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-16 (M8, PR #2598):** Metadata comparison handles ambiguous TJ word gaps without breaking intra-word kerning, collapses whitespace, and matches decoded XML values rather than markup names. Partial/reordered-name matching and resource limits are unchanged. Estimates unchanged. + - **2026-10-16 (M5, PR #2598 CI):** Pen preview regressions hold raster delivery while checking pending strokes, then wait for delivery by state. Adapted narrowly from #2812; no fullscreen or redaction behavior change. Estimates unchanged. - **2026-10-15 (M8, #2448):** Applying redactions clears what the marks removed from document metadata too: `/Info` entries whose value repeats a removed text run (case-insensitive) go with their mirrored XMP property, and an XMP stream that still holds removed text where per-entry editing cannot reach — custom schemas, entity-escaped values, page-level packets, or a stream that cannot be decoded and read — is removed whole, like the hidden-information metadata category. Unrelated entries and the rest of an editable packet survive. Text inside images and runs under two characters leave no searchable string, so Remove hidden information (metadata) remains the certain path. `redact` now uses `organize` (registered as a sideways edge). Estimates unchanged. - **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index d1cba3643f..3fcdb700ac 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -19,6 +19,8 @@ pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } pdfcraft-organize = { workspace = true } thiserror = { workspace = true } +# Already present in the lockfile: inspect XML values without searching markup names. +roxmltree = "0.20" [dev-dependencies] pdfcraft-testkit = { workspace = true } diff --git a/crates/redact/README.md b/crates/redact/README.md index 2866e00362..f184f902d1 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -48,14 +48,19 @@ as many generators do, still collects as one run — whenever the next removed g 1.5 glyph heights of where the last one's advance (character and word spacing included) landed in device space; a repositioning gap past 0.15 heights becomes a space, and kept glyphs, codes with no Unicode meaning, and page ends still split runs. -After verification, each `/Info` string entry whose value contains a fragment (lowercased, two -characters or more) — or -is contained in one, for a value shorter than the run; the author of a whole redacted sentence -is removed text too — is cleared with its mirrored XMP property; then every catalog and page -`/Metadata` stream is checked in its raw bytes **and** in an entity-unescaped view of them (XMP -stores `AT&T` as `AT&T`), and value by value (each tag's text, each quoted attribute), -and removed whole when a fragment or part of one survives there — custom schemas cannot be -edited per-entry — or when the stream cannot be decoded and read as UTF-8 within 16 MiB. A +Forward `TJ` displacements can also encode word spacing. Those gaps are optional spaces during +metadata comparison, retaining the unspaced interpretation for intra-word kerning. Mixed kerns +and word gaps are compared without enumerating combinations; literal spaces remain required. +After verification, fragments and values are lowercased with internal whitespace collapsed. +Each `/Info` string entry containing a complete fragment of two characters or more — or whose +complete value is contained in a fragment — is cleared with its mirrored XMP property. The +author of a whole redacted sentence is removed text too. Then every catalog and page +`/Metadata` stream is parsed as XML and checked value by value (decoded text, comments and +attribute values). Tag names, attribute names, namespace declarations and processing instructions +are not values. XML entities and CDATA are decoded by the parser. A packet is removed whole +when a fragment or part of one survives in a value — custom schemas cannot be +edited per-entry — or when the stream cannot be decoded and read as UTF-8 within 16 MiB or parsed +as XML. DTDs are unsupported and take this conservative whole-packet fallback. A trailer `/Info` that is a direct dictionary (the one shape an entry clear cannot edit) is removed whole when it still matches. All of this happens before the forced full save, so the previous revision's bytes leave the file. @@ -65,7 +70,8 @@ The match errs towards clearing, in both directions: - **over-clearing**: a fully redacted common word is a fragment, and an unrelated entry or packet merely containing it goes too; glyphs removed under *different* marks also join when they land within 1.5 em of each other in stream order, so two one-letter removals can form a two-character - needle; the wholesale fallback drops a whole stream for one custom-schema hit; + needle; the wholesale fallback drops a whole stream for one custom-schema hit. A short complete + XML value such as language `en` or version `1.7` can still match part of a longer removed run; - **under-matching**: text inside images never becomes a fragment, runs shorter than two characters are ignored, fragments split at gaps wider than 1.5 em and carry narrower repositioning gaps as spaces, so a value that spells the same text with or without one of those @@ -75,6 +81,13 @@ The match errs towards clearing, in both directions: them — and `/Metadata` on component objects (images, fonts, form XObjects) is not scanned — the same scope as Remove hidden information. +Matching still requires a complete removed fragment inside a value, or a complete value inside +a fragment. Partial phrases such as Title `Case: James Bond` after redacting `JAMES BOND IS A SPY`, +reordered names such as `Smith, John` versus `John Smith`, and Unicode normalization remain +unsupported. No word-term deletion is performed: unrelated Subject `Spy stories` stays. Fragment +count/length and matching-work limits are unchanged; the 16 MiB stream limit is not a bound on +total metadata-matching work. + Remove hidden information (metadata category, `doc_remove_hidden`) is therefore the certain follow-up when a document's metadata must be provably free of the redacted text. diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 36f03dbcb0..9b6d8bf5dc 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -92,9 +92,13 @@ pub(crate) struct Scope<'a> { /// repositioning gap past 0.15 heights becomes a space — and splits at kept glyphs, at codes /// with no Unicode meaning, and at page end. pub removed_fragments: Vec, + /// Comparison alternatives for ambiguous forward TJ gaps; the canonical runs above stay + /// unspaced so intra-word kerning retains its original interpretation. + pub kerned_fragments: Vec, /// The run being collected: its text, and where its last removed glyph's advance ended in /// device space with that glyph's height (`None` = no open run). run: String, + run_gaps: Vec, run_end: Option<((f64, f64), f64)>, fonts: HashMap, Rc>, used_names: Vec>, @@ -111,7 +115,9 @@ impl<'a> Scope<'a> { hidden_layers: Vec::new(), layer_blocks: 0, removed_fragments: Vec::new(), + kerned_fragments: Vec::new(), run: String::new(), + run_gaps: Vec::new(), run_end: None, fonts: HashMap::new(), used_names: Vec::new(), @@ -177,19 +183,32 @@ impl<'a> Scope<'a> { pub(crate) fn flush_run(&mut self) { self.run_end = None; if !self.run.is_empty() { + if !self.run_gaps.is_empty() { + self.kerned_fragments.push(crate::metadata::KernedFragment { text: self.run.clone(), gaps: std::mem::take(&mut self.run_gaps) }); + } self.removed_fragments.push(std::mem::take(&mut self.run)); } } /// A removed glyph: `kerned` says a `TJ` number displaced the text since the last removed - /// glyph — spacing inside one text run, which joins with no space and no split (a kern is not - /// a word gap). Otherwise the glyph continues the open run when it starts where the last + /// glyph: the canonical run joins with no space or split. A forward gap may also represent + /// a word space, recorded separately for metadata comparison. Otherwise the glyph continues + /// the open run when it starts where the last /// one's advance landed — device space, within 1.5 glyph heights, so text drawn one glyph per /// operator still collects as one run — and a gap past 0.15 heights is a word gap (a real /// space, or the joined text would not match the value it was drawn from). Degenerate /// (zero-height) glyphs and non-finite positions never join. fn removed_glyph(&mut self, text: &str, origin: (f64, f64), end: (f64, f64), height: f64, kerned: bool) { if kerned && self.run_end.is_some() { + if let Some(((x, y), previous_height)) = self.run_end { + let (dx, dy) = (origin.0 - x, origin.1 - y); + let h = previous_height.max(height); + let gap = dx.hypot(dy); + let forward = dx * (end.0 - origin.0) + dy * (end.1 - origin.1); + if h.is_finite() && h > 0.0 && gap.is_finite() && gap > 0.15 * h && forward.is_finite() && forward > 0.0 { + self.run_gaps.push(self.run.len()); + } + } self.run.push_str(text); self.run_end = Some((end, height)); return; diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index fcb3ca6724..80ba7b69bc 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -17,7 +17,6 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; @@ -25,6 +24,7 @@ use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; pub mod codes; mod image; mod interp; +mod metadata; mod overlay; pub mod patterns; pub mod sanitize; @@ -33,6 +33,7 @@ mod tags; mod tests; use interp::{Mode, Scope, process}; +use metadata::{KernedFragment, normalize, packet_hit}; pub type Rgb = [f64; 3]; @@ -349,6 +350,7 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result>()).collect(); // The text the marks removed (maximal runs), for the metadata pass below. let mut fragments: Vec = Vec::new(); + let mut kerned = Vec::new(); for &(pi, page_marks) in &groups { let page = pdfcraft_model::pages(doc).swap_remove(pi); @@ -362,6 +364,7 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result) -> Result) -> Result Cow<'_, str> { - if !s.contains('&') { - return Cow::Borrowed(s); - } - let bytes = s.as_bytes(); - let mut out = String::with_capacity(s.len()); - let mut i = 0; - while i < s.len() { - if bytes[i] == b'&' - && let Some(end) = s[i..].find(';').map(|e| i + e).filter(|e| *e - i <= 12) - { - let ent = &s[i + 1..end]; - let c = match ent { - "amp" => Some('&'), - "lt" => Some('<'), - "gt" => Some('>'), - "quot" => Some('"'), - "apos" => Some('\''), - _ => ent - .strip_prefix("#x") - .and_then(|h| u32::from_str_radix(h, 16).ok()) - .or_else(|| ent.strip_prefix('#').and_then(|d| d.parse::().ok())) - .and_then(char::from_u32), - }; - if let Some(c) = c { - out.push(c); - i = end + 1; - continue; - } - } - match s[i..].chars().next() { - Some(c) => { - out.push(c); - i += c.len_utf8(); - } - None => break, - } - } - Cow::Owned(out) -} - -/// Does the XMP packet `text` hold removed text? Scanned whole, raw and entity-unescaped, for a -/// removed run; and value by value — each tag's text and each quoted attribute, unescaped — for -/// a value that is only part of one (a name inside a redacted sentence). Structure is read from -/// the raw text, where an escaped `<` is not a tag, before each value is unescaped. -fn packet_hit(text: &str, hit: &impl Fn(&str) -> bool) -> bool { - hit(text) || hit(&xml_unescape(text)) || xml_chunks(text).into_iter().any(|c| hit(&xml_unescape(c))) -} - -/// The places an XML packet can hold a value: the text between tags and the quoted attribute -/// values inside them, as they stand (the caller unescapes). Lenient: malformed markup just -/// yields fewer places. -fn xml_chunks(xmp: &str) -> Vec<&str> { - let mut out = Vec::new(); - let mut rest = xmp; - while let Some(open) = rest.find('<') { - if open > 0 { - out.push(&rest[..open]); - } - // A tag with no `>` ends the scan; the tail is then text. - let Some(rel) = rest[open..].find('>') else { break }; - let tag = &rest[open + 1..open + rel]; - rest = &rest[open + rel + 1..]; - let bytes = tag.as_bytes(); - let mut i = 0; - while i < tag.len() { - let q = bytes[i]; - if (q == b'"' || q == b'\'') && i > 0 && bytes[i - 1] == b'=' { - let value = &tag[i + 1..]; - let len = value.find(q as char).unwrap_or(value.len()); - out.push(&value[..len]); - i += len + 2; - } else { - i += 1; - } - } - } - if !rest.is_empty() { - out.push(rest); - } - out -} - /// The metadata half of redaction (#2448): a string the marks took off the pages must not stay /// in the document information dictionary or the XMP metadata. Runs of removed text two -/// characters or longer are matched case-insensitively (a single clipped glyph is no evidence a +/// characters or longer are matched case-insensitively with collapsed whitespace (a single clipped glyph is no evidence a /// secret was removed, and would clear unrelated entries that merely contain that character). /// /// Per entry first, so unrelated values survive: each `/Info` string entry whose value contains @@ -635,13 +552,14 @@ fn xml_chunks(xmp: &str) -> Vec<&str> { /// which also removes the XMP property mirroring it and leaves the rest of the packet byte for /// byte. Then, wholesale where per-entry editing cannot reach: every catalog and page /// `/Metadata` stream is kept only when it decodes, reads as UTF-8, and holds no removed run or -/// part of one (raw, entity-unescaped, or value by value — custom schemas escape their values); +/// part of one in decoded XML values (markup names and namespace declarations are excluded); /// a trailer `/Info` that is a direct dictionary — the one shape an entry clear cannot edit — /// is removed whole when it still matches. Matching errs towards clearing; text that leaves no /// searchable run (image text, one surviving piece) needs `sanitize`'s metadata category to be /// provably gone. -fn clear_marked_metadata(doc: &mut Document, fragments: &[String], report: &mut Report) -> Result<(), RedactError> { - let needles: Vec = fragments.iter().map(|f| f.trim().to_lowercase()).filter(|f| f.chars().count() >= 2).collect(); +fn clear_marked_metadata(doc: &mut Document, fragments: &[String], kerned: &[KernedFragment], report: &mut Report) -> Result<(), RedactError> { + let needles: Vec = fragments.iter().map(|f| normalize(f)).filter(|f| f.chars().count() >= 2).collect(); + let kerned: Vec<_> = kerned.iter().map(KernedFragment::normalized).filter(|f| f.text.chars().count() >= 2).collect(); if needles.is_empty() { return Ok(()); } @@ -649,9 +567,11 @@ fn clear_marked_metadata(doc: &mut Document, fragments: &[String], report: &mut // direction never matches a value under two characters: one shared character is no evidence // a secret was removed. let hit = |v: &str| { - let v = v.to_lowercase(); + let v = normalize(v); let part = v.trim(); - needles.iter().any(|n| v.contains(n.as_str())) || (part.chars().count() >= 2 && needles.iter().any(|n| n.contains(part))) + needles.iter().any(|n| v.contains(n.as_str())) + || (part.chars().count() >= 2 && needles.iter().any(|n| n.contains(part))) + || (part.chars().count() >= 2 && kerned.iter().any(|n| n.matches(part))) }; // 1. Entry by entry: the value goes, and with it the XMP property that mirrors the entry. diff --git a/crates/redact/src/metadata.rs b/crates/redact/src/metadata.rs new file mode 100644 index 0000000000..0b6d4e9f0c --- /dev/null +++ b/crates/redact/src/metadata.rs @@ -0,0 +1,73 @@ +//! Comparison-only normalization; the document's original text is never rewritten here. + +pub(crate) fn normalize(text: &str) -> String { + text.split_whitespace().map(str::to_lowercase).collect::>().join(" ") +} + +/// A removed run with optional word spaces at forward `TJ` displacements. Kerning is ambiguous: +/// compare both interpretations without enumerating the exponentially many possible strings. +pub(crate) struct KernedFragment { + pub text: String, + pub gaps: Vec, +} + +impl KernedFragment { + pub fn normalized(&self) -> Self { + let mut text = String::new(); + let mut gaps = Vec::new(); + let (mut space, mut gap) = (false, false); + for (offset, c) in self.text.char_indices() { + gap |= self.gaps.binary_search(&offset).is_ok(); + if c.is_whitespace() { + space = !text.is_empty(); + continue; + } + if space { + text.push(' '); + } else if gap && !text.is_empty() { + gaps.push(text.len()); + } + text.extend(c.to_lowercase()); + space = false; + gap = false; + } + Self { text, gaps } + } + + pub fn matches(&self, value: &str) -> bool { + let Some(first) = self.text.chars().next() else { return false }; + // A complete removed run inside a metadata value. + if value.match_indices(first).any(|(start, _)| self.prefix(0, value.get(start..).unwrap_or_default(), true)) { + return true; + } + let Some(first) = value.chars().next() else { return false }; + // Or the complete metadata value inside a removed run. Never match isolated terms. + self.text.match_indices(first).any(|(start, _)| self.prefix(start, value, false)) + } + + fn prefix(&self, start: usize, value: &str, whole_run: bool) -> bool { + let Some(run) = self.text.get(start..) else { return false }; + let mut value = value.chars().peekable(); + for (offset, c) in run.char_indices() { + if value.peek().is_none() { + return !whole_run; + } + if offset > 0 && self.gaps.binary_search(&start.saturating_add(offset)).is_ok() && value.peek() == Some(&' ') { + value.next(); + } + if value.next() != Some(c) { + return false; + } + } + whole_run || value.next().is_none() + } +} + +/// XML parsers expose decoded text and attributes, excluding tag/attribute names and namespace +/// declarations. DTDs are disabled by the parser's defaults. Uninspectable XML fails closed. +pub(crate) fn packet_hit(text: &str, hit: &impl Fn(&str) -> bool) -> bool { + let Ok(doc) = roxmltree::Document::parse(text) else { return true }; + doc.descendants().any(|node| { + ((node.is_text() || node.is_comment()) && node.text().is_some_and(hit)) || node.attributes().any(|attribute| hit(attribute.value())) + }) +} diff --git a/crates/redact/src/metadata_tests.rs b/crates/redact/src/metadata_tests.rs new file mode 100644 index 0000000000..2683a9458c --- /dev/null +++ b/crates/redact/src/metadata_tests.rs @@ -0,0 +1,143 @@ +//! Metadata comparison regressions for PR #2598 comment 6213443550. +use super::*; + +fn packet(payload: &str, attributes: &str) -> String { + format!( + "{payload}" + ) +} + +fn collected(src: &[u8]) -> Vec { + let mut doc = one_page(src, "", vec![]); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let (_, data) = page_streams(&doc, &page.dict, 0).unwrap(); + let res = page.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + let mut rep = Report::default(); + let rects = [[-1_000_000.0, -1_000_000.0, 1_000_000.0, 1_000_000.0]]; + let mut scope = Scope::new(&rects, Mode::Apply, &mut rep); + process(&mut doc, &mut scope, &data, &res, pdfcraft_content::Matrix::IDENTITY); + scope.flush_run(); + scope.removed_fragments +} + +fn redact_all(src: &[u8], key: &str, value: &str, packet: Option<&str>) -> Document { + let mut doc = meta_doc(src, &[(key, value), ("Subject", "Spy stories")], packet, None); + let rects = [[0.0, 0.0, 300.0, 300.0]]; + assert!(under(&mut doc, 0, &rects) > 0, "fixture has marked glyphs"); + mark(&mut doc, 0, &rects, ""); + apply(&mut doc, None).unwrap(); + assert_eq!(under(&mut doc, 0, &rects), 0, "page content was removed"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(info_value(&saved, "Subject").as_deref(), Some("Spy stories")); + saved +} + +#[test] +fn tj_word_gaps_and_literal_spaces_clear_the_same_name() { + for (label, src, fragment, survives) in [ + ("TJ -333", "BT /F1 10 Tf 10 100 Td [(James)-333(Bond)]TJ ET", "JamesBond", false), + ("mixed kerns", "BT /F1 10 Tf 10 100 Td [(J)-200(ames)-333(Bond)]TJ ET", "JamesBond", false), + ("rotated", "BT /F1 10 Tf 0 1 -1 0 100 100 Tm [(James)-333(Bond)]TJ ET", "JamesBond", false), + ("literal space", "BT /F1 10 Tf 10 100 Td (James Bond)Tj ET", "James Bond", false), + ] { + assert_eq!(collected(src.as_bytes()), [fragment], "{label}"); + let packet = packet("James Bond", ""); + let saved = redact_all(src.as_bytes(), "Author", "James Bond", Some(&packet)); + let actual = info_value(&saved, "Author"); + assert_eq!(actual.as_deref(), survives.then_some("James Bond"), "the removed name must not survive"); + assert_eq!(xmp_text(&saved).is_some(), survives); + } +} + +#[test] +fn metadata_whitespace_is_normalized_without_broadening_name_matching() { + for (label, text, key, value) in [ + ("partial phrase", "JAMES BOND IS A SPY", "Title", "Case: James Bond"), + ("partial first name", "JAMES BOND IS A SPY", "Title", "Case: James"), + ("reordered", "John Smith", "Author", "Smith, John"), + ("double space metadata", "James Bond", "Author", "James Bond"), + ("tab metadata", "James Bond", "Author", "James\tBond"), + ("newline metadata", "James Bond", "Author", "James\nBond"), + ("double space page", "James Bond", "Author", "James Bond"), + ] { + let src = format!("BT /F1 10 Tf 10 100 Td ({text})Tj ET"); + let packet = packet(&format!("{value}"), ""); + let saved = redact_all(src.as_bytes(), key, value, Some(&packet)); + let actual = info_value(&saved, key); + let unsupported = label.starts_with("partial") || label == "reordered"; + assert_eq!(actual.as_deref(), unsupported.then_some(value), "whitespace normalizes; partial and reordered names remain unsupported"); + assert_eq!(xmp_text(&saved).as_deref(), unsupported.then_some(packet.as_str())); + } +} + +#[test] +fn xmp_matches_values_and_preserves_structure_only_matches() { + for (label, text, payload, attributes, removed) in [ + ("tag name", "DC", "Harmless", "", false), + ("attribute name", "ID", "Harmless", "z:id='safe'", false), + ("short language value", "AGENT", "en", "", true), + ("short version value", "RELEASE 1.7 APPROVED", "1.7", "", true), + ("unrelated control", "ZX", "Harmless", "z:id='safe'", false), + ] { + let packet = packet(payload, attributes); + let src = format!("BT /F1 10 Tf 10 100 Td ({text})Tj ET"); + if label.starts_with("short") { + assert!(!packet.to_lowercase().contains(&text.to_lowercase()), "the whole removed run is absent"); + } + let saved = redact_all(src.as_bytes(), "Author", "Ada", Some(&packet)); + let actual = xmp_text(&saved); + assert_eq!(actual.is_none(), removed, "{label}"); + assert_eq!(info_value(&saved, "Author").as_deref(), Some("Ada")); + } +} + +#[test] +fn xmp_values_are_decoded_and_markup_is_not_a_value() { + for (payload, attributes, removed) in [ + ("James Bond", "", true), + ("", "", true), + ("Harmless", "z:name = 'James Bond'", true), + ("Harmless", "", true), + ("", "", false), + ] { + let packet = packet(payload, attributes); + let text = if removed { "James Bond" } else { "JamesBond" }; + let src = format!("BT /F1 10 Tf 10 100 Td ({text})Tj ET"); + let saved = redact_all(src.as_bytes(), "Author", "Ada", Some(&packet)); + assert_eq!(xmp_text(&saved).is_none(), removed); + } + let packet = packet("Harmless", "xmlns:dc='https://example.org/dc/' z:id = 'safe > tail'"); + for text in ["DC", "ID"] { + let src = format!("BT /F1 10 Tf 10 100 Td ({text})Tj ET"); + assert!(xmp_text(&redact_all(src.as_bytes(), "Author", "Ada", Some(&packet))).is_some()); + } + for broken in ["", "]>&secret;"] { + assert!(xmp_text(&redact_all(b"BT /F1 10 Tf 10 100 Td (James Bond)Tj ET", "Author", "Ada", Some(broken))).is_none()); + } +} + +#[test] +fn optional_space_offsets_survive_unicode_lowercase_expansion() { + let pattern = KernedFragment { text: "İB".into(), gaps: vec![2] }.normalized(); + assert!(pattern.matches(&normalize("İ B"))); + assert!(pattern.matches(&normalize("İB"))); + assert!(!pattern.matches(&normalize("B İ"))); + assert!(pattern.matches(&normalize("Case: İ B, unrelated"))); +} + +#[test] +fn optional_tj_spaces_preserve_literal_boundaries_and_complete_value_matching() { + let src = b"BT /F1 10 Tf 10 100 Td [(J)-200(ames)-333(Bond) ( IS A SPY)]TJ ET"; + for (value, removed) in [("James Bond", true), ("JamesBond", true), ("Case: James Bond", false), ("Spy stories", false)] { + let saved = redact_all(src, "Title", value, None); + assert_eq!(info_value(&saved, "Title").is_none(), removed, "{value}"); + } + // Literal spaces must never become optional merely because the run also contains a kern. + let saved = redact_all(b"BT /F1 10 Tf 10 100 Td [(James Bond)-200(X)]TJ ET", "Author", "JamesBondX", None); + assert_eq!(info_value(&saved, "Author").as_deref(), Some("JamesBondX")); + for kern in [0, 333] { + let src = format!("BT /F1 10 Tf 10 100 Td [(James){kern}(Bond)]TJ ET"); + let saved = redact_all(src.as_bytes(), "Author", "James Bond", None); + assert_eq!(info_value(&saved, "Author").as_deref(), Some("James Bond"), "zero/backward kern {kern} is not a word gap"); + } +} diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 52ffd5a6cc..ba6e30026d 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -6,6 +6,9 @@ use pdfcraft_testkit::{Pdf, stream_obj as stream}; use super::*; +#[path = "metadata_tests.rs"] +mod metadata_tests; + fn pdf(objs: Vec>) -> Document { Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() } diff --git a/docs/roadmap.md b/docs/roadmap.md index 8b5b8ffeb0..9c921e405e 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -123,7 +123,7 @@ total matches the parity estimate. | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Existing simple field-font encodings and Differences are respected when generating appearances; an unencodable value uses a separate Helvetica appearance font without changing the source font. Synthetic reopen regressions cover punctuation, subset fallback and multiline text. Exact font metrics/layout and broader Unicode fallback remain incomplete. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Tab and Shift-Tab wrapping to the same field or another widget of that field refresh the newly opened editor from the accepted value before Save. Synthetic UI regressions cover both directions, same-frame Save, validation refusal and undo/redo. | | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | -| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Vertical-writing text is still refused, and the error no longer claims the text was under the mark (#1340). Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | +| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Vertical-writing text is still refused, and the error no longer claims the text was under the mark (#1340). Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding Metadata comparison handles TJ spacing, normalized whitespace and XML values (PR #2598); partial/reordered names remain unsupported. | | M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | | M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Missing: OCR languages beyond Latin, editable-text OCR output, full Office layout fidelity/import | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 24bf70340d..732f5219a3 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5680,8 +5680,8 @@ milestone = "M8" status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] -evidence = ["crates/ui-egui/tests/ui_group/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::redaction_is_refused_when_the_page_still_shows_text_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources", "crates/redact/src/tests.rs::redacted_text_is_cleared_from_info_and_xmp_on_apply", "crates/redact/src/tests.rs::an_xmp_stream_holding_redacted_text_is_removed_whole", "crates/redact/src/tests.rs::a_value_inside_a_longer_removed_run_is_cleared_too", "crates/automation/tests/automation.rs::redacting_clears_matching_document_info_through_tools"] -notes = "confirmation; one undo step; next save is a full rewrite. Refused, with nothing changed, when the page still shows text under a mark afterwards: text whose font is set through an ExtGState or taken by a form from its caller, some predefined CMaps and rotated Type 3 fonts are not removed yet, so those documents cannot be redacted. Removed images and forms are unlinked from page-tree resources (own, inherited or shared), not yet from a form's own resources, annotation appearances or patterns. /Info entries repeating a removed text run — or forming part of one, for a value shorter than the run — are cleared with their mirrored XMP property, and an XMP stream still holding removed text where per-entry editing cannot reach (custom schemas, entity-escaped values, page-level packets) or that cannot be decoded and read is removed whole; text inside images, runs under two characters and component-level /Metadata are not matched, so remove hidden information (metadata) remains the certain path" +evidence = ["crates/ui-egui/tests/ui_group/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::redaction_is_refused_when_the_page_still_shows_text_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources", "crates/redact/src/tests.rs::redacted_text_is_cleared_from_info_and_xmp_on_apply", "crates/redact/src/tests.rs::an_xmp_stream_holding_redacted_text_is_removed_whole", "crates/redact/src/tests.rs::a_value_inside_a_longer_removed_run_is_cleared_too", "crates/automation/tests/automation.rs::redacting_clears_matching_document_info_through_tools", "crates/redact/src/metadata_tests.rs::tj_word_gaps_and_literal_spaces_clear_the_same_name", "crates/redact/src/metadata_tests.rs::metadata_whitespace_is_normalized_without_broadening_name_matching", "crates/redact/src/metadata_tests.rs::xmp_matches_values_and_preserves_structure_only_matches"] +notes = "confirmation; one undo step; next save is a full rewrite. Refused, with nothing changed, when the page still shows text under a mark afterwards: text whose font is set through an ExtGState or taken by a form from its caller, some predefined CMaps and rotated Type 3 fonts are not removed yet, so those documents cannot be redacted. Removed images and forms are unlinked from page-tree resources (own, inherited or shared), not yet from a form's own resources, annotation appearances or patterns. /Info entries repeating a removed text run — or forming part of one, for a value shorter than the run — are cleared with their mirrored XMP property, and an XMP stream still holding removed text where per-entry editing cannot reach (custom schemas, entity-escaped values, page-level packets) or that cannot be decoded and parsed as XML is removed whole. Comparison collapses whitespace and permits spaces at ambiguous forward TJ gaps while preserving intra-word kerning; XML markup names and namespace declarations are excluded. Partial/reordered-name matching, Unicode normalization and fragment/work limits remain unsupported or unchanged; text inside images, runs under two characters and component-level /Metadata are not matched, so remove hidden information (metadata) remains the certain path" [[feature]] id = "protect.redact-glyph-removal" From 0b7184b4375470cb063c7c1c18e913e169a6515b Mon Sep 17 00:00:00 2001 From: jonathanwxh-cell Date: Fri, 16 Oct 2026 21:37:07 +0800 Subject: [PATCH 5/5] M8: bound metadata redaction work and inspect processing instructions --- ROADMAP.md | 1 + crates/redact/README.md | 31 ++++-- crates/redact/src/interp.rs | 44 +++++++- crates/redact/src/lib.rs | 125 ++++++++++++++++++--- crates/redact/src/metadata.rs | 58 +++++++++- crates/redact/src/metadata_tests.rs | 161 +++++++++++++++++++++++++++- crates/redact/src/tests.rs | 54 +++++++--- docs/roadmap.md | 2 +- parity/acrobat-features.toml | 4 +- 9 files changed, 429 insertions(+), 51 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 38d4517bcf..9509261b35 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -158,6 +158,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-16 (M8, PR #2598 review follow-up):** Match processing-instruction data without matching targets; bound fragment collection and total metadata comparison/decoding work, removing scoped metadata on exhaustion. Saved-secret regressions inspect reopened objects and decoded streams, with a compressed positive control. Whole-run matching policy and estimates unchanged. - **2026-10-16 (M8, PR #2598):** Metadata comparison handles ambiguous TJ word gaps without breaking intra-word kerning, collapses whitespace, and matches decoded XML values rather than markup names. Partial/reordered-name matching and resource limits are unchanged. Estimates unchanged. - **2026-10-16 (M5, PR #2598 CI):** Pen preview regressions hold raster delivery while checking pending strokes, then wait for delivery by state. Adapted narrowly from #2812; no fullscreen or redaction behavior change. Estimates unchanged. diff --git a/crates/redact/README.md b/crates/redact/README.md index f184f902d1..1c1f28ad5d 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -55,9 +55,9 @@ After verification, fragments and values are lowercased with internal whitespace Each `/Info` string entry containing a complete fragment of two characters or more — or whose complete value is contained in a fragment — is cleared with its mirrored XMP property. The author of a whole redacted sentence is removed text too. Then every catalog and page -`/Metadata` stream is parsed as XML and checked value by value (decoded text, comments and -attribute values). Tag names, attribute names, namespace declarations and processing instructions -are not values. XML entities and CDATA are decoded by the parser. A packet is removed whole +`/Metadata` stream is parsed as XML and checked value by value (decoded text, comments, +attribute values and processing-instruction data). Tag names, attribute names, namespace declarations +and processing-instruction targets are excluded. XML entities and CDATA are decoded by the parser. A packet is removed whole when a fragment or part of one survives in a value — custom schemas cannot be edited per-entry — or when the stream cannot be decoded and read as UTF-8 within 16 MiB or parsed as XML. DTDs are unsupported and take this conservative whole-packet fallback. A @@ -84,9 +84,28 @@ The match errs towards clearing, in both directions: Matching still requires a complete removed fragment inside a value, or a complete value inside a fragment. Partial phrases such as Title `Case: James Bond` after redacting `JAMES BOND IS A SPY`, reordered names such as `Smith, John` versus `John Smith`, and Unicode normalization remain -unsupported. No word-term deletion is performed: unrelated Subject `Spy stories` stays. Fragment -count/length and matching-work limits are unchanged; the 16 MiB stream limit is not a bound on -total metadata-matching work. +unsupported. A whole removed `James Bond` still matches `Case: James Bond`; the unsupported case +is a subphrase of a longer removed run. No word-term deletion is performed: unrelated Subject +`Spy stories` stays during selective matching. + +Collection is limited across all selected pages to 1,024 runs, 16 KiB per run and 256 KiB of +run text in total, checked before appending (optional-gap copies have the same bounds). +Metadata comparison has one shared budget of 8 Mi work units, including failed prefix attempts +in repetitive text, value normalization, key scanning and XML inspection. Each prefix character +costs 16 units, including its optional-gap lookup; scanning a string costs its byte length, +and normalization/XML input costs four units per byte. A value is limited to 1 MiB and an +information dictionary to 1,024 entries. Encoded packet bytes are charged before decoding, which +is limited before allocation by the remaining allowance as well as the existing 16 MiB ceiling. +Decoding failures conservatively exhaust the allowance because the decoder does not expose +consumed work. These are bounds on this metadata +pass, not on the whole PDF or the process's memory. + +If collection, a value or the shared work/decoding allowance exceeds its limit, the pass removes +**all** `/Info` and catalog/page `/Metadata`, including unrelated values and earlier clean packets. +It never silently truncates needles or treats unfinished matching as clean. The library report's +`metadata_cleared_on_limit` records this conservative fallback, alongside the actual removal counts; +the normal full rewrite removes retired objects. This does not expand the scan to component-level +metadata or change the partial-name matching policy. Remove hidden information (metadata category, `doc_remove_hidden`) is therefore the certain follow-up when a document's metadata must be provably free of the redacted text. diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 9b6d8bf5dc..a903740c0e 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -95,6 +95,10 @@ pub(crate) struct Scope<'a> { /// Comparison alternatives for ambiguous forward TJ gaps; the canonical runs above stay /// unspaced so intra-word kerning retains its original interpretation. pub kerned_fragments: Vec, + /// These totals travel with the fragments across pages. Once exceeded, collection stops + /// and apply removes all document/page metadata rather than matching truncated evidence. + pub metadata_bytes: usize, + pub metadata_limited: bool, /// The run being collected: its text, and where its last removed glyph's advance ended in /// device space with that glyph's height (`None` = no open run). run: String, @@ -116,6 +120,8 @@ impl<'a> Scope<'a> { layer_blocks: 0, removed_fragments: Vec::new(), kerned_fragments: Vec::new(), + metadata_bytes: 0, + metadata_limited: false, run: String::new(), run_gaps: Vec::new(), run_end: None, @@ -183,6 +189,10 @@ impl<'a> Scope<'a> { pub(crate) fn flush_run(&mut self) { self.run_end = None; if !self.run.is_empty() { + if self.removed_fragments.len() >= crate::metadata::MAX_FRAGMENTS { + self.limit_metadata(); + return; + } if !self.run_gaps.is_empty() { self.kerned_fragments.push(crate::metadata::KernedFragment { text: self.run.clone(), gaps: std::mem::take(&mut self.run_gaps) }); } @@ -190,6 +200,29 @@ impl<'a> Scope<'a> { } } + fn limit_metadata(&mut self) { + self.metadata_limited = true; + self.removed_fragments.clear(); + self.kerned_fragments.clear(); + self.run.clear(); + self.run_gaps.clear(); + self.run_end = None; + } + + fn append_removed(&mut self, text: &str) { + if self.metadata_limited { + return; + } + let run_bytes = self.run.len().saturating_add(text.len()); + let total = self.metadata_bytes.saturating_add(text.len()); + if run_bytes > crate::metadata::MAX_RUN_BYTES || total > crate::metadata::MAX_FRAGMENT_BYTES { + self.limit_metadata(); + return; + } + self.run.push_str(text); + self.metadata_bytes = total; + } + /// A removed glyph: `kerned` says a `TJ` number displaced the text since the last removed /// glyph: the canonical run joins with no space or split. A forward gap may also represent /// a word space, recorded separately for metadata comparison. Otherwise the glyph continues @@ -199,6 +232,9 @@ impl<'a> Scope<'a> { /// space, or the joined text would not match the value it was drawn from). Degenerate /// (zero-height) glyphs and non-finite positions never join. fn removed_glyph(&mut self, text: &str, origin: (f64, f64), end: (f64, f64), height: f64, kerned: bool) { + if self.metadata_limited || text.is_empty() { + return; + } if kerned && self.run_end.is_some() { if let Some(((x, y), previous_height)) = self.run_end { let (dx, dy) = (origin.0 - x, origin.1 - y); @@ -209,7 +245,7 @@ impl<'a> Scope<'a> { self.run_gaps.push(self.run.len()); } } - self.run.push_str(text); + self.append_removed(text); self.run_end = Some((end, height)); return; } @@ -221,15 +257,15 @@ impl<'a> Scope<'a> { let h = lh.max(height); if gap.is_finite() && h.is_finite() && gap <= 1.5 * h { if gap > 0.15 * h && !self.run.ends_with(char::is_whitespace) { - self.run.push(' '); + self.append_removed(" "); } - self.run.push_str(text); + self.append_removed(text); self.run_end = Some((end, height)); return; } } self.flush_run(); - self.run.push_str(text); + self.append_removed(text); self.run_end = Some((end, height)); } } diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 80ba7b69bc..e8d2c39621 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -136,6 +136,9 @@ pub struct Report { /// XMP metadata streams removed whole: they still held removed text where per-entry editing /// cannot reach (custom schemas, page-level packets), or could not be decoded and read. pub xmp_removed: usize, + /// A metadata collection or comparison limit was exceeded. All `/Info` and catalog/page + /// `/Metadata` entries were removed, including unrelated values, rather than risk a leak. + pub metadata_cleared_on_limit: bool, } fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { @@ -351,6 +354,7 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result = Vec::new(); let mut kerned = Vec::new(); + let (mut metadata_bytes, mut metadata_limited) = (0, false); for &(pi, page_marks) in &groups { let page = pdfcraft_model::pages(doc).swap_remove(pi); @@ -361,10 +365,16 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result) -> Result) -> Result Result<(), RedactError> { + report.metadata_cleared_on_limit = true; + let count = doc.trailer().get(b"Info").map(|o| doc.resolve(o)).and_then(|o| o.as_dict().map(Dict::len)).unwrap_or(0); + doc.trailer_mut().remove(b"Info"); + report.info_entries_cleared += count; + let mut owners: Vec<_> = doc.root().into_iter().collect(); + owners.extend(pdfcraft_model::pages(doc).iter().map(|p| p.obj)); + for owner in owners { + if doc.get(owner).as_dict().is_some_and(|d| d.contains(b"Metadata")) { + doc.update_dict(owner, |d| { + d.remove(b"Metadata"); + })?; + report.xmp_removed += 1; + } + } + Ok(()) +} + /// The metadata half of redaction (#2448): a string the marks took off the pages must not stay /// in the document information dictionary or the XMP metadata. Runs of removed text two /// characters or longer are matched case-insensitively with collapsed whitespace (a single clipped glyph is no evidence a @@ -558,8 +592,24 @@ const MAX_METADATA: usize = 16 << 20; /// searchable run (image text, one surviving piece) needs `sanitize`'s metadata category to be /// provably gone. fn clear_marked_metadata(doc: &mut Document, fragments: &[String], kerned: &[KernedFragment], report: &mut Report) -> Result<(), RedactError> { - let needles: Vec = fragments.iter().map(|f| normalize(f)).filter(|f| f.chars().count() >= 2).collect(); - let kerned: Vec<_> = kerned.iter().map(KernedFragment::normalized).filter(|f| f.text.chars().count() >= 2).collect(); + let work = metadata::Work::new(); + // Collection enforces these before allocating; guard this boundary too, before comparison + // normalization can expand Unicode. Optional-gap copies are bounded separately as well. + if fragments.len() > metadata::MAX_FRAGMENTS + || kerned.len() > metadata::MAX_FRAGMENTS + || fragments.iter().any(|f| f.len() > metadata::MAX_RUN_BYTES) + || kerned.iter().any(|f| f.text.len() > metadata::MAX_RUN_BYTES || f.gaps.len() > metadata::MAX_RUN_BYTES) + || fragments.iter().fold(0usize, |n, f| n.saturating_add(f.len())) > metadata::MAX_FRAGMENT_BYTES + || kerned.iter().fold(0usize, |n, f| n.saturating_add(f.text.len())) > metadata::MAX_FRAGMENT_BYTES + { + return clear_metadata_on_limit(doc, report); + } + let needles: Vec<_> = fragments + .iter() + .map(|f| KernedFragment { text: normalize(f), gaps: Vec::new() }) + .chain(kerned.iter().map(KernedFragment::normalized)) + .filter(|f| f.text.chars().take(2).count() >= 2) + .collect(); if needles.is_empty() { return Ok(()); } @@ -567,13 +617,32 @@ fn clear_marked_metadata(doc: &mut Document, fragments: &[String], kerned: &[Ker // direction never matches a value under two characters: one shared character is no evidence // a secret was removed. let hit = |v: &str| { + if v.len() > metadata::MAX_VALUE_BYTES || !work.charge(v.len().saturating_mul(4)) { + work.exhausted.set(true); + return true; + } let v = normalize(v); - let part = v.trim(); - needles.iter().any(|n| v.contains(n.as_str())) - || (part.chars().count() >= 2 && needles.iter().any(|n| n.contains(part))) - || (part.chars().count() >= 2 && kerned.iter().any(|n| n.matches(part))) + v.chars().take(2).count() >= 2 && needles.iter().any(|n| n.matches_with_budget(&v, &work)) }; + // Bound decoding a string value and building the key list before organize::info copies it. + let oversized_info = doc + .trailer() + .get(b"Info") + .map(|o| doc.resolve(o)) + .and_then(|o| { + o.as_dict().map(|d| { + d.len() > metadata::MAX_FRAGMENTS + || d.iter().any(|(key, v)| { + !work.charge(key.len()) || doc.resolve(v).as_string().is_some_and(|s| s.bytes.len() > metadata::MAX_VALUE_BYTES) + }) + }) + }) + .unwrap_or(false); + if oversized_info { + return clear_metadata_on_limit(doc, report); + } + // 1. Entry by entry: the value goes, and with it the XMP property that mirrors the entry. let keys: Vec = doc .trailer() @@ -582,7 +651,11 @@ fn clear_marked_metadata(doc: &mut Document, fragments: &[String], kerned: &[Ker .and_then(|o| o.as_dict().map(|d| d.iter().filter_map(|(k, _)| String::from_utf8(k.clone()).ok()).collect())) .unwrap_or_default(); for key in keys { - if pdfcraft_organize::info(doc, &key).is_some_and(|v| hit(&v)) { + let matches = pdfcraft_organize::info(doc, &key).is_some_and(|v| hit(&v)); + if work.exhausted.get() { + return clear_metadata_on_limit(doc, report); + } + if matches { pdfcraft_organize::set_info(doc, &key, "").map_err(|e| pdfcraft_edit::EditError::Invalid(e.to_string()))?; // A trailer `/Info` that is a direct dictionary is quietly left alone (only the // wholesale step below can take it); count what really went. @@ -598,16 +671,31 @@ fn clear_marked_metadata(doc: &mut Document, fragments: &[String], kerned: &[Ker owners.extend(pdfcraft_model::pages(doc).iter().map(|p| p.obj)); for owner in owners { let Some(entry) = doc.get(owner).as_dict().and_then(|d| d.get(b"Metadata").cloned()) else { continue }; + if !work.charge(1) { + return clear_metadata_on_limit(doc, report); + } let keeps = match &*doc.resolve(&entry) { - Object::Stream(s) => match s.decoded_within(MAX_METADATA) { - Ok(bytes) => match String::from_utf8(bytes) { - Ok(text) => !packet_hit(&text, &hit), - Err(_) => false, - }, - Err(_) => false, - }, + Object::Stream(s) => { + // Reserve room before decoding, even for invalid UTF-8 or failed filters. A + // packet that cannot be inspected within the remaining operation allowance + // exhausts it; do not repeatedly decode a fresh 16 MiB on every page. + if !work.charge(s.raw.len()) { + return clear_metadata_on_limit(doc, report); + } + let limit = MAX_METADATA.min(work.decode_limit()); + match s.decoded_within(limit) { + Ok(bytes) if work.charge(bytes.len().saturating_mul(4)) => String::from_utf8(bytes).is_ok_and(|text| !packet_hit(&text, &hit)), + _ => { + work.exhausted.set(true); + false + } + } + } _ => false, }; + if work.exhausted.get() { + return clear_metadata_on_limit(doc, report); + } if !keeps { doc.update_dict(owner, |d| { d.remove(b"Metadata"); @@ -629,6 +717,9 @@ fn clear_marked_metadata(doc: &mut Document, fragments: &[String], kerned: &[Ker doc.trailer_mut().remove(b"Info"); report.info_entries_cleared += n; } + if work.exhausted.get() { + return clear_metadata_on_limit(doc, report); + } Ok(()) } diff --git a/crates/redact/src/metadata.rs b/crates/redact/src/metadata.rs index 0b6d4e9f0c..6a522952fe 100644 --- a/crates/redact/src/metadata.rs +++ b/crates/redact/src/metadata.rs @@ -1,5 +1,41 @@ //! Comparison-only normalization; the document's original text is never rewritten here. +use std::cell::Cell; + +pub(crate) const MAX_FRAGMENTS: usize = 1_024; +pub(crate) const MAX_RUN_BYTES: usize = 16 << 10; +pub(crate) const MAX_FRAGMENT_BYTES: usize = 256 << 10; +pub(crate) const MAX_VALUE_BYTES: usize = 1 << 20; +const MAX_WORK: usize = 8 << 20; + +/// One shared operation budget, including failed comparisons. Exhaustion is not a no-match: +/// the caller must remove all metadata in its scope, including packets already found clean. +pub(crate) struct Work { + remaining: Cell, + pub exhausted: Cell, +} + +impl Work { + pub fn new() -> Self { + Self { remaining: Cell::new(MAX_WORK), exhausted: Cell::new(false) } + } + + pub fn charge(&self, amount: usize) -> bool { + if !self.exhausted.get() + && let Some(left) = self.remaining.get().checked_sub(amount) + { + self.remaining.set(left); + return true; + } + self.exhausted.set(true); + false + } + + pub fn decode_limit(&self) -> usize { + self.remaining.get() / 4 + } +} + pub(crate) fn normalize(text: &str) -> String { text.split_whitespace().map(str::to_lowercase).collect::>().join(" ") } @@ -34,21 +70,33 @@ impl KernedFragment { Self { text, gaps } } + #[cfg(test)] pub fn matches(&self, value: &str) -> bool { + self.matches_with_budget(value, &Work::new()) + } + + pub fn matches_with_budget(&self, value: &str, work: &Work) -> bool { let Some(first) = self.text.chars().next() else { return false }; // A complete removed run inside a metadata value. - if value.match_indices(first).any(|(start, _)| self.prefix(0, value.get(start..).unwrap_or_default(), true)) { + if !work.charge(value.len()) + || value.match_indices(first).any(|(start, _)| self.prefix(0, value.get(start..).unwrap_or_default(), true, work)) + { return true; } let Some(first) = value.chars().next() else { return false }; // Or the complete metadata value inside a removed run. Never match isolated terms. - self.text.match_indices(first).any(|(start, _)| self.prefix(start, value, false)) + !work.charge(self.text.len()) || self.text.match_indices(first).any(|(start, _)| self.prefix(start, value, false, work)) } - fn prefix(&self, start: usize, value: &str, whole_run: bool) -> bool { + fn prefix(&self, start: usize, value: &str, whole_run: bool, work: &Work) -> bool { let Some(run) = self.text.get(start..) else { return false }; let mut value = value.chars().peekable(); for (offset, c) in run.char_indices() { + // Charge repeated prefix attempts too. A collected run has at most 16 KiB of + // bytes/gaps, so 16 units cover the binary search and fixed character work. + if !work.charge(16) { + return true; + } if value.peek().is_none() { return !whole_run; } @@ -68,6 +116,8 @@ impl KernedFragment { pub(crate) fn packet_hit(text: &str, hit: &impl Fn(&str) -> bool) -> bool { let Ok(doc) = roxmltree::Document::parse(text) else { return true }; doc.descendants().any(|node| { - ((node.is_text() || node.is_comment()) && node.text().is_some_and(hit)) || node.attributes().any(|attribute| hit(attribute.value())) + ((node.is_text() || node.is_comment()) && node.text().is_some_and(hit)) + || node.pi().and_then(|pi| pi.value).is_some_and(hit) + || node.attributes().any(|attribute| hit(attribute.value())) }) } diff --git a/crates/redact/src/metadata_tests.rs b/crates/redact/src/metadata_tests.rs index 2683a9458c..089ee21ba5 100644 --- a/crates/redact/src/metadata_tests.rs +++ b/crates/redact/src/metadata_tests.rs @@ -1,4 +1,4 @@ -//! Metadata comparison regressions for PR #2598 comment 6213443550. +//! Metadata comparison regressions for PR #2598 comments 6213443550 and 6216325944. use super::*; fn packet(payload: &str, attributes: &str) -> String { @@ -141,3 +141,162 @@ fn optional_tj_spaces_preserve_literal_boundaries_and_complete_value_matching() assert_eq!(info_value(&saved, "Author").as_deref(), Some("James Bond"), "zero/backward kern {kern} is not a word gap"); } } + +#[test] +fn processing_instruction_data_is_metadata_but_its_target_is_not() { + for (payload, removed) in [ + ("Harmless", true), + ("Harmless", false), + ("Harmless", false), + ] { + let packet = packet(payload, ""); + let src = if removed { b"BT /F1 10 Tf 10 100 Td (James Bond)Tj ET".as_slice() } else { b"BT /F1 10 Tf 10 100 Td (JamesBond)Tj ET" }; + let saved = redact_all(src, "Author", "Ada", Some(&packet)); + assert_eq!(xmp_text(&saved).is_none(), removed, "{payload}"); + } +} + +#[test] +fn excessive_removed_text_clears_metadata_instead_of_truncating_needles() { + // The explicit character spacing keeps every synthetic glyph at the same origin. + // Each repositioning separates runs without making a huge page or a large fixture file. + for (label, run_len, runs) in [("run length", 16_385, 1), ("run count", 2, 1_025), ("total bytes", 16_384, 17)] { + let mut src = String::from("BT /F1 10 Tf -5 Tc "); + let run = "A".repeat(run_len); + for i in 0..runs { + let x = if i % 2 == 0 { 10 } else { 200 }; + src.push_str(&format!("1 0 0 1 {x} 100 Tm ({run})Tj ")); + } + src.push_str("ET"); + let clean = packet("Harmless", ""); + let mut doc = meta_doc(src.as_bytes(), &[("Author", "Ada"), ("Subject", "Spy stories")], Some(&clean), Some(&clean)); + let rects = [[0.0, 0.0, 300.0, 300.0]]; + mark(&mut doc, 0, &rects, ""); + let report = apply(&mut doc, None).unwrap(); + assert!(report.metadata_cleared_on_limit, "{label}"); + assert_eq!(report.glyphs, run_len * runs, "{label}"); + let saved = reopen(&doc); + assert_eq!(info_value(&saved, "Author"), None, "{label}: incomplete evidence must clear all metadata"); + assert_eq!(info_value(&saved, "Subject"), None, "{label}"); + assert!(xmp_text(&saved).is_none(), "{label}"); + assert!(pdfcraft_model::pages(&saved)[0].dict.get(b"Metadata").is_none(), "{label}"); + assert_eq!((report.info_entries_cleared, report.xmp_removed), (2, 2), "{label}"); + assert_no_secret(&doc, b"Ada"); + assert_no_secret(&doc, b"Spy stories"); + assert_no_secret(&doc, b"Harmless"); + } +} + +#[test] +fn repetitive_optional_gap_matching_has_a_shared_fail_closed_work_limit() { + let run = format!("{}b", "a".repeat(4_096)); + let value = format!("{}c", "a".repeat(4_096)); + let clean = packet("Harmless", ""); + let mut doc = meta_doc(b"", &[("Title", &value), ("Author", "Ada")], Some(&clean), Some(&clean)); + let mut report = Report::default(); + let pattern = KernedFragment { text: run.clone(), gaps: vec![1] }; + clear_marked_metadata(&mut doc, &[run], &[pattern], &mut report).unwrap(); + assert!(report.metadata_cleared_on_limit); + let saved = reopen(&doc); + assert_eq!(info_value(&saved, "Title"), None, "exhaustion must not be treated as no match"); + assert_eq!(info_value(&saved, "Author"), None, "the exhausted budget applies to the whole operation"); + assert!(xmp_text(&saved).is_none()); + assert!(pdfcraft_model::pages(&saved)[0].dict.get(b"Metadata").is_none()); +} + +#[test] +fn compressed_metadata_is_visible_to_the_secret_scan() { + let mut doc = meta_doc(b"", &[("Title", "HIDDEN-COMPRESSED-TITLE")], None, None); + let r = doc.trailer().reference(b"Info").unwrap(); + doc.update_dict(r, |d| { + d.set(b"Subject".to_vec(), Object::String(pdfcraft_cos::PdfString { bytes: vec![0xFE, 0xFF, 0, b'S', 0, b'E', 0, b'C'], hex: false })); + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!bytes.windows(b"HIDDEN-COMPRESSED-TITLE".len()).any(|w| w == b"HIDDEN-COMPRESSED-TITLE"), "the control really is compressed"); + assert!(secret_is_present(&doc, b"HIDDEN-COMPRESSED-TITLE")); + assert!(secret_is_present(&doc, &[0, b'S', 0, b'E', 0, b'C']), "PDF string bytes must be read before serialization escapes them"); +} + +#[test] +fn fragment_collection_limit_is_shared_across_selected_pages() { + let src = format!("BT /F1 10 Tf {} ET", "1 0 0 1 10 100 Tm (AB)Tj 1 0 0 1 200 100 Tm (AB)Tj ".repeat(300)); + for selected in [vec![0], vec![0, 1]] { + let clean = packet("Harmless", ""); + let mut doc = meta_doc(src.as_bytes(), &[("Author", "Ada"), ("Subject", "Spy stories")], Some(&clean), Some(&clean)); + let first = pdfcraft_model::pages(&doc).swap_remove(0); + let second = doc.add(Object::Dict(first.dict)); + let pages = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Pages").unwrap(); + doc.update_dict(pages, |d| { + d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(first.obj), Object::Ref(second)])); + d.set(b"Count".to_vec(), Object::Int(2)); + }) + .unwrap(); + for pi in &selected { + mark(&mut doc, *pi, &[[0.0, 0.0, 300.0, 300.0]], ""); + } + let report = apply(&mut doc, Some(&selected)).unwrap(); + let saved = reopen(&doc); + assert_eq!(report.metadata_cleared_on_limit, selected.len() == 2); + assert_eq!(info_value(&saved, "Author").is_none(), selected.len() == 2); + assert_eq!(xmp_text(&saved).is_none(), selected.len() == 2); + assert_eq!(report.glyphs, 1_200 * selected.len()); + if selected.len() == 2 { + assert_eq!((report.info_entries_cleared, report.xmp_removed), (2, 3)); + assert!(pdfcraft_model::pages(&saved).iter().all(|p| p.dict.get(b"Metadata").is_none())); + } + } +} + +#[test] +fn a_later_packet_exhausts_the_same_budget_and_retires_earlier_clean_metadata() { + let run = format!("{}b", "a".repeat(4_096)); + let value = format!("{}c", "a".repeat(4_096)); + let clean = packet("Harmless", ""); + let costly = packet(&format!("{value}"), ""); + let mut doc = meta_doc(b"", &[("Author", "Ada")], Some(&clean), Some(&costly)); + let mut report = Report::default(); + let pattern = KernedFragment { text: run.clone(), gaps: vec![1] }; + clear_marked_metadata(&mut doc, &[run], &[pattern], &mut report).unwrap(); + assert!(report.metadata_cleared_on_limit); + assert_eq!((report.info_entries_cleared, report.xmp_removed), (1, 2)); + let saved = reopen(&doc); + assert_eq!(info_value(&saved, "Author"), None); + assert!(xmp_text(&saved).is_none()); + assert!(pdfcraft_model::pages(&saved)[0].dict.get(b"Metadata").is_none()); +} + +#[test] +fn exact_collection_boundaries_keep_unrelated_metadata() { + for (run_len, runs) in [(16_384, 1), (2, 1_024), (16_384, 16)] { + let run = "A".repeat(run_len); + let mut src = String::from("BT /F1 10 Tf -5 Tc "); + for i in 0..runs { + src.push_str(&format!("1 0 0 1 {} 100 Tm ({run})Tj ", if i % 2 == 0 { 10 } else { 200 })); + } + src.push_str("ET"); + let mut doc = meta_doc(src.as_bytes(), &[("Author", "Someone")], None, None); + mark(&mut doc, 0, &[[0.0, 0.0, 300.0, 300.0]], ""); + let report = apply(&mut doc, None).unwrap(); + assert!(!report.metadata_cleared_on_limit, "{run_len} bytes by {runs} runs"); + assert_eq!(info_value(&reopen(&doc), "Author").as_deref(), Some("Someone")); + } +} + +#[test] +fn metadata_values_and_packet_decoding_share_the_operation_allowance() { + for packet_limit in [false, true] { + let large = "Z".repeat(if packet_limit { 2 << 20 } else { (1 << 20) + 1 }); + let clean = packet("Harmless", ""); + let costly = packet(&format!("{large}"), ""); + let mut doc = meta_doc(b"", &[("Title", if packet_limit { "Ada" } else { &large })], Some(&clean), Some(&costly)); + let mut report = Report::default(); + clear_marked_metadata(&mut doc, &["AB".into()], &[], &mut report).unwrap(); + assert!(report.metadata_cleared_on_limit); + assert_eq!((report.info_entries_cleared, report.xmp_removed), (1, 2)); + let saved = reopen(&doc); + assert_eq!(info_value(&saved, "Title"), None); + assert!(xmp_text(&saved).is_none()); + assert!(pdfcraft_model::pages(&saved)[0].dict.get(b"Metadata").is_none()); + } +} diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index ba6e30026d..29db78c7d4 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -706,8 +706,8 @@ fn redacted_text_is_cleared_from_info_and_xmp_on_apply() { assert!(xmp.contains("Writer 1.0"), "the rest of the packet stays: {xmp}"); let page = &pdfcraft_model::pages(&doc)[0]; assert!(page.dict.get(b"Metadata").is_some(), "a clean page-level stream stays"); - let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); - assert!(!String::from_utf8_lossy(&bytes).to_lowercase().contains("topsecret"), "no trace in the saved file"); + assert_no_secret(&doc, b"TOPSECRET"); + assert_no_secret(&doc, b"TopSecret"); } /// Issue #2448, review follow-up: a value SHORTER than the removed run is still removed text — @@ -731,7 +731,8 @@ fn a_value_inside_a_longer_removed_run_is_cleared_too() { let cat = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); assert!(!cat.contains(b"Metadata"), "the custom-schema value took the packet with it"); let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); - assert!(!String::from_utf8_lossy(&bytes).to_lowercase().contains("james bond"), "no trace in the saved file"); + assert_no_secret(&doc, b"JAMES BOND"); + assert_no_secret(&doc, b"James Bond"); let reread = Document::open(Arc::new(bytes)).unwrap(); assert_eq!(info_value(&reread, "Subject").as_deref(), Some("Spy stories"), "unrelated metadata survives in the saved file"); } @@ -758,10 +759,9 @@ fn an_xmp_stream_holding_redacted_text_is_removed_whole() { assert!(!cat.contains(b"Metadata"), "the escaped secret took the catalog stream with it"); let page = &pdfcraft_model::pages(&doc)[0]; assert!(page.dict.get(b"Metadata").is_none(), "the page-level stream went too"); - let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); - let text = String::from_utf8_lossy(&bytes).to_lowercase(); - assert!(!text.contains("secret&plan") && !text.contains("secret&plan"), "no escaped trace in the saved file"); - assert!(!text.contains("othertext"), "the page-level secret is gone too"); + assert_no_secret(&doc, b"SECRET&PLAN"); + assert_no_secret(&doc, b"SECRET&PLAN"); + assert_no_secret(&doc, b"OTHERTEXT"); } /// Issue #2448: a trailer `/Info` that is a direct dictionary is the one shape an entry-by-entry @@ -783,8 +783,8 @@ fn a_direct_info_dictionary_holding_redacted_text_is_removed_whole() { if let Some(d) = &info { assert!(!d.contains(b"Title") && !d.contains(b"Author"), "nothing of it came back: {d:?}"); } - let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); - assert!(!String::from_utf8_lossy(&bytes).to_lowercase().contains("topsecret"), "no trace in the saved file"); + assert_no_secret(&doc, b"TOPSECRET"); + assert_no_secret(&doc, b"TopSecret"); } /// PR #2598: many generators draw text one glyph per operator (`(T) Tj 5 0 Td (O) Tj …`). The @@ -936,7 +936,7 @@ fn utf16_info_entries_holding_redacted_text_are_cleared() { assert_eq!(info_value(&doc, "Author").as_deref(), Some("Ada"), "unrelated entries stay"); let needle: Vec = "SECRET€".encode_utf16().flat_map(u16::to_be_bytes).collect(); let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); - assert!(!bytes.windows(needle.len()).any(|w| w == needle), "no UTF-16BE trace in the saved file"); + assert_no_secret(&doc, &needle); // A full write may pack /Info into a compressed object stream, so survival is checked by // reading the file back, not by scanning its bytes. let reread = Document::open(Arc::new(bytes)).unwrap(); @@ -1217,16 +1217,38 @@ fn tags_lose_what_redaction_removed() { } fn assert_no_secret(doc: &Document, secret: &[u8]) { - let saved = reopen(doc); - for (r, object) in saved.scan_objects() { + assert!(!secret_is_present(doc, secret), "secret remains in reopened objects or decoded streams"); +} + +fn secret_is_present(doc: &Document, secret: &[u8]) -> bool { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let saved = Document::open(Arc::new(bytes)).unwrap(); + let has = |bytes: &[u8]| bytes.windows(secret.len()).any(|w| w == secret); + for (_, object) in saved.scan_objects() { let mut bytes = Vec::new(); pdfcraft_cos::serialize(&object, &mut bytes); - assert!(!bytes.windows(secret.len()).any(|w| w == secret), "secret remains in object {r:?}"); - if let Object::Stream(s) = &*object { - let data = s.decoded().unwrap(); - assert!(!data.windows(secret.len()).any(|w| w == secret), "secret remains in stream {r:?}"); + if has(&bytes) { + return true; + } + if let Object::Stream(s) = &*object + && has(&s.decoded().unwrap()) + { + return true; + } + // Serialization can escape PDF string bytes (notably UTF-16BE). Inspect strings in + // reopened arrays/dictionaries directly as well, without following reference cycles. + let mut pending = vec![&*object]; + while let Some(value) = pending.pop() { + match value { + Object::String(s) if has(&s.bytes) => return true, + Object::Array(a) => pending.extend(a), + Object::Dict(d) => pending.extend(d.iter().map(|(_, value)| value)), + Object::Stream(s) => pending.extend(s.dict.iter().map(|(_, value)| value)), + _ => {} + } } } + false } #[test] diff --git a/docs/roadmap.md b/docs/roadmap.md index 8390d44dd3..31c42d4480 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -124,7 +124,7 @@ total matches the parity estimate. | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Existing simple field-font encodings and Differences are respected when generating appearances; an unencodable value uses a separate Helvetica appearance font without changing the source font. Synthetic reopen regressions cover punctuation, subset fallback and multiline text. Exact font metrics/layout and broader Unicode fallback remain incomplete. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Tab and Shift-Tab wrapping to the same field or another widget of that field refresh the newly opened editor from the accepted value before Save. Synthetic UI regressions cover both directions, same-frame Save, validation refusal and undo/redo. | | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | -| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding Metadata comparison handles TJ spacing, normalized whitespace and XML values (PR #2598); partial/reordered names remain unsupported. | +| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding Metadata comparison handles TJ spacing, normalized whitespace and XML values (PR #2598); partial/reordered names remain unsupported. Metadata processing-instruction data is checked; collection and comparison limits clear scoped metadata on exhaustion, with reopened-object regression checks. | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites, page validation overlays separate from original signature artwork (baked Not Verified captions remain; #79 open), snapshot tabs that reuse instead of multiplying. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | | M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Missing: OCR languages beyond Latin, editable-text OCR output, full Office layout fidelity/import | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 902f7b3a3f..276edd9130 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5680,8 +5680,8 @@ milestone = "M8" status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] -evidence = ["crates/ui-egui/tests/ui_group/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::redaction_is_refused_when_the_page_still_shows_text_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources", "crates/redact/src/tests.rs::redacted_text_is_cleared_from_info_and_xmp_on_apply", "crates/redact/src/tests.rs::an_xmp_stream_holding_redacted_text_is_removed_whole", "crates/redact/src/tests.rs::a_value_inside_a_longer_removed_run_is_cleared_too", "crates/automation/tests/automation.rs::redacting_clears_matching_document_info_through_tools", "crates/redact/src/metadata_tests.rs::tj_word_gaps_and_literal_spaces_clear_the_same_name", "crates/redact/src/metadata_tests.rs::metadata_whitespace_is_normalized_without_broadening_name_matching", "crates/redact/src/metadata_tests.rs::xmp_matches_values_and_preserves_structure_only_matches"] -notes = "confirmation; one undo step; next save is a full rewrite. Refused, with nothing changed, when the page still shows text under a mark afterwards: text whose font is set through an ExtGState or taken by a form from its caller, some predefined CMaps and rotated Type 3 fonts are not removed yet, so those documents cannot be redacted. Removed images and forms are unlinked from page-tree resources (own, inherited or shared), not yet from a form's own resources, annotation appearances or patterns. /Info entries repeating a removed text run — or forming part of one, for a value shorter than the run — are cleared with their mirrored XMP property, and an XMP stream still holding removed text where per-entry editing cannot reach (custom schemas, entity-escaped values, page-level packets) or that cannot be decoded and parsed as XML is removed whole. Comparison collapses whitespace and permits spaces at ambiguous forward TJ gaps while preserving intra-word kerning; XML markup names and namespace declarations are excluded. Partial/reordered-name matching, Unicode normalization and fragment/work limits remain unsupported or unchanged; text inside images, runs under two characters and component-level /Metadata are not matched, so remove hidden information (metadata) remains the certain path" +evidence = ["crates/ui-egui/tests/ui_group/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::redaction_is_refused_when_the_page_still_shows_text_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources", "crates/redact/src/tests.rs::redacted_text_is_cleared_from_info_and_xmp_on_apply", "crates/redact/src/tests.rs::an_xmp_stream_holding_redacted_text_is_removed_whole", "crates/redact/src/tests.rs::a_value_inside_a_longer_removed_run_is_cleared_too", "crates/automation/tests/automation.rs::redacting_clears_matching_document_info_through_tools", "crates/redact/src/metadata_tests.rs::tj_word_gaps_and_literal_spaces_clear_the_same_name", "crates/redact/src/metadata_tests.rs::metadata_whitespace_is_normalized_without_broadening_name_matching", "crates/redact/src/metadata_tests.rs::xmp_matches_values_and_preserves_structure_only_matches", "crates/redact/src/metadata_tests.rs::processing_instruction_data_is_metadata_but_its_target_is_not", "crates/redact/src/metadata_tests.rs::excessive_removed_text_clears_metadata_instead_of_truncating_needles", "crates/redact/src/metadata_tests.rs::repetitive_optional_gap_matching_has_a_shared_fail_closed_work_limit", "crates/redact/src/metadata_tests.rs::compressed_metadata_is_visible_to_the_secret_scan"] +notes = "confirmation; one undo step; next save is a full rewrite. Refused, with nothing changed, when the page still shows text under a mark afterwards: text whose font is set through an ExtGState or taken by a form from its caller, some predefined CMaps and rotated Type 3 fonts are not removed yet, so those documents cannot be redacted. Removed images and forms are unlinked from page-tree resources (own, inherited or shared), not yet from a form's own resources, annotation appearances or patterns. /Info entries repeating a removed text run — or forming part of one, for a value shorter than the run — are cleared with their mirrored XMP property, and an XMP stream still holding removed text where per-entry editing cannot reach (custom schemas, entity-escaped values, page-level packets) or that cannot be decoded and parsed as XML is removed whole. Comparison collapses whitespace and permits spaces at ambiguous forward TJ gaps while preserving intra-word kerning; XML markup names and namespace declarations are excluded. Processing-instruction data is matched, targets are excluded. Collection and matching have shared limits; exhausting one clears all /Info and catalog/page metadata and is recorded in the library report. Partial/reordered-name matching and Unicode normalization remain unsupported; text inside images, runs under two characters and component-level /Metadata are not matched, so remove hidden information (metadata) remains the certain path" [[feature]] id = "protect.redact-glyph-removal"