Skip to content

perf(reflow): merge emphasis and code span extraction into a single cmark pass #732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rvben merged 3 commits into from
Jul 14, 2026
Merged

perf(reflow): merge emphasis and code span extraction into a single cmark pass #732

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
450ab23
perf(reflow): optimize MyST inline role parsing using pre-extracted c…
chandlerc Jul 11, 2026
11329b2
fix(reflow): ensure safe UTF-8 slicing on fallback paths and merge te…
chandlerc Jul 11, 2026
94e88b7
perf(reflow): merge emphasis and code span extraction into a single c…
chandlerc Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 67 additions & 77 deletions src/utils/text_reflow.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -746,16 +746,21 @@ struct EmphasisSpan {
/// - GFM strikethrough (~~text~~)
///
/// Returns spans sorted by start position.
fn extract_emphasis_spans(text: &str) -> Vec<EmphasisSpan> {
// Every emphasis, strong, or strikethrough marker starts with one of these
// bytes, so their absence rules out any span without running the parser.
if !text.contains(['*', '_', '~']) {
return Vec::new();
fn extract_emphasis_and_code_spans(text: &str) -> (Vec<EmphasisSpan>, Vec<CodeSpan>) {
// If neither marker is present, skip the parser entirely.
let has_emphasis = text.contains(['*', '_', '~']);
let has_code = text.contains('`');
if !has_emphasis && !has_code {
return (Vec::new(), Vec::new());
}

let mut spans = Vec::new();
let mut emphasis_spans = Vec::new();
let mut code_spans = Vec::new();

let mut options = Options::empty();
options.insert(Options::ENABLE_STRIKETHROUGH);
if has_emphasis {
options.insert(Options::ENABLE_STRIKETHROUGH);
}

// Stacks to track nested formatting with their start positions
let mut emphasis_stack: Vec<(usize, bool)> = Vec::new(); // (start_byte, uses_underscore)
Expand All @@ -766,20 +771,25 @@ fn extract_emphasis_spans(text: &str) -> Vec<EmphasisSpan> {

for (event, range) in parser {
match event {
Event::Code(_) => {
code_spans.push(CodeSpan {
start: range.start,
end: range.end,
});
}
Event::Start(Tag::Emphasis) => {
// Check if this uses underscore by looking at the original text
let uses_underscore = text.get(range.start..range.start + 1) == Some("_");
emphasis_stack.push((range.start, uses_underscore));
}
Event::End(TagEnd::Emphasis) => {
if let Some((start_byte, uses_underscore)) = emphasis_stack.pop() {
// Extract content between the markers (1 char marker on each side)
let content_start = start_byte + 1;
let content_end = range.end - 1;
if content_end > content_start
&& let Some(content) = text.get(content_start..content_end)
{
spans.push(EmphasisSpan {
emphasis_spans.push(EmphasisSpan {
start: start_byte,
end: range.end,
content: content.to_string(),
Expand All @@ -792,19 +802,17 @@ fn extract_emphasis_spans(text: &str) -> Vec<EmphasisSpan> {
}
}
Event::Start(Tag::Strong) => {
// Check if this uses underscore by looking at the original text
let uses_underscore = text.get(range.start..range.start + 2) == Some("__");
strong_stack.push((range.start, uses_underscore));
}
Event::End(TagEnd::Strong) => {
if let Some((start_byte, uses_underscore)) = strong_stack.pop() {
// Extract content between the markers (2 char marker on each side)
let content_start = start_byte + 2;
let content_end = range.end - 2;
if content_end > content_start
&& let Some(content) = text.get(content_start..content_end)
{
spans.push(EmphasisSpan {
emphasis_spans.push(EmphasisSpan {
start: start_byte,
end: range.end,
content: content.to_string(),
Expand All @@ -821,17 +829,14 @@ fn extract_emphasis_spans(text: &str) -> Vec<EmphasisSpan> {
}
Event::End(TagEnd::Strikethrough) => {
if let Some(start_byte) = strikethrough_stack.pop() {
// pulldown-cmark's GFM strikethrough accepts both ~~text~~ and
// ~text~. Detect the actual marker width so single-tilde spans
// keep their first and last content character.
let double = text.get(start_byte..start_byte + 2) == Some("~~");
let marker_len = if double { 2 } else { 1 };
let content_start = start_byte + marker_len;
let content_end = range.end - marker_len;
if content_end > content_start
&& let Some(content) = text.get(content_start..content_end)
{
spans.push(EmphasisSpan {
emphasis_spans.push(EmphasisSpan {
start: start_byte,
end: range.end,
content: content.to_string(),
Expand All @@ -847,9 +852,8 @@ fn extract_emphasis_spans(text: &str) -> Vec<EmphasisSpan> {
}
}

// Sort by start position
spans.sort_by_key(|s| s.start);
spans
emphasis_spans.sort_by_key(|s| s.start);
(emphasis_spans, code_spans)
}

#[derive(Debug, Clone)]
Expand Down Expand Up @@ -988,7 +992,7 @@ fn extract_link_spans(text: &str, defined_references: Option<&HashSet<String>>)
/// a `{`, a name starting with an ASCII letter or `_` and continuing with
/// alphanumerics / `-` / `_` / `:` / `.`, a closing `}`, then a balanced inline
/// code span using one or more backticks. Returns `None` when any part is missing.
fn myst_role_len_at(text: &str) -> Option<usize> {
fn myst_role_len_at(text: &str, absolute_pos: usize, code_spans: &[CodeSpan]) -> Option<usize> {
let bytes = text.as_bytes();
if bytes.first() != Some(&b'{') {
return None;
Expand All @@ -1013,26 +1017,11 @@ fn myst_role_len_at(text: &str) -> Option<usize> {
j += 1; // past '}'

// Must be immediately followed by an inline code span.
if bytes.get(j) != Some(&b'`') {
return None;
}
let backtick_start = j;
while bytes.get(j) == Some(&b'`') {
j += 1;
}
let backtick_count = j - backtick_start;

// Find the matching run of `backtick_count` backticks.
while j + backtick_count <= bytes.len() {
if bytes[j] == b'`' {
let close_count = bytes[j..].iter().take_while(|&&b| b == b'`').count();
if close_count == backtick_count {
return Some(j + close_count);
}
j += close_count;
} else {
j += 1;
}
let code_span_start = absolute_pos + j;
if let Ok(idx) = code_spans.binary_search_by_key(&code_span_start, |span| span.start) {
let span = &code_spans[idx];
let code_span_len = span.end - span.start;
return Some(j + code_span_len);
}

None
Expand Down Expand Up @@ -1065,15 +1054,11 @@ fn parse_markdown_elements_inner(
let mut remaining = text;

// Pre-extract emphasis spans, link spans, and code spans using pulldown-cmark.
// These run as separate parses rather than one shared parser: link resolution
// (the broken-link callback, needed to keep undefined shortcuts/references
// atomic) changes which bracket runs collapse into link nodes, which shifts
// where nearby emphasis delimiters pair up in edge cases (e.g. an unresolved
// shortcut containing `**`). Sharing a parser would leak that shift into
// emphasis_spans; each parse keeps its own event stream self-consistent.
let emphasis_spans = extract_emphasis_spans(text);
// Emphasis and code spans are extracted in a single shared parse to reduce cmark overhead.
// Link spans must run as a separate parse because link resolution (the broken-link
// callback) changes bracket collapses, which shifts delimiter range boundaries.
let (emphasis_spans, code_spans) = extract_emphasis_and_code_spans(text);
let link_spans = extract_link_spans(text, defined_references);
let code_spans = extract_code_spans(text);

// Caching state to avoid O(N^2) worst case on long inputs
let mut cached_wiki_link: Option<Option<PatternMatch>> = None;
Expand Down Expand Up @@ -1331,7 +1316,7 @@ fn parse_markdown_elements_inner(
if myst_roles
&& let Some(pos) = next_curly_pos
&& pos < next_special
&& let Some(role_len) = myst_role_len_at(&remaining[pos..])
&& let Some(role_len) = myst_role_len_at(&remaining[pos..], current_offset + pos, &code_spans)
{
next_special = pos;
special_type = "myst_role";
Expand Down Expand Up @@ -1482,11 +1467,7 @@ fn parse_markdown_elements_inner(
elements.push(Element::HtmlTag(remaining[pos..match_end].to_string()));
remaining = &remaining[match_end..];
}
_ => {
// Unknown pattern, treat as text
elements.push(Element::Text("[".to_string()));
remaining = &remaining[1..];
}
_ => unreachable!("unknown pattern type: {}", pattern_type),
}
} else {
// Process non-link special characters
Expand Down Expand Up @@ -1516,30 +1497,25 @@ fn parse_markdown_elements_inner(
}
"pulldown_emphasis" => {
// Use pre-extracted emphasis/strikethrough span from pulldown-cmark
if let Some(span) = pulldown_emphasis {
let span_len = span.end - span.start;
if span.is_strikethrough {
elements.push(Element::Strikethrough {
content: span.content.clone(),
double: span.strikethrough_double,
});
} else if span.is_strong {
elements.push(Element::Bold {
content: span.content.clone(),
underscore: span.uses_underscore,
});
} else {
elements.push(Element::Italic {
content: span.content.clone(),
underscore: span.uses_underscore,
});
}
remaining = &remaining[span_len..];
let span = pulldown_emphasis.expect("pulldown_emphasis must be set");
let span_len = span.end - span.start;
if span.is_strikethrough {
elements.push(Element::Strikethrough {
content: span.content.clone(),
double: span.strikethrough_double,
});
} else if span.is_strong {
elements.push(Element::Bold {
content: span.content.clone(),
underscore: span.uses_underscore,
});
} else {
// Fallback - shouldn't happen
elements.push(Element::Text(remaining[..1].to_string()));
remaining = &remaining[1..];
elements.push(Element::Italic {
content: span.content.clone(),
underscore: span.uses_underscore,
});
}
remaining = &remaining[span_len..];
}
_ => {
// No special elements found, add all remaining text
Expand All @@ -1550,7 +1526,21 @@ fn parse_markdown_elements_inner(
}
}

elements
// Merge contiguous text elements to clean up the output.
let mut merged_elements = Vec::new();
for el in elements {
match el {
Element::Text(s) => {
if let Some(Element::Text(last_s)) = merged_elements.last_mut() {
last_s.push_str(&s);
} else {
merged_elements.push(Element::Text(s));
}
}
other => merged_elements.push(other),
}
}
merged_elements
}

fn should_insert_space_before_join(current: &str) -> bool {
Expand Down
Loading