feat(markdown): freeze the Markdown profile and canonical AST view

P0 baseline: one Markdown parser, one frozen option set, no silent upgrades.

- profile: CommonMark 0.31.2 core + GFM tables/strikethrough/task lists +
  dollar math + YAML front matter at byte 0, with every other extension
  disabled and asserted by tests. comrak is pinned to exactly 1.0.0.
- ast::MdNode: owned, serializable node tree carrying kind, byte Span and
  literal payloads. Nodes outside the profile are preserved as MdKind::Other
  instead of being dropped; task list items keep their checkbox state.
- frontmatter: extracts the YAML body from the parser node, parses it once,
  and exposes only the explicit `import` alias mapping; invalid YAML is
  reported, never guessed.
- text: strict executable payload extraction (Text/Code only, everything else
  is a hard error) and inert plain-text extraction for heading names.
- UTF-8 BOM is tolerated; all spans are re-based onto the original file.

No semantic mapping or execution exists in this crate by design.
This commit is contained in:
cflmy committed 2026-11-20 10:25:02 +08:00
1 parent 1520244923
commit 10c0c0a013
5 files changed
+1006

No files matched your search

+230
View File
@@ -0,0 +1,230 @@
//! Owned, serializable view of the CommonMark/GFM/Math AST.
//!
//! The profile uses exactly one Markdown parser (comrak). This module converts
//! the parser's arena tree into an owned tree that keeps:
//!
//! * the node kind, so program roles can be derived from AST structure alone;
//! * the byte [`Span`] in the original file, for diagnostics and IDE support;
//! * literal payloads for inert leaves (text, code, math, HTML, front matter);
//! * the full child structure, including nodes outside the frozen profile,
//! which are preserved as [`MdKind::Other`] rather than dropped.
use marqdo_span::Span;
use serde::{Deserialize, Serialize};
/// A node of the canonical Markdown AST view.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MdNode {
/// Structural kind and its small attribute set.
pub kind: MdKind,
/// Byte span in the original source (UTF-8, half open).
pub span: Span,
/// Literal payload for leaves that carry text (Text, Code, Math, ...).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub literal: Option<String>,
/// Child nodes in document order.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub children: Vec<MdNode>,
}
/// Structural kind of a Markdown AST node.
///
/// The variants mirror the frozen Markdown profile. Nodes produced by parser
/// extensions that are not part of the profile are preserved as `Other` so the
/// AST view never hides syntax, but they are inert for the language.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum MdKind {
/// Document root.
Document,
/// YAML front matter block at the start of the document.
FrontMatter,
/// Block quote (an invocation in executable position).
BlockQuote,
/// Ordered or bullet list.
List {
/// `true` for ordered lists, `false` for bullet lists.
ordered: bool,
/// Ordered list start number; `None` for bullet lists.
#[serde(default, skip_serializing_if = "Option::is_none")]
start: Option<usize>,
/// CommonMark tight/loose flag (a rendering distinction only).
tight: bool,
},
/// One list item. Task list items (GFM) keep their checkbox state.
Item {
/// `Some(checked)` for a GFM task list item, `None` otherwise.
#[serde(default, skip_serializing_if = "Option::is_none")]
task: Option<bool>,
},
/// Heading of level 1-6.
Heading {
/// Level 1-6.
level: u8,
/// Whether the source used a Setext underline.
setext: bool,
},
/// Paragraph.
Paragraph,
/// Literal text.
Text,
/// Strong emphasis: an execution statement in executable position.
Strong,
/// Emphasis: a return statement in executable position.
Emph,
/// GFM strikethrough.
Strikethrough,
/// Inline code span.
Code,
/// Fenced or indented code block.
CodeBlock {
/// Whether the block was fenced.
fenced: bool,
/// Info string (language) without surrounding whitespace.
info: String,
},
/// Inline math.
Math {
/// Whether display math (`$$...$$`).
display: bool,
},
/// Block display math (`$$` fences on their own lines).
MathBlock,
/// Link with a destination.
Link {
/// Destination as written.
url: String,
/// Optional title.
title: String,
},
/// Image with a source.
Image {
/// Source as written.
url: String,
/// Optional title.
title: String,
},
/// GFM table.
Table {
/// Column alignments as `none`/`left`/`center`/`right`.
alignments: Vec<String>,
},
/// Table row.
TableRow {
/// Whether this is the header row.
header: bool,
},
/// Table cell.
TableCell,
/// Horizontal rule.
ThematicBreak,
/// Soft line break inside a paragraph.
SoftBreak,
/// Hard line break.
LineBreak,
/// Raw HTML block.
HtmlBlock,
/// Raw inline HTML.
HtmlInline,
/// Footnote definition (profile leaves footnotes disabled; kept for completeness).
FootnoteDefinition {
/// Footnote label.
name: String,
},
/// Footnote reference (profile leaves footnotes disabled).
FootnoteReference {
/// Footnote label.
name: String,
},
/// A node outside the frozen profile, preserved for inspection only.
Other {
/// Parser tag name.
tag: String,
},
}
impl MdNode {
/// Create a leaf node.
pub fn leaf(kind: MdKind, span: Span, literal: Option<String>) -> Self {
Self {
kind,
span,
literal,
children: Vec::new(),
}
}
/// Depth-first pre-order traversal including `self`.
pub fn walk<'a>(&'a self, f: &mut impl FnMut(&'a MdNode)) {
f(self);
for child in &self.children {
child.walk(f);
}
}
/// Direct child nodes of a given kind.
pub fn children_of<'a>(&'a self, kind: &MdKind) -> impl Iterator<Item = &'a MdNode> {
let kind = kind.clone();
self.children.iter().filter(move |child| child.kind == kind)
}
/// Whether this node is an inline node.
pub fn is_inline(&self) -> bool {
matches!(
self.kind,
MdKind::Text
| MdKind::Strong
| MdKind::Emph
| MdKind::Strikethrough
| MdKind::Code
| MdKind::Math { .. }
| MdKind::Link { .. }
| MdKind::Image { .. }
| MdKind::SoftBreak
| MdKind::LineBreak
| MdKind::HtmlInline
| MdKind::FootnoteReference { .. }
)
}
/// Whether this node is a block node.
pub fn is_block(&self) -> bool {
!self.is_inline()
}
}
impl MdKind {
/// Short stable tag name used in diagnostics.
pub const fn tag(&self) -> &'static str {
match self {
MdKind::Document => "document",
MdKind::FrontMatter => "front_matter",
MdKind::BlockQuote => "block_quote",
MdKind::List { .. } => "list",
MdKind::Item { .. } => "item",
MdKind::Heading { .. } => "heading",
MdKind::Paragraph => "paragraph",
MdKind::Text => "text",
MdKind::Strong => "strong",
MdKind::Emph => "emph",
MdKind::Strikethrough => "strikethrough",
MdKind::Code => "code",
MdKind::CodeBlock { .. } => "code_block",
MdKind::Math { .. } => "math",
MdKind::MathBlock => "math_block",
MdKind::Link { .. } => "link",
MdKind::Image { .. } => "image",
MdKind::Table { .. } => "table",
MdKind::TableRow { .. } => "table_row",
MdKind::TableCell => "table_cell",
MdKind::ThematicBreak => "thematic_break",
MdKind::SoftBreak => "soft_break",
MdKind::LineBreak => "line_break",
MdKind::HtmlBlock => "html_block",
MdKind::HtmlInline => "html_inline",
MdKind::FootnoteDefinition { .. } => "footnote_definition",
MdKind::FootnoteReference { .. } => "footnote_reference",
MdKind::Other { .. } => "other",
}
}
}
+163
View File
@@ -0,0 +1,163 @@
//! YAML front matter handling.
//!
//! Front matter is only recognised at byte offset 0 (after an optional UTF-8
//! BOM) and only when the Markdown parser itself reports a `FrontMatter` node.
//! The parser node keeps the raw delimiters; this module extracts the YAML body
//! and exposes the explicit `import` mapping without interpreting anything
//! else. Front matter never executes.
use std::collections::BTreeMap;
use marqdo_span::Span;
/// Parsed (or unparseable) front matter.
#[derive(Debug, Clone, PartialEq)]
pub struct FrontMatter {
/// Raw YAML body between the delimiter lines, excluding the delimiters.
pub raw: String,
/// Parsed YAML value; `Err` carries a human readable reason.
pub yaml: Result<serde_yaml::Value, String>,
/// Span of the whole front matter node (including delimiters).
pub span: Span,
/// Span of the YAML body only.
pub yaml_span: Span,
}
impl FrontMatter {
/// The explicit `import` mapping from front matter.
///
/// The supported shape is a YAML mapping of local alias to module address:
///
/// ```yaml
/// import:
/// geometry: "@geometry"
/// ```
///
/// Anything else is an error; imports are never inferred.
pub fn imports(&self) -> Result<BTreeMap<String, String>, String> {
let yaml = self
.yaml
.as_ref()
.map_err(|error| format!("front matter is not valid YAML: {error}"))?;
let Some(imports) = yaml.get("import") else {
return Ok(BTreeMap::new());
};
let Some(mapping) = imports.as_mapping() else {
return Err("`import` must be a mapping of alias to module address".to_string());
};
let mut result = BTreeMap::new();
for (alias, target) in mapping {
let Some(alias) = alias.as_str() else {
return Err("`import` aliases must be strings".to_string());
};
let Some(target) = target.as_str() else {
return Err(format!("import target for `{alias}` must be a string"));
};
if alias.is_empty() {
return Err("`import` alias must not be empty".to_string());
}
if result
.insert(alias.to_string(), target.to_string())
.is_some()
{
return Err(format!("duplicate import alias `{alias}`"));
}
}
Ok(result)
}
/// The raw YAML parses successfully.
pub fn is_valid(&self) -> bool {
self.yaml.is_ok()
}
}
/// Split a `FrontMatter` parser literal into the YAML body.
///
/// The parser literal starts with the opening delimiter line and includes the
/// closing delimiter line plus trailing newlines, e.g.
/// `"---\nimport:\n a: b\n---\n\n"`.
pub fn parse_literal(literal: &str, span: Span) -> FrontMatter {
let (raw, body_offset) = split_delimiters(literal);
let yaml = if raw.trim().is_empty() {
Ok(serde_yaml::Value::Null)
} else {
serde_yaml::from_str::<serde_yaml::Value>(&raw).map_err(|error| error.to_string())
};
let start = span.start.saturating_add(body_offset);
let end = start.saturating_add(raw.len()).min(span.end.max(start));
FrontMatter {
raw,
yaml,
span,
yaml_span: Span::new(start, end),
}
}
/// Return the YAML body and its byte offset within `literal`.
fn split_delimiters(literal: &str) -> (String, usize) {
let mut offset = 0usize;
let mut lines = literal.split_inclusive('\n');
let Some(first) = lines.next() else {
return (String::new(), 0);
};
offset += first.len();
let mut current = offset;
let body_start = offset;
for line in lines {
let trimmed = line.trim_end_matches(['\r', '\n']);
if trimmed == "---" || trimmed == "..." {
let body = literal[body_start..current].to_string();
return (body, body_start);
}
current += line.len();
}
// Unterminated front matter cannot normally reach here: comrak only emits a
// FrontMatter node when a closing delimiter exists. Be forgiving and treat
// the rest as YAML so diagnostics can still point at it.
(literal[body_start..].to_string(), body_start)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_import_mapping() {
let literal = "---\nimport:\n geometry: \"@geometry\"\n---\n\n";
let fm = parse_literal(literal, Span::new(0, literal.len()));
assert!(fm.is_valid());
let imports = fm.imports().unwrap();
assert_eq!(
imports.get("geometry").map(String::as_str),
Some("@geometry")
);
assert_eq!(fm.raw, "import:\n geometry: \"@geometry\"\n");
assert_eq!(fm.yaml_span.start, 4);
assert_eq!(&literal[fm.yaml_span.start..fm.yaml_span.end], fm.raw);
}
#[test]
fn empty_frontmatter_is_null() {
let literal = "---\n---\n";
let fm = parse_literal(literal, Span::new(0, literal.len()));
assert_eq!(fm.raw, "");
assert!(fm.imports().unwrap().is_empty());
}
#[test]
fn invalid_yaml_is_reported_not_panicked() {
let literal = "---\nimport: [unclosed\n---\n";
let fm = parse_literal(literal, Span::new(0, literal.len()));
assert!(!fm.is_valid());
assert!(fm.imports().is_err());
}
#[test]
fn import_shapes_are_validated() {
let literal = "---\nimport: geometry\n---\n";
let fm = parse_literal(literal, Span::new(0, literal.len()));
assert!(fm.imports().is_err());
assert!(fm.imports().unwrap_err().contains("mapping"));
}
}
+19
View File
@@ -0,0 +1,19 @@
//! Canonical Markdown AST for Marqdo.
//!
//! This crate is the only place that talks to the Markdown parser. It freezes
//! the CommonMark/GFM/Math profile (see [`profile`]), converts the parser tree
//! into an owned [`ast::MdNode`] tree with byte spans, and extracts YAML front
//! matter. It performs **no** semantic mapping and **no** execution: deciding
//! which AST nodes are executable is the frontend's job and is based on the
//! node kind and structural position only.
pub mod ast;
pub mod frontmatter;
pub mod profile;
pub mod text;
pub use ast::{MdKind, MdNode};
pub use frontmatter::FrontMatter;
pub use marqdo_span::{LineCol, Span};
pub use profile::{parse, MarkdownDocument, COMRAK_VERSION, PROFILE_ID};
pub use text::{executable_payload, normalize_name, plain_text, PayloadError};
+407
View File
@@ -0,0 +1,407 @@
//! The frozen Markdown profile: one parser, one option set, no implicit
//! upgrades.
//!
//! The parser version is pinned in `Cargo.toml` (`=1.0.0`) and the exact option
//! set is asserted by tests. Changing either is a language change and requires
//! an ADR plus regenerated goldens.
use comrak::nodes::{ListType, Node, NodeValue, Sourcepos, TableAlignment};
use comrak::{parse_document, Arena, Options};
use marqdo_span::{LineIndex, Span};
use crate::ast::{MdKind, MdNode};
use crate::frontmatter::{self, FrontMatter};
/// Identifier of the frozen Markdown profile.
pub const PROFILE_ID: &str = "marqdo.md.v1";
/// Exact comrak version the profile is validated against.
pub const COMRAK_VERSION: &str = "1.0.0";
/// Build the single supported parser configuration.
///
/// Enabled: CommonMark 0.31.2 core, GFM tables, GFM strikethrough, GFM task
/// lists, `$...$` / `$$...$$` math, and YAML front matter delimited by `---`
/// at the very start of the document.
///
/// Disabled: autolinks, footnotes, description lists, wikilinks, alerts,
/// directives, superscript/subscript, raw-HTML suppression changes and every
/// other optional extension. These defaults are asserted in tests.
pub fn options() -> Options<'static> {
let mut options = Options::default();
options.extension.table = true;
options.extension.strikethrough = true;
options.extension.tasklist = true;
options.extension.math_dollars = true;
options.extension.math_dollars_block = true;
options.extension.front_matter_delimiter = Some("---".to_string());
options
}
/// A parsed Markdown document under the frozen profile.
#[derive(Debug, Clone)]
pub struct MarkdownDocument {
/// Original source bytes as UTF-8 (including a BOM if present).
pub source: String,
/// Number of BOM bytes stripped before parsing; all spans are re-based onto
/// the original file so they stay file offsets.
pub bom_len: usize,
/// Canonical AST root.
pub root: MdNode,
/// Parsed front matter, when present at the document start.
pub frontmatter: Option<FrontMatter>,
/// Line index over the original source, for diagnostics.
pub line_index: LineIndex,
}
impl MarkdownDocument {
/// Map a byte offset in the original source to a 1-based line/column.
pub fn line_col(&self, offset: usize) -> marqdo_span::LineCol {
self.line_index.line_col(offset)
}
/// Borrow the front matter, if the document has any.
pub fn frontmatter(&self) -> Option<&FrontMatter> {
self.frontmatter.as_ref()
}
}
/// Parse a source document with the frozen profile.
///
/// Parsing never executes anything and never fails: malformed constructs that
/// the Markdown grammar itself accepts are reported later by the semantic
/// layer, which knows which AST positions are executable.
pub fn parse(source: &str) -> MarkdownDocument {
let (bom_len, body) = strip_bom(source);
let body_index = LineIndex::new(body);
let arena = Arena::new();
let root = parse_document(&arena, body, &options());
let root = convert(&root, &body_index, bom_len, body.len());
let frontmatter = root
.children
.iter()
.find(|child| child.kind == MdKind::FrontMatter)
.map(|node| {
frontmatter::parse_literal(node.literal.as_deref().unwrap_or_default(), node.span)
});
MarkdownDocument {
source: source.to_string(),
bom_len,
root,
frontmatter,
line_index: LineIndex::new(source),
}
}
fn strip_bom(source: &str) -> (usize, &str) {
match source.strip_prefix('\u{feff}') {
Some(rest) => (source.len() - rest.len(), rest),
None => (0, source),
}
}
fn convert(node: &Node<'_>, index: &LineIndex, shift: usize, source_len: usize) -> MdNode {
let data = node.data();
let span = span_of(data.sourcepos, index, shift, source_len);
let (kind, literal) = map_value(&data.value);
let children = node
.children()
.map(|child| convert(&child, index, shift, source_len))
.collect();
MdNode {
kind,
span,
literal,
children,
}
}
/// Convert a comrak source position (1-based line, 1-based byte column,
/// inclusive end) into a half-open byte span in the original file.
fn span_of(pos: Sourcepos, index: &LineIndex, shift: usize, source_len: usize) -> Span {
// Offsets are computed in the BOM-stripped body, clamped there, and only
// then re-based onto the original file.
let start = offset_of(pos.start.line, pos.start.column, index, false)
.map(|o| o.min(source_len))
.unwrap_or(0);
let end = offset_of(pos.end.line, pos.end.column, index, true)
.map(|o| o.min(source_len))
.unwrap_or(start);
let end = end.max(start);
Span::new(start + shift, end + shift)
}
fn offset_of(line: usize, column: usize, index: &LineIndex, inclusive: bool) -> Option<usize> {
if line == 0 {
return None;
}
let line_start = index.line_start(line)?;
let zero_based = column.saturating_sub(1);
Some(line_start + zero_based + usize::from(inclusive))
}
fn map_value(value: &NodeValue) -> (MdKind, Option<String>) {
match value {
NodeValue::Document => (MdKind::Document, None),
NodeValue::FrontMatter(raw) => (MdKind::FrontMatter, Some(raw.clone())),
NodeValue::BlockQuote | NodeValue::MultilineBlockQuote(_) => (MdKind::BlockQuote, None),
NodeValue::List(list) => (
MdKind::List {
ordered: list.list_type == ListType::Ordered,
start: if list.list_type == ListType::Ordered {
Some(list.start)
} else {
None
},
tight: list.tight,
},
None,
),
NodeValue::Item(_) => (MdKind::Item { task: None }, None),
NodeValue::Heading(heading) => (
MdKind::Heading {
level: heading.level,
setext: heading.setext,
},
None,
),
NodeValue::Paragraph => (MdKind::Paragraph, None),
NodeValue::Text(text) => (MdKind::Text, Some(text.to_string())),
NodeValue::Strong(_) => (MdKind::Strong, None),
NodeValue::Emph(_) => (MdKind::Emph, None),
NodeValue::Strikethrough(_) => (MdKind::Strikethrough, None),
NodeValue::Code(code) => (MdKind::Code, Some(code.literal.clone())),
NodeValue::CodeBlock(block) => (
MdKind::CodeBlock {
fenced: block.fenced,
info: block.info.trim().to_string(),
},
Some(block.literal.clone()),
),
NodeValue::Math(math) => (
MdKind::Math {
display: math.display_math,
},
Some(math.literal.clone()),
),
NodeValue::MathBlock(block) => (MdKind::MathBlock, Some(block.literal.clone())),
NodeValue::Link(link) => (
MdKind::Link {
url: link.url.clone(),
title: link.title.clone(),
},
None,
),
NodeValue::Image(image) => (
MdKind::Image {
url: image.url.clone(),
title: image.title.clone(),
},
None,
),
NodeValue::Table(table) => (
MdKind::Table {
alignments: table
.alignments
.iter()
.map(|alignment| {
match alignment {
TableAlignment::Left => "left",
TableAlignment::Center => "center",
TableAlignment::Right => "right",
TableAlignment::None => "none",
_ => "none",
}
.to_string()
})
.collect(),
},
None,
),
NodeValue::TableRow(row) => (MdKind::TableRow { header: row.header }, None),
NodeValue::TableCell => (MdKind::TableCell, None),
NodeValue::ThematicBreak(_) => (MdKind::ThematicBreak, None),
NodeValue::SoftBreak => (MdKind::SoftBreak, None),
NodeValue::LineBreak => (MdKind::LineBreak, None),
NodeValue::HtmlBlock(block) => (MdKind::HtmlBlock, Some(block.literal.clone())),
NodeValue::HtmlInline(raw) => (MdKind::HtmlInline, Some(raw.clone())),
NodeValue::TaskItem(item) => (
MdKind::Item {
task: Some(item.symbol.is_some()),
},
None,
),
NodeValue::FootnoteDefinition(definition) => (
MdKind::FootnoteDefinition {
name: definition.name.clone(),
},
None,
),
NodeValue::FootnoteReference(reference) => (
MdKind::FootnoteReference {
name: reference.name.clone(),
},
None,
),
other => (
MdKind::Other {
tag: tag_name(other),
},
literal_of(other),
),
}
}
fn literal_of(value: &NodeValue) -> Option<String> {
match value {
NodeValue::Raw(raw) => Some(raw.clone()),
_ => None,
}
}
/// Turn an unmodeled parser node into a stable snake_case tag. The node is
/// inert for the language but must remain visible in `inspect --ast`.
fn tag_name(value: &NodeValue) -> String {
let debug = format!("{value:?}");
let head = debug
.split(|c: char| !c.is_alphanumeric() && c != '_')
.next()
.unwrap_or("unknown");
let mut out = String::new();
for (i, ch) in head.chars().enumerate() {
if ch.is_uppercase() {
if i != 0 {
out.push('_');
}
out.extend(ch.to_lowercase());
} else {
out.push(ch);
}
}
if out.is_empty() {
"unknown".to_string()
} else {
out
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ast::MdKind;
fn kind_tree(source: &str) -> MdNode {
parse(source).root
}
fn find<'a>(node: &'a MdNode, kind: &MdKind) -> Option<&'a MdNode> {
if &node.kind == kind {
return Some(node);
}
node.children.iter().find_map(|child| find(child, kind))
}
#[test]
fn profile_options_are_frozen() {
let options = options();
assert!(options.extension.table);
assert!(options.extension.strikethrough);
assert!(options.extension.tasklist);
assert!(options.extension.math_dollars);
assert!(options.extension.math_dollars_block);
assert_eq!(
options.extension.front_matter_delimiter.as_deref(),
Some("---")
);
assert!(!options.extension.autolink);
assert!(!options.extension.footnotes);
assert!(!options.extension.description_lists);
assert!(!options.extension.wikilinks_title_after_pipe);
assert!(!options.extension.wikilinks_title_before_pipe);
assert!(!options.extension.alerts);
assert!(!options.parse.smart);
assert!(!options.parse.ignore_setext);
assert!(!options.extension.inline_footnotes);
}
#[test]
fn parse_maps_core_constructs() {
let root = kind_tree("# T\n\n## F\n\n**x = 1**\n\n*y*\n\n> 打印(1)\n");
assert_eq!(root.kind, MdKind::Document);
assert!(find(
&root,
&MdKind::Heading {
level: 1,
setext: false
}
)
.is_some());
assert!(find(&root, &MdKind::Strong).is_some());
assert!(find(&root, &MdKind::Emph).is_some());
assert!(find(&root, &MdKind::BlockQuote).is_some());
}
#[test]
fn spans_are_byte_offsets_into_original_source() {
let source = "# 数学\n\n正文\n";
let doc = parse(source);
let heading = &doc.root.children[0];
assert_eq!(
heading.kind,
MdKind::Heading {
level: 1,
setext: false
}
);
assert_eq!(&source[heading.span.start..heading.span.end], "# 数学");
assert_eq!(doc.line_col(heading.span.start).line, 1);
}
#[test]
fn bom_is_tolerated_and_spans_shift() {
let source = "\u{feff}# T\n";
let doc = parse(source);
assert_eq!(doc.bom_len, 3);
let heading = &doc.root.children[0];
assert_eq!(heading.span.start, 3);
assert_eq!(&source[heading.span.start..heading.span.end], "# T");
}
#[test]
fn math_nodes_are_native_and_inert() {
let doc = parse("inline $a+b$ math\n\n$$\nx+y\n$$\n");
assert!(find(&doc.root, &MdKind::Math { display: false }).is_some());
assert!(find(&doc.root, &MdKind::MathBlock).is_some());
}
#[test]
fn setext_headings_are_heading_nodes() {
let doc = parse("函数\n---\n");
let heading = &doc.root.children[0];
assert_eq!(
heading.kind,
MdKind::Heading {
level: 2,
setext: true
}
);
}
#[test]
fn tables_and_task_lists_are_supported() {
let doc = parse("| a | b |\n|---|---|\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n");
assert!(find(
&doc.root,
&MdKind::Table {
alignments: vec!["none".into(), "none".into()]
}
)
.is_some());
let mut checked = Vec::new();
doc.root.walk(&mut |node| {
if let MdKind::Item { task: Some(state) } = node.kind {
checked.push(state);
}
});
assert_eq!(checked, vec![true, false]);
}
}
+187
View File
@@ -0,0 +1,187 @@
//! Inert text extraction and strict executable payload extraction.
//!
//! Two different operations live here on purpose:
//!
//! * [`plain_text`] renders inert content (heading names, table labels); it
//! never decides that something is executable.
//! * [`executable_payload`] extracts the payload of an executable `Strong` or
//! `Emph` node. It accepts only `Text` and `Code` children and fails on any
//! other inline structure: an executable node with an unexpected shape is a
//! hard compile error, never a silent fallback to prose.
use unicode_normalization::UnicodeNormalization;
use marqdo_span::Span;
use crate::ast::{MdKind, MdNode};
/// Why a payload could not be extracted.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PayloadError {
/// Span of the offending node.
pub span: Span,
/// Human readable reason.
pub reason: String,
}
impl PayloadError {
fn new(span: Span, reason: impl Into<String>) -> Self {
Self {
span,
reason: reason.into(),
}
}
}
/// Extract the strict payload of an executable inline node.
///
/// Only `Text` and `Code` children are allowed; their literals are concatenated
/// exactly as written (a code span keeps its payload verbatim). Nested
/// `Strong`/`Emph`/`Link` and line breaks are rejected so that one AST node can
/// never be executed twice or hide control content.
pub fn executable_payload(node: &MdNode) -> Result<String, PayloadError> {
let mut payload = String::new();
for child in &node.children {
match &child.kind {
MdKind::Text => payload.push_str(child.literal.as_deref().unwrap_or_default()),
MdKind::Code => payload.push_str(child.literal.as_deref().unwrap_or_default()),
other => {
return Err(PayloadError::new(
child.span,
format!(
"`{}` is not a valid payload inside an executable node",
other.tag()
),
));
}
}
}
Ok(payload)
}
/// Render inert content to plain text.
///
/// Used for heading names and other inert labels. Images use their alt text,
/// links use their label, code and math use their literal source, and line
/// breaks become a single space.
pub fn plain_text(node: &MdNode) -> String {
let mut out = String::new();
collect_plain_text(node, &mut out);
out
}
fn collect_plain_text(node: &MdNode, out: &mut String) {
match &node.kind {
MdKind::Text | MdKind::Code | MdKind::Math { .. } | MdKind::MathBlock => {
out.push_str(node.literal.as_deref().unwrap_or_default());
}
MdKind::SoftBreak | MdKind::LineBreak => out.push(' '),
MdKind::HtmlBlock | MdKind::HtmlInline | MdKind::FrontMatter => {
// Raw HTML and front matter are not part of visible prose.
}
_ => {
for child in &node.children {
collect_plain_text(child, out);
}
}
}
}
/// Normalise a heading name: NFC, trim, and collapse internal whitespace runs
/// to a single space. Returns `None` when nothing visible remains.
pub fn normalize_name(text: &str) -> Option<String> {
let normalized: String = text.nfc().collect();
let mut collapsed = String::new();
let mut pending_space = false;
for ch in normalized.trim().chars() {
if ch.is_whitespace() {
pending_space = true;
} else {
if pending_space && !collapsed.is_empty() {
collapsed.push(' ');
}
pending_space = false;
collapsed.push(ch);
}
}
if collapsed.is_empty() {
None
} else {
Some(collapsed)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn node(kind: MdKind, literal: Option<&str>, children: Vec<MdNode>) -> MdNode {
MdNode {
kind,
span: Span::new(0, 0),
literal: literal.map(str::to_string),
children,
}
}
#[test]
fn payload_accepts_text_and_code() {
let strong = node(
MdKind::Strong,
None,
vec![
node(MdKind::Text, Some("面积 = "), vec![]),
node(MdKind::Code, Some("计算(r=1)"), vec![]),
],
);
assert_eq!(executable_payload(&strong).unwrap(), "面积 = 计算(r=1)");
}
#[test]
fn payload_rejects_nested_markup() {
let strong = node(
MdKind::Strong,
None,
vec![node(
MdKind::Emph,
None,
vec![node(MdKind::Text, Some("x"), vec![])],
)],
);
let error = executable_payload(&strong).unwrap_err();
assert!(error.reason.contains("emph"));
}
#[test]
fn plain_text_uses_alt_and_label() {
let image = node(
MdKind::Image {
url: "a.png".into(),
title: String::new(),
},
None,
vec![node(MdKind::Text, Some("示意图"), vec![])],
);
let heading = node(
MdKind::Heading {
level: 1,
setext: false,
},
None,
vec![
image,
node(MdKind::SoftBreak, None, vec![]),
node(MdKind::Text, Some("名字"), vec![]),
],
);
assert_eq!(plain_text(&heading), "示意图 名字");
}
#[test]
fn normalize_collapses_whitespace_and_nfc() {
assert_eq!(normalize_name(" 圆 面积 ").as_deref(), Some("圆 面积"));
// Decomposed e + combining acute becomes NFC single code point.
assert_eq!(normalize_name("e\u{301}").as_deref(), Some("é"));
assert_eq!(normalize_name(" "), None);
}
}