diff --git a/crates/marqdo-markdown/src/ast.rs b/crates/marqdo-markdown/src/ast.rs new file mode 100644 index 0000000..58f7570 --- /dev/null +++ b/crates/marqdo-markdown/src/ast.rs @@ -0,0 +1,230 @@ +//! Owned, serializable view of the CommonMark/GFM/Math AST. +//! +//! The profile uses exactly one Markdown parser (comrak). This module converts +//! the parser's arena tree into an owned tree that keeps: +//! +//! * the node kind, so program roles can be derived from AST structure alone; +//! * the byte [`Span`] in the original file, for diagnostics and IDE support; +//! * literal payloads for inert leaves (text, code, math, HTML, front matter); +//! * the full child structure, including nodes outside the frozen profile, +//! which are preserved as [`MdKind::Other`] rather than dropped. + +use marqdo_span::Span; +use serde::{Deserialize, Serialize}; + +/// A node of the canonical Markdown AST view. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MdNode { + /// Structural kind and its small attribute set. + pub kind: MdKind, + /// Byte span in the original source (UTF-8, half open). + pub span: Span, + /// Literal payload for leaves that carry text (Text, Code, Math, ...). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub literal: Option, + /// Child nodes in document order. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub children: Vec, +} + +/// Structural kind of a Markdown AST node. +/// +/// The variants mirror the frozen Markdown profile. Nodes produced by parser +/// extensions that are not part of the profile are preserved as `Other` so the +/// AST view never hides syntax, but they are inert for the language. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum MdKind { + /// Document root. + Document, + /// YAML front matter block at the start of the document. + FrontMatter, + /// Block quote (an invocation in executable position). + BlockQuote, + /// Ordered or bullet list. + List { + /// `true` for ordered lists, `false` for bullet lists. + ordered: bool, + /// Ordered list start number; `None` for bullet lists. + #[serde(default, skip_serializing_if = "Option::is_none")] + start: Option, + /// CommonMark tight/loose flag (a rendering distinction only). + tight: bool, + }, + /// One list item. Task list items (GFM) keep their checkbox state. + Item { + /// `Some(checked)` for a GFM task list item, `None` otherwise. + #[serde(default, skip_serializing_if = "Option::is_none")] + task: Option, + }, + /// Heading of level 1-6. + Heading { + /// Level 1-6. + level: u8, + /// Whether the source used a Setext underline. + setext: bool, + }, + /// Paragraph. + Paragraph, + /// Literal text. + Text, + /// Strong emphasis: an execution statement in executable position. + Strong, + /// Emphasis: a return statement in executable position. + Emph, + /// GFM strikethrough. + Strikethrough, + /// Inline code span. + Code, + /// Fenced or indented code block. + CodeBlock { + /// Whether the block was fenced. + fenced: bool, + /// Info string (language) without surrounding whitespace. + info: String, + }, + /// Inline math. + Math { + /// Whether display math (`$$...$$`). + display: bool, + }, + /// Block display math (`$$` fences on their own lines). + MathBlock, + /// Link with a destination. + Link { + /// Destination as written. + url: String, + /// Optional title. + title: String, + }, + /// Image with a source. + Image { + /// Source as written. + url: String, + /// Optional title. + title: String, + }, + /// GFM table. + Table { + /// Column alignments as `none`/`left`/`center`/`right`. + alignments: Vec, + }, + /// Table row. + TableRow { + /// Whether this is the header row. + header: bool, + }, + /// Table cell. + TableCell, + /// Horizontal rule. + ThematicBreak, + /// Soft line break inside a paragraph. + SoftBreak, + /// Hard line break. + LineBreak, + /// Raw HTML block. + HtmlBlock, + /// Raw inline HTML. + HtmlInline, + /// Footnote definition (profile leaves footnotes disabled; kept for completeness). + FootnoteDefinition { + /// Footnote label. + name: String, + }, + /// Footnote reference (profile leaves footnotes disabled). + FootnoteReference { + /// Footnote label. + name: String, + }, + /// A node outside the frozen profile, preserved for inspection only. + Other { + /// Parser tag name. + tag: String, + }, +} + +impl MdNode { + /// Create a leaf node. + pub fn leaf(kind: MdKind, span: Span, literal: Option) -> Self { + Self { + kind, + span, + literal, + children: Vec::new(), + } + } + + /// Depth-first pre-order traversal including `self`. + pub fn walk<'a>(&'a self, f: &mut impl FnMut(&'a MdNode)) { + f(self); + for child in &self.children { + child.walk(f); + } + } + + /// Direct child nodes of a given kind. + pub fn children_of<'a>(&'a self, kind: &MdKind) -> impl Iterator { + let kind = kind.clone(); + self.children.iter().filter(move |child| child.kind == kind) + } + + /// Whether this node is an inline node. + pub fn is_inline(&self) -> bool { + matches!( + self.kind, + MdKind::Text + | MdKind::Strong + | MdKind::Emph + | MdKind::Strikethrough + | MdKind::Code + | MdKind::Math { .. } + | MdKind::Link { .. } + | MdKind::Image { .. } + | MdKind::SoftBreak + | MdKind::LineBreak + | MdKind::HtmlInline + | MdKind::FootnoteReference { .. } + ) + } + + /// Whether this node is a block node. + pub fn is_block(&self) -> bool { + !self.is_inline() + } +} + +impl MdKind { + /// Short stable tag name used in diagnostics. + pub const fn tag(&self) -> &'static str { + match self { + MdKind::Document => "document", + MdKind::FrontMatter => "front_matter", + MdKind::BlockQuote => "block_quote", + MdKind::List { .. } => "list", + MdKind::Item { .. } => "item", + MdKind::Heading { .. } => "heading", + MdKind::Paragraph => "paragraph", + MdKind::Text => "text", + MdKind::Strong => "strong", + MdKind::Emph => "emph", + MdKind::Strikethrough => "strikethrough", + MdKind::Code => "code", + MdKind::CodeBlock { .. } => "code_block", + MdKind::Math { .. } => "math", + MdKind::MathBlock => "math_block", + MdKind::Link { .. } => "link", + MdKind::Image { .. } => "image", + MdKind::Table { .. } => "table", + MdKind::TableRow { .. } => "table_row", + MdKind::TableCell => "table_cell", + MdKind::ThematicBreak => "thematic_break", + MdKind::SoftBreak => "soft_break", + MdKind::LineBreak => "line_break", + MdKind::HtmlBlock => "html_block", + MdKind::HtmlInline => "html_inline", + MdKind::FootnoteDefinition { .. } => "footnote_definition", + MdKind::FootnoteReference { .. } => "footnote_reference", + MdKind::Other { .. } => "other", + } + } +} diff --git a/crates/marqdo-markdown/src/frontmatter.rs b/crates/marqdo-markdown/src/frontmatter.rs new file mode 100644 index 0000000..f4c1761 --- /dev/null +++ b/crates/marqdo-markdown/src/frontmatter.rs @@ -0,0 +1,163 @@ +//! YAML front matter handling. +//! +//! Front matter is only recognised at byte offset 0 (after an optional UTF-8 +//! BOM) and only when the Markdown parser itself reports a `FrontMatter` node. +//! The parser node keeps the raw delimiters; this module extracts the YAML body +//! and exposes the explicit `import` mapping without interpreting anything +//! else. Front matter never executes. + +use std::collections::BTreeMap; + +use marqdo_span::Span; + +/// Parsed (or unparseable) front matter. +#[derive(Debug, Clone, PartialEq)] +pub struct FrontMatter { + /// Raw YAML body between the delimiter lines, excluding the delimiters. + pub raw: String, + /// Parsed YAML value; `Err` carries a human readable reason. + pub yaml: Result, + /// Span of the whole front matter node (including delimiters). + pub span: Span, + /// Span of the YAML body only. + pub yaml_span: Span, +} + +impl FrontMatter { + /// The explicit `import` mapping from front matter. + /// + /// The supported shape is a YAML mapping of local alias to module address: + /// + /// ```yaml + /// import: + /// geometry: "@geometry" + /// ``` + /// + /// Anything else is an error; imports are never inferred. + pub fn imports(&self) -> Result, String> { + let yaml = self + .yaml + .as_ref() + .map_err(|error| format!("front matter is not valid YAML: {error}"))?; + let Some(imports) = yaml.get("import") else { + return Ok(BTreeMap::new()); + }; + let Some(mapping) = imports.as_mapping() else { + return Err("`import` must be a mapping of alias to module address".to_string()); + }; + let mut result = BTreeMap::new(); + for (alias, target) in mapping { + let Some(alias) = alias.as_str() else { + return Err("`import` aliases must be strings".to_string()); + }; + let Some(target) = target.as_str() else { + return Err(format!("import target for `{alias}` must be a string")); + }; + if alias.is_empty() { + return Err("`import` alias must not be empty".to_string()); + } + if result + .insert(alias.to_string(), target.to_string()) + .is_some() + { + return Err(format!("duplicate import alias `{alias}`")); + } + } + Ok(result) + } + + /// The raw YAML parses successfully. + pub fn is_valid(&self) -> bool { + self.yaml.is_ok() + } +} + +/// Split a `FrontMatter` parser literal into the YAML body. +/// +/// The parser literal starts with the opening delimiter line and includes the +/// closing delimiter line plus trailing newlines, e.g. +/// `"---\nimport:\n a: b\n---\n\n"`. +pub fn parse_literal(literal: &str, span: Span) -> FrontMatter { + let (raw, body_offset) = split_delimiters(literal); + let yaml = if raw.trim().is_empty() { + Ok(serde_yaml::Value::Null) + } else { + serde_yaml::from_str::(&raw).map_err(|error| error.to_string()) + }; + let start = span.start.saturating_add(body_offset); + let end = start.saturating_add(raw.len()).min(span.end.max(start)); + FrontMatter { + raw, + yaml, + span, + yaml_span: Span::new(start, end), + } +} + +/// Return the YAML body and its byte offset within `literal`. +fn split_delimiters(literal: &str) -> (String, usize) { + let mut offset = 0usize; + let mut lines = literal.split_inclusive('\n'); + let Some(first) = lines.next() else { + return (String::new(), 0); + }; + offset += first.len(); + let mut current = offset; + let body_start = offset; + for line in lines { + let trimmed = line.trim_end_matches(['\r', '\n']); + if trimmed == "---" || trimmed == "..." { + let body = literal[body_start..current].to_string(); + return (body, body_start); + } + current += line.len(); + } + // Unterminated front matter cannot normally reach here: comrak only emits a + // FrontMatter node when a closing delimiter exists. Be forgiving and treat + // the rest as YAML so diagnostics can still point at it. + (literal[body_start..].to_string(), body_start) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_import_mapping() { + let literal = "---\nimport:\n geometry: \"@geometry\"\n---\n\n"; + let fm = parse_literal(literal, Span::new(0, literal.len())); + assert!(fm.is_valid()); + let imports = fm.imports().unwrap(); + assert_eq!( + imports.get("geometry").map(String::as_str), + Some("@geometry") + ); + assert_eq!(fm.raw, "import:\n geometry: \"@geometry\"\n"); + assert_eq!(fm.yaml_span.start, 4); + assert_eq!(&literal[fm.yaml_span.start..fm.yaml_span.end], fm.raw); + } + + #[test] + fn empty_frontmatter_is_null() { + let literal = "---\n---\n"; + let fm = parse_literal(literal, Span::new(0, literal.len())); + assert_eq!(fm.raw, ""); + assert!(fm.imports().unwrap().is_empty()); + } + + #[test] + fn invalid_yaml_is_reported_not_panicked() { + let literal = "---\nimport: [unclosed\n---\n"; + let fm = parse_literal(literal, Span::new(0, literal.len())); + assert!(!fm.is_valid()); + assert!(fm.imports().is_err()); + } + + #[test] + fn import_shapes_are_validated() { + let literal = "---\nimport: geometry\n---\n"; + let fm = parse_literal(literal, Span::new(0, literal.len())); + assert!(fm.imports().is_err()); + assert!(fm.imports().unwrap_err().contains("mapping")); + } +} diff --git a/crates/marqdo-markdown/src/lib.rs b/crates/marqdo-markdown/src/lib.rs new file mode 100644 index 0000000..856ce61 --- /dev/null +++ b/crates/marqdo-markdown/src/lib.rs @@ -0,0 +1,19 @@ +//! Canonical Markdown AST for Marqdo. +//! +//! This crate is the only place that talks to the Markdown parser. It freezes +//! the CommonMark/GFM/Math profile (see [`profile`]), converts the parser tree +//! into an owned [`ast::MdNode`] tree with byte spans, and extracts YAML front +//! matter. It performs **no** semantic mapping and **no** execution: deciding +//! which AST nodes are executable is the frontend's job and is based on the +//! node kind and structural position only. + +pub mod ast; +pub mod frontmatter; +pub mod profile; +pub mod text; + +pub use ast::{MdKind, MdNode}; +pub use frontmatter::FrontMatter; +pub use marqdo_span::{LineCol, Span}; +pub use profile::{parse, MarkdownDocument, COMRAK_VERSION, PROFILE_ID}; +pub use text::{executable_payload, normalize_name, plain_text, PayloadError}; diff --git a/crates/marqdo-markdown/src/profile.rs b/crates/marqdo-markdown/src/profile.rs new file mode 100644 index 0000000..304f7d7 --- /dev/null +++ b/crates/marqdo-markdown/src/profile.rs @@ -0,0 +1,407 @@ +//! The frozen Markdown profile: one parser, one option set, no implicit +//! upgrades. +//! +//! The parser version is pinned in `Cargo.toml` (`=1.0.0`) and the exact option +//! set is asserted by tests. Changing either is a language change and requires +//! an ADR plus regenerated goldens. + +use comrak::nodes::{ListType, Node, NodeValue, Sourcepos, TableAlignment}; +use comrak::{parse_document, Arena, Options}; +use marqdo_span::{LineIndex, Span}; + +use crate::ast::{MdKind, MdNode}; +use crate::frontmatter::{self, FrontMatter}; + +/// Identifier of the frozen Markdown profile. +pub const PROFILE_ID: &str = "marqdo.md.v1"; +/// Exact comrak version the profile is validated against. +pub const COMRAK_VERSION: &str = "1.0.0"; + +/// Build the single supported parser configuration. +/// +/// Enabled: CommonMark 0.31.2 core, GFM tables, GFM strikethrough, GFM task +/// lists, `$...$` / `$$...$$` math, and YAML front matter delimited by `---` +/// at the very start of the document. +/// +/// Disabled: autolinks, footnotes, description lists, wikilinks, alerts, +/// directives, superscript/subscript, raw-HTML suppression changes and every +/// other optional extension. These defaults are asserted in tests. +pub fn options() -> Options<'static> { + let mut options = Options::default(); + options.extension.table = true; + options.extension.strikethrough = true; + options.extension.tasklist = true; + options.extension.math_dollars = true; + options.extension.math_dollars_block = true; + options.extension.front_matter_delimiter = Some("---".to_string()); + options +} + +/// A parsed Markdown document under the frozen profile. +#[derive(Debug, Clone)] +pub struct MarkdownDocument { + /// Original source bytes as UTF-8 (including a BOM if present). + pub source: String, + /// Number of BOM bytes stripped before parsing; all spans are re-based onto + /// the original file so they stay file offsets. + pub bom_len: usize, + /// Canonical AST root. + pub root: MdNode, + /// Parsed front matter, when present at the document start. + pub frontmatter: Option, + /// Line index over the original source, for diagnostics. + pub line_index: LineIndex, +} + +impl MarkdownDocument { + /// Map a byte offset in the original source to a 1-based line/column. + pub fn line_col(&self, offset: usize) -> marqdo_span::LineCol { + self.line_index.line_col(offset) + } + + /// Borrow the front matter, if the document has any. + pub fn frontmatter(&self) -> Option<&FrontMatter> { + self.frontmatter.as_ref() + } +} + +/// Parse a source document with the frozen profile. +/// +/// Parsing never executes anything and never fails: malformed constructs that +/// the Markdown grammar itself accepts are reported later by the semantic +/// layer, which knows which AST positions are executable. +pub fn parse(source: &str) -> MarkdownDocument { + let (bom_len, body) = strip_bom(source); + let body_index = LineIndex::new(body); + let arena = Arena::new(); + let root = parse_document(&arena, body, &options()); + let root = convert(&root, &body_index, bom_len, body.len()); + let frontmatter = root + .children + .iter() + .find(|child| child.kind == MdKind::FrontMatter) + .map(|node| { + frontmatter::parse_literal(node.literal.as_deref().unwrap_or_default(), node.span) + }); + MarkdownDocument { + source: source.to_string(), + bom_len, + root, + frontmatter, + line_index: LineIndex::new(source), + } +} + +fn strip_bom(source: &str) -> (usize, &str) { + match source.strip_prefix('\u{feff}') { + Some(rest) => (source.len() - rest.len(), rest), + None => (0, source), + } +} + +fn convert(node: &Node<'_>, index: &LineIndex, shift: usize, source_len: usize) -> MdNode { + let data = node.data(); + let span = span_of(data.sourcepos, index, shift, source_len); + let (kind, literal) = map_value(&data.value); + let children = node + .children() + .map(|child| convert(&child, index, shift, source_len)) + .collect(); + MdNode { + kind, + span, + literal, + children, + } +} + +/// Convert a comrak source position (1-based line, 1-based byte column, +/// inclusive end) into a half-open byte span in the original file. +fn span_of(pos: Sourcepos, index: &LineIndex, shift: usize, source_len: usize) -> Span { + // Offsets are computed in the BOM-stripped body, clamped there, and only + // then re-based onto the original file. + let start = offset_of(pos.start.line, pos.start.column, index, false) + .map(|o| o.min(source_len)) + .unwrap_or(0); + let end = offset_of(pos.end.line, pos.end.column, index, true) + .map(|o| o.min(source_len)) + .unwrap_or(start); + let end = end.max(start); + Span::new(start + shift, end + shift) +} + +fn offset_of(line: usize, column: usize, index: &LineIndex, inclusive: bool) -> Option { + if line == 0 { + return None; + } + let line_start = index.line_start(line)?; + let zero_based = column.saturating_sub(1); + Some(line_start + zero_based + usize::from(inclusive)) +} + +fn map_value(value: &NodeValue) -> (MdKind, Option) { + match value { + NodeValue::Document => (MdKind::Document, None), + NodeValue::FrontMatter(raw) => (MdKind::FrontMatter, Some(raw.clone())), + NodeValue::BlockQuote | NodeValue::MultilineBlockQuote(_) => (MdKind::BlockQuote, None), + NodeValue::List(list) => ( + MdKind::List { + ordered: list.list_type == ListType::Ordered, + start: if list.list_type == ListType::Ordered { + Some(list.start) + } else { + None + }, + tight: list.tight, + }, + None, + ), + NodeValue::Item(_) => (MdKind::Item { task: None }, None), + NodeValue::Heading(heading) => ( + MdKind::Heading { + level: heading.level, + setext: heading.setext, + }, + None, + ), + NodeValue::Paragraph => (MdKind::Paragraph, None), + NodeValue::Text(text) => (MdKind::Text, Some(text.to_string())), + NodeValue::Strong(_) => (MdKind::Strong, None), + NodeValue::Emph(_) => (MdKind::Emph, None), + NodeValue::Strikethrough(_) => (MdKind::Strikethrough, None), + NodeValue::Code(code) => (MdKind::Code, Some(code.literal.clone())), + NodeValue::CodeBlock(block) => ( + MdKind::CodeBlock { + fenced: block.fenced, + info: block.info.trim().to_string(), + }, + Some(block.literal.clone()), + ), + NodeValue::Math(math) => ( + MdKind::Math { + display: math.display_math, + }, + Some(math.literal.clone()), + ), + NodeValue::MathBlock(block) => (MdKind::MathBlock, Some(block.literal.clone())), + NodeValue::Link(link) => ( + MdKind::Link { + url: link.url.clone(), + title: link.title.clone(), + }, + None, + ), + NodeValue::Image(image) => ( + MdKind::Image { + url: image.url.clone(), + title: image.title.clone(), + }, + None, + ), + NodeValue::Table(table) => ( + MdKind::Table { + alignments: table + .alignments + .iter() + .map(|alignment| { + match alignment { + TableAlignment::Left => "left", + TableAlignment::Center => "center", + TableAlignment::Right => "right", + TableAlignment::None => "none", + _ => "none", + } + .to_string() + }) + .collect(), + }, + None, + ), + NodeValue::TableRow(row) => (MdKind::TableRow { header: row.header }, None), + NodeValue::TableCell => (MdKind::TableCell, None), + NodeValue::ThematicBreak(_) => (MdKind::ThematicBreak, None), + NodeValue::SoftBreak => (MdKind::SoftBreak, None), + NodeValue::LineBreak => (MdKind::LineBreak, None), + NodeValue::HtmlBlock(block) => (MdKind::HtmlBlock, Some(block.literal.clone())), + NodeValue::HtmlInline(raw) => (MdKind::HtmlInline, Some(raw.clone())), + NodeValue::TaskItem(item) => ( + MdKind::Item { + task: Some(item.symbol.is_some()), + }, + None, + ), + NodeValue::FootnoteDefinition(definition) => ( + MdKind::FootnoteDefinition { + name: definition.name.clone(), + }, + None, + ), + NodeValue::FootnoteReference(reference) => ( + MdKind::FootnoteReference { + name: reference.name.clone(), + }, + None, + ), + other => ( + MdKind::Other { + tag: tag_name(other), + }, + literal_of(other), + ), + } +} + +fn literal_of(value: &NodeValue) -> Option { + match value { + NodeValue::Raw(raw) => Some(raw.clone()), + _ => None, + } +} + +/// Turn an unmodeled parser node into a stable snake_case tag. The node is +/// inert for the language but must remain visible in `inspect --ast`. +fn tag_name(value: &NodeValue) -> String { + let debug = format!("{value:?}"); + let head = debug + .split(|c: char| !c.is_alphanumeric() && c != '_') + .next() + .unwrap_or("unknown"); + let mut out = String::new(); + for (i, ch) in head.chars().enumerate() { + if ch.is_uppercase() { + if i != 0 { + out.push('_'); + } + out.extend(ch.to_lowercase()); + } else { + out.push(ch); + } + } + if out.is_empty() { + "unknown".to_string() + } else { + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::MdKind; + + fn kind_tree(source: &str) -> MdNode { + parse(source).root + } + + fn find<'a>(node: &'a MdNode, kind: &MdKind) -> Option<&'a MdNode> { + if &node.kind == kind { + return Some(node); + } + node.children.iter().find_map(|child| find(child, kind)) + } + + #[test] + fn profile_options_are_frozen() { + let options = options(); + assert!(options.extension.table); + assert!(options.extension.strikethrough); + assert!(options.extension.tasklist); + assert!(options.extension.math_dollars); + assert!(options.extension.math_dollars_block); + assert_eq!( + options.extension.front_matter_delimiter.as_deref(), + Some("---") + ); + assert!(!options.extension.autolink); + assert!(!options.extension.footnotes); + assert!(!options.extension.description_lists); + assert!(!options.extension.wikilinks_title_after_pipe); + assert!(!options.extension.wikilinks_title_before_pipe); + assert!(!options.extension.alerts); + assert!(!options.parse.smart); + assert!(!options.parse.ignore_setext); + assert!(!options.extension.inline_footnotes); + } + + #[test] + fn parse_maps_core_constructs() { + let root = kind_tree("# T\n\n## F\n\n**x = 1**\n\n*y*\n\n> 打印(1)\n"); + assert_eq!(root.kind, MdKind::Document); + assert!(find( + &root, + &MdKind::Heading { + level: 1, + setext: false + } + ) + .is_some()); + assert!(find(&root, &MdKind::Strong).is_some()); + assert!(find(&root, &MdKind::Emph).is_some()); + assert!(find(&root, &MdKind::BlockQuote).is_some()); + } + + #[test] + fn spans_are_byte_offsets_into_original_source() { + let source = "# 数学\n\n正文\n"; + let doc = parse(source); + let heading = &doc.root.children[0]; + assert_eq!( + heading.kind, + MdKind::Heading { + level: 1, + setext: false + } + ); + assert_eq!(&source[heading.span.start..heading.span.end], "# 数学"); + assert_eq!(doc.line_col(heading.span.start).line, 1); + } + + #[test] + fn bom_is_tolerated_and_spans_shift() { + let source = "\u{feff}# T\n"; + let doc = parse(source); + assert_eq!(doc.bom_len, 3); + let heading = &doc.root.children[0]; + assert_eq!(heading.span.start, 3); + assert_eq!(&source[heading.span.start..heading.span.end], "# T"); + } + + #[test] + fn math_nodes_are_native_and_inert() { + let doc = parse("inline $a+b$ math\n\n$$\nx+y\n$$\n"); + assert!(find(&doc.root, &MdKind::Math { display: false }).is_some()); + assert!(find(&doc.root, &MdKind::MathBlock).is_some()); + } + + #[test] + fn setext_headings_are_heading_nodes() { + let doc = parse("函数\n---\n"); + let heading = &doc.root.children[0]; + assert_eq!( + heading.kind, + MdKind::Heading { + level: 2, + setext: true + } + ); + } + + #[test] + fn tables_and_task_lists_are_supported() { + let doc = parse("| a | b |\n|---|---|\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n"); + assert!(find( + &doc.root, + &MdKind::Table { + alignments: vec!["none".into(), "none".into()] + } + ) + .is_some()); + let mut checked = Vec::new(); + doc.root.walk(&mut |node| { + if let MdKind::Item { task: Some(state) } = node.kind { + checked.push(state); + } + }); + assert_eq!(checked, vec![true, false]); + } +} diff --git a/crates/marqdo-markdown/src/text.rs b/crates/marqdo-markdown/src/text.rs new file mode 100644 index 0000000..320fc0b --- /dev/null +++ b/crates/marqdo-markdown/src/text.rs @@ -0,0 +1,187 @@ +//! Inert text extraction and strict executable payload extraction. +//! +//! Two different operations live here on purpose: +//! +//! * [`plain_text`] renders inert content (heading names, table labels); it +//! never decides that something is executable. +//! * [`executable_payload`] extracts the payload of an executable `Strong` or +//! `Emph` node. It accepts only `Text` and `Code` children and fails on any +//! other inline structure: an executable node with an unexpected shape is a +//! hard compile error, never a silent fallback to prose. + +use unicode_normalization::UnicodeNormalization; + +use marqdo_span::Span; + +use crate::ast::{MdKind, MdNode}; + +/// Why a payload could not be extracted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PayloadError { + /// Span of the offending node. + pub span: Span, + /// Human readable reason. + pub reason: String, +} + +impl PayloadError { + fn new(span: Span, reason: impl Into) -> Self { + Self { + span, + reason: reason.into(), + } + } +} + +/// Extract the strict payload of an executable inline node. +/// +/// Only `Text` and `Code` children are allowed; their literals are concatenated +/// exactly as written (a code span keeps its payload verbatim). Nested +/// `Strong`/`Emph`/`Link` and line breaks are rejected so that one AST node can +/// never be executed twice or hide control content. +pub fn executable_payload(node: &MdNode) -> Result { + let mut payload = String::new(); + for child in &node.children { + match &child.kind { + MdKind::Text => payload.push_str(child.literal.as_deref().unwrap_or_default()), + MdKind::Code => payload.push_str(child.literal.as_deref().unwrap_or_default()), + other => { + return Err(PayloadError::new( + child.span, + format!( + "`{}` is not a valid payload inside an executable node", + other.tag() + ), + )); + } + } + } + Ok(payload) +} + +/// Render inert content to plain text. +/// +/// Used for heading names and other inert labels. Images use their alt text, +/// links use their label, code and math use their literal source, and line +/// breaks become a single space. +pub fn plain_text(node: &MdNode) -> String { + let mut out = String::new(); + collect_plain_text(node, &mut out); + out +} + +fn collect_plain_text(node: &MdNode, out: &mut String) { + match &node.kind { + MdKind::Text | MdKind::Code | MdKind::Math { .. } | MdKind::MathBlock => { + out.push_str(node.literal.as_deref().unwrap_or_default()); + } + MdKind::SoftBreak | MdKind::LineBreak => out.push(' '), + MdKind::HtmlBlock | MdKind::HtmlInline | MdKind::FrontMatter => { + // Raw HTML and front matter are not part of visible prose. + } + _ => { + for child in &node.children { + collect_plain_text(child, out); + } + } + } +} + +/// Normalise a heading name: NFC, trim, and collapse internal whitespace runs +/// to a single space. Returns `None` when nothing visible remains. +pub fn normalize_name(text: &str) -> Option { + let normalized: String = text.nfc().collect(); + let mut collapsed = String::new(); + let mut pending_space = false; + for ch in normalized.trim().chars() { + if ch.is_whitespace() { + pending_space = true; + } else { + if pending_space && !collapsed.is_empty() { + collapsed.push(' '); + } + pending_space = false; + collapsed.push(ch); + } + } + if collapsed.is_empty() { + None + } else { + Some(collapsed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(kind: MdKind, literal: Option<&str>, children: Vec) -> MdNode { + MdNode { + kind, + span: Span::new(0, 0), + literal: literal.map(str::to_string), + children, + } + } + + #[test] + fn payload_accepts_text_and_code() { + let strong = node( + MdKind::Strong, + None, + vec![ + node(MdKind::Text, Some("面积 = "), vec![]), + node(MdKind::Code, Some("计算(r=1)"), vec![]), + ], + ); + assert_eq!(executable_payload(&strong).unwrap(), "面积 = 计算(r=1)"); + } + + #[test] + fn payload_rejects_nested_markup() { + let strong = node( + MdKind::Strong, + None, + vec![node( + MdKind::Emph, + None, + vec![node(MdKind::Text, Some("x"), vec![])], + )], + ); + let error = executable_payload(&strong).unwrap_err(); + assert!(error.reason.contains("emph")); + } + + #[test] + fn plain_text_uses_alt_and_label() { + let image = node( + MdKind::Image { + url: "a.png".into(), + title: String::new(), + }, + None, + vec![node(MdKind::Text, Some("示意图"), vec![])], + ); + let heading = node( + MdKind::Heading { + level: 1, + setext: false, + }, + None, + vec![ + image, + node(MdKind::SoftBreak, None, vec![]), + node(MdKind::Text, Some("名字"), vec![]), + ], + ); + assert_eq!(plain_text(&heading), "示意图 名字"); + } + + #[test] + fn normalize_collapses_whitespace_and_nfc() { + assert_eq!(normalize_name(" 圆 面积 ").as_deref(), Some("圆 面积")); + // Decomposed e + combining acute becomes NFC single code point. + assert_eq!(normalize_name("e\u{301}").as_deref(), Some("é")); + assert_eq!(normalize_name(" "), None); + } +}