feat(markdown): freeze the Markdown profile and canonical AST view
P0 baseline: one Markdown parser, one frozen option set, no silent upgrades. - profile: CommonMark 0.31.2 core + GFM tables/strikethrough/task lists + dollar math + YAML front matter at byte 0, with every other extension disabled and asserted by tests. comrak is pinned to exactly 1.0.0. - ast::MdNode: owned, serializable node tree carrying kind, byte Span and literal payloads. Nodes outside the profile are preserved as MdKind::Other instead of being dropped; task list items keep their checkbox state. - frontmatter: extracts the YAML body from the parser node, parses it once, and exposes only the explicit `import` alias mapping; invalid YAML is reported, never guessed. - text: strict executable payload extraction (Text/Code only, everything else is a hard error) and inert plain-text extraction for heading names. - UTF-8 BOM is tolerated; all spans are re-based onto the original file. No semantic mapping or execution exists in this crate by design.
This commit is contained in:
1 parent
1520244923
commit
10c0c0a013
5 files changed
+1006
No files matched your search
@@ -0,0 +1,230 @@
|
||||
//! Owned, serializable view of the CommonMark/GFM/Math AST.
|
||||
//!
|
||||
//! The profile uses exactly one Markdown parser (comrak). This module converts
|
||||
//! the parser's arena tree into an owned tree that keeps:
|
||||
//!
|
||||
//! * the node kind, so program roles can be derived from AST structure alone;
|
||||
//! * the byte [`Span`] in the original file, for diagnostics and IDE support;
|
||||
//! * literal payloads for inert leaves (text, code, math, HTML, front matter);
|
||||
//! * the full child structure, including nodes outside the frozen profile,
|
||||
//! which are preserved as [`MdKind::Other`] rather than dropped.
|
||||
|
||||
use marqdo_span::Span;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// A node of the canonical Markdown AST view.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct MdNode {
|
||||
/// Structural kind and its small attribute set.
|
||||
pub kind: MdKind,
|
||||
/// Byte span in the original source (UTF-8, half open).
|
||||
pub span: Span,
|
||||
/// Literal payload for leaves that carry text (Text, Code, Math, ...).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub literal: Option<String>,
|
||||
/// Child nodes in document order.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub children: Vec<MdNode>,
|
||||
}
|
||||
|
||||
/// Structural kind of a Markdown AST node.
|
||||
///
|
||||
/// The variants mirror the frozen Markdown profile. Nodes produced by parser
|
||||
/// extensions that are not part of the profile are preserved as `Other` so the
|
||||
/// AST view never hides syntax, but they are inert for the language.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum MdKind {
|
||||
/// Document root.
|
||||
Document,
|
||||
/// YAML front matter block at the start of the document.
|
||||
FrontMatter,
|
||||
/// Block quote (an invocation in executable position).
|
||||
BlockQuote,
|
||||
/// Ordered or bullet list.
|
||||
List {
|
||||
/// `true` for ordered lists, `false` for bullet lists.
|
||||
ordered: bool,
|
||||
/// Ordered list start number; `None` for bullet lists.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
start: Option<usize>,
|
||||
/// CommonMark tight/loose flag (a rendering distinction only).
|
||||
tight: bool,
|
||||
},
|
||||
/// One list item. Task list items (GFM) keep their checkbox state.
|
||||
Item {
|
||||
/// `Some(checked)` for a GFM task list item, `None` otherwise.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
task: Option<bool>,
|
||||
},
|
||||
/// Heading of level 1-6.
|
||||
Heading {
|
||||
/// Level 1-6.
|
||||
level: u8,
|
||||
/// Whether the source used a Setext underline.
|
||||
setext: bool,
|
||||
},
|
||||
/// Paragraph.
|
||||
Paragraph,
|
||||
/// Literal text.
|
||||
Text,
|
||||
/// Strong emphasis: an execution statement in executable position.
|
||||
Strong,
|
||||
/// Emphasis: a return statement in executable position.
|
||||
Emph,
|
||||
/// GFM strikethrough.
|
||||
Strikethrough,
|
||||
/// Inline code span.
|
||||
Code,
|
||||
/// Fenced or indented code block.
|
||||
CodeBlock {
|
||||
/// Whether the block was fenced.
|
||||
fenced: bool,
|
||||
/// Info string (language) without surrounding whitespace.
|
||||
info: String,
|
||||
},
|
||||
/// Inline math.
|
||||
Math {
|
||||
/// Whether display math (`$$...$$`).
|
||||
display: bool,
|
||||
},
|
||||
/// Block display math (`$$` fences on their own lines).
|
||||
MathBlock,
|
||||
/// Link with a destination.
|
||||
Link {
|
||||
/// Destination as written.
|
||||
url: String,
|
||||
/// Optional title.
|
||||
title: String,
|
||||
},
|
||||
/// Image with a source.
|
||||
Image {
|
||||
/// Source as written.
|
||||
url: String,
|
||||
/// Optional title.
|
||||
title: String,
|
||||
},
|
||||
/// GFM table.
|
||||
Table {
|
||||
/// Column alignments as `none`/`left`/`center`/`right`.
|
||||
alignments: Vec<String>,
|
||||
},
|
||||
/// Table row.
|
||||
TableRow {
|
||||
/// Whether this is the header row.
|
||||
header: bool,
|
||||
},
|
||||
/// Table cell.
|
||||
TableCell,
|
||||
/// Horizontal rule.
|
||||
ThematicBreak,
|
||||
/// Soft line break inside a paragraph.
|
||||
SoftBreak,
|
||||
/// Hard line break.
|
||||
LineBreak,
|
||||
/// Raw HTML block.
|
||||
HtmlBlock,
|
||||
/// Raw inline HTML.
|
||||
HtmlInline,
|
||||
/// Footnote definition (profile leaves footnotes disabled; kept for completeness).
|
||||
FootnoteDefinition {
|
||||
/// Footnote label.
|
||||
name: String,
|
||||
},
|
||||
/// Footnote reference (profile leaves footnotes disabled).
|
||||
FootnoteReference {
|
||||
/// Footnote label.
|
||||
name: String,
|
||||
},
|
||||
/// A node outside the frozen profile, preserved for inspection only.
|
||||
Other {
|
||||
/// Parser tag name.
|
||||
tag: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl MdNode {
|
||||
/// Create a leaf node.
|
||||
pub fn leaf(kind: MdKind, span: Span, literal: Option<String>) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
span,
|
||||
literal,
|
||||
children: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Depth-first pre-order traversal including `self`.
|
||||
pub fn walk<'a>(&'a self, f: &mut impl FnMut(&'a MdNode)) {
|
||||
f(self);
|
||||
for child in &self.children {
|
||||
child.walk(f);
|
||||
}
|
||||
}
|
||||
|
||||
/// Direct child nodes of a given kind.
|
||||
pub fn children_of<'a>(&'a self, kind: &MdKind) -> impl Iterator<Item = &'a MdNode> {
|
||||
let kind = kind.clone();
|
||||
self.children.iter().filter(move |child| child.kind == kind)
|
||||
}
|
||||
|
||||
/// Whether this node is an inline node.
|
||||
pub fn is_inline(&self) -> bool {
|
||||
matches!(
|
||||
self.kind,
|
||||
MdKind::Text
|
||||
| MdKind::Strong
|
||||
| MdKind::Emph
|
||||
| MdKind::Strikethrough
|
||||
| MdKind::Code
|
||||
| MdKind::Math { .. }
|
||||
| MdKind::Link { .. }
|
||||
| MdKind::Image { .. }
|
||||
| MdKind::SoftBreak
|
||||
| MdKind::LineBreak
|
||||
| MdKind::HtmlInline
|
||||
| MdKind::FootnoteReference { .. }
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether this node is a block node.
|
||||
pub fn is_block(&self) -> bool {
|
||||
!self.is_inline()
|
||||
}
|
||||
}
|
||||
|
||||
impl MdKind {
|
||||
/// Short stable tag name used in diagnostics.
|
||||
pub const fn tag(&self) -> &'static str {
|
||||
match self {
|
||||
MdKind::Document => "document",
|
||||
MdKind::FrontMatter => "front_matter",
|
||||
MdKind::BlockQuote => "block_quote",
|
||||
MdKind::List { .. } => "list",
|
||||
MdKind::Item { .. } => "item",
|
||||
MdKind::Heading { .. } => "heading",
|
||||
MdKind::Paragraph => "paragraph",
|
||||
MdKind::Text => "text",
|
||||
MdKind::Strong => "strong",
|
||||
MdKind::Emph => "emph",
|
||||
MdKind::Strikethrough => "strikethrough",
|
||||
MdKind::Code => "code",
|
||||
MdKind::CodeBlock { .. } => "code_block",
|
||||
MdKind::Math { .. } => "math",
|
||||
MdKind::MathBlock => "math_block",
|
||||
MdKind::Link { .. } => "link",
|
||||
MdKind::Image { .. } => "image",
|
||||
MdKind::Table { .. } => "table",
|
||||
MdKind::TableRow { .. } => "table_row",
|
||||
MdKind::TableCell => "table_cell",
|
||||
MdKind::ThematicBreak => "thematic_break",
|
||||
MdKind::SoftBreak => "soft_break",
|
||||
MdKind::LineBreak => "line_break",
|
||||
MdKind::HtmlBlock => "html_block",
|
||||
MdKind::HtmlInline => "html_inline",
|
||||
MdKind::FootnoteDefinition { .. } => "footnote_definition",
|
||||
MdKind::FootnoteReference { .. } => "footnote_reference",
|
||||
MdKind::Other { .. } => "other",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
//! YAML front matter handling.
|
||||
//!
|
||||
//! Front matter is only recognised at byte offset 0 (after an optional UTF-8
|
||||
//! BOM) and only when the Markdown parser itself reports a `FrontMatter` node.
|
||||
//! The parser node keeps the raw delimiters; this module extracts the YAML body
|
||||
//! and exposes the explicit `import` mapping without interpreting anything
|
||||
//! else. Front matter never executes.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use marqdo_span::Span;
|
||||
|
||||
/// Parsed (or unparseable) front matter.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct FrontMatter {
|
||||
/// Raw YAML body between the delimiter lines, excluding the delimiters.
|
||||
pub raw: String,
|
||||
/// Parsed YAML value; `Err` carries a human readable reason.
|
||||
pub yaml: Result<serde_yaml::Value, String>,
|
||||
/// Span of the whole front matter node (including delimiters).
|
||||
pub span: Span,
|
||||
/// Span of the YAML body only.
|
||||
pub yaml_span: Span,
|
||||
}
|
||||
|
||||
impl FrontMatter {
|
||||
/// The explicit `import` mapping from front matter.
|
||||
///
|
||||
/// The supported shape is a YAML mapping of local alias to module address:
|
||||
///
|
||||
/// ```yaml
|
||||
/// import:
|
||||
/// geometry: "@geometry"
|
||||
/// ```
|
||||
///
|
||||
/// Anything else is an error; imports are never inferred.
|
||||
pub fn imports(&self) -> Result<BTreeMap<String, String>, String> {
|
||||
let yaml = self
|
||||
.yaml
|
||||
.as_ref()
|
||||
.map_err(|error| format!("front matter is not valid YAML: {error}"))?;
|
||||
let Some(imports) = yaml.get("import") else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
let Some(mapping) = imports.as_mapping() else {
|
||||
return Err("`import` must be a mapping of alias to module address".to_string());
|
||||
};
|
||||
let mut result = BTreeMap::new();
|
||||
for (alias, target) in mapping {
|
||||
let Some(alias) = alias.as_str() else {
|
||||
return Err("`import` aliases must be strings".to_string());
|
||||
};
|
||||
let Some(target) = target.as_str() else {
|
||||
return Err(format!("import target for `{alias}` must be a string"));
|
||||
};
|
||||
if alias.is_empty() {
|
||||
return Err("`import` alias must not be empty".to_string());
|
||||
}
|
||||
if result
|
||||
.insert(alias.to_string(), target.to_string())
|
||||
.is_some()
|
||||
{
|
||||
return Err(format!("duplicate import alias `{alias}`"));
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// The raw YAML parses successfully.
|
||||
pub fn is_valid(&self) -> bool {
|
||||
self.yaml.is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a `FrontMatter` parser literal into the YAML body.
|
||||
///
|
||||
/// The parser literal starts with the opening delimiter line and includes the
|
||||
/// closing delimiter line plus trailing newlines, e.g.
|
||||
/// `"---\nimport:\n a: b\n---\n\n"`.
|
||||
pub fn parse_literal(literal: &str, span: Span) -> FrontMatter {
|
||||
let (raw, body_offset) = split_delimiters(literal);
|
||||
let yaml = if raw.trim().is_empty() {
|
||||
Ok(serde_yaml::Value::Null)
|
||||
} else {
|
||||
serde_yaml::from_str::<serde_yaml::Value>(&raw).map_err(|error| error.to_string())
|
||||
};
|
||||
let start = span.start.saturating_add(body_offset);
|
||||
let end = start.saturating_add(raw.len()).min(span.end.max(start));
|
||||
FrontMatter {
|
||||
raw,
|
||||
yaml,
|
||||
span,
|
||||
yaml_span: Span::new(start, end),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the YAML body and its byte offset within `literal`.
|
||||
fn split_delimiters(literal: &str) -> (String, usize) {
|
||||
let mut offset = 0usize;
|
||||
let mut lines = literal.split_inclusive('\n');
|
||||
let Some(first) = lines.next() else {
|
||||
return (String::new(), 0);
|
||||
};
|
||||
offset += first.len();
|
||||
let mut current = offset;
|
||||
let body_start = offset;
|
||||
for line in lines {
|
||||
let trimmed = line.trim_end_matches(['\r', '\n']);
|
||||
if trimmed == "---" || trimmed == "..." {
|
||||
let body = literal[body_start..current].to_string();
|
||||
return (body, body_start);
|
||||
}
|
||||
current += line.len();
|
||||
}
|
||||
// Unterminated front matter cannot normally reach here: comrak only emits a
|
||||
// FrontMatter node when a closing delimiter exists. Be forgiving and treat
|
||||
// the rest as YAML so diagnostics can still point at it.
|
||||
(literal[body_start..].to_string(), body_start)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_import_mapping() {
|
||||
let literal = "---\nimport:\n geometry: \"@geometry\"\n---\n\n";
|
||||
let fm = parse_literal(literal, Span::new(0, literal.len()));
|
||||
assert!(fm.is_valid());
|
||||
let imports = fm.imports().unwrap();
|
||||
assert_eq!(
|
||||
imports.get("geometry").map(String::as_str),
|
||||
Some("@geometry")
|
||||
);
|
||||
assert_eq!(fm.raw, "import:\n geometry: \"@geometry\"\n");
|
||||
assert_eq!(fm.yaml_span.start, 4);
|
||||
assert_eq!(&literal[fm.yaml_span.start..fm.yaml_span.end], fm.raw);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_frontmatter_is_null() {
|
||||
let literal = "---\n---\n";
|
||||
let fm = parse_literal(literal, Span::new(0, literal.len()));
|
||||
assert_eq!(fm.raw, "");
|
||||
assert!(fm.imports().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_yaml_is_reported_not_panicked() {
|
||||
let literal = "---\nimport: [unclosed\n---\n";
|
||||
let fm = parse_literal(literal, Span::new(0, literal.len()));
|
||||
assert!(!fm.is_valid());
|
||||
assert!(fm.imports().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_shapes_are_validated() {
|
||||
let literal = "---\nimport: geometry\n---\n";
|
||||
let fm = parse_literal(literal, Span::new(0, literal.len()));
|
||||
assert!(fm.imports().is_err());
|
||||
assert!(fm.imports().unwrap_err().contains("mapping"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
//! Canonical Markdown AST for Marqdo.
|
||||
//!
|
||||
//! This crate is the only place that talks to the Markdown parser. It freezes
|
||||
//! the CommonMark/GFM/Math profile (see [`profile`]), converts the parser tree
|
||||
//! into an owned [`ast::MdNode`] tree with byte spans, and extracts YAML front
|
||||
//! matter. It performs **no** semantic mapping and **no** execution: deciding
|
||||
//! which AST nodes are executable is the frontend's job and is based on the
|
||||
//! node kind and structural position only.
|
||||
|
||||
pub mod ast;
|
||||
pub mod frontmatter;
|
||||
pub mod profile;
|
||||
pub mod text;
|
||||
|
||||
pub use ast::{MdKind, MdNode};
|
||||
pub use frontmatter::FrontMatter;
|
||||
pub use marqdo_span::{LineCol, Span};
|
||||
pub use profile::{parse, MarkdownDocument, COMRAK_VERSION, PROFILE_ID};
|
||||
pub use text::{executable_payload, normalize_name, plain_text, PayloadError};
|
||||
@@ -0,0 +1,407 @@
|
||||
//! The frozen Markdown profile: one parser, one option set, no implicit
|
||||
//! upgrades.
|
||||
//!
|
||||
//! The parser version is pinned in `Cargo.toml` (`=1.0.0`) and the exact option
|
||||
//! set is asserted by tests. Changing either is a language change and requires
|
||||
//! an ADR plus regenerated goldens.
|
||||
|
||||
use comrak::nodes::{ListType, Node, NodeValue, Sourcepos, TableAlignment};
|
||||
use comrak::{parse_document, Arena, Options};
|
||||
use marqdo_span::{LineIndex, Span};
|
||||
|
||||
use crate::ast::{MdKind, MdNode};
|
||||
use crate::frontmatter::{self, FrontMatter};
|
||||
|
||||
/// Identifier of the frozen Markdown profile.
|
||||
pub const PROFILE_ID: &str = "marqdo.md.v1";
|
||||
/// Exact comrak version the profile is validated against.
|
||||
pub const COMRAK_VERSION: &str = "1.0.0";
|
||||
|
||||
/// Build the single supported parser configuration.
|
||||
///
|
||||
/// Enabled: CommonMark 0.31.2 core, GFM tables, GFM strikethrough, GFM task
|
||||
/// lists, `$...$` / `$$...$$` math, and YAML front matter delimited by `---`
|
||||
/// at the very start of the document.
|
||||
///
|
||||
/// Disabled: autolinks, footnotes, description lists, wikilinks, alerts,
|
||||
/// directives, superscript/subscript, raw-HTML suppression changes and every
|
||||
/// other optional extension. These defaults are asserted in tests.
|
||||
pub fn options() -> Options<'static> {
|
||||
let mut options = Options::default();
|
||||
options.extension.table = true;
|
||||
options.extension.strikethrough = true;
|
||||
options.extension.tasklist = true;
|
||||
options.extension.math_dollars = true;
|
||||
options.extension.math_dollars_block = true;
|
||||
options.extension.front_matter_delimiter = Some("---".to_string());
|
||||
options
|
||||
}
|
||||
|
||||
/// A parsed Markdown document under the frozen profile.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MarkdownDocument {
|
||||
/// Original source bytes as UTF-8 (including a BOM if present).
|
||||
pub source: String,
|
||||
/// Number of BOM bytes stripped before parsing; all spans are re-based onto
|
||||
/// the original file so they stay file offsets.
|
||||
pub bom_len: usize,
|
||||
/// Canonical AST root.
|
||||
pub root: MdNode,
|
||||
/// Parsed front matter, when present at the document start.
|
||||
pub frontmatter: Option<FrontMatter>,
|
||||
/// Line index over the original source, for diagnostics.
|
||||
pub line_index: LineIndex,
|
||||
}
|
||||
|
||||
impl MarkdownDocument {
|
||||
/// Map a byte offset in the original source to a 1-based line/column.
|
||||
pub fn line_col(&self, offset: usize) -> marqdo_span::LineCol {
|
||||
self.line_index.line_col(offset)
|
||||
}
|
||||
|
||||
/// Borrow the front matter, if the document has any.
|
||||
pub fn frontmatter(&self) -> Option<&FrontMatter> {
|
||||
self.frontmatter.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a source document with the frozen profile.
|
||||
///
|
||||
/// Parsing never executes anything and never fails: malformed constructs that
|
||||
/// the Markdown grammar itself accepts are reported later by the semantic
|
||||
/// layer, which knows which AST positions are executable.
|
||||
pub fn parse(source: &str) -> MarkdownDocument {
|
||||
let (bom_len, body) = strip_bom(source);
|
||||
let body_index = LineIndex::new(body);
|
||||
let arena = Arena::new();
|
||||
let root = parse_document(&arena, body, &options());
|
||||
let root = convert(&root, &body_index, bom_len, body.len());
|
||||
let frontmatter = root
|
||||
.children
|
||||
.iter()
|
||||
.find(|child| child.kind == MdKind::FrontMatter)
|
||||
.map(|node| {
|
||||
frontmatter::parse_literal(node.literal.as_deref().unwrap_or_default(), node.span)
|
||||
});
|
||||
MarkdownDocument {
|
||||
source: source.to_string(),
|
||||
bom_len,
|
||||
root,
|
||||
frontmatter,
|
||||
line_index: LineIndex::new(source),
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_bom(source: &str) -> (usize, &str) {
|
||||
match source.strip_prefix('\u{feff}') {
|
||||
Some(rest) => (source.len() - rest.len(), rest),
|
||||
None => (0, source),
|
||||
}
|
||||
}
|
||||
|
||||
fn convert(node: &Node<'_>, index: &LineIndex, shift: usize, source_len: usize) -> MdNode {
|
||||
let data = node.data();
|
||||
let span = span_of(data.sourcepos, index, shift, source_len);
|
||||
let (kind, literal) = map_value(&data.value);
|
||||
let children = node
|
||||
.children()
|
||||
.map(|child| convert(&child, index, shift, source_len))
|
||||
.collect();
|
||||
MdNode {
|
||||
kind,
|
||||
span,
|
||||
literal,
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a comrak source position (1-based line, 1-based byte column,
|
||||
/// inclusive end) into a half-open byte span in the original file.
|
||||
fn span_of(pos: Sourcepos, index: &LineIndex, shift: usize, source_len: usize) -> Span {
|
||||
// Offsets are computed in the BOM-stripped body, clamped there, and only
|
||||
// then re-based onto the original file.
|
||||
let start = offset_of(pos.start.line, pos.start.column, index, false)
|
||||
.map(|o| o.min(source_len))
|
||||
.unwrap_or(0);
|
||||
let end = offset_of(pos.end.line, pos.end.column, index, true)
|
||||
.map(|o| o.min(source_len))
|
||||
.unwrap_or(start);
|
||||
let end = end.max(start);
|
||||
Span::new(start + shift, end + shift)
|
||||
}
|
||||
|
||||
fn offset_of(line: usize, column: usize, index: &LineIndex, inclusive: bool) -> Option<usize> {
|
||||
if line == 0 {
|
||||
return None;
|
||||
}
|
||||
let line_start = index.line_start(line)?;
|
||||
let zero_based = column.saturating_sub(1);
|
||||
Some(line_start + zero_based + usize::from(inclusive))
|
||||
}
|
||||
|
||||
fn map_value(value: &NodeValue) -> (MdKind, Option<String>) {
|
||||
match value {
|
||||
NodeValue::Document => (MdKind::Document, None),
|
||||
NodeValue::FrontMatter(raw) => (MdKind::FrontMatter, Some(raw.clone())),
|
||||
NodeValue::BlockQuote | NodeValue::MultilineBlockQuote(_) => (MdKind::BlockQuote, None),
|
||||
NodeValue::List(list) => (
|
||||
MdKind::List {
|
||||
ordered: list.list_type == ListType::Ordered,
|
||||
start: if list.list_type == ListType::Ordered {
|
||||
Some(list.start)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
tight: list.tight,
|
||||
},
|
||||
None,
|
||||
),
|
||||
NodeValue::Item(_) => (MdKind::Item { task: None }, None),
|
||||
NodeValue::Heading(heading) => (
|
||||
MdKind::Heading {
|
||||
level: heading.level,
|
||||
setext: heading.setext,
|
||||
},
|
||||
None,
|
||||
),
|
||||
NodeValue::Paragraph => (MdKind::Paragraph, None),
|
||||
NodeValue::Text(text) => (MdKind::Text, Some(text.to_string())),
|
||||
NodeValue::Strong(_) => (MdKind::Strong, None),
|
||||
NodeValue::Emph(_) => (MdKind::Emph, None),
|
||||
NodeValue::Strikethrough(_) => (MdKind::Strikethrough, None),
|
||||
NodeValue::Code(code) => (MdKind::Code, Some(code.literal.clone())),
|
||||
NodeValue::CodeBlock(block) => (
|
||||
MdKind::CodeBlock {
|
||||
fenced: block.fenced,
|
||||
info: block.info.trim().to_string(),
|
||||
},
|
||||
Some(block.literal.clone()),
|
||||
),
|
||||
NodeValue::Math(math) => (
|
||||
MdKind::Math {
|
||||
display: math.display_math,
|
||||
},
|
||||
Some(math.literal.clone()),
|
||||
),
|
||||
NodeValue::MathBlock(block) => (MdKind::MathBlock, Some(block.literal.clone())),
|
||||
NodeValue::Link(link) => (
|
||||
MdKind::Link {
|
||||
url: link.url.clone(),
|
||||
title: link.title.clone(),
|
||||
},
|
||||
None,
|
||||
),
|
||||
NodeValue::Image(image) => (
|
||||
MdKind::Image {
|
||||
url: image.url.clone(),
|
||||
title: image.title.clone(),
|
||||
},
|
||||
None,
|
||||
),
|
||||
NodeValue::Table(table) => (
|
||||
MdKind::Table {
|
||||
alignments: table
|
||||
.alignments
|
||||
.iter()
|
||||
.map(|alignment| {
|
||||
match alignment {
|
||||
TableAlignment::Left => "left",
|
||||
TableAlignment::Center => "center",
|
||||
TableAlignment::Right => "right",
|
||||
TableAlignment::None => "none",
|
||||
_ => "none",
|
||||
}
|
||||
.to_string()
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
None,
|
||||
),
|
||||
NodeValue::TableRow(row) => (MdKind::TableRow { header: row.header }, None),
|
||||
NodeValue::TableCell => (MdKind::TableCell, None),
|
||||
NodeValue::ThematicBreak(_) => (MdKind::ThematicBreak, None),
|
||||
NodeValue::SoftBreak => (MdKind::SoftBreak, None),
|
||||
NodeValue::LineBreak => (MdKind::LineBreak, None),
|
||||
NodeValue::HtmlBlock(block) => (MdKind::HtmlBlock, Some(block.literal.clone())),
|
||||
NodeValue::HtmlInline(raw) => (MdKind::HtmlInline, Some(raw.clone())),
|
||||
NodeValue::TaskItem(item) => (
|
||||
MdKind::Item {
|
||||
task: Some(item.symbol.is_some()),
|
||||
},
|
||||
None,
|
||||
),
|
||||
NodeValue::FootnoteDefinition(definition) => (
|
||||
MdKind::FootnoteDefinition {
|
||||
name: definition.name.clone(),
|
||||
},
|
||||
None,
|
||||
),
|
||||
NodeValue::FootnoteReference(reference) => (
|
||||
MdKind::FootnoteReference {
|
||||
name: reference.name.clone(),
|
||||
},
|
||||
None,
|
||||
),
|
||||
other => (
|
||||
MdKind::Other {
|
||||
tag: tag_name(other),
|
||||
},
|
||||
literal_of(other),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn literal_of(value: &NodeValue) -> Option<String> {
|
||||
match value {
|
||||
NodeValue::Raw(raw) => Some(raw.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Turn an unmodeled parser node into a stable snake_case tag. The node is
|
||||
/// inert for the language but must remain visible in `inspect --ast`.
|
||||
fn tag_name(value: &NodeValue) -> String {
|
||||
let debug = format!("{value:?}");
|
||||
let head = debug
|
||||
.split(|c: char| !c.is_alphanumeric() && c != '_')
|
||||
.next()
|
||||
.unwrap_or("unknown");
|
||||
let mut out = String::new();
|
||||
for (i, ch) in head.chars().enumerate() {
|
||||
if ch.is_uppercase() {
|
||||
if i != 0 {
|
||||
out.push('_');
|
||||
}
|
||||
out.extend(ch.to_lowercase());
|
||||
} else {
|
||||
out.push(ch);
|
||||
}
|
||||
}
|
||||
if out.is_empty() {
|
||||
"unknown".to_string()
|
||||
} else {
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ast::MdKind;
|
||||
|
||||
fn kind_tree(source: &str) -> MdNode {
|
||||
parse(source).root
|
||||
}
|
||||
|
||||
fn find<'a>(node: &'a MdNode, kind: &MdKind) -> Option<&'a MdNode> {
|
||||
if &node.kind == kind {
|
||||
return Some(node);
|
||||
}
|
||||
node.children.iter().find_map(|child| find(child, kind))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_options_are_frozen() {
|
||||
let options = options();
|
||||
assert!(options.extension.table);
|
||||
assert!(options.extension.strikethrough);
|
||||
assert!(options.extension.tasklist);
|
||||
assert!(options.extension.math_dollars);
|
||||
assert!(options.extension.math_dollars_block);
|
||||
assert_eq!(
|
||||
options.extension.front_matter_delimiter.as_deref(),
|
||||
Some("---")
|
||||
);
|
||||
assert!(!options.extension.autolink);
|
||||
assert!(!options.extension.footnotes);
|
||||
assert!(!options.extension.description_lists);
|
||||
assert!(!options.extension.wikilinks_title_after_pipe);
|
||||
assert!(!options.extension.wikilinks_title_before_pipe);
|
||||
assert!(!options.extension.alerts);
|
||||
assert!(!options.parse.smart);
|
||||
assert!(!options.parse.ignore_setext);
|
||||
assert!(!options.extension.inline_footnotes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_maps_core_constructs() {
|
||||
let root = kind_tree("# T\n\n## F\n\n**x = 1**\n\n*y*\n\n> 打印(1)\n");
|
||||
assert_eq!(root.kind, MdKind::Document);
|
||||
assert!(find(
|
||||
&root,
|
||||
&MdKind::Heading {
|
||||
level: 1,
|
||||
setext: false
|
||||
}
|
||||
)
|
||||
.is_some());
|
||||
assert!(find(&root, &MdKind::Strong).is_some());
|
||||
assert!(find(&root, &MdKind::Emph).is_some());
|
||||
assert!(find(&root, &MdKind::BlockQuote).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spans_are_byte_offsets_into_original_source() {
|
||||
let source = "# 数学\n\n正文\n";
|
||||
let doc = parse(source);
|
||||
let heading = &doc.root.children[0];
|
||||
assert_eq!(
|
||||
heading.kind,
|
||||
MdKind::Heading {
|
||||
level: 1,
|
||||
setext: false
|
||||
}
|
||||
);
|
||||
assert_eq!(&source[heading.span.start..heading.span.end], "# 数学");
|
||||
assert_eq!(doc.line_col(heading.span.start).line, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bom_is_tolerated_and_spans_shift() {
|
||||
let source = "\u{feff}# T\n";
|
||||
let doc = parse(source);
|
||||
assert_eq!(doc.bom_len, 3);
|
||||
let heading = &doc.root.children[0];
|
||||
assert_eq!(heading.span.start, 3);
|
||||
assert_eq!(&source[heading.span.start..heading.span.end], "# T");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn math_nodes_are_native_and_inert() {
|
||||
let doc = parse("inline $a+b$ math\n\n$$\nx+y\n$$\n");
|
||||
assert!(find(&doc.root, &MdKind::Math { display: false }).is_some());
|
||||
assert!(find(&doc.root, &MdKind::MathBlock).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setext_headings_are_heading_nodes() {
|
||||
let doc = parse("函数\n---\n");
|
||||
let heading = &doc.root.children[0];
|
||||
assert_eq!(
|
||||
heading.kind,
|
||||
MdKind::Heading {
|
||||
level: 2,
|
||||
setext: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tables_and_task_lists_are_supported() {
|
||||
let doc = parse("| a | b |\n|---|---|\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n");
|
||||
assert!(find(
|
||||
&doc.root,
|
||||
&MdKind::Table {
|
||||
alignments: vec!["none".into(), "none".into()]
|
||||
}
|
||||
)
|
||||
.is_some());
|
||||
let mut checked = Vec::new();
|
||||
doc.root.walk(&mut |node| {
|
||||
if let MdKind::Item { task: Some(state) } = node.kind {
|
||||
checked.push(state);
|
||||
}
|
||||
});
|
||||
assert_eq!(checked, vec![true, false]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
//! Inert text extraction and strict executable payload extraction.
|
||||
//!
|
||||
//! Two different operations live here on purpose:
|
||||
//!
|
||||
//! * [`plain_text`] renders inert content (heading names, table labels); it
|
||||
//! never decides that something is executable.
|
||||
//! * [`executable_payload`] extracts the payload of an executable `Strong` or
|
||||
//! `Emph` node. It accepts only `Text` and `Code` children and fails on any
|
||||
//! other inline structure: an executable node with an unexpected shape is a
|
||||
//! hard compile error, never a silent fallback to prose.
|
||||
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
use marqdo_span::Span;
|
||||
|
||||
use crate::ast::{MdKind, MdNode};
|
||||
|
||||
/// Why a payload could not be extracted.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PayloadError {
|
||||
/// Span of the offending node.
|
||||
pub span: Span,
|
||||
/// Human readable reason.
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
impl PayloadError {
|
||||
fn new(span: Span, reason: impl Into<String>) -> Self {
|
||||
Self {
|
||||
span,
|
||||
reason: reason.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the strict payload of an executable inline node.
|
||||
///
|
||||
/// Only `Text` and `Code` children are allowed; their literals are concatenated
|
||||
/// exactly as written (a code span keeps its payload verbatim). Nested
|
||||
/// `Strong`/`Emph`/`Link` and line breaks are rejected so that one AST node can
|
||||
/// never be executed twice or hide control content.
|
||||
pub fn executable_payload(node: &MdNode) -> Result<String, PayloadError> {
|
||||
let mut payload = String::new();
|
||||
for child in &node.children {
|
||||
match &child.kind {
|
||||
MdKind::Text => payload.push_str(child.literal.as_deref().unwrap_or_default()),
|
||||
MdKind::Code => payload.push_str(child.literal.as_deref().unwrap_or_default()),
|
||||
other => {
|
||||
return Err(PayloadError::new(
|
||||
child.span,
|
||||
format!(
|
||||
"`{}` is not a valid payload inside an executable node",
|
||||
other.tag()
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
/// Render inert content to plain text.
|
||||
///
|
||||
/// Used for heading names and other inert labels. Images use their alt text,
|
||||
/// links use their label, code and math use their literal source, and line
|
||||
/// breaks become a single space.
|
||||
pub fn plain_text(node: &MdNode) -> String {
|
||||
let mut out = String::new();
|
||||
collect_plain_text(node, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn collect_plain_text(node: &MdNode, out: &mut String) {
|
||||
match &node.kind {
|
||||
MdKind::Text | MdKind::Code | MdKind::Math { .. } | MdKind::MathBlock => {
|
||||
out.push_str(node.literal.as_deref().unwrap_or_default());
|
||||
}
|
||||
MdKind::SoftBreak | MdKind::LineBreak => out.push(' '),
|
||||
MdKind::HtmlBlock | MdKind::HtmlInline | MdKind::FrontMatter => {
|
||||
// Raw HTML and front matter are not part of visible prose.
|
||||
}
|
||||
_ => {
|
||||
for child in &node.children {
|
||||
collect_plain_text(child, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalise a heading name: NFC, trim, and collapse internal whitespace runs
|
||||
/// to a single space. Returns `None` when nothing visible remains.
|
||||
pub fn normalize_name(text: &str) -> Option<String> {
|
||||
let normalized: String = text.nfc().collect();
|
||||
let mut collapsed = String::new();
|
||||
let mut pending_space = false;
|
||||
for ch in normalized.trim().chars() {
|
||||
if ch.is_whitespace() {
|
||||
pending_space = true;
|
||||
} else {
|
||||
if pending_space && !collapsed.is_empty() {
|
||||
collapsed.push(' ');
|
||||
}
|
||||
pending_space = false;
|
||||
collapsed.push(ch);
|
||||
}
|
||||
}
|
||||
if collapsed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(collapsed)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn node(kind: MdKind, literal: Option<&str>, children: Vec<MdNode>) -> MdNode {
|
||||
MdNode {
|
||||
kind,
|
||||
span: Span::new(0, 0),
|
||||
literal: literal.map(str::to_string),
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn payload_accepts_text_and_code() {
|
||||
let strong = node(
|
||||
MdKind::Strong,
|
||||
None,
|
||||
vec![
|
||||
node(MdKind::Text, Some("面积 = "), vec![]),
|
||||
node(MdKind::Code, Some("计算(r=1)"), vec![]),
|
||||
],
|
||||
);
|
||||
assert_eq!(executable_payload(&strong).unwrap(), "面积 = 计算(r=1)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn payload_rejects_nested_markup() {
|
||||
let strong = node(
|
||||
MdKind::Strong,
|
||||
None,
|
||||
vec![node(
|
||||
MdKind::Emph,
|
||||
None,
|
||||
vec![node(MdKind::Text, Some("x"), vec![])],
|
||||
)],
|
||||
);
|
||||
let error = executable_payload(&strong).unwrap_err();
|
||||
assert!(error.reason.contains("emph"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_text_uses_alt_and_label() {
|
||||
let image = node(
|
||||
MdKind::Image {
|
||||
url: "a.png".into(),
|
||||
title: String::new(),
|
||||
},
|
||||
None,
|
||||
vec![node(MdKind::Text, Some("示意图"), vec![])],
|
||||
);
|
||||
let heading = node(
|
||||
MdKind::Heading {
|
||||
level: 1,
|
||||
setext: false,
|
||||
},
|
||||
None,
|
||||
vec![
|
||||
image,
|
||||
node(MdKind::SoftBreak, None, vec![]),
|
||||
node(MdKind::Text, Some("名字"), vec![]),
|
||||
],
|
||||
);
|
||||
assert_eq!(plain_text(&heading), "示意图 名字");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_collapses_whitespace_and_nfc() {
|
||||
assert_eq!(normalize_name(" 圆 面积 ").as_deref(), Some("圆 面积"));
|
||||
// Decomposed e + combining acute becomes NFC single code point.
|
||||
assert_eq!(normalize_name("e\u{301}").as_deref(), Some("é"));
|
||||
assert_eq!(normalize_name(" "), None);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user