feat(expression): strict expression and statement grammar for executable payloads

Implement the independent expression parser frozen in spec/expression.ebnf:

- Unicode XID identifiers (Chinese names supported), reserved keywords
  true/false/None/and/or/not.
- Arbitrary-precision Int literals, binary64 Float literals (digits required on
  both sides, overflow rejected), double-quoted Text with \u{...} escapes.
- List and record literals, calls with positional-then-named arguments and
  trailing commas, member access, indexing, chained postfix operators.
- Precedence exactly as specified: right-associative ** with unary minus
  binding outside, multiplicative/additive, single non-associative comparison,
  short-circuit and/or.
- Assignment only as `identifier = expression` at statement start; no comments,
  no second Markdown parser, max nesting depth 128 with structured errors.
- parse_expression/parse_statement/parse_arguments never panic; ParseError has
  stable code expr.parse, payload-relative byte spans and Display/Error.

50 unit tests + 2 doctests, cargo fmt/clippy clean.
This commit is contained in:
cflmy committed 2026-11-20 10:35:49 +08:00
1 parent 55a2b2d82d
commit 9897ecc37f
6 files changed
+2649 -5

No files matched your search

Generated
+86 -4
View File
@@ -58,6 +58,12 @@ version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "bitflags"
version = "2.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
[[package]]
name = "block-buffer"
version = "0.10.4"
@@ -186,6 +192,16 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
[[package]]
name = "errno"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "fastrand"
version = "2.5.0"
@@ -225,6 +241,23 @@ dependencies = [
"version_check",
]
[[package]]
name = "getrandom"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
dependencies = [
"cfg-if",
"libc",
"r-efi",
]
[[package]]
name = "glob"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -281,6 +314,12 @@ dependencies = [
"windows-link",
]
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "marqdo-cli"
version = "0.1.0"
@@ -309,15 +348,16 @@ name = "marqdo-expression"
version = "0.1.0"
dependencies = [
"marqdo-span",
"num-bigint 0.5.1",
"serde",
"thiserror",
"serde_json",
"unicode-ident",
]
[[package]]
name = "marqdo-foreign"
version = "0.1.0"
dependencies = [
"marqdo-span",
"marqdo-value",
"serde_json",
"thiserror",
@@ -334,7 +374,8 @@ dependencies = [
"marqdo-value",
"serde",
"serde_json",
"thiserror",
"sha2",
"unicode-ident",
"unicode-normalization",
]
@@ -342,6 +383,7 @@ dependencies = [
name = "marqdo-host"
version = "0.1.0"
dependencies = [
"glob",
"marqdo-foreign",
"marqdo-math",
"marqdo-plugin-abi",
@@ -350,6 +392,7 @@ dependencies = [
"marqdo-value",
"serde",
"serde_json",
"tempfile",
"thiserror",
]
@@ -359,7 +402,6 @@ version = "0.1.0"
dependencies = [
"marqdo-expression",
"marqdo-span",
"marqdo-value",
"serde",
"serde_json",
]
@@ -406,12 +448,14 @@ dependencies = [
name = "marqdo-plugin-abi"
version = "0.1.0"
dependencies = [
"hex",
"libloading",
"marqdo-span",
"marqdo-value",
"serde",
"serde_json",
"serde_yaml",
"sha2",
"thiserror",
]
@@ -530,6 +574,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "once_cell_polyfill"
version = "1.70.2"
@@ -593,12 +643,31 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "rustc-hash"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
[[package]]
name = "rustix"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "ryu"
version = "1.0.23"
@@ -710,6 +779,19 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom",
"once_cell",
"rustix",
"windows-sys",
]
[[package]]
name = "thiserror"
version = "2.0.21"
+4 -1
View File
@@ -11,5 +11,8 @@ authors.workspace = true
[dependencies]
marqdo-span = { workspace = true }
serde = { workspace = true }
thiserror = { workspace = true }
num-bigint = { workspace = true }
unicode-ident = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
+292
View File
@@ -0,0 +1,292 @@
//! Abstract syntax tree for the Marqdo expression grammar v1.
//!
//! All spans are byte offsets relative to the payload string that was handed to
//! [`crate::parse_expression`], [`crate::parse_statement`] or
//! [`crate::parse_arguments`]. The frontend re-bases them onto the Markdown
//! source through the text/code segments that made up the payload.
//!
//! # Span coverage
//!
//! Every node that carries a stored span exposes it through [`Expr::span`] /
//! [`Statement::span`]. The frozen v1 AST shape stores a span only on the
//! structural variants (`Unary`, `Binary`, `Call`, `Member`, `Index`), on
//! [`Ident`], [`Arg`] and [`RecordEntry`]. Scalar and container literals
//! (`Int`, `Float`, `Text`, `Bool`, `None`, `List`, `Record`) intentionally
//! have no span field: [`Expr::span`] returns the empty span `0..0` for them.
//! Consumers that need a location for a literal should use the nearest
//! enclosing structural node or the enclosing [`Statement::span`].
use marqdo_span::Span;
use num_bigint::BigInt;
use serde::{Deserialize, Serialize};
/// A bare identifier together with its source span.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct Ident {
/// Identifier text as written (Unicode XID, decoded verbatim).
pub name: String,
/// Span of the identifier token.
pub span: Span,
}
/// One argument of a call, optionally named.
///
/// ```text
/// f(1, r=3)
/// ^ ^ named argument, `name == Some("r")`
/// ```
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Arg {
/// `Some` when the argument was written as `name = value`.
pub name: Option<String>,
/// Argument value expression.
pub value: Expr,
/// Span of the whole argument (name through value).
pub span: Span,
}
/// One key/value entry of a record literal.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct RecordEntry {
/// Entry key (identifier text or decoded text-literal contents).
pub key: String,
/// Span of the key token (including the quotes for a text-literal key).
pub key_span: Span,
/// Entry value expression.
pub value: Expr,
}
/// Binary operators, listed from loosest to tightest binding.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum BinaryOp {
/// Short-circuiting logical or (`or`).
Or,
/// Short-circuiting logical and (`and`).
And,
/// `==`
Eq,
/// `!=`
Ne,
/// `<`
Lt,
/// `<=`
Le,
/// `>`
Gt,
/// `>=`
Ge,
/// `+`
Add,
/// `-`
Sub,
/// `*`
Mul,
/// `/` (always yields `Float` at runtime).
Div,
/// `//` (floor division, `Int` only).
FloorDiv,
/// `%` (remainder, `Int` only).
Mod,
/// `**` (right associative).
Pow,
}
/// Prefix operators.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum UnaryOp {
/// `not`
Not,
/// unary `-`
Neg,
/// unary `+`
Pos,
}
/// An expression node.
///
/// The variant set and payloads are the frozen public contract of this crate.
/// See the module docs for the span policy of literal variants.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum Expr {
/// Arbitrary-precision decimal integer literal.
Int(BigInt),
/// IEEE-754 binary64 float literal.
Float(f64),
/// Decoded text literal (escapes already resolved).
Text(String),
/// `true` / `false`.
Bool(bool),
/// `None`.
None,
/// A variable, parameter or function name reference.
Ident(Ident),
/// `[a, b, c]`.
List(Vec<Expr>),
/// `{key: value}`.
Record(Vec<RecordEntry>),
/// Prefix operator application.
Unary {
/// Applied prefix operator.
op: UnaryOp,
/// Operand.
operand: Box<Expr>,
/// Span from the operator through the end of the operand.
span: Span,
},
/// Infix operator application.
Binary {
/// Applied infix operator.
op: BinaryOp,
/// Left operand.
left: Box<Expr>,
/// Right operand.
right: Box<Expr>,
/// Span of the whole infix expression.
span: Span,
},
/// Call expression.
Call {
/// Callee expression.
callee: Box<Expr>,
/// Arguments in source order.
args: Vec<Arg>,
/// Span from the callee start through the closing `)`.
span: Span,
},
/// Member access (`base.name`).
Member {
/// Object being projected.
base: Box<Expr>,
/// Member name (never a keyword).
name: String,
/// Span from the base start through the member name.
span: Span,
},
/// Index access (`base[index]`).
Index {
/// Object being indexed.
base: Box<Expr>,
/// Index expression.
index: Box<Expr>,
/// Span from the base start through the closing `]`.
span: Span,
},
}
impl Expr {
/// Best known source span of this expression.
///
/// Returns the empty span `0..0` for variants that do not store a span
/// (`Int`, `Float`, `Text`, `Bool`, `None`, `List`, `Record`); see the
/// module documentation for the rationale.
pub fn span(&self) -> Span {
match self {
Expr::Ident(ident) => ident.span,
Expr::Unary { span, .. }
| Expr::Binary { span, .. }
| Expr::Call { span, .. }
| Expr::Member { span, .. }
| Expr::Index { span, .. } => *span,
Expr::Int(_) | Expr::Float(_) | Expr::Text(_) | Expr::Bool(_) | Expr::None => {
Span::default()
}
Expr::List(_) | Expr::Record(_) => Span::default(),
}
}
}
/// The payload of a statement.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum StatementKind {
/// `<bare identifier> = <expression>`; the only assignment form in v1.
Assign {
/// Assigned identifier.
name: String,
/// Span of the identifier token (not the whole assignment).
name_span: Span,
/// Right-hand side expression.
value: Expr,
},
/// A bare expression statement (typically a call).
Expr(Expr),
}
/// A complete parsed statement: either an assignment or an expression.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Statement {
/// Statement payload.
pub kind: StatementKind,
/// Span of the whole statement, from the first token through the last.
pub span: Span,
}
impl Statement {
/// Span of the whole statement, from the first token through the last.
pub fn span(&self) -> Span {
self.span
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::parse_expression;
#[test]
fn span_accessor_covers_structural_nodes() {
let expr = parse_expression("f(1, r=2)").unwrap();
assert_eq!(expr.span(), Span::new(0, 9));
let Expr::Call { callee, args, .. } = &expr else {
panic!("expected call");
};
assert_eq!(callee.span(), Span::new(0, 1));
assert_eq!(args[0].span, Span::new(2, 3));
assert_eq!(args[1].span, Span::new(5, 8));
assert_eq!(args[1].name.as_deref(), Some("r"));
}
#[test]
fn spanless_variants_return_the_empty_span() {
for expr in [
Expr::Int(BigInt::from(1)),
Expr::Float(1.0),
Expr::Text("t".to_owned()),
Expr::Bool(true),
Expr::None,
Expr::List(Vec::new()),
Expr::Record(Vec::new()),
] {
assert_eq!(expr.span(), Span::default());
}
}
#[test]
fn ast_types_serde_round_trip() {
let expr = parse_expression(
r#"f(1, [true, None], {键: 9.5e1}, 2 ** -1, a.b[0] == not false, 名="值")"#,
)
.unwrap();
let json = serde_json::to_string(&expr).unwrap();
let decoded: Expr = serde_json::from_str(&json).unwrap();
assert_eq!(decoded, expr);
let stmt = crate::parse_statement("结果 = 打印(阶乘(n=3))").unwrap();
let json = serde_json::to_string(&stmt).unwrap();
let decoded: Statement = serde_json::from_str(&json).unwrap();
assert_eq!(decoded, stmt);
assert_eq!(decoded.span(), stmt.span());
let args = crate::parse_arguments("(r=3, precision=2)").unwrap();
let json = serde_json::to_string(&args).unwrap();
let decoded: Vec<Arg> = serde_json::from_str(&json).unwrap();
assert_eq!(decoded, args);
}
#[test]
fn span_serde_round_trips_inside_ast() {
let expr = parse_expression("a.b").unwrap();
let json = serde_json::to_string(&expr).unwrap();
assert!(json.contains("\"start\":0"));
assert!(json.contains("\"end\":3"));
}
}
+608
View File
@@ -0,0 +1,608 @@
//! Hand written lexer for the Marqdo expression grammar v1.
//!
//! This module is an implementation detail that is exposed only to make
//! diagnostics and tests easier; the contractual API of the crate lives in
//! [`crate::parse_expression`], [`crate::parse_statement`] and
//! [`crate::parse_arguments`] plus the [`crate::ast`] types.
//!
//! Lexical rules (see `spec/expression.ebnf`):
//!
//! * Unicode whitespace separates tokens (newlines included).
//! * Identifiers use UAX #31 `XID_Start` / `XID_Continue`.
//! * Integers are arbitrary-precision decimal digit runs.
//! * Floats require digits on both sides of `.` and may carry an exponent.
//! * Text literals are double quoted with the escapes `\\`, `\"`, `\n`, `\t`,
//! `\r` and `\u{...}`.
//! * There are no comments in v1.
use crate::ParseError;
use marqdo_span::Span;
/// A lexical token kind.
#[derive(Debug, Clone, PartialEq)]
pub enum TokenKind {
/// Decimal integer literal, kept as raw digits (arbitrary precision).
Int(String),
/// Float literal, kept as raw source text (parsed to `f64` by the parser).
Float(String),
/// Text literal whose escapes have already been decoded.
Text(String),
/// Identifier text (never a keyword).
Ident(String),
/// `true`
True,
/// `false`
False,
/// `None`
NoneKw,
/// `and`
And,
/// `or`
Or,
/// `not`
Not,
/// `(`
LParen,
/// `)`
RParen,
/// `[`
LBracket,
/// `]`
RBracket,
/// `{`
LBrace,
/// `}`
RBrace,
/// `,`
Comma,
/// `:`
Colon,
/// `.`
Dot,
/// `=` (assignment)
Assign,
/// `==`
EqEq,
/// `!=`
Ne,
/// `<`
Lt,
/// `<=`
Le,
/// `>`
Gt,
/// `>=`
Ge,
/// `+`
Plus,
/// `-`
Minus,
/// `*`
Star,
/// `**`
StarStar,
/// `/`
Slash,
/// `//`
SlashSlash,
/// `%`
Percent,
/// End of input.
Eof,
}
/// A token with its byte span in the payload.
#[derive(Debug, Clone, PartialEq)]
pub struct Token {
/// Token kind.
pub kind: TokenKind,
/// Byte span of the token.
pub span: Span,
}
/// Tokenize a payload string.
///
/// The returned vector always ends with exactly one [`TokenKind::Eof`] token
/// whose span is the empty span at the end of the input.
pub fn tokenize(source: &str) -> Result<Vec<Token>, ParseError> {
Lexer::new(source).tokenize()
}
struct Lexer<'a> {
source: &'a str,
pos: usize,
}
impl<'a> Lexer<'a> {
fn new(source: &'a str) -> Self {
Self { source, pos: 0 }
}
fn peek(&self) -> Option<char> {
self.source[self.pos..].chars().next()
}
fn bump(&mut self) -> Option<char> {
let c = self.peek()?;
self.pos += c.len_utf8();
Some(c)
}
fn error(&self, start: usize, message: impl Into<String>) -> ParseError {
ParseError::new(Span::new(start, self.pos), message)
}
fn tokenize(mut self) -> Result<Vec<Token>, ParseError> {
let mut tokens = Vec::new();
loop {
self.skip_whitespace();
let start = self.pos;
let Some(c) = self.peek() else {
tokens.push(Token {
kind: TokenKind::Eof,
span: Span::empty(self.source.len()),
});
return Ok(tokens);
};
let kind = match c {
'"' => TokenKind::Text(self.scan_text()?),
'0'..='9' => self.scan_number()?,
_ if unicode_ident::is_xid_start(c) => self.scan_identifier(),
'(' => {
self.bump();
TokenKind::LParen
}
')' => {
self.bump();
TokenKind::RParen
}
'[' => {
self.bump();
TokenKind::LBracket
}
']' => {
self.bump();
TokenKind::RBracket
}
'{' => {
self.bump();
TokenKind::LBrace
}
'}' => {
self.bump();
TokenKind::RBrace
}
',' => {
self.bump();
TokenKind::Comma
}
':' => {
self.bump();
TokenKind::Colon
}
'.' => {
self.bump();
if matches!(self.peek(), Some(next) if next.is_ascii_digit()) {
return Err(self.error(
start,
"invalid float literal: a decimal point must be preceded by a digit",
));
}
TokenKind::Dot
}
'=' => {
self.bump();
if matches!(self.peek(), Some('=')) {
self.bump();
TokenKind::EqEq
} else {
TokenKind::Assign
}
}
'!' => {
self.bump();
if matches!(self.peek(), Some('=')) {
self.bump();
TokenKind::Ne
} else {
return Err(self.error(
start,
"unexpected character `!`; logical negation is written `not`",
));
}
}
'<' => {
self.bump();
if matches!(self.peek(), Some('=')) {
self.bump();
TokenKind::Le
} else {
TokenKind::Lt
}
}
'>' => {
self.bump();
if matches!(self.peek(), Some('=')) {
self.bump();
TokenKind::Ge
} else {
TokenKind::Gt
}
}
'+' => {
self.bump();
TokenKind::Plus
}
'-' => {
self.bump();
TokenKind::Minus
}
'*' => {
self.bump();
if matches!(self.peek(), Some('*')) {
self.bump();
TokenKind::StarStar
} else {
TokenKind::Star
}
}
'/' => {
self.bump();
if matches!(self.peek(), Some('/')) {
self.bump();
TokenKind::SlashSlash
} else {
TokenKind::Slash
}
}
'%' => {
self.bump();
TokenKind::Percent
}
'#' => {
self.bump();
return Err(self.error(
start,
"unexpected character `#`; comments are not supported in v1",
));
}
other => {
self.bump();
return Err(self.error(start, format!("unexpected character {:?}", other)));
}
};
tokens.push(Token {
kind,
span: Span::new(start, self.pos),
});
}
}
fn skip_whitespace(&mut self) {
while let Some(c) = self.peek() {
if c.is_whitespace() {
self.bump();
} else {
break;
}
}
}
fn scan_identifier(&mut self) -> TokenKind {
let start = self.pos;
// The caller guarantees `is_xid_start` for the first character.
self.bump();
while let Some(c) = self.peek() {
if unicode_ident::is_xid_continue(c) {
self.bump();
} else {
break;
}
}
let text = &self.source[start..self.pos];
match text {
"true" => TokenKind::True,
"false" => TokenKind::False,
"None" => TokenKind::NoneKw,
"and" => TokenKind::And,
"or" => TokenKind::Or,
"not" => TokenKind::Not,
_ => TokenKind::Ident(text.to_owned()),
}
}
fn scan_number(&mut self) -> Result<TokenKind, ParseError> {
let start = self.pos;
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
self.bump();
}
let mut is_float = false;
if matches!(self.peek(), Some('.')) {
self.bump();
if !matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
return Err(self.error(
start,
"invalid float literal: expected at least one digit after `.`",
));
}
is_float = true;
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
self.bump();
}
}
if matches!(self.peek(), Some('e') | Some('E')) {
if !is_float {
return Err(self.error(
start,
"invalid float literal: an exponent requires digits on both sides of `.`",
));
}
self.bump();
if matches!(self.peek(), Some('+') | Some('-')) {
self.bump();
}
if !matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
return Err(self.error(
start,
"invalid float literal: expected at least one exponent digit",
));
}
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
self.bump();
}
}
if let Some(c) = self.peek() {
if c == '.' || unicode_ident::is_xid_continue(c) {
return Err(self.error(
start,
format!("invalid numeric literal: unexpected `{c}` immediately after a number"),
));
}
}
let raw = self.source[start..self.pos].to_owned();
Ok(if is_float {
TokenKind::Float(raw)
} else {
TokenKind::Int(raw)
})
}
fn scan_text(&mut self) -> Result<String, ParseError> {
let start = self.pos;
self.bump(); // opening quote
let mut out = String::new();
loop {
let Some(c) = self.bump() else {
return Err(ParseError::new(
Span::new(start, self.source.len()),
"unterminated text literal: expected a closing `\"`",
));
};
match c {
'"' => return Ok(out),
'\\' => {
let escape_start = self.pos - 1;
match self.bump() {
Some('"') => out.push('"'),
Some('\\') => out.push('\\'),
Some('n') => out.push('\n'),
Some('t') => out.push('\t'),
Some('r') => out.push('\r'),
Some('u') => self.scan_unicode_escape(escape_start, &mut out)?,
Some(other) => {
return Err(self.error(
escape_start,
format!(
"invalid escape sequence `\\{other}`; supported escapes are \
`\\\\`, `\\\"`, `\\n`, `\\t`, `\\r` and `\\u{{...}}`"
),
));
}
None => {
return Err(ParseError::new(
Span::new(start, self.source.len()),
"unterminated text literal: expected a closing `\"`",
));
}
}
}
other => out.push(other),
}
}
}
fn scan_unicode_escape(
&mut self,
escape_start: usize,
out: &mut String,
) -> Result<(), ParseError> {
if !matches!(self.peek(), Some('{')) {
return Err(self.error(
escape_start,
"invalid unicode escape: expected `{` after `\\u`",
));
}
self.bump();
let digits_start = self.pos;
let mut value: u32 = 0;
let mut overflow = false;
while let Some(c) = self.peek() {
let Some(digit) = c.to_digit(16) else {
break;
};
self.bump();
value = match value.checked_mul(16).and_then(|v| v.checked_add(digit)) {
Some(v) => v,
None => {
overflow = true;
0
}
};
}
if self.pos == digits_start {
return Err(self.error(
escape_start,
"invalid unicode escape: expected at least one hexadecimal digit",
));
}
if !matches!(self.peek(), Some('}')) {
return Err(self.error(
escape_start,
"invalid unicode escape: expected a closing `}`",
));
}
self.bump();
if overflow {
return Err(self.error(
escape_start,
"invalid unicode escape: code point is out of range",
));
}
match char::from_u32(value) {
Some(c) => {
out.push(c);
Ok(())
}
None => Err(self.error(
escape_start,
format!("invalid unicode escape: U+{value:04X} is not a Unicode scalar value"),
)),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn kinds(source: &str) -> Vec<TokenKind> {
tokenize(source)
.unwrap()
.into_iter()
.map(|t| t.kind)
.collect()
}
#[test]
fn tokenizes_operators_and_punctuation() {
assert_eq!(
kinds("== != < <= > >= = + - * ** / // % ( ) [ ] { } , : ."),
vec![
TokenKind::EqEq,
TokenKind::Ne,
TokenKind::Lt,
TokenKind::Le,
TokenKind::Gt,
TokenKind::Ge,
TokenKind::Assign,
TokenKind::Plus,
TokenKind::Minus,
TokenKind::Star,
TokenKind::StarStar,
TokenKind::Slash,
TokenKind::SlashSlash,
TokenKind::Percent,
TokenKind::LParen,
TokenKind::RParen,
TokenKind::LBracket,
TokenKind::RBracket,
TokenKind::LBrace,
TokenKind::RBrace,
TokenKind::Comma,
TokenKind::Colon,
TokenKind::Dot,
TokenKind::Eof,
]
);
}
#[test]
fn tokenizes_keywords_separately_from_identifiers() {
assert_eq!(
kinds("true false None and or not truex None_ 真实"),
vec![
TokenKind::True,
TokenKind::False,
TokenKind::NoneKw,
TokenKind::And,
TokenKind::Or,
TokenKind::Not,
TokenKind::Ident("truex".into()),
TokenKind::Ident("None_".into()),
TokenKind::Ident("真实".into()),
TokenKind::Eof,
]
);
}
#[test]
fn token_remembers_multibyte_spans() {
let tokens = tokenize("张三 + 1").unwrap();
assert_eq!(tokens[0].kind, TokenKind::Ident("张三".into()));
assert_eq!(tokens[0].span, Span::new(0, 6));
assert_eq!(tokens[1].span, Span::new(7, 8));
assert_eq!(tokens[2].kind, TokenKind::Int("1".into()));
assert_eq!(tokens[2].span, Span::new(9, 10));
assert_eq!(tokens[3].span, Span::empty(10));
}
#[test]
fn unicode_whitespace_is_skipped() {
assert_eq!(
kinds("1\u{00a0}+\u{2003}2\n"),
vec![
TokenKind::Int("1".into()),
TokenKind::Plus,
TokenKind::Int("2".into()),
TokenKind::Eof
]
);
}
#[test]
fn number_lexing_rejects_invalid_forms() {
for bad in [".5", "1.", "1e5", "1.2.3", "1.5e", "1.5e+", "1a", "1_2"] {
let err = tokenize(bad).unwrap_err();
assert_eq!(err.code, "expr.parse", "input {bad:?}");
}
assert_eq!(
kinds("1.5e+3 0.0 007 1.25E-2"),
vec![
TokenKind::Float("1.5e+3".into()),
TokenKind::Float("0.0".into()),
TokenKind::Int("007".into()),
TokenKind::Float("1.25E-2".into()),
TokenKind::Eof
]
);
}
#[test]
fn text_escapes_decode() {
let tokens = tokenize(r#""a\\b\"c\n\t\r\u{4e2d}""#).unwrap();
assert_eq!(tokens[0].kind, TokenKind::Text("a\\b\"c\n\t\r中".into()));
}
#[test]
fn bad_text_lexing_reports_errors() {
for bad in [
r#""unterminated"#,
r#""bad \q escape""#,
r#""bad \u{} escape""#,
r#""bad \u{D800} escape""#,
r#""bad \u{110000} escape""#,
r#""bad \u12 escape""#,
] {
let err = tokenize(bad).unwrap_err();
assert_eq!(err.code, "expr.parse", "input {bad:?}");
}
}
#[test]
fn comments_are_not_supported() {
let err = tokenize("# comment").unwrap_err();
assert!(err.message.contains("comments"));
let err = tokenize("1 # 2").unwrap_err();
assert!(err.message.contains("comments"));
}
}
+225
View File
@@ -0,0 +1,225 @@
//! Strict, independent expression and statement parser for executable Markdown
//! payloads.
//!
//! This crate is the only grammar for Marqdo executable payloads. It knows
//! nothing about Markdown and nothing about the runtime: the Markdown AST
//! decides *where* a payload exists, this crate decides *what the payload
//! means* (see `spec/expression.ebnf` and `spec/evaluation.md`).
//!
//! # Public API
//!
//! ```rust
//! use marqdo_expression::{parse_expression, parse_statement};
//!
//! let stmt = parse_statement("结果 = 阶乘(n=3)").unwrap();
//! assert_eq!(stmt.span().start, 0);
//!
//! let expr = parse_expression("2 ** -1").unwrap();
//! assert_eq!(expr.span().start, 0);
//! ```
//!
//! The contractual surface is:
//!
//! * [`parse_expression`] — parse a complete expression payload.
//! * [`parse_statement`] — parse a complete statement payload (assignment or
//! expression).
//! * [`parse_arguments`] — parse a leading `( ... )` argument list, used for
//! Markdown-link calls such as `> [计算面积](./math.mq.md#计算面积)(r=3)`.
//! * the [`ast`] module (re-exported at the crate root).
//! * [`ParseError`] — structured `expr.parse` diagnostics with byte spans.
//!
//! Everything else, including [`lexer`], is an implementation detail that is
//! exposed only to make diagnostics and tests easier.
//!
//! # Grammar summary
//!
//! The grammar is frozen in `spec/expression.ebnf`. In order of decreasing
//! precedence: prefix `not` / `-` / `+`, `**` (right associative), postfix
//! calls `.name` / `[index]`, `* / // %`, `+ -`, a single non-associative
//! comparison, `and`, `or`. Assignment is only available at statement start
//! and only for a bare identifier target.
//!
//! Notable lexical rules:
//!
//! * identifiers use Unicode XID (`XID_Start XID_Continue*`), so `阶乘` and
//! `café` are identifiers;
//! * `true`, `false`, `None`, `and`, `or`, `not` are reserved keywords;
//! * integers are arbitrary-precision decimal literals;
//! * floats require digits on both sides of `.` (`.5` and `1.` are invalid)
//! and may carry an exponent;
//! * text literals are double quoted and support `\\`, `\"`, `\n`, `\t`, `\r`
//! and `\u{...}` escapes;
//! * comments are not supported;
//! * nesting deeper than [`MAX_NESTING_DEPTH`] is a structured parse error, so
//! hostile payloads cannot overflow the parser stack.
//!
//! # Spans and errors
//!
//! Spans are half-open byte offsets relative to the payload string. Errors use
//! the stable diagnostic code [`EXPR_PARSE`] (`"expr.parse"`) and follow the
//! same shape used elsewhere in the toolchain (code + message + span).
//!
//! # Example payloads
//!
//! ```rust
//! use marqdo_expression::parse_statement;
//!
//! let stmt = parse_statement("打印(\"面积 = \" + 文本(area))").unwrap();
//! assert!(matches!(
//! stmt.kind,
//! marqdo_expression::StatementKind::Expr(_)
//! ));
//! ```
//!
//! The parser never panics on malformed input: every failure is a
//! [`ParseError`].
#![forbid(unsafe_code)]
pub mod ast;
pub mod lexer;
mod parser;
use std::fmt;
use serde::Serialize;
pub use ast::{Arg, BinaryOp, Expr, Ident, RecordEntry, Statement, StatementKind, UnaryOp};
pub use marqdo_span::Span;
pub use parser::MAX_NESTING_DEPTH;
/// Stable diagnostic code for every expression parse failure.
pub const EXPR_PARSE: &str = "expr.parse";
/// A structured expression/statement parse failure.
///
/// `code` is always [`EXPR_PARSE`]. `span` is a byte range into the payload
/// string and may be empty (for example at end of input).
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ParseError {
/// Stable diagnostic code, always `"expr.parse"`.
pub code: &'static str,
/// Human readable explanation of the failure.
pub message: String,
/// Byte range in the payload where the failure was detected.
pub span: Span,
}
impl ParseError {
/// Create a parse error with the fixed `expr.parse` code.
pub fn new(span: Span, message: impl Into<String>) -> Self {
Self {
code: EXPR_PARSE,
message: message.into(),
span,
}
}
/// Stable diagnostic code (always `"expr.parse"`).
pub fn code(&self) -> &'static str {
self.code
}
}
impl fmt::Display for ParseError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}: {} at {}", self.code, self.message, self.span)
}
}
impl std::error::Error for ParseError {}
/// Parse a complete expression payload.
///
/// The whole input must be consumed; empty input, trailing tokens, unsupported
/// constructs and assignments are [`ParseError`]s.
pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
parser::parse_expression(source)
}
/// Parse a complete statement payload.
///
/// A statement is either `<bare identifier> = <expression>` or a bare
/// expression (typically a call). The whole input must be consumed.
pub fn parse_statement(source: &str) -> Result<Statement, ParseError> {
parser::parse_statement(source)
}
/// Parse a leading parenthesised argument list.
///
/// `source` must start with `(` and contain exactly one argument list, e.g.
/// `(r=3, precision=2)`. Argument rules are identical to call arguments:
/// positional arguments precede named arguments, named arguments are unique,
/// and a single trailing comma is allowed. Trailing tokens are an error.
///
/// This exists so the frontend can parse Markdown-link calls of the form
/// `> [计算面积](./math.mq.md#计算面积)(r=3)` without re-lexing the payload.
pub fn parse_arguments(source: &str) -> Result<Vec<Arg>, ParseError> {
parser::parse_arguments(source)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn public_api_parses_expression_and_statement() {
assert!(parse_expression("1 + 2 * 3").is_ok());
assert!(parse_statement("x = 1").is_ok());
assert!(parse_arguments("(r=3)").is_ok());
}
#[test]
fn empty_payloads_are_errors() {
for source in ["", " ", "\n\t"] {
let err = parse_expression(source).unwrap_err();
assert_eq!(err.code, EXPR_PARSE);
assert_eq!(err.code(), "expr.parse");
assert!(err.message.contains("expression"));
let err = parse_statement(source).unwrap_err();
assert_eq!(err.code, EXPR_PARSE);
assert!(err.message.contains("statement"));
}
}
#[test]
fn parse_error_displays_code_and_span_and_implements_error() {
let err = parse_expression("1 2").unwrap_err();
let rendered = err.to_string();
assert!(rendered.starts_with("expr.parse: "), "{rendered}");
assert!(rendered.contains("2..3"), "{rendered}");
assert!(rendered.contains("unexpected"), "{rendered}");
let dyn_error: &dyn std::error::Error = &err;
assert!(dyn_error.source().is_none());
let custom = ParseError::new(Span::new(4, 9), "boom");
assert_eq!(custom.code, "expr.parse");
assert_eq!(custom.message, "boom");
assert_eq!(custom.span, Span::new(4, 9));
}
#[test]
fn ast_types_are_reachable_from_the_crate_root() {
let expr: Expr = parse_expression("f(1)").unwrap();
let statement: Statement = parse_statement("f(1)").unwrap();
let _: StatementKind = statement.kind;
let _: BinaryOp = BinaryOp::Add;
let _: UnaryOp = UnaryOp::Not;
let _: Arg = Arg {
name: None,
value: expr,
span: Span::default(),
};
let _: RecordEntry = RecordEntry {
key: "k".to_owned(),
key_span: Span::default(),
value: Expr::None,
};
let _: Ident = Ident {
name: "x".to_owned(),
span: Span::default(),
};
}
}
File diff suppressed because it is too large. Load diff