feat(expression): strict expression and statement grammar for executable payloads
Implement the independent expression parser frozen in spec/expression.ebnf:
- Unicode XID identifiers (Chinese names supported), reserved keywords
true/false/None/and/or/not.
- Arbitrary-precision Int literals, binary64 Float literals (digits required on
both sides, overflow rejected), double-quoted Text with \u{...} escapes.
- List and record literals, calls with positional-then-named arguments and
trailing commas, member access, indexing, chained postfix operators.
- Precedence exactly as specified: right-associative ** with unary minus
binding outside, multiplicative/additive, single non-associative comparison,
short-circuit and/or.
- Assignment only as `identifier = expression` at statement start; no comments,
no second Markdown parser, max nesting depth 128 with structured errors.
- parse_expression/parse_statement/parse_arguments never panic; ParseError has
stable code expr.parse, payload-relative byte spans and Display/Error.
50 unit tests + 2 doctests, cargo fmt/clippy clean.
This commit is contained in:
1 parent
55a2b2d82d
commit
9897ecc37f
6 files changed
+2649
-5
No files matched your search
Generated
+86
-4
@@ -58,6 +58,12 @@ version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
@@ -186,6 +192,16 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.5.0"
|
||||
@@ -225,6 +241,23 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
@@ -281,6 +314,12 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "marqdo-cli"
|
||||
version = "0.1.0"
|
||||
@@ -309,15 +348,16 @@ name = "marqdo-expression"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"marqdo-span",
|
||||
"num-bigint 0.5.1",
|
||||
"serde",
|
||||
"thiserror",
|
||||
"serde_json",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "marqdo-foreign"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"marqdo-span",
|
||||
"marqdo-value",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
@@ -334,7 +374,8 @@ dependencies = [
|
||||
"marqdo-value",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"sha2",
|
||||
"unicode-ident",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
@@ -342,6 +383,7 @@ dependencies = [
|
||||
name = "marqdo-host"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"marqdo-foreign",
|
||||
"marqdo-math",
|
||||
"marqdo-plugin-abi",
|
||||
@@ -350,6 +392,7 @@ dependencies = [
|
||||
"marqdo-value",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
@@ -359,7 +402,6 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"marqdo-expression",
|
||||
"marqdo-span",
|
||||
"marqdo-value",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
@@ -406,12 +448,14 @@ dependencies = [
|
||||
name = "marqdo-plugin-abi"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"hex",
|
||||
"libloading",
|
||||
"marqdo-span",
|
||||
"marqdo-value",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"sha2",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
@@ -530,6 +574,12 @@ dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
@@ -593,12 +643,31 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.23"
|
||||
@@ -710,6 +779,19 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.21"
|
||||
|
||||
@@ -11,5 +11,8 @@ authors.workspace = true
|
||||
[dependencies]
|
||||
marqdo-span = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
num-bigint = { workspace = true }
|
||||
unicode-ident = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,292 @@
|
||||
//! Abstract syntax tree for the Marqdo expression grammar v1.
|
||||
//!
|
||||
//! All spans are byte offsets relative to the payload string that was handed to
|
||||
//! [`crate::parse_expression`], [`crate::parse_statement`] or
|
||||
//! [`crate::parse_arguments`]. The frontend re-bases them onto the Markdown
|
||||
//! source through the text/code segments that made up the payload.
|
||||
//!
|
||||
//! # Span coverage
|
||||
//!
|
||||
//! Every node that carries a stored span exposes it through [`Expr::span`] /
|
||||
//! [`Statement::span`]. The frozen v1 AST shape stores a span only on the
|
||||
//! structural variants (`Unary`, `Binary`, `Call`, `Member`, `Index`), on
|
||||
//! [`Ident`], [`Arg`] and [`RecordEntry`]. Scalar and container literals
|
||||
//! (`Int`, `Float`, `Text`, `Bool`, `None`, `List`, `Record`) intentionally
|
||||
//! have no span field: [`Expr::span`] returns the empty span `0..0` for them.
|
||||
//! Consumers that need a location for a literal should use the nearest
|
||||
//! enclosing structural node or the enclosing [`Statement::span`].
|
||||
|
||||
use marqdo_span::Span;
|
||||
use num_bigint::BigInt;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// A bare identifier together with its source span.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct Ident {
|
||||
/// Identifier text as written (Unicode XID, decoded verbatim).
|
||||
pub name: String,
|
||||
/// Span of the identifier token.
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
/// One argument of a call, optionally named.
|
||||
///
|
||||
/// ```text
|
||||
/// f(1, r=3)
|
||||
/// ^ ^ named argument, `name == Some("r")`
|
||||
/// ```
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Arg {
|
||||
/// `Some` when the argument was written as `name = value`.
|
||||
pub name: Option<String>,
|
||||
/// Argument value expression.
|
||||
pub value: Expr,
|
||||
/// Span of the whole argument (name through value).
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
/// One key/value entry of a record literal.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct RecordEntry {
|
||||
/// Entry key (identifier text or decoded text-literal contents).
|
||||
pub key: String,
|
||||
/// Span of the key token (including the quotes for a text-literal key).
|
||||
pub key_span: Span,
|
||||
/// Entry value expression.
|
||||
pub value: Expr,
|
||||
}
|
||||
|
||||
/// Binary operators, listed from loosest to tightest binding.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum BinaryOp {
|
||||
/// Short-circuiting logical or (`or`).
|
||||
Or,
|
||||
/// Short-circuiting logical and (`and`).
|
||||
And,
|
||||
/// `==`
|
||||
Eq,
|
||||
/// `!=`
|
||||
Ne,
|
||||
/// `<`
|
||||
Lt,
|
||||
/// `<=`
|
||||
Le,
|
||||
/// `>`
|
||||
Gt,
|
||||
/// `>=`
|
||||
Ge,
|
||||
/// `+`
|
||||
Add,
|
||||
/// `-`
|
||||
Sub,
|
||||
/// `*`
|
||||
Mul,
|
||||
/// `/` (always yields `Float` at runtime).
|
||||
Div,
|
||||
/// `//` (floor division, `Int` only).
|
||||
FloorDiv,
|
||||
/// `%` (remainder, `Int` only).
|
||||
Mod,
|
||||
/// `**` (right associative).
|
||||
Pow,
|
||||
}
|
||||
|
||||
/// Prefix operators.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum UnaryOp {
|
||||
/// `not`
|
||||
Not,
|
||||
/// unary `-`
|
||||
Neg,
|
||||
/// unary `+`
|
||||
Pos,
|
||||
}
|
||||
|
||||
/// An expression node.
|
||||
///
|
||||
/// The variant set and payloads are the frozen public contract of this crate.
|
||||
/// See the module docs for the span policy of literal variants.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum Expr {
|
||||
/// Arbitrary-precision decimal integer literal.
|
||||
Int(BigInt),
|
||||
/// IEEE-754 binary64 float literal.
|
||||
Float(f64),
|
||||
/// Decoded text literal (escapes already resolved).
|
||||
Text(String),
|
||||
/// `true` / `false`.
|
||||
Bool(bool),
|
||||
/// `None`.
|
||||
None,
|
||||
/// A variable, parameter or function name reference.
|
||||
Ident(Ident),
|
||||
/// `[a, b, c]`.
|
||||
List(Vec<Expr>),
|
||||
/// `{key: value}`.
|
||||
Record(Vec<RecordEntry>),
|
||||
/// Prefix operator application.
|
||||
Unary {
|
||||
/// Applied prefix operator.
|
||||
op: UnaryOp,
|
||||
/// Operand.
|
||||
operand: Box<Expr>,
|
||||
/// Span from the operator through the end of the operand.
|
||||
span: Span,
|
||||
},
|
||||
/// Infix operator application.
|
||||
Binary {
|
||||
/// Applied infix operator.
|
||||
op: BinaryOp,
|
||||
/// Left operand.
|
||||
left: Box<Expr>,
|
||||
/// Right operand.
|
||||
right: Box<Expr>,
|
||||
/// Span of the whole infix expression.
|
||||
span: Span,
|
||||
},
|
||||
/// Call expression.
|
||||
Call {
|
||||
/// Callee expression.
|
||||
callee: Box<Expr>,
|
||||
/// Arguments in source order.
|
||||
args: Vec<Arg>,
|
||||
/// Span from the callee start through the closing `)`.
|
||||
span: Span,
|
||||
},
|
||||
/// Member access (`base.name`).
|
||||
Member {
|
||||
/// Object being projected.
|
||||
base: Box<Expr>,
|
||||
/// Member name (never a keyword).
|
||||
name: String,
|
||||
/// Span from the base start through the member name.
|
||||
span: Span,
|
||||
},
|
||||
/// Index access (`base[index]`).
|
||||
Index {
|
||||
/// Object being indexed.
|
||||
base: Box<Expr>,
|
||||
/// Index expression.
|
||||
index: Box<Expr>,
|
||||
/// Span from the base start through the closing `]`.
|
||||
span: Span,
|
||||
},
|
||||
}
|
||||
|
||||
impl Expr {
|
||||
/// Best known source span of this expression.
|
||||
///
|
||||
/// Returns the empty span `0..0` for variants that do not store a span
|
||||
/// (`Int`, `Float`, `Text`, `Bool`, `None`, `List`, `Record`); see the
|
||||
/// module documentation for the rationale.
|
||||
pub fn span(&self) -> Span {
|
||||
match self {
|
||||
Expr::Ident(ident) => ident.span,
|
||||
Expr::Unary { span, .. }
|
||||
| Expr::Binary { span, .. }
|
||||
| Expr::Call { span, .. }
|
||||
| Expr::Member { span, .. }
|
||||
| Expr::Index { span, .. } => *span,
|
||||
Expr::Int(_) | Expr::Float(_) | Expr::Text(_) | Expr::Bool(_) | Expr::None => {
|
||||
Span::default()
|
||||
}
|
||||
Expr::List(_) | Expr::Record(_) => Span::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The payload of a statement.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum StatementKind {
|
||||
/// `<bare identifier> = <expression>`; the only assignment form in v1.
|
||||
Assign {
|
||||
/// Assigned identifier.
|
||||
name: String,
|
||||
/// Span of the identifier token (not the whole assignment).
|
||||
name_span: Span,
|
||||
/// Right-hand side expression.
|
||||
value: Expr,
|
||||
},
|
||||
/// A bare expression statement (typically a call).
|
||||
Expr(Expr),
|
||||
}
|
||||
|
||||
/// A complete parsed statement: either an assignment or an expression.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Statement {
|
||||
/// Statement payload.
|
||||
pub kind: StatementKind,
|
||||
/// Span of the whole statement, from the first token through the last.
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
impl Statement {
|
||||
/// Span of the whole statement, from the first token through the last.
|
||||
pub fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::parse_expression;
|
||||
|
||||
#[test]
|
||||
fn span_accessor_covers_structural_nodes() {
|
||||
let expr = parse_expression("f(1, r=2)").unwrap();
|
||||
assert_eq!(expr.span(), Span::new(0, 9));
|
||||
let Expr::Call { callee, args, .. } = &expr else {
|
||||
panic!("expected call");
|
||||
};
|
||||
assert_eq!(callee.span(), Span::new(0, 1));
|
||||
assert_eq!(args[0].span, Span::new(2, 3));
|
||||
assert_eq!(args[1].span, Span::new(5, 8));
|
||||
assert_eq!(args[1].name.as_deref(), Some("r"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spanless_variants_return_the_empty_span() {
|
||||
for expr in [
|
||||
Expr::Int(BigInt::from(1)),
|
||||
Expr::Float(1.0),
|
||||
Expr::Text("t".to_owned()),
|
||||
Expr::Bool(true),
|
||||
Expr::None,
|
||||
Expr::List(Vec::new()),
|
||||
Expr::Record(Vec::new()),
|
||||
] {
|
||||
assert_eq!(expr.span(), Span::default());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ast_types_serde_round_trip() {
|
||||
let expr = parse_expression(
|
||||
r#"f(1, [true, None], {键: 9.5e1}, 2 ** -1, a.b[0] == not false, 名="值")"#,
|
||||
)
|
||||
.unwrap();
|
||||
let json = serde_json::to_string(&expr).unwrap();
|
||||
let decoded: Expr = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(decoded, expr);
|
||||
|
||||
let stmt = crate::parse_statement("结果 = 打印(阶乘(n=3))").unwrap();
|
||||
let json = serde_json::to_string(&stmt).unwrap();
|
||||
let decoded: Statement = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(decoded, stmt);
|
||||
assert_eq!(decoded.span(), stmt.span());
|
||||
|
||||
let args = crate::parse_arguments("(r=3, precision=2)").unwrap();
|
||||
let json = serde_json::to_string(&args).unwrap();
|
||||
let decoded: Vec<Arg> = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(decoded, args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn span_serde_round_trips_inside_ast() {
|
||||
let expr = parse_expression("a.b").unwrap();
|
||||
let json = serde_json::to_string(&expr).unwrap();
|
||||
assert!(json.contains("\"start\":0"));
|
||||
assert!(json.contains("\"end\":3"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,608 @@
|
||||
//! Hand written lexer for the Marqdo expression grammar v1.
|
||||
//!
|
||||
//! This module is an implementation detail that is exposed only to make
|
||||
//! diagnostics and tests easier; the contractual API of the crate lives in
|
||||
//! [`crate::parse_expression`], [`crate::parse_statement`] and
|
||||
//! [`crate::parse_arguments`] plus the [`crate::ast`] types.
|
||||
//!
|
||||
//! Lexical rules (see `spec/expression.ebnf`):
|
||||
//!
|
||||
//! * Unicode whitespace separates tokens (newlines included).
|
||||
//! * Identifiers use UAX #31 `XID_Start` / `XID_Continue`.
|
||||
//! * Integers are arbitrary-precision decimal digit runs.
|
||||
//! * Floats require digits on both sides of `.` and may carry an exponent.
|
||||
//! * Text literals are double quoted with the escapes `\\`, `\"`, `\n`, `\t`,
|
||||
//! `\r` and `\u{...}`.
|
||||
//! * There are no comments in v1.
|
||||
|
||||
use crate::ParseError;
|
||||
use marqdo_span::Span;
|
||||
|
||||
/// A lexical token kind.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum TokenKind {
|
||||
/// Decimal integer literal, kept as raw digits (arbitrary precision).
|
||||
Int(String),
|
||||
/// Float literal, kept as raw source text (parsed to `f64` by the parser).
|
||||
Float(String),
|
||||
/// Text literal whose escapes have already been decoded.
|
||||
Text(String),
|
||||
/// Identifier text (never a keyword).
|
||||
Ident(String),
|
||||
/// `true`
|
||||
True,
|
||||
/// `false`
|
||||
False,
|
||||
/// `None`
|
||||
NoneKw,
|
||||
/// `and`
|
||||
And,
|
||||
/// `or`
|
||||
Or,
|
||||
/// `not`
|
||||
Not,
|
||||
/// `(`
|
||||
LParen,
|
||||
/// `)`
|
||||
RParen,
|
||||
/// `[`
|
||||
LBracket,
|
||||
/// `]`
|
||||
RBracket,
|
||||
/// `{`
|
||||
LBrace,
|
||||
/// `}`
|
||||
RBrace,
|
||||
/// `,`
|
||||
Comma,
|
||||
/// `:`
|
||||
Colon,
|
||||
/// `.`
|
||||
Dot,
|
||||
/// `=` (assignment)
|
||||
Assign,
|
||||
/// `==`
|
||||
EqEq,
|
||||
/// `!=`
|
||||
Ne,
|
||||
/// `<`
|
||||
Lt,
|
||||
/// `<=`
|
||||
Le,
|
||||
/// `>`
|
||||
Gt,
|
||||
/// `>=`
|
||||
Ge,
|
||||
/// `+`
|
||||
Plus,
|
||||
/// `-`
|
||||
Minus,
|
||||
/// `*`
|
||||
Star,
|
||||
/// `**`
|
||||
StarStar,
|
||||
/// `/`
|
||||
Slash,
|
||||
/// `//`
|
||||
SlashSlash,
|
||||
/// `%`
|
||||
Percent,
|
||||
/// End of input.
|
||||
Eof,
|
||||
}
|
||||
|
||||
/// A token with its byte span in the payload.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Token {
|
||||
/// Token kind.
|
||||
pub kind: TokenKind,
|
||||
/// Byte span of the token.
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
/// Tokenize a payload string.
|
||||
///
|
||||
/// The returned vector always ends with exactly one [`TokenKind::Eof`] token
|
||||
/// whose span is the empty span at the end of the input.
|
||||
pub fn tokenize(source: &str) -> Result<Vec<Token>, ParseError> {
|
||||
Lexer::new(source).tokenize()
|
||||
}
|
||||
|
||||
struct Lexer<'a> {
|
||||
source: &'a str,
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl<'a> Lexer<'a> {
|
||||
fn new(source: &'a str) -> Self {
|
||||
Self { source, pos: 0 }
|
||||
}
|
||||
|
||||
fn peek(&self) -> Option<char> {
|
||||
self.source[self.pos..].chars().next()
|
||||
}
|
||||
|
||||
fn bump(&mut self) -> Option<char> {
|
||||
let c = self.peek()?;
|
||||
self.pos += c.len_utf8();
|
||||
Some(c)
|
||||
}
|
||||
|
||||
fn error(&self, start: usize, message: impl Into<String>) -> ParseError {
|
||||
ParseError::new(Span::new(start, self.pos), message)
|
||||
}
|
||||
|
||||
fn tokenize(mut self) -> Result<Vec<Token>, ParseError> {
|
||||
let mut tokens = Vec::new();
|
||||
loop {
|
||||
self.skip_whitespace();
|
||||
let start = self.pos;
|
||||
let Some(c) = self.peek() else {
|
||||
tokens.push(Token {
|
||||
kind: TokenKind::Eof,
|
||||
span: Span::empty(self.source.len()),
|
||||
});
|
||||
return Ok(tokens);
|
||||
};
|
||||
let kind = match c {
|
||||
'"' => TokenKind::Text(self.scan_text()?),
|
||||
'0'..='9' => self.scan_number()?,
|
||||
_ if unicode_ident::is_xid_start(c) => self.scan_identifier(),
|
||||
'(' => {
|
||||
self.bump();
|
||||
TokenKind::LParen
|
||||
}
|
||||
')' => {
|
||||
self.bump();
|
||||
TokenKind::RParen
|
||||
}
|
||||
'[' => {
|
||||
self.bump();
|
||||
TokenKind::LBracket
|
||||
}
|
||||
']' => {
|
||||
self.bump();
|
||||
TokenKind::RBracket
|
||||
}
|
||||
'{' => {
|
||||
self.bump();
|
||||
TokenKind::LBrace
|
||||
}
|
||||
'}' => {
|
||||
self.bump();
|
||||
TokenKind::RBrace
|
||||
}
|
||||
',' => {
|
||||
self.bump();
|
||||
TokenKind::Comma
|
||||
}
|
||||
':' => {
|
||||
self.bump();
|
||||
TokenKind::Colon
|
||||
}
|
||||
'.' => {
|
||||
self.bump();
|
||||
if matches!(self.peek(), Some(next) if next.is_ascii_digit()) {
|
||||
return Err(self.error(
|
||||
start,
|
||||
"invalid float literal: a decimal point must be preceded by a digit",
|
||||
));
|
||||
}
|
||||
TokenKind::Dot
|
||||
}
|
||||
'=' => {
|
||||
self.bump();
|
||||
if matches!(self.peek(), Some('=')) {
|
||||
self.bump();
|
||||
TokenKind::EqEq
|
||||
} else {
|
||||
TokenKind::Assign
|
||||
}
|
||||
}
|
||||
'!' => {
|
||||
self.bump();
|
||||
if matches!(self.peek(), Some('=')) {
|
||||
self.bump();
|
||||
TokenKind::Ne
|
||||
} else {
|
||||
return Err(self.error(
|
||||
start,
|
||||
"unexpected character `!`; logical negation is written `not`",
|
||||
));
|
||||
}
|
||||
}
|
||||
'<' => {
|
||||
self.bump();
|
||||
if matches!(self.peek(), Some('=')) {
|
||||
self.bump();
|
||||
TokenKind::Le
|
||||
} else {
|
||||
TokenKind::Lt
|
||||
}
|
||||
}
|
||||
'>' => {
|
||||
self.bump();
|
||||
if matches!(self.peek(), Some('=')) {
|
||||
self.bump();
|
||||
TokenKind::Ge
|
||||
} else {
|
||||
TokenKind::Gt
|
||||
}
|
||||
}
|
||||
'+' => {
|
||||
self.bump();
|
||||
TokenKind::Plus
|
||||
}
|
||||
'-' => {
|
||||
self.bump();
|
||||
TokenKind::Minus
|
||||
}
|
||||
'*' => {
|
||||
self.bump();
|
||||
if matches!(self.peek(), Some('*')) {
|
||||
self.bump();
|
||||
TokenKind::StarStar
|
||||
} else {
|
||||
TokenKind::Star
|
||||
}
|
||||
}
|
||||
'/' => {
|
||||
self.bump();
|
||||
if matches!(self.peek(), Some('/')) {
|
||||
self.bump();
|
||||
TokenKind::SlashSlash
|
||||
} else {
|
||||
TokenKind::Slash
|
||||
}
|
||||
}
|
||||
'%' => {
|
||||
self.bump();
|
||||
TokenKind::Percent
|
||||
}
|
||||
'#' => {
|
||||
self.bump();
|
||||
return Err(self.error(
|
||||
start,
|
||||
"unexpected character `#`; comments are not supported in v1",
|
||||
));
|
||||
}
|
||||
other => {
|
||||
self.bump();
|
||||
return Err(self.error(start, format!("unexpected character {:?}", other)));
|
||||
}
|
||||
};
|
||||
tokens.push(Token {
|
||||
kind,
|
||||
span: Span::new(start, self.pos),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn skip_whitespace(&mut self) {
|
||||
while let Some(c) = self.peek() {
|
||||
if c.is_whitespace() {
|
||||
self.bump();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_identifier(&mut self) -> TokenKind {
|
||||
let start = self.pos;
|
||||
// The caller guarantees `is_xid_start` for the first character.
|
||||
self.bump();
|
||||
while let Some(c) = self.peek() {
|
||||
if unicode_ident::is_xid_continue(c) {
|
||||
self.bump();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let text = &self.source[start..self.pos];
|
||||
match text {
|
||||
"true" => TokenKind::True,
|
||||
"false" => TokenKind::False,
|
||||
"None" => TokenKind::NoneKw,
|
||||
"and" => TokenKind::And,
|
||||
"or" => TokenKind::Or,
|
||||
"not" => TokenKind::Not,
|
||||
_ => TokenKind::Ident(text.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_number(&mut self) -> Result<TokenKind, ParseError> {
|
||||
let start = self.pos;
|
||||
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
||||
self.bump();
|
||||
}
|
||||
let mut is_float = false;
|
||||
if matches!(self.peek(), Some('.')) {
|
||||
self.bump();
|
||||
if !matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
||||
return Err(self.error(
|
||||
start,
|
||||
"invalid float literal: expected at least one digit after `.`",
|
||||
));
|
||||
}
|
||||
is_float = true;
|
||||
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
||||
self.bump();
|
||||
}
|
||||
}
|
||||
if matches!(self.peek(), Some('e') | Some('E')) {
|
||||
if !is_float {
|
||||
return Err(self.error(
|
||||
start,
|
||||
"invalid float literal: an exponent requires digits on both sides of `.`",
|
||||
));
|
||||
}
|
||||
self.bump();
|
||||
if matches!(self.peek(), Some('+') | Some('-')) {
|
||||
self.bump();
|
||||
}
|
||||
if !matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
||||
return Err(self.error(
|
||||
start,
|
||||
"invalid float literal: expected at least one exponent digit",
|
||||
));
|
||||
}
|
||||
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
||||
self.bump();
|
||||
}
|
||||
}
|
||||
if let Some(c) = self.peek() {
|
||||
if c == '.' || unicode_ident::is_xid_continue(c) {
|
||||
return Err(self.error(
|
||||
start,
|
||||
format!("invalid numeric literal: unexpected `{c}` immediately after a number"),
|
||||
));
|
||||
}
|
||||
}
|
||||
let raw = self.source[start..self.pos].to_owned();
|
||||
Ok(if is_float {
|
||||
TokenKind::Float(raw)
|
||||
} else {
|
||||
TokenKind::Int(raw)
|
||||
})
|
||||
}
|
||||
|
||||
fn scan_text(&mut self) -> Result<String, ParseError> {
|
||||
let start = self.pos;
|
||||
self.bump(); // opening quote
|
||||
let mut out = String::new();
|
||||
loop {
|
||||
let Some(c) = self.bump() else {
|
||||
return Err(ParseError::new(
|
||||
Span::new(start, self.source.len()),
|
||||
"unterminated text literal: expected a closing `\"`",
|
||||
));
|
||||
};
|
||||
match c {
|
||||
'"' => return Ok(out),
|
||||
'\\' => {
|
||||
let escape_start = self.pos - 1;
|
||||
match self.bump() {
|
||||
Some('"') => out.push('"'),
|
||||
Some('\\') => out.push('\\'),
|
||||
Some('n') => out.push('\n'),
|
||||
Some('t') => out.push('\t'),
|
||||
Some('r') => out.push('\r'),
|
||||
Some('u') => self.scan_unicode_escape(escape_start, &mut out)?,
|
||||
Some(other) => {
|
||||
return Err(self.error(
|
||||
escape_start,
|
||||
format!(
|
||||
"invalid escape sequence `\\{other}`; supported escapes are \
|
||||
`\\\\`, `\\\"`, `\\n`, `\\t`, `\\r` and `\\u{{...}}`"
|
||||
),
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(ParseError::new(
|
||||
Span::new(start, self.source.len()),
|
||||
"unterminated text literal: expected a closing `\"`",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
other => out.push(other),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_unicode_escape(
|
||||
&mut self,
|
||||
escape_start: usize,
|
||||
out: &mut String,
|
||||
) -> Result<(), ParseError> {
|
||||
if !matches!(self.peek(), Some('{')) {
|
||||
return Err(self.error(
|
||||
escape_start,
|
||||
"invalid unicode escape: expected `{` after `\\u`",
|
||||
));
|
||||
}
|
||||
self.bump();
|
||||
let digits_start = self.pos;
|
||||
let mut value: u32 = 0;
|
||||
let mut overflow = false;
|
||||
while let Some(c) = self.peek() {
|
||||
let Some(digit) = c.to_digit(16) else {
|
||||
break;
|
||||
};
|
||||
self.bump();
|
||||
value = match value.checked_mul(16).and_then(|v| v.checked_add(digit)) {
|
||||
Some(v) => v,
|
||||
None => {
|
||||
overflow = true;
|
||||
0
|
||||
}
|
||||
};
|
||||
}
|
||||
if self.pos == digits_start {
|
||||
return Err(self.error(
|
||||
escape_start,
|
||||
"invalid unicode escape: expected at least one hexadecimal digit",
|
||||
));
|
||||
}
|
||||
if !matches!(self.peek(), Some('}')) {
|
||||
return Err(self.error(
|
||||
escape_start,
|
||||
"invalid unicode escape: expected a closing `}`",
|
||||
));
|
||||
}
|
||||
self.bump();
|
||||
if overflow {
|
||||
return Err(self.error(
|
||||
escape_start,
|
||||
"invalid unicode escape: code point is out of range",
|
||||
));
|
||||
}
|
||||
match char::from_u32(value) {
|
||||
Some(c) => {
|
||||
out.push(c);
|
||||
Ok(())
|
||||
}
|
||||
None => Err(self.error(
|
||||
escape_start,
|
||||
format!("invalid unicode escape: U+{value:04X} is not a Unicode scalar value"),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn kinds(source: &str) -> Vec<TokenKind> {
|
||||
tokenize(source)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|t| t.kind)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenizes_operators_and_punctuation() {
|
||||
assert_eq!(
|
||||
kinds("== != < <= > >= = + - * ** / // % ( ) [ ] { } , : ."),
|
||||
vec![
|
||||
TokenKind::EqEq,
|
||||
TokenKind::Ne,
|
||||
TokenKind::Lt,
|
||||
TokenKind::Le,
|
||||
TokenKind::Gt,
|
||||
TokenKind::Ge,
|
||||
TokenKind::Assign,
|
||||
TokenKind::Plus,
|
||||
TokenKind::Minus,
|
||||
TokenKind::Star,
|
||||
TokenKind::StarStar,
|
||||
TokenKind::Slash,
|
||||
TokenKind::SlashSlash,
|
||||
TokenKind::Percent,
|
||||
TokenKind::LParen,
|
||||
TokenKind::RParen,
|
||||
TokenKind::LBracket,
|
||||
TokenKind::RBracket,
|
||||
TokenKind::LBrace,
|
||||
TokenKind::RBrace,
|
||||
TokenKind::Comma,
|
||||
TokenKind::Colon,
|
||||
TokenKind::Dot,
|
||||
TokenKind::Eof,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenizes_keywords_separately_from_identifiers() {
|
||||
assert_eq!(
|
||||
kinds("true false None and or not truex None_ 真实"),
|
||||
vec![
|
||||
TokenKind::True,
|
||||
TokenKind::False,
|
||||
TokenKind::NoneKw,
|
||||
TokenKind::And,
|
||||
TokenKind::Or,
|
||||
TokenKind::Not,
|
||||
TokenKind::Ident("truex".into()),
|
||||
TokenKind::Ident("None_".into()),
|
||||
TokenKind::Ident("真实".into()),
|
||||
TokenKind::Eof,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_remembers_multibyte_spans() {
|
||||
let tokens = tokenize("张三 + 1").unwrap();
|
||||
assert_eq!(tokens[0].kind, TokenKind::Ident("张三".into()));
|
||||
assert_eq!(tokens[0].span, Span::new(0, 6));
|
||||
assert_eq!(tokens[1].span, Span::new(7, 8));
|
||||
assert_eq!(tokens[2].kind, TokenKind::Int("1".into()));
|
||||
assert_eq!(tokens[2].span, Span::new(9, 10));
|
||||
assert_eq!(tokens[3].span, Span::empty(10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_whitespace_is_skipped() {
|
||||
assert_eq!(
|
||||
kinds("1\u{00a0}+\u{2003}2\n"),
|
||||
vec![
|
||||
TokenKind::Int("1".into()),
|
||||
TokenKind::Plus,
|
||||
TokenKind::Int("2".into()),
|
||||
TokenKind::Eof
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn number_lexing_rejects_invalid_forms() {
|
||||
for bad in [".5", "1.", "1e5", "1.2.3", "1.5e", "1.5e+", "1a", "1_2"] {
|
||||
let err = tokenize(bad).unwrap_err();
|
||||
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
||||
}
|
||||
assert_eq!(
|
||||
kinds("1.5e+3 0.0 007 1.25E-2"),
|
||||
vec![
|
||||
TokenKind::Float("1.5e+3".into()),
|
||||
TokenKind::Float("0.0".into()),
|
||||
TokenKind::Int("007".into()),
|
||||
TokenKind::Float("1.25E-2".into()),
|
||||
TokenKind::Eof
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_escapes_decode() {
|
||||
let tokens = tokenize(r#""a\\b\"c\n\t\r\u{4e2d}""#).unwrap();
|
||||
assert_eq!(tokens[0].kind, TokenKind::Text("a\\b\"c\n\t\r中".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_text_lexing_reports_errors() {
|
||||
for bad in [
|
||||
r#""unterminated"#,
|
||||
r#""bad \q escape""#,
|
||||
r#""bad \u{} escape""#,
|
||||
r#""bad \u{D800} escape""#,
|
||||
r#""bad \u{110000} escape""#,
|
||||
r#""bad \u12 escape""#,
|
||||
] {
|
||||
let err = tokenize(bad).unwrap_err();
|
||||
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comments_are_not_supported() {
|
||||
let err = tokenize("# comment").unwrap_err();
|
||||
assert!(err.message.contains("comments"));
|
||||
let err = tokenize("1 # 2").unwrap_err();
|
||||
assert!(err.message.contains("comments"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
//! Strict, independent expression and statement parser for executable Markdown
|
||||
//! payloads.
|
||||
//!
|
||||
//! This crate is the only grammar for Marqdo executable payloads. It knows
|
||||
//! nothing about Markdown and nothing about the runtime: the Markdown AST
|
||||
//! decides *where* a payload exists, this crate decides *what the payload
|
||||
//! means* (see `spec/expression.ebnf` and `spec/evaluation.md`).
|
||||
//!
|
||||
//! # Public API
|
||||
//!
|
||||
//! ```rust
|
||||
//! use marqdo_expression::{parse_expression, parse_statement};
|
||||
//!
|
||||
//! let stmt = parse_statement("结果 = 阶乘(n=3)").unwrap();
|
||||
//! assert_eq!(stmt.span().start, 0);
|
||||
//!
|
||||
//! let expr = parse_expression("2 ** -1").unwrap();
|
||||
//! assert_eq!(expr.span().start, 0);
|
||||
//! ```
|
||||
//!
|
||||
//! The contractual surface is:
|
||||
//!
|
||||
//! * [`parse_expression`] — parse a complete expression payload.
|
||||
//! * [`parse_statement`] — parse a complete statement payload (assignment or
|
||||
//! expression).
|
||||
//! * [`parse_arguments`] — parse a leading `( ... )` argument list, used for
|
||||
//! Markdown-link calls such as `> [计算面积](./math.mq.md#计算面积)(r=3)`.
|
||||
//! * the [`ast`] module (re-exported at the crate root).
|
||||
//! * [`ParseError`] — structured `expr.parse` diagnostics with byte spans.
|
||||
//!
|
||||
//! Everything else, including [`lexer`], is an implementation detail that is
|
||||
//! exposed only to make diagnostics and tests easier.
|
||||
//!
|
||||
//! # Grammar summary
|
||||
//!
|
||||
//! The grammar is frozen in `spec/expression.ebnf`. In order of decreasing
|
||||
//! precedence: prefix `not` / `-` / `+`, `**` (right associative), postfix
|
||||
//! calls `.name` / `[index]`, `* / // %`, `+ -`, a single non-associative
|
||||
//! comparison, `and`, `or`. Assignment is only available at statement start
|
||||
//! and only for a bare identifier target.
|
||||
//!
|
||||
//! Notable lexical rules:
|
||||
//!
|
||||
//! * identifiers use Unicode XID (`XID_Start XID_Continue*`), so `阶乘` and
|
||||
//! `café` are identifiers;
|
||||
//! * `true`, `false`, `None`, `and`, `or`, `not` are reserved keywords;
|
||||
//! * integers are arbitrary-precision decimal literals;
|
||||
//! * floats require digits on both sides of `.` (`.5` and `1.` are invalid)
|
||||
//! and may carry an exponent;
|
||||
//! * text literals are double quoted and support `\\`, `\"`, `\n`, `\t`, `\r`
|
||||
//! and `\u{...}` escapes;
|
||||
//! * comments are not supported;
|
||||
//! * nesting deeper than [`MAX_NESTING_DEPTH`] is a structured parse error, so
|
||||
//! hostile payloads cannot overflow the parser stack.
|
||||
//!
|
||||
//! # Spans and errors
|
||||
//!
|
||||
//! Spans are half-open byte offsets relative to the payload string. Errors use
|
||||
//! the stable diagnostic code [`EXPR_PARSE`] (`"expr.parse"`) and follow the
|
||||
//! same shape used elsewhere in the toolchain (code + message + span).
|
||||
//!
|
||||
//! # Example payloads
|
||||
//!
|
||||
//! ```rust
|
||||
//! use marqdo_expression::parse_statement;
|
||||
//!
|
||||
//! let stmt = parse_statement("打印(\"面积 = \" + 文本(area))").unwrap();
|
||||
//! assert!(matches!(
|
||||
//! stmt.kind,
|
||||
//! marqdo_expression::StatementKind::Expr(_)
|
||||
//! ));
|
||||
//! ```
|
||||
//!
|
||||
//! The parser never panics on malformed input: every failure is a
|
||||
//! [`ParseError`].
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod ast;
|
||||
pub mod lexer;
|
||||
mod parser;
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
pub use ast::{Arg, BinaryOp, Expr, Ident, RecordEntry, Statement, StatementKind, UnaryOp};
|
||||
pub use marqdo_span::Span;
|
||||
pub use parser::MAX_NESTING_DEPTH;
|
||||
|
||||
/// Stable diagnostic code for every expression parse failure.
|
||||
pub const EXPR_PARSE: &str = "expr.parse";
|
||||
|
||||
/// A structured expression/statement parse failure.
|
||||
///
|
||||
/// `code` is always [`EXPR_PARSE`]. `span` is a byte range into the payload
|
||||
/// string and may be empty (for example at end of input).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ParseError {
|
||||
/// Stable diagnostic code, always `"expr.parse"`.
|
||||
pub code: &'static str,
|
||||
/// Human readable explanation of the failure.
|
||||
pub message: String,
|
||||
/// Byte range in the payload where the failure was detected.
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
impl ParseError {
|
||||
/// Create a parse error with the fixed `expr.parse` code.
|
||||
pub fn new(span: Span, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
code: EXPR_PARSE,
|
||||
message: message.into(),
|
||||
span,
|
||||
}
|
||||
}
|
||||
|
||||
/// Stable diagnostic code (always `"expr.parse"`).
|
||||
pub fn code(&self) -> &'static str {
|
||||
self.code
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ParseError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}: {} at {}", self.code, self.message, self.span)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ParseError {}
|
||||
|
||||
/// Parse a complete expression payload.
|
||||
///
|
||||
/// The whole input must be consumed; empty input, trailing tokens, unsupported
|
||||
/// constructs and assignments are [`ParseError`]s.
|
||||
pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
|
||||
parser::parse_expression(source)
|
||||
}
|
||||
|
||||
/// Parse a complete statement payload.
|
||||
///
|
||||
/// A statement is either `<bare identifier> = <expression>` or a bare
|
||||
/// expression (typically a call). The whole input must be consumed.
|
||||
pub fn parse_statement(source: &str) -> Result<Statement, ParseError> {
|
||||
parser::parse_statement(source)
|
||||
}
|
||||
|
||||
/// Parse a leading parenthesised argument list.
|
||||
///
|
||||
/// `source` must start with `(` and contain exactly one argument list, e.g.
|
||||
/// `(r=3, precision=2)`. Argument rules are identical to call arguments:
|
||||
/// positional arguments precede named arguments, named arguments are unique,
|
||||
/// and a single trailing comma is allowed. Trailing tokens are an error.
|
||||
///
|
||||
/// This exists so the frontend can parse Markdown-link calls of the form
|
||||
/// `> [计算面积](./math.mq.md#计算面积)(r=3)` without re-lexing the payload.
|
||||
pub fn parse_arguments(source: &str) -> Result<Vec<Arg>, ParseError> {
|
||||
parser::parse_arguments(source)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn public_api_parses_expression_and_statement() {
|
||||
assert!(parse_expression("1 + 2 * 3").is_ok());
|
||||
assert!(parse_statement("x = 1").is_ok());
|
||||
assert!(parse_arguments("(r=3)").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_payloads_are_errors() {
|
||||
for source in ["", " ", "\n\t"] {
|
||||
let err = parse_expression(source).unwrap_err();
|
||||
assert_eq!(err.code, EXPR_PARSE);
|
||||
assert_eq!(err.code(), "expr.parse");
|
||||
assert!(err.message.contains("expression"));
|
||||
|
||||
let err = parse_statement(source).unwrap_err();
|
||||
assert_eq!(err.code, EXPR_PARSE);
|
||||
assert!(err.message.contains("statement"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_error_displays_code_and_span_and_implements_error() {
|
||||
let err = parse_expression("1 2").unwrap_err();
|
||||
let rendered = err.to_string();
|
||||
assert!(rendered.starts_with("expr.parse: "), "{rendered}");
|
||||
assert!(rendered.contains("2..3"), "{rendered}");
|
||||
assert!(rendered.contains("unexpected"), "{rendered}");
|
||||
|
||||
let dyn_error: &dyn std::error::Error = &err;
|
||||
assert!(dyn_error.source().is_none());
|
||||
|
||||
let custom = ParseError::new(Span::new(4, 9), "boom");
|
||||
assert_eq!(custom.code, "expr.parse");
|
||||
assert_eq!(custom.message, "boom");
|
||||
assert_eq!(custom.span, Span::new(4, 9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ast_types_are_reachable_from_the_crate_root() {
|
||||
let expr: Expr = parse_expression("f(1)").unwrap();
|
||||
let statement: Statement = parse_statement("f(1)").unwrap();
|
||||
let _: StatementKind = statement.kind;
|
||||
let _: BinaryOp = BinaryOp::Add;
|
||||
let _: UnaryOp = UnaryOp::Not;
|
||||
let _: Arg = Arg {
|
||||
name: None,
|
||||
value: expr,
|
||||
span: Span::default(),
|
||||
};
|
||||
let _: RecordEntry = RecordEntry {
|
||||
key: "k".to_owned(),
|
||||
key_span: Span::default(),
|
||||
value: Expr::None,
|
||||
};
|
||||
let _: Ident = Ident {
|
||||
name: "x".to_owned(),
|
||||
span: Span::default(),
|
||||
};
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user