diff --git a/Cargo.lock b/Cargo.lock index 944ea91..d9b9323 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + [[package]] name = "block-buffer" version = "0.10.4" @@ -186,6 +192,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" +[[package]] +name = "errno" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -225,6 +241,23 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "hashbrown" version = "0.17.1" @@ -281,6 +314,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "marqdo-cli" version = "0.1.0" @@ -309,15 +348,16 @@ name = "marqdo-expression" version = "0.1.0" dependencies = [ "marqdo-span", + "num-bigint 0.5.1", "serde", - "thiserror", + "serde_json", + "unicode-ident", ] [[package]] name = "marqdo-foreign" version = "0.1.0" dependencies = [ - "marqdo-span", "marqdo-value", "serde_json", "thiserror", @@ -334,7 +374,8 @@ dependencies = [ "marqdo-value", "serde", "serde_json", - "thiserror", + "sha2", + "unicode-ident", "unicode-normalization", ] @@ -342,6 +383,7 @@ dependencies = [ name = "marqdo-host" version = "0.1.0" dependencies = [ + "glob", "marqdo-foreign", "marqdo-math", "marqdo-plugin-abi", @@ -350,6 +392,7 @@ dependencies = [ "marqdo-value", "serde", "serde_json", + "tempfile", "thiserror", ] @@ -359,7 +402,6 @@ version = "0.1.0" dependencies = [ "marqdo-expression", "marqdo-span", - "marqdo-value", "serde", "serde_json", ] @@ -406,12 +448,14 @@ dependencies = [ name = "marqdo-plugin-abi" version = "0.1.0" dependencies = [ + "hex", "libloading", "marqdo-span", "marqdo-value", "serde", "serde_json", "serde_yaml", + "sha2", "thiserror", ] @@ -530,6 +574,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "once_cell_polyfill" version = "1.70.2" @@ -593,12 +643,31 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rustc-hash" version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "ryu" version = "1.0.23" @@ -710,6 +779,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "2.0.21" diff --git a/crates/marqdo-expression/Cargo.toml b/crates/marqdo-expression/Cargo.toml index a918d02..bc10d00 100644 --- a/crates/marqdo-expression/Cargo.toml +++ b/crates/marqdo-expression/Cargo.toml @@ -11,5 +11,8 @@ authors.workspace = true [dependencies] marqdo-span = { workspace = true } serde = { workspace = true } -thiserror = { workspace = true } +num-bigint = { workspace = true } +unicode-ident = { workspace = true } +[dev-dependencies] +serde_json = { workspace = true } diff --git a/crates/marqdo-expression/src/ast.rs b/crates/marqdo-expression/src/ast.rs new file mode 100644 index 0000000..c041b28 --- /dev/null +++ b/crates/marqdo-expression/src/ast.rs @@ -0,0 +1,292 @@ +//! Abstract syntax tree for the Marqdo expression grammar v1. +//! +//! All spans are byte offsets relative to the payload string that was handed to +//! [`crate::parse_expression`], [`crate::parse_statement`] or +//! [`crate::parse_arguments`]. The frontend re-bases them onto the Markdown +//! source through the text/code segments that made up the payload. +//! +//! # Span coverage +//! +//! Every node that carries a stored span exposes it through [`Expr::span`] / +//! [`Statement::span`]. The frozen v1 AST shape stores a span only on the +//! structural variants (`Unary`, `Binary`, `Call`, `Member`, `Index`), on +//! [`Ident`], [`Arg`] and [`RecordEntry`]. Scalar and container literals +//! (`Int`, `Float`, `Text`, `Bool`, `None`, `List`, `Record`) intentionally +//! have no span field: [`Expr::span`] returns the empty span `0..0` for them. +//! Consumers that need a location for a literal should use the nearest +//! enclosing structural node or the enclosing [`Statement::span`]. + +use marqdo_span::Span; +use num_bigint::BigInt; +use serde::{Deserialize, Serialize}; + +/// A bare identifier together with its source span. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Ident { + /// Identifier text as written (Unicode XID, decoded verbatim). + pub name: String, + /// Span of the identifier token. + pub span: Span, +} + +/// One argument of a call, optionally named. +/// +/// ```text +/// f(1, r=3) +/// ^ ^ named argument, `name == Some("r")` +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Arg { + /// `Some` when the argument was written as `name = value`. + pub name: Option, + /// Argument value expression. + pub value: Expr, + /// Span of the whole argument (name through value). + pub span: Span, +} + +/// One key/value entry of a record literal. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecordEntry { + /// Entry key (identifier text or decoded text-literal contents). + pub key: String, + /// Span of the key token (including the quotes for a text-literal key). + pub key_span: Span, + /// Entry value expression. + pub value: Expr, +} + +/// Binary operators, listed from loosest to tightest binding. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum BinaryOp { + /// Short-circuiting logical or (`or`). + Or, + /// Short-circuiting logical and (`and`). + And, + /// `==` + Eq, + /// `!=` + Ne, + /// `<` + Lt, + /// `<=` + Le, + /// `>` + Gt, + /// `>=` + Ge, + /// `+` + Add, + /// `-` + Sub, + /// `*` + Mul, + /// `/` (always yields `Float` at runtime). + Div, + /// `//` (floor division, `Int` only). + FloorDiv, + /// `%` (remainder, `Int` only). + Mod, + /// `**` (right associative). + Pow, +} + +/// Prefix operators. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum UnaryOp { + /// `not` + Not, + /// unary `-` + Neg, + /// unary `+` + Pos, +} + +/// An expression node. +/// +/// The variant set and payloads are the frozen public contract of this crate. +/// See the module docs for the span policy of literal variants. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum Expr { + /// Arbitrary-precision decimal integer literal. + Int(BigInt), + /// IEEE-754 binary64 float literal. + Float(f64), + /// Decoded text literal (escapes already resolved). + Text(String), + /// `true` / `false`. + Bool(bool), + /// `None`. + None, + /// A variable, parameter or function name reference. + Ident(Ident), + /// `[a, b, c]`. + List(Vec), + /// `{key: value}`. + Record(Vec), + /// Prefix operator application. + Unary { + /// Applied prefix operator. + op: UnaryOp, + /// Operand. + operand: Box, + /// Span from the operator through the end of the operand. + span: Span, + }, + /// Infix operator application. + Binary { + /// Applied infix operator. + op: BinaryOp, + /// Left operand. + left: Box, + /// Right operand. + right: Box, + /// Span of the whole infix expression. + span: Span, + }, + /// Call expression. + Call { + /// Callee expression. + callee: Box, + /// Arguments in source order. + args: Vec, + /// Span from the callee start through the closing `)`. + span: Span, + }, + /// Member access (`base.name`). + Member { + /// Object being projected. + base: Box, + /// Member name (never a keyword). + name: String, + /// Span from the base start through the member name. + span: Span, + }, + /// Index access (`base[index]`). + Index { + /// Object being indexed. + base: Box, + /// Index expression. + index: Box, + /// Span from the base start through the closing `]`. + span: Span, + }, +} + +impl Expr { + /// Best known source span of this expression. + /// + /// Returns the empty span `0..0` for variants that do not store a span + /// (`Int`, `Float`, `Text`, `Bool`, `None`, `List`, `Record`); see the + /// module documentation for the rationale. + pub fn span(&self) -> Span { + match self { + Expr::Ident(ident) => ident.span, + Expr::Unary { span, .. } + | Expr::Binary { span, .. } + | Expr::Call { span, .. } + | Expr::Member { span, .. } + | Expr::Index { span, .. } => *span, + Expr::Int(_) | Expr::Float(_) | Expr::Text(_) | Expr::Bool(_) | Expr::None => { + Span::default() + } + Expr::List(_) | Expr::Record(_) => Span::default(), + } + } +} + +/// The payload of a statement. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum StatementKind { + /// ` = `; the only assignment form in v1. + Assign { + /// Assigned identifier. + name: String, + /// Span of the identifier token (not the whole assignment). + name_span: Span, + /// Right-hand side expression. + value: Expr, + }, + /// A bare expression statement (typically a call). + Expr(Expr), +} + +/// A complete parsed statement: either an assignment or an expression. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Statement { + /// Statement payload. + pub kind: StatementKind, + /// Span of the whole statement, from the first token through the last. + pub span: Span, +} + +impl Statement { + /// Span of the whole statement, from the first token through the last. + pub fn span(&self) -> Span { + self.span + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parse_expression; + + #[test] + fn span_accessor_covers_structural_nodes() { + let expr = parse_expression("f(1, r=2)").unwrap(); + assert_eq!(expr.span(), Span::new(0, 9)); + let Expr::Call { callee, args, .. } = &expr else { + panic!("expected call"); + }; + assert_eq!(callee.span(), Span::new(0, 1)); + assert_eq!(args[0].span, Span::new(2, 3)); + assert_eq!(args[1].span, Span::new(5, 8)); + assert_eq!(args[1].name.as_deref(), Some("r")); + } + + #[test] + fn spanless_variants_return_the_empty_span() { + for expr in [ + Expr::Int(BigInt::from(1)), + Expr::Float(1.0), + Expr::Text("t".to_owned()), + Expr::Bool(true), + Expr::None, + Expr::List(Vec::new()), + Expr::Record(Vec::new()), + ] { + assert_eq!(expr.span(), Span::default()); + } + } + + #[test] + fn ast_types_serde_round_trip() { + let expr = parse_expression( + r#"f(1, [true, None], {键: 9.5e1}, 2 ** -1, a.b[0] == not false, 名="值")"#, + ) + .unwrap(); + let json = serde_json::to_string(&expr).unwrap(); + let decoded: Expr = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded, expr); + + let stmt = crate::parse_statement("结果 = 打印(阶乘(n=3))").unwrap(); + let json = serde_json::to_string(&stmt).unwrap(); + let decoded: Statement = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded, stmt); + assert_eq!(decoded.span(), stmt.span()); + + let args = crate::parse_arguments("(r=3, precision=2)").unwrap(); + let json = serde_json::to_string(&args).unwrap(); + let decoded: Vec = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded, args); + } + + #[test] + fn span_serde_round_trips_inside_ast() { + let expr = parse_expression("a.b").unwrap(); + let json = serde_json::to_string(&expr).unwrap(); + assert!(json.contains("\"start\":0")); + assert!(json.contains("\"end\":3")); + } +} diff --git a/crates/marqdo-expression/src/lexer.rs b/crates/marqdo-expression/src/lexer.rs new file mode 100644 index 0000000..2a63c56 --- /dev/null +++ b/crates/marqdo-expression/src/lexer.rs @@ -0,0 +1,608 @@ +//! Hand written lexer for the Marqdo expression grammar v1. +//! +//! This module is an implementation detail that is exposed only to make +//! diagnostics and tests easier; the contractual API of the crate lives in +//! [`crate::parse_expression`], [`crate::parse_statement`] and +//! [`crate::parse_arguments`] plus the [`crate::ast`] types. +//! +//! Lexical rules (see `spec/expression.ebnf`): +//! +//! * Unicode whitespace separates tokens (newlines included). +//! * Identifiers use UAX #31 `XID_Start` / `XID_Continue`. +//! * Integers are arbitrary-precision decimal digit runs. +//! * Floats require digits on both sides of `.` and may carry an exponent. +//! * Text literals are double quoted with the escapes `\\`, `\"`, `\n`, `\t`, +//! `\r` and `\u{...}`. +//! * There are no comments in v1. + +use crate::ParseError; +use marqdo_span::Span; + +/// A lexical token kind. +#[derive(Debug, Clone, PartialEq)] +pub enum TokenKind { + /// Decimal integer literal, kept as raw digits (arbitrary precision). + Int(String), + /// Float literal, kept as raw source text (parsed to `f64` by the parser). + Float(String), + /// Text literal whose escapes have already been decoded. + Text(String), + /// Identifier text (never a keyword). + Ident(String), + /// `true` + True, + /// `false` + False, + /// `None` + NoneKw, + /// `and` + And, + /// `or` + Or, + /// `not` + Not, + /// `(` + LParen, + /// `)` + RParen, + /// `[` + LBracket, + /// `]` + RBracket, + /// `{` + LBrace, + /// `}` + RBrace, + /// `,` + Comma, + /// `:` + Colon, + /// `.` + Dot, + /// `=` (assignment) + Assign, + /// `==` + EqEq, + /// `!=` + Ne, + /// `<` + Lt, + /// `<=` + Le, + /// `>` + Gt, + /// `>=` + Ge, + /// `+` + Plus, + /// `-` + Minus, + /// `*` + Star, + /// `**` + StarStar, + /// `/` + Slash, + /// `//` + SlashSlash, + /// `%` + Percent, + /// End of input. + Eof, +} + +/// A token with its byte span in the payload. +#[derive(Debug, Clone, PartialEq)] +pub struct Token { + /// Token kind. + pub kind: TokenKind, + /// Byte span of the token. + pub span: Span, +} + +/// Tokenize a payload string. +/// +/// The returned vector always ends with exactly one [`TokenKind::Eof`] token +/// whose span is the empty span at the end of the input. +pub fn tokenize(source: &str) -> Result, ParseError> { + Lexer::new(source).tokenize() +} + +struct Lexer<'a> { + source: &'a str, + pos: usize, +} + +impl<'a> Lexer<'a> { + fn new(source: &'a str) -> Self { + Self { source, pos: 0 } + } + + fn peek(&self) -> Option { + self.source[self.pos..].chars().next() + } + + fn bump(&mut self) -> Option { + let c = self.peek()?; + self.pos += c.len_utf8(); + Some(c) + } + + fn error(&self, start: usize, message: impl Into) -> ParseError { + ParseError::new(Span::new(start, self.pos), message) + } + + fn tokenize(mut self) -> Result, ParseError> { + let mut tokens = Vec::new(); + loop { + self.skip_whitespace(); + let start = self.pos; + let Some(c) = self.peek() else { + tokens.push(Token { + kind: TokenKind::Eof, + span: Span::empty(self.source.len()), + }); + return Ok(tokens); + }; + let kind = match c { + '"' => TokenKind::Text(self.scan_text()?), + '0'..='9' => self.scan_number()?, + _ if unicode_ident::is_xid_start(c) => self.scan_identifier(), + '(' => { + self.bump(); + TokenKind::LParen + } + ')' => { + self.bump(); + TokenKind::RParen + } + '[' => { + self.bump(); + TokenKind::LBracket + } + ']' => { + self.bump(); + TokenKind::RBracket + } + '{' => { + self.bump(); + TokenKind::LBrace + } + '}' => { + self.bump(); + TokenKind::RBrace + } + ',' => { + self.bump(); + TokenKind::Comma + } + ':' => { + self.bump(); + TokenKind::Colon + } + '.' => { + self.bump(); + if matches!(self.peek(), Some(next) if next.is_ascii_digit()) { + return Err(self.error( + start, + "invalid float literal: a decimal point must be preceded by a digit", + )); + } + TokenKind::Dot + } + '=' => { + self.bump(); + if matches!(self.peek(), Some('=')) { + self.bump(); + TokenKind::EqEq + } else { + TokenKind::Assign + } + } + '!' => { + self.bump(); + if matches!(self.peek(), Some('=')) { + self.bump(); + TokenKind::Ne + } else { + return Err(self.error( + start, + "unexpected character `!`; logical negation is written `not`", + )); + } + } + '<' => { + self.bump(); + if matches!(self.peek(), Some('=')) { + self.bump(); + TokenKind::Le + } else { + TokenKind::Lt + } + } + '>' => { + self.bump(); + if matches!(self.peek(), Some('=')) { + self.bump(); + TokenKind::Ge + } else { + TokenKind::Gt + } + } + '+' => { + self.bump(); + TokenKind::Plus + } + '-' => { + self.bump(); + TokenKind::Minus + } + '*' => { + self.bump(); + if matches!(self.peek(), Some('*')) { + self.bump(); + TokenKind::StarStar + } else { + TokenKind::Star + } + } + '/' => { + self.bump(); + if matches!(self.peek(), Some('/')) { + self.bump(); + TokenKind::SlashSlash + } else { + TokenKind::Slash + } + } + '%' => { + self.bump(); + TokenKind::Percent + } + '#' => { + self.bump(); + return Err(self.error( + start, + "unexpected character `#`; comments are not supported in v1", + )); + } + other => { + self.bump(); + return Err(self.error(start, format!("unexpected character {:?}", other))); + } + }; + tokens.push(Token { + kind, + span: Span::new(start, self.pos), + }); + } + } + + fn skip_whitespace(&mut self) { + while let Some(c) = self.peek() { + if c.is_whitespace() { + self.bump(); + } else { + break; + } + } + } + + fn scan_identifier(&mut self) -> TokenKind { + let start = self.pos; + // The caller guarantees `is_xid_start` for the first character. + self.bump(); + while let Some(c) = self.peek() { + if unicode_ident::is_xid_continue(c) { + self.bump(); + } else { + break; + } + } + let text = &self.source[start..self.pos]; + match text { + "true" => TokenKind::True, + "false" => TokenKind::False, + "None" => TokenKind::NoneKw, + "and" => TokenKind::And, + "or" => TokenKind::Or, + "not" => TokenKind::Not, + _ => TokenKind::Ident(text.to_owned()), + } + } + + fn scan_number(&mut self) -> Result { + let start = self.pos; + while matches!(self.peek(), Some(c) if c.is_ascii_digit()) { + self.bump(); + } + let mut is_float = false; + if matches!(self.peek(), Some('.')) { + self.bump(); + if !matches!(self.peek(), Some(c) if c.is_ascii_digit()) { + return Err(self.error( + start, + "invalid float literal: expected at least one digit after `.`", + )); + } + is_float = true; + while matches!(self.peek(), Some(c) if c.is_ascii_digit()) { + self.bump(); + } + } + if matches!(self.peek(), Some('e') | Some('E')) { + if !is_float { + return Err(self.error( + start, + "invalid float literal: an exponent requires digits on both sides of `.`", + )); + } + self.bump(); + if matches!(self.peek(), Some('+') | Some('-')) { + self.bump(); + } + if !matches!(self.peek(), Some(c) if c.is_ascii_digit()) { + return Err(self.error( + start, + "invalid float literal: expected at least one exponent digit", + )); + } + while matches!(self.peek(), Some(c) if c.is_ascii_digit()) { + self.bump(); + } + } + if let Some(c) = self.peek() { + if c == '.' || unicode_ident::is_xid_continue(c) { + return Err(self.error( + start, + format!("invalid numeric literal: unexpected `{c}` immediately after a number"), + )); + } + } + let raw = self.source[start..self.pos].to_owned(); + Ok(if is_float { + TokenKind::Float(raw) + } else { + TokenKind::Int(raw) + }) + } + + fn scan_text(&mut self) -> Result { + let start = self.pos; + self.bump(); // opening quote + let mut out = String::new(); + loop { + let Some(c) = self.bump() else { + return Err(ParseError::new( + Span::new(start, self.source.len()), + "unterminated text literal: expected a closing `\"`", + )); + }; + match c { + '"' => return Ok(out), + '\\' => { + let escape_start = self.pos - 1; + match self.bump() { + Some('"') => out.push('"'), + Some('\\') => out.push('\\'), + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some('u') => self.scan_unicode_escape(escape_start, &mut out)?, + Some(other) => { + return Err(self.error( + escape_start, + format!( + "invalid escape sequence `\\{other}`; supported escapes are \ + `\\\\`, `\\\"`, `\\n`, `\\t`, `\\r` and `\\u{{...}}`" + ), + )); + } + None => { + return Err(ParseError::new( + Span::new(start, self.source.len()), + "unterminated text literal: expected a closing `\"`", + )); + } + } + } + other => out.push(other), + } + } + } + + fn scan_unicode_escape( + &mut self, + escape_start: usize, + out: &mut String, + ) -> Result<(), ParseError> { + if !matches!(self.peek(), Some('{')) { + return Err(self.error( + escape_start, + "invalid unicode escape: expected `{` after `\\u`", + )); + } + self.bump(); + let digits_start = self.pos; + let mut value: u32 = 0; + let mut overflow = false; + while let Some(c) = self.peek() { + let Some(digit) = c.to_digit(16) else { + break; + }; + self.bump(); + value = match value.checked_mul(16).and_then(|v| v.checked_add(digit)) { + Some(v) => v, + None => { + overflow = true; + 0 + } + }; + } + if self.pos == digits_start { + return Err(self.error( + escape_start, + "invalid unicode escape: expected at least one hexadecimal digit", + )); + } + if !matches!(self.peek(), Some('}')) { + return Err(self.error( + escape_start, + "invalid unicode escape: expected a closing `}`", + )); + } + self.bump(); + if overflow { + return Err(self.error( + escape_start, + "invalid unicode escape: code point is out of range", + )); + } + match char::from_u32(value) { + Some(c) => { + out.push(c); + Ok(()) + } + None => Err(self.error( + escape_start, + format!("invalid unicode escape: U+{value:04X} is not a Unicode scalar value"), + )), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kinds(source: &str) -> Vec { + tokenize(source) + .unwrap() + .into_iter() + .map(|t| t.kind) + .collect() + } + + #[test] + fn tokenizes_operators_and_punctuation() { + assert_eq!( + kinds("== != < <= > >= = + - * ** / // % ( ) [ ] { } , : ."), + vec![ + TokenKind::EqEq, + TokenKind::Ne, + TokenKind::Lt, + TokenKind::Le, + TokenKind::Gt, + TokenKind::Ge, + TokenKind::Assign, + TokenKind::Plus, + TokenKind::Minus, + TokenKind::Star, + TokenKind::StarStar, + TokenKind::Slash, + TokenKind::SlashSlash, + TokenKind::Percent, + TokenKind::LParen, + TokenKind::RParen, + TokenKind::LBracket, + TokenKind::RBracket, + TokenKind::LBrace, + TokenKind::RBrace, + TokenKind::Comma, + TokenKind::Colon, + TokenKind::Dot, + TokenKind::Eof, + ] + ); + } + + #[test] + fn tokenizes_keywords_separately_from_identifiers() { + assert_eq!( + kinds("true false None and or not truex None_ 真实"), + vec![ + TokenKind::True, + TokenKind::False, + TokenKind::NoneKw, + TokenKind::And, + TokenKind::Or, + TokenKind::Not, + TokenKind::Ident("truex".into()), + TokenKind::Ident("None_".into()), + TokenKind::Ident("真实".into()), + TokenKind::Eof, + ] + ); + } + + #[test] + fn token_remembers_multibyte_spans() { + let tokens = tokenize("张三 + 1").unwrap(); + assert_eq!(tokens[0].kind, TokenKind::Ident("张三".into())); + assert_eq!(tokens[0].span, Span::new(0, 6)); + assert_eq!(tokens[1].span, Span::new(7, 8)); + assert_eq!(tokens[2].kind, TokenKind::Int("1".into())); + assert_eq!(tokens[2].span, Span::new(9, 10)); + assert_eq!(tokens[3].span, Span::empty(10)); + } + + #[test] + fn unicode_whitespace_is_skipped() { + assert_eq!( + kinds("1\u{00a0}+\u{2003}2\n"), + vec![ + TokenKind::Int("1".into()), + TokenKind::Plus, + TokenKind::Int("2".into()), + TokenKind::Eof + ] + ); + } + + #[test] + fn number_lexing_rejects_invalid_forms() { + for bad in [".5", "1.", "1e5", "1.2.3", "1.5e", "1.5e+", "1a", "1_2"] { + let err = tokenize(bad).unwrap_err(); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + assert_eq!( + kinds("1.5e+3 0.0 007 1.25E-2"), + vec![ + TokenKind::Float("1.5e+3".into()), + TokenKind::Float("0.0".into()), + TokenKind::Int("007".into()), + TokenKind::Float("1.25E-2".into()), + TokenKind::Eof + ] + ); + } + + #[test] + fn text_escapes_decode() { + let tokens = tokenize(r#""a\\b\"c\n\t\r\u{4e2d}""#).unwrap(); + assert_eq!(tokens[0].kind, TokenKind::Text("a\\b\"c\n\t\r中".into())); + } + + #[test] + fn bad_text_lexing_reports_errors() { + for bad in [ + r#""unterminated"#, + r#""bad \q escape""#, + r#""bad \u{} escape""#, + r#""bad \u{D800} escape""#, + r#""bad \u{110000} escape""#, + r#""bad \u12 escape""#, + ] { + let err = tokenize(bad).unwrap_err(); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + } + + #[test] + fn comments_are_not_supported() { + let err = tokenize("# comment").unwrap_err(); + assert!(err.message.contains("comments")); + let err = tokenize("1 # 2").unwrap_err(); + assert!(err.message.contains("comments")); + } +} diff --git a/crates/marqdo-expression/src/lib.rs b/crates/marqdo-expression/src/lib.rs new file mode 100644 index 0000000..f26748a --- /dev/null +++ b/crates/marqdo-expression/src/lib.rs @@ -0,0 +1,225 @@ +//! Strict, independent expression and statement parser for executable Markdown +//! payloads. +//! +//! This crate is the only grammar for Marqdo executable payloads. It knows +//! nothing about Markdown and nothing about the runtime: the Markdown AST +//! decides *where* a payload exists, this crate decides *what the payload +//! means* (see `spec/expression.ebnf` and `spec/evaluation.md`). +//! +//! # Public API +//! +//! ```rust +//! use marqdo_expression::{parse_expression, parse_statement}; +//! +//! let stmt = parse_statement("结果 = 阶乘(n=3)").unwrap(); +//! assert_eq!(stmt.span().start, 0); +//! +//! let expr = parse_expression("2 ** -1").unwrap(); +//! assert_eq!(expr.span().start, 0); +//! ``` +//! +//! The contractual surface is: +//! +//! * [`parse_expression`] — parse a complete expression payload. +//! * [`parse_statement`] — parse a complete statement payload (assignment or +//! expression). +//! * [`parse_arguments`] — parse a leading `( ... )` argument list, used for +//! Markdown-link calls such as `> [计算面积](./math.mq.md#计算面积)(r=3)`. +//! * the [`ast`] module (re-exported at the crate root). +//! * [`ParseError`] — structured `expr.parse` diagnostics with byte spans. +//! +//! Everything else, including [`lexer`], is an implementation detail that is +//! exposed only to make diagnostics and tests easier. +//! +//! # Grammar summary +//! +//! The grammar is frozen in `spec/expression.ebnf`. In order of decreasing +//! precedence: prefix `not` / `-` / `+`, `**` (right associative), postfix +//! calls `.name` / `[index]`, `* / // %`, `+ -`, a single non-associative +//! comparison, `and`, `or`. Assignment is only available at statement start +//! and only for a bare identifier target. +//! +//! Notable lexical rules: +//! +//! * identifiers use Unicode XID (`XID_Start XID_Continue*`), so `阶乘` and +//! `café` are identifiers; +//! * `true`, `false`, `None`, `and`, `or`, `not` are reserved keywords; +//! * integers are arbitrary-precision decimal literals; +//! * floats require digits on both sides of `.` (`.5` and `1.` are invalid) +//! and may carry an exponent; +//! * text literals are double quoted and support `\\`, `\"`, `\n`, `\t`, `\r` +//! and `\u{...}` escapes; +//! * comments are not supported; +//! * nesting deeper than [`MAX_NESTING_DEPTH`] is a structured parse error, so +//! hostile payloads cannot overflow the parser stack. +//! +//! # Spans and errors +//! +//! Spans are half-open byte offsets relative to the payload string. Errors use +//! the stable diagnostic code [`EXPR_PARSE`] (`"expr.parse"`) and follow the +//! same shape used elsewhere in the toolchain (code + message + span). +//! +//! # Example payloads +//! +//! ```rust +//! use marqdo_expression::parse_statement; +//! +//! let stmt = parse_statement("打印(\"面积 = \" + 文本(area))").unwrap(); +//! assert!(matches!( +//! stmt.kind, +//! marqdo_expression::StatementKind::Expr(_) +//! )); +//! ``` +//! +//! The parser never panics on malformed input: every failure is a +//! [`ParseError`]. + +#![forbid(unsafe_code)] + +pub mod ast; +pub mod lexer; +mod parser; + +use std::fmt; + +use serde::Serialize; + +pub use ast::{Arg, BinaryOp, Expr, Ident, RecordEntry, Statement, StatementKind, UnaryOp}; +pub use marqdo_span::Span; +pub use parser::MAX_NESTING_DEPTH; + +/// Stable diagnostic code for every expression parse failure. +pub const EXPR_PARSE: &str = "expr.parse"; + +/// A structured expression/statement parse failure. +/// +/// `code` is always [`EXPR_PARSE`]. `span` is a byte range into the payload +/// string and may be empty (for example at end of input). +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ParseError { + /// Stable diagnostic code, always `"expr.parse"`. + pub code: &'static str, + /// Human readable explanation of the failure. + pub message: String, + /// Byte range in the payload where the failure was detected. + pub span: Span, +} + +impl ParseError { + /// Create a parse error with the fixed `expr.parse` code. + pub fn new(span: Span, message: impl Into) -> Self { + Self { + code: EXPR_PARSE, + message: message.into(), + span, + } + } + + /// Stable diagnostic code (always `"expr.parse"`). + pub fn code(&self) -> &'static str { + self.code + } +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}: {} at {}", self.code, self.message, self.span) + } +} + +impl std::error::Error for ParseError {} + +/// Parse a complete expression payload. +/// +/// The whole input must be consumed; empty input, trailing tokens, unsupported +/// constructs and assignments are [`ParseError`]s. +pub fn parse_expression(source: &str) -> Result { + parser::parse_expression(source) +} + +/// Parse a complete statement payload. +/// +/// A statement is either ` = ` or a bare +/// expression (typically a call). The whole input must be consumed. +pub fn parse_statement(source: &str) -> Result { + parser::parse_statement(source) +} + +/// Parse a leading parenthesised argument list. +/// +/// `source` must start with `(` and contain exactly one argument list, e.g. +/// `(r=3, precision=2)`. Argument rules are identical to call arguments: +/// positional arguments precede named arguments, named arguments are unique, +/// and a single trailing comma is allowed. Trailing tokens are an error. +/// +/// This exists so the frontend can parse Markdown-link calls of the form +/// `> [计算面积](./math.mq.md#计算面积)(r=3)` without re-lexing the payload. +pub fn parse_arguments(source: &str) -> Result, ParseError> { + parser::parse_arguments(source) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn public_api_parses_expression_and_statement() { + assert!(parse_expression("1 + 2 * 3").is_ok()); + assert!(parse_statement("x = 1").is_ok()); + assert!(parse_arguments("(r=3)").is_ok()); + } + + #[test] + fn empty_payloads_are_errors() { + for source in ["", " ", "\n\t"] { + let err = parse_expression(source).unwrap_err(); + assert_eq!(err.code, EXPR_PARSE); + assert_eq!(err.code(), "expr.parse"); + assert!(err.message.contains("expression")); + + let err = parse_statement(source).unwrap_err(); + assert_eq!(err.code, EXPR_PARSE); + assert!(err.message.contains("statement")); + } + } + + #[test] + fn parse_error_displays_code_and_span_and_implements_error() { + let err = parse_expression("1 2").unwrap_err(); + let rendered = err.to_string(); + assert!(rendered.starts_with("expr.parse: "), "{rendered}"); + assert!(rendered.contains("2..3"), "{rendered}"); + assert!(rendered.contains("unexpected"), "{rendered}"); + + let dyn_error: &dyn std::error::Error = &err; + assert!(dyn_error.source().is_none()); + + let custom = ParseError::new(Span::new(4, 9), "boom"); + assert_eq!(custom.code, "expr.parse"); + assert_eq!(custom.message, "boom"); + assert_eq!(custom.span, Span::new(4, 9)); + } + + #[test] + fn ast_types_are_reachable_from_the_crate_root() { + let expr: Expr = parse_expression("f(1)").unwrap(); + let statement: Statement = parse_statement("f(1)").unwrap(); + let _: StatementKind = statement.kind; + let _: BinaryOp = BinaryOp::Add; + let _: UnaryOp = UnaryOp::Not; + let _: Arg = Arg { + name: None, + value: expr, + span: Span::default(), + }; + let _: RecordEntry = RecordEntry { + key: "k".to_owned(), + key_span: Span::default(), + value: Expr::None, + }; + let _: Ident = Ident { + name: "x".to_owned(), + span: Span::default(), + }; + } +} diff --git a/crates/marqdo-expression/src/parser.rs b/crates/marqdo-expression/src/parser.rs new file mode 100644 index 0000000..239bacf --- /dev/null +++ b/crates/marqdo-expression/src/parser.rs @@ -0,0 +1,1434 @@ +//! Recursive-descent parser for the Marqdo expression grammar v1. +//! +//! The parser is a straightforward implementation of `spec/expression.ebnf`: +//! +//! | level | operators | associativity | +//! |---|---|---| +//! | `or_expr` | `or` | left | +//! | `and_expr` | `and` | left | +//! | `compare_expr` | `== != < <= > >=` | non-associative | +//! | `add_expr` | `+ -` | left | +//! | `mul_expr` | `* / // %` | left | +//! | `unary_expr` | `not - +` | prefix, right | +//! | `power_expr` | `**` | right | +//! | `postfix_expr` | call, `.name`, `[expr]` | left | +//! +//! Nesting is bounded by [`MAX_NESTING_DEPTH`]; the guard is checked before +//! recursing into any sub-expression so adversarial inputs fail with a +//! structured [`ParseError`] instead of overflowing the stack. + +use crate::ast::{Arg, BinaryOp, Expr, Ident, RecordEntry, Statement, StatementKind, UnaryOp}; +use crate::lexer::{self, Token, TokenKind}; +use crate::ParseError; +use marqdo_span::Span; +use num_bigint::BigInt; + +/// Maximum number of nested expression constructs accepted by the parser. +/// +/// The top-level expression is depth 0; every nested grouping, list/record +/// element, call argument, index expression, prefix operator and power +/// exponent adds one. The 129th nested construct is rejected with an +/// `expr.parse` error. +pub const MAX_NESTING_DEPTH: usize = 128; + +pub(crate) fn parse_expression(source: &str) -> Result { + let mut parser = Parser::new(source)?; + if parser.at_eof() { + return Err(parser.error_here("expected an expression, found end of input")); + } + let expr = parser.expression()?; + parser.expect_end_of_expression()?; + Ok(expr) +} + +pub(crate) fn parse_statement(source: &str) -> Result { + let mut parser = Parser::new(source)?; + if parser.at_eof() { + return Err(parser.error_here("expected a statement, found end of input")); + } + let start = parser.peek().span.start; + if let TokenKind::Ident(_) = parser.peek().kind { + if matches!( + parser.peek_n(1).map(|token| &token.kind), + Some(TokenKind::Assign) + ) { + return parser.assignment(start); + } + } + let expr = parser.expression()?; + if !parser.at_eof() { + return Err(parser.trailing_statement_error(&expr)); + } + let end = parser.previous_end(); + Ok(Statement { + kind: StatementKind::Expr(expr), + span: Span::new(start, end), + }) +} + +pub(crate) fn parse_arguments(source: &str) -> Result, ParseError> { + let mut parser = Parser::new(source)?; + if parser.at_eof() { + return Err( + parser.error_here("expected an argument list starting with `(`, found end of input") + ); + } + if !matches!(parser.peek().kind, TokenKind::LParen) { + return Err(parser.error_here(format!( + "expected an argument list starting with `(`, found {}", + parser.describe_current() + ))); + } + let args = parser.call_arguments()?; + if !parser.at_eof() { + return Err(parser.error_here(format!( + "unexpected {} after argument list", + parser.describe_current() + ))); + } + Ok(args) +} + +struct Parser { + tokens: Vec, + index: usize, + depth: usize, +} + +impl Parser { + fn new(source: &str) -> Result { + Ok(Self { + tokens: lexer::tokenize(source)?, + index: 0, + depth: 0, + }) + } + + // ------------------------------------------------------------------ + // Token cursor helpers + // ------------------------------------------------------------------ + + fn peek(&self) -> &Token { + // `tokens` always contains at least the EOF token, and `bump` never + // advances past it, so this index is always in bounds. + &self.tokens[self.index] + } + + fn peek_n(&self, offset: usize) -> Option<&Token> { + self.tokens.get(self.index + offset) + } + + fn bump(&mut self) -> Token { + let token = self.tokens[self.index].clone(); + if self.index + 1 < self.tokens.len() { + self.index += 1; + } + token + } + + fn at_eof(&self) -> bool { + matches!(self.peek().kind, TokenKind::Eof) + } + + fn previous_end(&self) -> usize { + if self.index == 0 { + 0 + } else { + self.tokens[self.index - 1].span.end + } + } + + fn error_here(&self, message: impl Into) -> ParseError { + ParseError::new(self.peek().span, message) + } + + fn describe_current(&self) -> String { + describe_token(&self.peek().kind) + } + + fn expect(&mut self, kind: TokenKind, what: &str) -> Result { + if self.peek().kind == kind { + Ok(self.bump()) + } else { + Err(self.error_here(format!( + "expected {what}, found {}", + self.describe_current() + ))) + } + } + + fn expect_identifier(&mut self, context: &str) -> Result { + match self.peek().kind.clone() { + TokenKind::Ident(name) => { + let token = self.bump(); + Ok(Ident { + name, + span: token.span, + }) + } + _ if is_keyword(&self.peek().kind) => Err(self.error_here(format!( + "keywords are reserved and cannot be used as identifiers; found {}", + self.describe_current() + ))), + _ => Err(self.error_here(format!( + "expected {context}, found {}", + self.describe_current() + ))), + } + } + + // ------------------------------------------------------------------ + // Nesting guard + // ------------------------------------------------------------------ + + fn nested( + &mut self, + parse: impl FnOnce(&mut Self) -> Result, + ) -> Result { + self.depth += 1; + if self.depth > MAX_NESTING_DEPTH { + self.depth -= 1; + return Err(self.error_here(format!( + "expression nesting exceeds the maximum depth of {MAX_NESTING_DEPTH}" + ))); + } + let result = parse(self); + self.depth -= 1; + result + } + + // ------------------------------------------------------------------ + // Statements + // ------------------------------------------------------------------ + + fn assignment(&mut self, start: usize) -> Result { + let TokenKind::Ident(name) = self.peek().kind.clone() else { + return Err(self.error_here("expected an identifier on the left of `=`")); + }; + let name_token = self.bump(); + self.bump(); // `=` + if self.at_eof() { + return Err(self.error_here("expected an expression after `=`")); + } + let value = self.expression()?; + if !self.at_eof() { + return Err(if matches!(self.peek().kind, TokenKind::Assign) { + self.error_here( + "chained assignment is not supported; assignment is only allowed at \ + statement start", + ) + } else { + self.error_here(format!( + "unexpected {} after assignment value", + self.describe_current() + )) + }); + } + let end = self.previous_end(); + Ok(Statement { + kind: StatementKind::Assign { + name, + name_span: name_token.span, + value, + }, + span: Span::new(start, end), + }) + } + + fn trailing_statement_error(&self, expr: &Expr) -> ParseError { + if matches!(self.peek().kind, TokenKind::Assign) { + return if matches!(expr, Expr::Member { .. } | Expr::Index { .. }) { + self.error_here( + "invalid assignment target: member and index assignment are not supported \ + in v1; assign a bare identifier", + ) + } else { + self.error_here( + "invalid assignment target: assignment requires a bare identifier on the \ + left of `=`", + ) + }; + } + self.error_here(format!( + "unexpected {} after expression", + self.describe_current() + )) + } + + fn expect_end_of_expression(&self) -> Result<(), ParseError> { + if self.at_eof() { + Ok(()) + } else if matches!(self.peek().kind, TokenKind::Assign) { + Err(self.error_here( + "unexpected `=`; assignment is only supported as a statement, not inside an \ + expression", + )) + } else { + Err(self.error_here(format!( + "unexpected {} after expression", + self.describe_current() + ))) + } + } + + // ------------------------------------------------------------------ + // Expression levels, loosest to tightest + // ------------------------------------------------------------------ + + fn expression(&mut self) -> Result { + self.parse_or() + } + + fn parse_or(&mut self) -> Result { + let start = self.peek().span.start; + let mut left = self.parse_and()?; + while matches!(self.peek().kind, TokenKind::Or) { + self.bump(); + let right = self.parse_and()?; + let end = self.previous_end(); + left = binary(BinaryOp::Or, left, right, Span::new(start, end)); + } + Ok(left) + } + + fn parse_and(&mut self) -> Result { + let start = self.peek().span.start; + let mut left = self.parse_compare()?; + while matches!(self.peek().kind, TokenKind::And) { + self.bump(); + let right = self.parse_compare()?; + let end = self.previous_end(); + left = binary(BinaryOp::And, left, right, Span::new(start, end)); + } + Ok(left) + } + + fn parse_compare(&mut self) -> Result { + let start = self.peek().span.start; + let left = self.parse_add()?; + let Some(op) = compare_op(&self.peek().kind) else { + return Ok(left); + }; + self.bump(); + let right = self.parse_add()?; + let end = self.previous_end(); + let expr = binary(op, left, right, Span::new(start, end)); + if compare_op(&self.peek().kind).is_some() { + return Err(self.error_here(format!( + "comparison operators are non-associative; found {} after a comparison (join \ + comparisons with `and`)", + self.describe_current() + ))); + } + Ok(expr) + } + + fn parse_add(&mut self) -> Result { + let start = self.peek().span.start; + let mut left = self.parse_mul()?; + loop { + let op = match self.peek().kind { + TokenKind::Plus => BinaryOp::Add, + TokenKind::Minus => BinaryOp::Sub, + _ => break, + }; + self.bump(); + let right = self.parse_mul()?; + let end = self.previous_end(); + left = binary(op, left, right, Span::new(start, end)); + } + Ok(left) + } + + fn parse_mul(&mut self) -> Result { + let start = self.peek().span.start; + let mut left = self.parse_unary()?; + loop { + let op = match self.peek().kind { + TokenKind::Star => BinaryOp::Mul, + TokenKind::Slash => BinaryOp::Div, + TokenKind::SlashSlash => BinaryOp::FloorDiv, + TokenKind::Percent => BinaryOp::Mod, + _ => break, + }; + self.bump(); + let right = self.parse_unary()?; + let end = self.previous_end(); + left = binary(op, left, right, Span::new(start, end)); + } + Ok(left) + } + + fn parse_unary(&mut self) -> Result { + let op = match self.peek().kind { + TokenKind::Not => Some(UnaryOp::Not), + TokenKind::Minus => Some(UnaryOp::Neg), + TokenKind::Plus => Some(UnaryOp::Pos), + _ => None, + }; + let Some(op) = op else { + return self.parse_power(); + }; + let start = self.peek().span.start; + self.bump(); + let operand = self.nested(|parser| parser.parse_unary())?; + let end = self.previous_end(); + Ok(Expr::Unary { + op, + operand: Box::new(operand), + span: Span::new(start, end), + }) + } + + fn parse_power(&mut self) -> Result { + let start = self.peek().span.start; + let base = self.parse_postfix()?; + if matches!(self.peek().kind, TokenKind::StarStar) { + self.bump(); + let exponent = self.nested(|parser| parser.parse_unary())?; + let end = self.previous_end(); + return Ok(binary(BinaryOp::Pow, base, exponent, Span::new(start, end))); + } + Ok(base) + } + + // ------------------------------------------------------------------ + // Postfix chain + // ------------------------------------------------------------------ + + fn parse_postfix(&mut self) -> Result { + let start = self.peek().span.start; + let mut expr = self.parse_primary()?; + loop { + if matches!(self.peek().kind, TokenKind::LParen) { + let args = self.call_arguments()?; + let end = self.previous_end(); + expr = Expr::Call { + callee: Box::new(expr), + args, + span: Span::new(start, end), + }; + } else if matches!(self.peek().kind, TokenKind::Dot) { + self.bump(); + let member = self.expect_identifier("a member name after `.`")?; + let end = member.span.end; + expr = Expr::Member { + base: Box::new(expr), + name: member.name, + span: Span::new(start, end), + }; + } else if matches!(self.peek().kind, TokenKind::LBracket) { + self.bump(); + let index = self.nested(|parser| parser.expression())?; + let close = self.expect(TokenKind::RBracket, "`]`")?; + expr = Expr::Index { + base: Box::new(expr), + index: Box::new(index), + span: Span::new(start, close.span.end), + }; + } else { + break; + } + } + Ok(expr) + } + + /// Parse `( [argument {"," argument} [","]] )`. + /// + /// Shared by call postfixes and by [`crate::parse_arguments`]. + fn call_arguments(&mut self) -> Result, ParseError> { + self.expect(TokenKind::LParen, "`(`")?; + let mut args = Vec::new(); + if matches!(self.peek().kind, TokenKind::RParen) { + self.bump(); + return Ok(args); + } + let mut seen_named = false; + let mut names: Vec = Vec::new(); + loop { + let start = self.peek().span.start; + let mut name = None; + if let Some((candidate, name_span)) = self.peek_named_argument() { + seen_named = true; + if names.iter().any(|existing| existing == &candidate) { + return Err(ParseError::new( + name_span, + format!("duplicate named argument `{candidate}`"), + )); + } + self.bump(); // identifier + self.bump(); // `=` + names.push(candidate.clone()); + name = Some(candidate); + } else if seen_named { + return Err( + self.error_here("positional arguments must all precede named arguments") + ); + } + if matches!(self.peek().kind, TokenKind::Comma | TokenKind::RParen) { + return Err(self.error_here(format!( + "expected an argument expression, found {}", + self.describe_current() + ))); + } + let value = self.nested(|parser| parser.expression())?; + let end = self.previous_end(); + args.push(Arg { + name, + value, + span: Span::new(start, end), + }); + if matches!(self.peek().kind, TokenKind::Comma) { + self.bump(); + if matches!(self.peek().kind, TokenKind::RParen) { + self.bump(); + break; + } + } else if matches!(self.peek().kind, TokenKind::RParen) { + self.bump(); + break; + } else { + return Err(self.error_here(format!( + "expected `,` or `)` in argument list, found {}", + self.describe_current() + ))); + } + } + Ok(args) + } + + fn peek_named_argument(&self) -> Option<(String, Span)> { + let name = match &self.peek().kind { + TokenKind::Ident(name) => name.clone(), + _ => return None, + }; + match self.peek_n(1).map(|token| &token.kind) { + Some(TokenKind::Assign) => Some((name, self.peek().span)), + _ => None, + } + } + + // ------------------------------------------------------------------ + // Primaries + // ------------------------------------------------------------------ + + fn parse_primary(&mut self) -> Result { + let token = self.peek().clone(); + match token.kind { + TokenKind::Int(raw) => { + self.bump(); + let value = BigInt::parse_bytes(raw.as_bytes(), 10) + .ok_or_else(|| ParseError::new(token.span, "invalid integer literal"))?; + Ok(Expr::Int(value)) + } + TokenKind::Float(raw) => { + self.bump(); + self.float_literal(&raw, token.span) + } + TokenKind::Text(text) => { + self.bump(); + Ok(Expr::Text(text)) + } + TokenKind::True => { + self.bump(); + Ok(Expr::Bool(true)) + } + TokenKind::False => { + self.bump(); + Ok(Expr::Bool(false)) + } + TokenKind::NoneKw => { + self.bump(); + Ok(Expr::None) + } + TokenKind::Ident(name) => { + self.bump(); + Ok(Expr::Ident(Ident { + name, + span: token.span, + })) + } + TokenKind::LParen => { + self.bump(); + let inner = self.nested(|parser| parser.expression())?; + self.expect(TokenKind::RParen, "`)`")?; + Ok(inner) + } + TokenKind::LBracket => self.list_literal(), + TokenKind::LBrace => self.record_literal(), + _ => Err(self.error_here(format!( + "expected an expression, found {}", + self.describe_current() + ))), + } + } + + fn float_literal(&self, raw: &str, span: Span) -> Result { + let value = raw + .parse::() + .map_err(|_| ParseError::new(span, format!("invalid float literal `{raw}`")))?; + if !value.is_finite() { + return Err(ParseError::new( + span, + format!( + "float literal `{raw}` is out of range for IEEE-754 binary64; infinities are \ + only allowed as operation results" + ), + )); + } + Ok(Expr::Float(value)) + } + + fn list_literal(&mut self) -> Result { + self.bump(); // `[` + let mut items = Vec::new(); + if matches!(self.peek().kind, TokenKind::RBracket) { + self.bump(); + return Ok(Expr::List(items)); + } + loop { + let item = self.nested(|parser| parser.expression())?; + items.push(item); + if matches!(self.peek().kind, TokenKind::Comma) { + self.bump(); + if matches!(self.peek().kind, TokenKind::RBracket) { + self.bump(); + break; + } + } else if matches!(self.peek().kind, TokenKind::RBracket) { + self.bump(); + break; + } else { + return Err(self.error_here(format!( + "expected `,` or `]` in list literal, found {}", + self.describe_current() + ))); + } + } + Ok(Expr::List(items)) + } + + fn record_literal(&mut self) -> Result { + self.bump(); // `{` + let mut entries = Vec::new(); + if matches!(self.peek().kind, TokenKind::RBrace) { + self.bump(); + return Ok(Expr::Record(entries)); + } + loop { + let (key, key_span) = self.record_key()?; + self.expect(TokenKind::Colon, "`:` after a record key")?; + if matches!(self.peek().kind, TokenKind::RBrace | TokenKind::Comma) { + return Err(self.error_here(format!( + "expected a record value expression, found {}", + self.describe_current() + ))); + } + let value = self.nested(|parser| parser.expression())?; + entries.push(RecordEntry { + key, + key_span, + value, + }); + if matches!(self.peek().kind, TokenKind::Comma) { + self.bump(); + if matches!(self.peek().kind, TokenKind::RBrace) { + self.bump(); + break; + } + } else if matches!(self.peek().kind, TokenKind::RBrace) { + self.bump(); + break; + } else { + return Err(self.error_here(format!( + "expected `,` or `}}` in record literal, found {}", + self.describe_current() + ))); + } + } + Ok(Expr::Record(entries)) + } + + fn record_key(&mut self) -> Result<(String, Span), ParseError> { + match self.peek().kind.clone() { + TokenKind::Ident(name) => { + let token = self.bump(); + Ok((name, token.span)) + } + TokenKind::Text(text) => { + let token = self.bump(); + Ok((text, token.span)) + } + _ if is_keyword(&self.peek().kind) => Err(self.error_here(format!( + "record keys must be identifiers or text literals; {} is a reserved keyword \ + (quote it as a text literal)", + self.describe_current() + ))), + _ => Err(self.error_here(format!( + "expected a record key (identifier or text literal), found {}", + self.describe_current() + ))), + } + } +} + +fn binary(op: BinaryOp, left: Expr, right: Expr, span: Span) -> Expr { + Expr::Binary { + op, + left: Box::new(left), + right: Box::new(right), + span, + } +} + +fn compare_op(kind: &TokenKind) -> Option { + match kind { + TokenKind::EqEq => Some(BinaryOp::Eq), + TokenKind::Ne => Some(BinaryOp::Ne), + TokenKind::Lt => Some(BinaryOp::Lt), + TokenKind::Le => Some(BinaryOp::Le), + TokenKind::Gt => Some(BinaryOp::Gt), + TokenKind::Ge => Some(BinaryOp::Ge), + _ => None, + } +} + +fn is_keyword(kind: &TokenKind) -> bool { + matches!( + kind, + TokenKind::True + | TokenKind::False + | TokenKind::NoneKw + | TokenKind::And + | TokenKind::Or + | TokenKind::Not + ) +} + +fn describe_token(kind: &TokenKind) -> String { + match kind { + TokenKind::Int(raw) => format!("integer literal `{raw}`"), + TokenKind::Float(raw) => format!("float literal `{raw}`"), + TokenKind::Text(_) => "text literal".to_owned(), + TokenKind::Ident(name) => format!("identifier `{name}`"), + TokenKind::True => "`true`".to_owned(), + TokenKind::False => "`false`".to_owned(), + TokenKind::NoneKw => "`None`".to_owned(), + TokenKind::And => "`and`".to_owned(), + TokenKind::Or => "`or`".to_owned(), + TokenKind::Not => "`not`".to_owned(), + TokenKind::LParen => "`(`".to_owned(), + TokenKind::RParen => "`)`".to_owned(), + TokenKind::LBracket => "`[`".to_owned(), + TokenKind::RBracket => "`]`".to_owned(), + TokenKind::LBrace => "`{`".to_owned(), + TokenKind::RBrace => "`}`".to_owned(), + TokenKind::Comma => "`,`".to_owned(), + TokenKind::Colon => "`:`".to_owned(), + TokenKind::Dot => "`.`".to_owned(), + TokenKind::Assign => "`=`".to_owned(), + TokenKind::EqEq => "`==`".to_owned(), + TokenKind::Ne => "`!=`".to_owned(), + TokenKind::Lt => "`<`".to_owned(), + TokenKind::Le => "`<=`".to_owned(), + TokenKind::Gt => "`>`".to_owned(), + TokenKind::Ge => "`>=`".to_owned(), + TokenKind::Plus => "`+`".to_owned(), + TokenKind::Minus => "`-`".to_owned(), + TokenKind::Star => "`*`".to_owned(), + TokenKind::StarStar => "`**`".to_owned(), + TokenKind::Slash => "`/`".to_owned(), + TokenKind::SlashSlash => "`//`".to_owned(), + TokenKind::Percent => "`%`".to_owned(), + TokenKind::Eof => "end of input".to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{parse_arguments, parse_expression, parse_statement, ParseError}; + use std::str::FromStr; + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + fn ex(source: &str) -> Expr { + parse_expression(source) + .unwrap_or_else(|err| panic!("{source:?} should parse but failed: {err}")) + } + + fn ex_err(source: &str) -> ParseError { + match parse_expression(source) { + Ok(expr) => panic!("{source:?} should fail but parsed as {expr:?}"), + Err(err) => err, + } + } + + fn st(source: &str) -> Statement { + parse_statement(source) + .unwrap_or_else(|err| panic!("{source:?} should parse as a statement: {err}")) + } + + fn st_err(source: &str) -> ParseError { + match parse_statement(source) { + Ok(stmt) => panic!("{source:?} should fail but parsed as {stmt:?}"), + Err(err) => err, + } + } + + /// Compact span-insensitive rendering, used to assert tree shape. + fn sx(source: &str) -> String { + sexpr(&ex(source)) + } + + fn sexpr(expr: &Expr) -> String { + match expr { + Expr::Int(value) => value.to_string(), + Expr::Float(value) => format!("{value:?}"), + Expr::Text(value) => format!("{value:?}"), + Expr::Bool(value) => value.to_string(), + Expr::None => "None".to_owned(), + Expr::Ident(ident) => ident.name.clone(), + Expr::List(items) => { + let mut out = String::from("(list"); + for item in items { + out.push(' '); + out.push_str(&sexpr(item)); + } + out.push(')'); + out + } + Expr::Record(entries) => { + let mut out = String::from("(record"); + for entry in entries { + out.push_str(&format!(" {}: {}", entry.key, sexpr(&entry.value))); + } + out.push(')'); + out + } + Expr::Unary { op, operand, .. } => { + format!("({} {})", unary_symbol(*op), sexpr(operand)) + } + Expr::Binary { + op, left, right, .. + } => format!("({} {} {})", binary_symbol(*op), sexpr(left), sexpr(right)), + Expr::Call { callee, args, .. } => { + let mut out = format!("(call {}", sexpr(callee)); + for arg in args { + match &arg.name { + Some(name) => out.push_str(&format!(" {name}={}", sexpr(&arg.value))), + None => { + out.push(' '); + out.push_str(&sexpr(&arg.value)); + } + } + } + out.push(')'); + out + } + Expr::Member { base, name, .. } => format!("(. {} {name})", sexpr(base)), + Expr::Index { base, index, .. } => { + format!("(index {} {})", sexpr(base), sexpr(index)) + } + } + } + + fn binary_symbol(op: BinaryOp) -> &'static str { + match op { + BinaryOp::Or => "or", + BinaryOp::And => "and", + BinaryOp::Eq => "==", + BinaryOp::Ne => "!=", + BinaryOp::Lt => "<", + BinaryOp::Le => "<=", + BinaryOp::Gt => ">", + BinaryOp::Ge => ">=", + BinaryOp::Add => "+", + BinaryOp::Sub => "-", + BinaryOp::Mul => "*", + BinaryOp::Div => "/", + BinaryOp::FloorDiv => "//", + BinaryOp::Mod => "%", + BinaryOp::Pow => "**", + } + } + + fn unary_symbol(op: UnaryOp) -> &'static str { + match op { + UnaryOp::Not => "not", + UnaryOp::Neg => "neg", + UnaryOp::Pos => "pos", + } + } + + // ------------------------------------------------------------------ + // Literals + // ------------------------------------------------------------------ + + #[test] + fn integer_literals_are_arbitrary_precision() { + assert_eq!(ex("0"), Expr::Int(BigInt::from(0))); + assert_eq!(ex("007"), Expr::Int(BigInt::from(7))); + assert_eq!( + ex("9223372036854775808"), + Expr::Int(BigInt::from(9_223_372_036_854_775_808u64)) + ); + let huge = "123456789012345678901234567890123456789012345678901234567890"; + assert_eq!(ex(huge), Expr::Int(BigInt::from_str(huge).unwrap())); + assert_eq!(sx("-9223372036854775809"), "(neg 9223372036854775809)"); + } + + #[test] + fn float_literals_cover_the_grammar() { + assert_eq!(ex("1.5"), Expr::Float(1.5)); + assert_eq!(ex("0.0"), Expr::Float(0.0)); + assert_eq!(ex("1.5e3"), Expr::Float(1500.0)); + assert_eq!(ex("1.25E-2"), Expr::Float(0.0125)); + assert_eq!(ex("0.5e+2"), Expr::Float(50.0)); + assert_eq!( + ex("-2.5"), + Expr::Unary { + op: UnaryOp::Neg, + operand: Box::new(Expr::Float(2.5)), + span: Span::new(0, 4), + } + ); + } + + #[test] + fn invalid_float_literals_are_rejected() { + for bad in [ + ".5", "1.", "1e5", "1.2.3", "1.5e", "1.5e+", "1_000", "1.0e999", + ] { + let err = ex_err(bad); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + assert!(!err.message.is_empty(), "input {bad:?}"); + } + // `.5` gets the specific "must be preceded" diagnostic from the lexer. + assert!(ex_err(".5").message.contains("preceded")); + assert!(ex_err("1.0e999").message.contains("out of range")); + } + + #[test] + fn text_literals_decode_escapes() { + assert_eq!(ex(r#""hello""#), Expr::Text("hello".to_owned())); + assert_eq!(ex(r#""""#), Expr::Text(String::new())); + assert_eq!( + ex(r#""a\\b\"c\n\t\r\u{4e2d}""#), + Expr::Text("a\\b\"c\n\t\r中".to_owned()) + ); + assert_eq!(ex(r#""\u{4E2D}\u{1F600}""#), Expr::Text("中😀".to_owned())); + // A raw newline is a valid text character inside a literal. + assert_eq!(ex("\"a\nb\""), Expr::Text("a\nb".to_owned())); + assert_eq!(ex(r#""{}()[],:.""#), Expr::Text("{}()[],:.".to_owned())); + } + + #[test] + fn invalid_text_literals_are_rejected() { + for bad in [ + r#""unterminated"#, + r#""bad \q escape""#, + r#""bad \u{} escape""#, + r#""bad \u{D800} escape""#, + r#""bad \u{110000} escape""#, + r#""bad \u12 escape""#, + ] { + let err = ex_err(bad); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + // `\\` at the end of the input is an unterminated literal. + let err = parse_expression("\"abc\\").unwrap_err(); + assert!(err.message.contains("unterminated")); + } + + #[test] + fn boolean_and_none_literals_parse() { + assert_eq!(ex("true"), Expr::Bool(true)); + assert_eq!(ex("false"), Expr::Bool(false)); + assert_eq!(ex("None"), Expr::None); + // `truex` is an identifier, not the keyword `true` plus `x`. + assert_eq!(sx("truex"), "truex"); + assert_eq!(sx("None_"), "None_"); + } + + #[test] + fn unicode_identifiers_are_accepted() { + assert_eq!(sx("阶乘"), "阶乘"); + assert_eq!(sx("计算_面积2"), "计算_面积2"); + assert_eq!(sx("café"), "café"); + assert!(matches!(ex("π"), Expr::Ident(_))); + // U+005F `_` is not XID_Start, so it cannot start an identifier. + let _ = st("变量 = 1").kind; + for bad in ["_x", "__", "1abc"] { + assert_eq!(ex_err(bad).code, "expr.parse", "input {bad:?}"); + } + } + + #[test] + fn reserved_keywords_cannot_be_identifiers() { + for bad in ["and", "or", "not", "true = 1"] { + let err = ex_err(bad); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + assert!(ex_err("a.and").message.contains("reserved")); + assert!(ex_err("a.true").message.contains("reserved")); + } + + // ------------------------------------------------------------------ + // Operator precedence and associativity + // ------------------------------------------------------------------ + + #[test] + fn additive_and_multiplicative_precedence() { + assert_eq!(sx("1 + 2 * 3"), "(+ 1 (* 2 3))"); + assert_eq!(sx("(1 + 2) * 3"), "(* (+ 1 2) 3)"); + assert_eq!(sx("1 * 2 + 3"), "(+ (* 1 2) 3)"); + assert_eq!(sx("1 - 2 - 3"), "(- (- 1 2) 3)"); + assert_eq!(sx("8 / 4 // 2 % 3"), "(% (// (/ 8 4) 2) 3)"); + assert_eq!(sx("1 + 2 + 3"), "(+ (+ 1 2) 3)"); + } + + #[test] + fn power_is_right_associative_and_binds_tighter_than_unary_minus() { + assert_eq!(sx("2 ** 3 ** 2"), "(** 2 (** 3 2))"); + assert_eq!(sx("-x ** 2"), "(neg (** x 2))"); + assert_eq!(sx("(-x) ** 2"), "(** (neg x) 2)"); + assert_eq!(sx("2 ** -1"), "(** 2 (neg 1))"); + assert_eq!(sx("2 ** -1 ** 2"), "(** 2 (neg (** 1 2)))"); + assert_eq!(sx("+x ** 2"), "(pos (** x 2))"); + assert_eq!(sx("a * b ** c * d"), "(* (* a (** b c)) d)"); + } + + #[test] + fn unary_operators_stack() { + assert_eq!(sx("--1"), "(neg (neg 1))"); + assert_eq!(sx("not not true"), "(not (not true))"); + assert_eq!(sx("+1"), "(pos 1)"); + assert_eq!(sx("-+1"), "(neg (pos 1))"); + assert_eq!(sx("not a"), "(not a)"); + assert!(ex_err("not").message.contains("expected an expression")); + assert!(ex_err("!a").message.contains("`not`")); + } + + #[test] + fn comparisons_bind_looser_than_arithmetic() { + assert_eq!(sx("a < b + c"), "(< a (+ b c))"); + assert_eq!(sx("a + b < c * d"), "(< (+ a b) (* c d))"); + assert_eq!(sx("a == b"), "(== a b)"); + assert_eq!(sx("a != b"), "(!= a b)"); + assert_eq!(sx("a <= b"), "(<= a b)"); + assert_eq!(sx("a > b"), "(> a b)"); + assert_eq!(sx("a >= b"), "(>= a b)"); + // `not` binds tighter than comparison. + assert_eq!(sx("not a == b"), "(== (not a) b)"); + assert_eq!(sx("-a < b"), "(< (neg a) b)"); + } + + #[test] + fn comparisons_are_non_associative() { + let err = ex_err("a < b < c"); + assert!(err.message.contains("non-associative")); + assert_eq!(err.span, Span::new(6, 7)); + assert!(ex_err("a == b == c").message.contains("non-associative")); + assert!(ex_err("a <= b > c").message.contains("non-associative")); + // The documented workaround parses. + assert_eq!(sx("a < b and b < c"), "(and (< a b) (< b c))"); + } + + #[test] + fn logical_operators_chain_left_associatively() { + assert_eq!(sx("a and b or c"), "(or (and a b) c)"); + assert_eq!(sx("a or b and c"), "(or a (and b c))"); + assert_eq!(sx("a or b or c"), "(or (or a b) c)"); + assert_eq!(sx("a and b and c"), "(and (and a b) c)"); + assert_eq!(sx("not a and not b"), "(and (not a) (not b))"); + assert_eq!(sx("a and b == c"), "(and a (== b c))"); + assert_eq!(sx("a == b and c != d"), "(and (== a b) (!= c d))"); + } + + // ------------------------------------------------------------------ + // Calls, members, indexes + // ------------------------------------------------------------------ + + #[test] + fn calls_parse_positional_and_named_arguments() { + assert_eq!(sx("f()"), "(call f)"); + assert_eq!(sx("f(1, 2)"), "(call f 1 2)"); + assert_eq!(sx("f(1,)"), "(call f 1)"); + assert_eq!(sx("f(x=1)"), "(call f x=1)"); + assert_eq!(sx("f(1, y=2,)"), "(call f 1 y=2)"); + assert_eq!(sx("f(g(1), h(x=2))"), "(call f (call g 1) (call h x=2))"); + assert_eq!(sx(r#"打印("你好")"#), r#"(call 打印 "你好")"#); + } + + #[test] + fn call_argument_errors_are_rejected() { + let err = ex_err("f(a=1, 2)"); + assert!(err.message.contains("positional arguments")); + let err = ex_err("f(a=1, a=2)"); + assert!(err.message.contains("duplicate named argument")); + for bad in ["f(,)", "f(1,,2)", "f(1 2)", "f(", "f(x=)", "f(a=1, b)"] { + let err = ex_err(bad); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + } + + #[test] + fn member_and_index_access_chain() { + assert_eq!(sx("a.b"), "(. a b)"); + assert_eq!(sx("a.b.c"), "(. (. a b) c)"); + assert_eq!(sx("a.b[0].c(d)"), "(call (. (index (. a b) 0) c) d)"); + assert_eq!(sx("a[0][1]"), "(index (index a 0) 1)"); + assert_eq!(sx("a[1 + 2]"), "(index a (+ 1 2))"); + assert_eq!(sx("f(1).name"), "(. (call f 1) name)"); + } + + #[test] + fn member_and_index_errors_are_rejected() { + for bad in ["a.", "a[]", "a[1", "a[1,2]", "a.1", "a[1]]"] { + let err = ex_err(bad); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + assert!(ex_err("a.true").message.contains("reserved")); + } + + // ------------------------------------------------------------------ + // Collection literals + // ------------------------------------------------------------------ + + #[test] + fn list_literals_parse() { + assert_eq!(sx("[]"), "(list)"); + assert_eq!(sx("[1]"), "(list 1)"); + assert_eq!(sx("[1, 2, 3]"), "(list 1 2 3)"); + assert_eq!(sx("[1,]"), "(list 1)"); + assert_eq!(sx("[a, [b, c]]"), "(list a (list b c))"); + assert_eq!(sx(r#"["文本", true, None]"#), r#"(list "文本" true None)"#); + } + + #[test] + fn list_literal_errors_are_rejected() { + for bad in ["[1 2]", "[,]", "[1,,2]", "[1", "[1, ", "[1; 2]"] { + let err = ex_err(bad); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + } + + #[test] + fn record_literals_parse() { + assert_eq!(sx("{}"), "(record)"); + assert_eq!(sx("{a: 1}"), "(record a: 1)"); + assert_eq!(sx("{a: 1, b: 2,}"), "(record a: 1 b: 2)"); + assert_eq!(sx(r#"{"键": "值"}"#), r#"(record 键: "值")"#); + assert_eq!(sx("{a: {b: [1]}}"), "(record a: (record b: (list 1)))"); + let expr = ex("{答案: 42}"); + match expr { + Expr::Record(entries) => { + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].key, "答案"); + assert_eq!(entries[0].key_span, Span::new(1, 7)); + } + other => panic!("expected a record, got {other:?}"), + } + } + + #[test] + fn record_literal_errors_are_rejected() { + for bad in ["{a 1}", "{true: 1}", "{a: }", "{1: 2}", "{a: 1,,}", "{"] { + let err = ex_err(bad); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + let err = ex_err("{true: 1}"); + assert!(err.message.contains("reserved keyword")); + } + + // ------------------------------------------------------------------ + // Statements + // ------------------------------------------------------------------ + + #[test] + fn assignment_statements_parse() { + let stmt = st("x = 1"); + assert_eq!( + stmt.kind, + StatementKind::Assign { + name: "x".to_owned(), + name_span: Span::new(0, 1), + value: Expr::Int(BigInt::from(1)), + } + ); + let stmt = st("结果 = 阶乘(n=3)"); + match &stmt.kind { + StatementKind::Assign { name, value, .. } => { + assert_eq!(name, "结果"); + assert_eq!(sexpr(value), "(call 阶乘 n=3)"); + } + other => panic!("expected assignment, got {other:?}"), + } + assert_eq!(st("x = y + 1").span(), Span::new(0, 9)); + } + + #[test] + fn expression_statements_parse() { + assert_eq!(st("打印(1)").kind, StatementKind::Expr(ex("打印(1)"))); + assert_eq!(st("x").kind, StatementKind::Expr(ex("x"))); + assert_eq!(st("x == 1").kind, StatementKind::Expr(ex("x == 1"))); + } + + #[test] + fn invalid_statements_are_rejected() { + for bad in [ + "", + " ", + "x =", + "x = 1 = 2", + "a.b = 1", + "a[0] = 1", + "1 = 2", + "= 1", + "x = 1 2", + ] { + let err = st_err(bad); + assert_eq!(err.code, "expr.parse", "input {bad:?}"); + } + assert!(st_err("x = 1 = 2").message.contains("chained assignment")); + assert!(st_err("a.b = 1").message.contains("member and index")); + assert!(st_err("x =").message.contains("after `=`")); + } + + #[test] + fn expressions_reject_assignment_syntax() { + assert!(ex_err("x = 1") + .message + .contains("assignment is only supported as a statement")); + } + + #[test] + fn comments_are_not_supported() { + for bad in ["# comment", "1 # 2", "f() # tail"] { + let err = ex_err(bad); + assert!(err.message.contains("comments"), "input {bad:?}"); + } + } + + // ------------------------------------------------------------------ + // Spans + // ------------------------------------------------------------------ + + #[test] + fn spans_are_byte_offsets_into_the_payload() { + let expr = ex("1 + 2"); + let Expr::Binary { span, .. } = expr else { + panic!("expected binary"); + }; + assert_eq!(span, Span::new(0, 5)); + + let stmt = st(" x = 1 "); + assert_eq!(stmt.span(), Span::new(2, 7)); + match stmt.kind { + StatementKind::Assign { name_span, .. } => assert_eq!(name_span, Span::new(2, 3)), + other => panic!("expected assignment, got {other:?}"), + } + + let Expr::Call { + callee, span, args, .. + } = ex("f(1)") + else { + panic!("expected call"); + }; + assert_eq!(span, Span::new(0, 4)); + assert_eq!(callee.span(), Span::new(0, 1)); + assert_eq!(args[0].span, Span::new(2, 3)); + + let Expr::Member { span, .. } = ex("a.b") else { + panic!("expected member"); + }; + assert_eq!(span, Span::new(0, 3)); + + let Expr::Index { span, .. } = ex("a[0]") else { + panic!("expected index"); + }; + assert_eq!(span, Span::new(0, 4)); + + let Expr::Unary { span, .. } = ex("-x") else { + panic!("expected unary"); + }; + assert_eq!(span, Span::new(0, 2)); + + let Expr::Binary { span, .. } = ex("2 ** 3") else { + panic!("expected power"); + }; + assert_eq!(span, Span::new(0, 6)); + + let Expr::Binary { span, .. } = ex("(1) + 2") else { + panic!("expected binary"); + }; + assert_eq!(span, Span::new(0, 7)); + + // Multibyte payloads use byte offsets (张三 is three bytes per scalar). + let Expr::Binary { span, .. } = ex("张三 + 1") else { + panic!("expected binary"); + }; + assert_eq!(span, Span::new(0, 10)); + + // Literal variants have no stored span in the frozen AST shape. + assert_eq!(ex("42").span(), Span::default()); + assert_eq!(ex("[1]").span(), Span::default()); + assert_eq!(ex("{a: 1}").span(), Span::default()); + assert_eq!(ex(r#""s""#).span(), Span::default()); + } + + #[test] + fn error_spans_point_at_the_offending_token() { + assert_eq!(ex_err("1 2").span, Span::new(2, 3)); + assert_eq!(ex_err("f(1 2)").span, Span::new(4, 5)); + assert_eq!(ex_err("1 +").span, Span::empty(3)); + assert_eq!(ex_err("f(").span, Span::empty(2)); + assert_eq!(ex_err("a < b < c").span, Span::new(6, 7)); + let err = ex_err("f(a=1, a=2)"); + assert_eq!(err.span, Span::new(7, 8)); // span of the duplicate name token + } + + // ------------------------------------------------------------------ + // Nesting limit + // ------------------------------------------------------------------ + + fn nested_parens(depth: usize) -> String { + format!("{}1{}", "(".repeat(depth), ")".repeat(depth)) + } + + #[test] + fn nesting_depth_limit_is_enforced() { + assert!(parse_expression(&nested_parens(MAX_NESTING_DEPTH)).is_ok()); + let err = parse_expression(&nested_parens(MAX_NESTING_DEPTH + 1)).unwrap_err(); + assert_eq!(err.code, "expr.parse"); + assert!(err.message.contains("nesting")); + assert_eq!( + err.span, + Span::new(MAX_NESTING_DEPTH + 1, MAX_NESTING_DEPTH + 2) + ); + + // Each prefix operator nests its operand, so exactly + // MAX_NESTING_DEPTH prefix operators are accepted. + let ok_unary = format!("{}1", "-".repeat(MAX_NESTING_DEPTH)); + assert!(parse_expression(&ok_unary).is_ok()); + let bad_unary = format!("{}1", "-".repeat(MAX_NESTING_DEPTH + 1)); + assert!(parse_expression(&bad_unary) + .unwrap_err() + .message + .contains("nesting")); + + let deep_list = format!("{}1{}", "[".repeat(1_000), "]".repeat(1_000)); + let err = parse_expression(&deep_list).unwrap_err(); + assert!(err.message.contains("nesting")); + + let deep_record = format!("{}1{}", "{a: ".repeat(1_000), "}".repeat(1_000)); + assert!(parse_expression(&deep_record) + .unwrap_err() + .message + .contains("nesting")); + } + + // ------------------------------------------------------------------ + // parse_arguments() + // ------------------------------------------------------------------ + + #[test] + fn parse_arguments_parses_a_leading_argument_list() { + assert!(parse_arguments("()").unwrap().is_empty()); + + let args = parse_arguments("(r=3, precision=2)").unwrap(); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name.as_deref(), Some("r")); + assert_eq!(sexpr(&args[0].value), "3"); + assert_eq!(args[0].span, Span::new(1, 4)); + assert_eq!(args[1].name.as_deref(), Some("precision")); + assert_eq!(args[1].span, Span::new(6, 17)); + + let mixed = parse_arguments("(1, r=2,)").unwrap(); + assert_eq!(mixed.len(), 2); + assert_eq!(mixed[0].name, None); + assert_eq!(sexpr(&mixed[0].value), "1"); + assert_eq!(mixed[1].name.as_deref(), Some("r")); + assert_eq!(mixed[1].value, Expr::Int(BigInt::from(2))); + } + + #[test] + fn parse_arguments_matches_call_argument_semantics() { + let standalone = parse_arguments("(1, x=2)").unwrap(); + let via_call = match ex("f(1, x=2)") { + Expr::Call { args, .. } => args, + other => panic!("expected call, got {other:?}"), + }; + assert_eq!(standalone.len(), via_call.len()); + assert_eq!(standalone[0].name, via_call[0].name); + assert_eq!(sexpr(&standalone[0].value), sexpr(&via_call[0].value)); + assert_eq!(standalone[1].name, via_call[1].name); + assert_eq!(sexpr(&standalone[1].value), sexpr(&via_call[1].value)); + } + + #[test] + fn parse_arguments_rejects_invalid_lists() { + assert!(parse_arguments("").is_err()); + assert!(parse_arguments("r=3").is_err()); + assert!(parse_arguments("(r=3) junk").is_err()); + assert!(parse_arguments("(1, r=2, 3)").is_err()); + assert!(parse_arguments("(").is_err()); + assert!(parse_arguments("()extra").is_err()); + assert!(parse_arguments("(1 2)").is_err()); + let err = parse_arguments("(a=1, a=2)").unwrap_err(); + assert!(err.message.contains("duplicate named argument")); + let err = parse_arguments("(1, r=2, 3)").unwrap_err(); + assert!(err.message.contains("positional arguments")); + } + + // ------------------------------------------------------------------ + // Adversarial inputs must never panic + // ------------------------------------------------------------------ + + #[test] + fn adversarial_inputs_never_panic() { + let inputs = [ + "", + " ", + "\t\n", + "\\", + "\"", + "\u{0}", + "\u{10FFFF}", + "\u{feff}1", + "1.2.3.4", + "((((((((((", + "))))))", + "**", + "//", + "==", + "!", + "not not not", + "{}[]()", + "a..b", + "a[[[", + "[\\u{", + "😀", + "1e", + "0x10", + "...", + "a = = b", + "f(((((((((", + "🎉 + 1", + "{\"a\": }", + "1234567890123456789012345678901234567890.0e-99999", + ]; + for input in inputs { + let _ = parse_expression(input); + let _ = parse_statement(input); + let _ = parse_arguments(input); + } + + // A long flat expression parses without exhausting the stack. + let long = format!("1{}", " + 1".repeat(2_000)); + assert!(parse_expression(&long).is_ok()); + } +}