Implements the first vertical slices of RFC-CM-0001/0002 on top of a113d89:
- P1: opaque Memory regions (64 KiB pages, budgets, grow-only), byte
read/write, Memory-index place assignment, wire/host-boundary rejection.
- CM-A04/A05: value-semantics List/Map/Text/Table primitives and the
recoverable Result map convention.
- CM-A06: opaque Resource handles, resource.open/read/write/close, stdout
write, granted project-file read and buffered write, strict fail-closed.
- CM-A07 Compat-A: explicit .mq.md format/utf8/io chain matching builtin
print for the covered value types; builtin remains default.
- P2 fixtures: allocators, list/tree/graph/hash, integer codecs, UTF-8
encode/decode, CSV/JSON subsets, table filter/sort/join, bytecode VM.
- ADRs 0022-0027, spec updates, evidence reports and 68 semantic goldens.
Gate: cargo fmt --check, clippy -D warnings, 407 tests passed.
1501 lines
53 KiB
Rust
1501 lines
53 KiB
Rust
//! Recursive-descent parser for the Marqdo expression grammar v1.
|
|
//!
|
|
//! The parser is a straightforward implementation of `spec/expression.ebnf`:
|
|
//!
|
|
//! | level | operators | associativity |
|
|
//! |---|---|---|
|
|
//! | `or_expr` | `or` | left |
|
|
//! | `and_expr` | `and` | left |
|
|
//! | `compare_expr` | `== != < <= > >=` | non-associative |
|
|
//! | `add_expr` | `+ -` | left |
|
|
//! | `mul_expr` | `* / // %` | left |
|
|
//! | `unary_expr` | `not - +` | prefix, right |
|
|
//! | `power_expr` | `**` | right |
|
|
//! | `postfix_expr` | call, `.name`, `[expr]` | left |
|
|
//!
|
|
//! Nesting is bounded by [`MAX_NESTING_DEPTH`]; the guard is checked before
|
|
//! recursing into any sub-expression so adversarial inputs fail with a
|
|
//! structured [`ParseError`] instead of overflowing the stack.
|
|
|
|
use crate::ast::{
|
|
Arg, AssignTarget, BinaryOp, Expr, Ident, RecordEntry, Statement, StatementKind, UnaryOp,
|
|
};
|
|
use crate::lexer::{self, Token, TokenKind};
|
|
use crate::ParseError;
|
|
use marqdo_span::Span;
|
|
use num_bigint::BigInt;
|
|
|
|
/// Maximum number of nested expression constructs accepted by the parser.
|
|
///
|
|
/// The top-level expression is depth 0; every nested grouping, list/record
|
|
/// element, call argument, index expression, prefix operator and power
|
|
/// exponent adds one. The 129th nested construct is rejected with an
|
|
/// `expr.parse` error.
|
|
pub const MAX_NESTING_DEPTH: usize = 128;
|
|
|
|
pub(crate) fn parse_expression(source: &str) -> Result<Expr, ParseError> {
|
|
let mut parser = Parser::new(source)?;
|
|
if parser.at_eof() {
|
|
return Err(parser.error_here("expected an expression, found end of input"));
|
|
}
|
|
let expr = parser.expression()?;
|
|
parser.expect_end_of_expression()?;
|
|
Ok(expr)
|
|
}
|
|
|
|
pub(crate) fn parse_statement(source: &str) -> Result<Statement, ParseError> {
|
|
let mut parser = Parser::new(source)?;
|
|
if parser.at_eof() {
|
|
return Err(parser.error_here("expected a statement, found end of input"));
|
|
}
|
|
let start = parser.peek().span.start;
|
|
let left = parser.expression()?;
|
|
if matches!(parser.peek().kind, TokenKind::Assign) {
|
|
return parser.assignment(start, left);
|
|
}
|
|
if !parser.at_eof() {
|
|
return Err(parser.trailing_statement_error());
|
|
}
|
|
let end = parser.previous_end();
|
|
Ok(Statement {
|
|
kind: StatementKind::Expr(left),
|
|
span: Span::new(start, end),
|
|
})
|
|
}
|
|
|
|
pub(crate) fn parse_arguments(source: &str) -> Result<Vec<Arg>, ParseError> {
|
|
let mut parser = Parser::new(source)?;
|
|
if parser.at_eof() {
|
|
return Err(
|
|
parser.error_here("expected an argument list starting with `(`, found end of input")
|
|
);
|
|
}
|
|
if !matches!(parser.peek().kind, TokenKind::LParen) {
|
|
return Err(parser.error_here(format!(
|
|
"expected an argument list starting with `(`, found {}",
|
|
parser.describe_current()
|
|
)));
|
|
}
|
|
let args = parser.call_arguments()?;
|
|
if !parser.at_eof() {
|
|
return Err(parser.error_here(format!(
|
|
"unexpected {} after argument list",
|
|
parser.describe_current()
|
|
)));
|
|
}
|
|
Ok(args)
|
|
}
|
|
|
|
struct Parser {
|
|
tokens: Vec<Token>,
|
|
index: usize,
|
|
depth: usize,
|
|
}
|
|
|
|
impl Parser {
|
|
fn new(source: &str) -> Result<Self, ParseError> {
|
|
Ok(Self {
|
|
tokens: lexer::tokenize(source)?,
|
|
index: 0,
|
|
depth: 0,
|
|
})
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Token cursor helpers
|
|
// ------------------------------------------------------------------
|
|
|
|
fn peek(&self) -> &Token {
|
|
// `tokens` always contains at least the EOF token, and `bump` never
|
|
// advances past it, so this index is always in bounds.
|
|
&self.tokens[self.index]
|
|
}
|
|
|
|
fn peek_n(&self, offset: usize) -> Option<&Token> {
|
|
self.tokens.get(self.index + offset)
|
|
}
|
|
|
|
fn bump(&mut self) -> Token {
|
|
let token = self.tokens[self.index].clone();
|
|
if self.index + 1 < self.tokens.len() {
|
|
self.index += 1;
|
|
}
|
|
token
|
|
}
|
|
|
|
fn at_eof(&self) -> bool {
|
|
matches!(self.peek().kind, TokenKind::Eof)
|
|
}
|
|
|
|
fn previous_end(&self) -> usize {
|
|
if self.index == 0 {
|
|
0
|
|
} else {
|
|
self.tokens[self.index - 1].span.end
|
|
}
|
|
}
|
|
|
|
fn error_here(&self, message: impl Into<String>) -> ParseError {
|
|
ParseError::new(self.peek().span, message)
|
|
}
|
|
|
|
fn describe_current(&self) -> String {
|
|
describe_token(&self.peek().kind)
|
|
}
|
|
|
|
fn expect(&mut self, kind: TokenKind, what: &str) -> Result<Token, ParseError> {
|
|
if self.peek().kind == kind {
|
|
Ok(self.bump())
|
|
} else {
|
|
Err(self.error_here(format!(
|
|
"expected {what}, found {}",
|
|
self.describe_current()
|
|
)))
|
|
}
|
|
}
|
|
|
|
fn expect_identifier(&mut self, context: &str) -> Result<Ident, ParseError> {
|
|
match self.peek().kind.clone() {
|
|
TokenKind::Ident(name) => {
|
|
let token = self.bump();
|
|
Ok(Ident {
|
|
name,
|
|
span: token.span,
|
|
})
|
|
}
|
|
_ if is_keyword(&self.peek().kind) => Err(self.error_here(format!(
|
|
"keywords are reserved and cannot be used as identifiers; found {}",
|
|
self.describe_current()
|
|
))),
|
|
_ => Err(self.error_here(format!(
|
|
"expected {context}, found {}",
|
|
self.describe_current()
|
|
))),
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Nesting guard
|
|
// ------------------------------------------------------------------
|
|
|
|
fn nested<T>(
|
|
&mut self,
|
|
parse: impl FnOnce(&mut Self) -> Result<T, ParseError>,
|
|
) -> Result<T, ParseError> {
|
|
self.depth += 1;
|
|
if self.depth > MAX_NESTING_DEPTH {
|
|
self.depth -= 1;
|
|
return Err(self.error_here(format!(
|
|
"expression nesting exceeds the maximum depth of {MAX_NESTING_DEPTH}"
|
|
)));
|
|
}
|
|
let result = parse(self);
|
|
self.depth -= 1;
|
|
result
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Statements
|
|
// ------------------------------------------------------------------
|
|
|
|
fn assignment(&mut self, start: usize, left: Expr) -> Result<Statement, ParseError> {
|
|
let target = self.assign_target(left)?;
|
|
self.bump(); // `=`
|
|
if self.at_eof() {
|
|
return Err(self.error_here("expected an expression after `=`"));
|
|
}
|
|
let value = self.expression()?;
|
|
if !self.at_eof() {
|
|
return Err(if matches!(self.peek().kind, TokenKind::Assign) {
|
|
self.error_here(
|
|
"chained assignment is not supported; assignment is only allowed at \
|
|
statement start",
|
|
)
|
|
} else {
|
|
self.error_here(format!(
|
|
"unexpected {} after assignment value",
|
|
self.describe_current()
|
|
))
|
|
});
|
|
}
|
|
let end = self.previous_end();
|
|
Ok(Statement {
|
|
kind: StatementKind::Assign { target, value },
|
|
span: Span::new(start, end),
|
|
})
|
|
}
|
|
|
|
/// Turn the already-parsed left expression into a writable target.
|
|
///
|
|
/// v1 only writes local slots. Addressable `Memory` adds index targets;
|
|
/// member assignment and every other shape stay hard parse errors so an
|
|
/// invalid executable shape never silently becomes prose (ADR-0022).
|
|
fn assign_target(&self, left: Expr) -> Result<AssignTarget, ParseError> {
|
|
match left {
|
|
Expr::Ident(Ident { name, span }) => Ok(AssignTarget::Local {
|
|
name,
|
|
name_span: span,
|
|
}),
|
|
Expr::Index { base, index, span } => Ok(AssignTarget::Index { base, index, span }),
|
|
Expr::Member { span, .. } => Err(ParseError::new(
|
|
span,
|
|
"member assignment is not supported in v1; only a local name or a \
|
|
Memory index is writable",
|
|
)),
|
|
other => Err(ParseError::new(
|
|
other.span(),
|
|
format!(
|
|
"invalid assignment target: {} is not a writable place; assign a local \
|
|
name or a Memory index",
|
|
describe_expr(&other)
|
|
),
|
|
)),
|
|
}
|
|
}
|
|
|
|
fn trailing_statement_error(&self) -> ParseError {
|
|
self.error_here(format!(
|
|
"unexpected {} after expression",
|
|
self.describe_current()
|
|
))
|
|
}
|
|
|
|
fn expect_end_of_expression(&self) -> Result<(), ParseError> {
|
|
if self.at_eof() {
|
|
Ok(())
|
|
} else if matches!(self.peek().kind, TokenKind::Assign) {
|
|
Err(self.error_here(
|
|
"unexpected `=`; assignment is only supported as a statement, not inside an \
|
|
expression",
|
|
))
|
|
} else {
|
|
Err(self.error_here(format!(
|
|
"unexpected {} after expression",
|
|
self.describe_current()
|
|
)))
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Expression levels, loosest to tightest
|
|
// ------------------------------------------------------------------
|
|
|
|
fn expression(&mut self) -> Result<Expr, ParseError> {
|
|
self.parse_or()
|
|
}
|
|
|
|
fn parse_or(&mut self) -> Result<Expr, ParseError> {
|
|
let start = self.peek().span.start;
|
|
let mut left = self.parse_and()?;
|
|
while matches!(self.peek().kind, TokenKind::Or) {
|
|
self.bump();
|
|
let right = self.parse_and()?;
|
|
let end = self.previous_end();
|
|
left = binary(BinaryOp::Or, left, right, Span::new(start, end));
|
|
}
|
|
Ok(left)
|
|
}
|
|
|
|
fn parse_and(&mut self) -> Result<Expr, ParseError> {
|
|
let start = self.peek().span.start;
|
|
let mut left = self.parse_compare()?;
|
|
while matches!(self.peek().kind, TokenKind::And) {
|
|
self.bump();
|
|
let right = self.parse_compare()?;
|
|
let end = self.previous_end();
|
|
left = binary(BinaryOp::And, left, right, Span::new(start, end));
|
|
}
|
|
Ok(left)
|
|
}
|
|
|
|
fn parse_compare(&mut self) -> Result<Expr, ParseError> {
|
|
let start = self.peek().span.start;
|
|
let left = self.parse_add()?;
|
|
let Some(op) = compare_op(&self.peek().kind) else {
|
|
return Ok(left);
|
|
};
|
|
self.bump();
|
|
let right = self.parse_add()?;
|
|
let end = self.previous_end();
|
|
let expr = binary(op, left, right, Span::new(start, end));
|
|
if compare_op(&self.peek().kind).is_some() {
|
|
return Err(self.error_here(format!(
|
|
"comparison operators are non-associative; found {} after a comparison (join \
|
|
comparisons with `and`)",
|
|
self.describe_current()
|
|
)));
|
|
}
|
|
Ok(expr)
|
|
}
|
|
|
|
fn parse_add(&mut self) -> Result<Expr, ParseError> {
|
|
let start = self.peek().span.start;
|
|
let mut left = self.parse_mul()?;
|
|
loop {
|
|
let op = match self.peek().kind {
|
|
TokenKind::Plus => BinaryOp::Add,
|
|
TokenKind::Minus => BinaryOp::Sub,
|
|
_ => break,
|
|
};
|
|
self.bump();
|
|
let right = self.parse_mul()?;
|
|
let end = self.previous_end();
|
|
left = binary(op, left, right, Span::new(start, end));
|
|
}
|
|
Ok(left)
|
|
}
|
|
|
|
fn parse_mul(&mut self) -> Result<Expr, ParseError> {
|
|
let start = self.peek().span.start;
|
|
let mut left = self.parse_unary()?;
|
|
loop {
|
|
let op = match self.peek().kind {
|
|
TokenKind::Star => BinaryOp::Mul,
|
|
TokenKind::Slash => BinaryOp::Div,
|
|
TokenKind::SlashSlash => BinaryOp::FloorDiv,
|
|
TokenKind::Percent => BinaryOp::Mod,
|
|
_ => break,
|
|
};
|
|
self.bump();
|
|
let right = self.parse_unary()?;
|
|
let end = self.previous_end();
|
|
left = binary(op, left, right, Span::new(start, end));
|
|
}
|
|
Ok(left)
|
|
}
|
|
|
|
fn parse_unary(&mut self) -> Result<Expr, ParseError> {
|
|
let op = match self.peek().kind {
|
|
TokenKind::Not => Some(UnaryOp::Not),
|
|
TokenKind::Minus => Some(UnaryOp::Neg),
|
|
TokenKind::Plus => Some(UnaryOp::Pos),
|
|
_ => None,
|
|
};
|
|
let Some(op) = op else {
|
|
return self.parse_power();
|
|
};
|
|
let start = self.peek().span.start;
|
|
self.bump();
|
|
let operand = self.nested(|parser| parser.parse_unary())?;
|
|
let end = self.previous_end();
|
|
Ok(Expr::Unary {
|
|
op,
|
|
operand: Box::new(operand),
|
|
span: Span::new(start, end),
|
|
})
|
|
}
|
|
|
|
fn parse_power(&mut self) -> Result<Expr, ParseError> {
|
|
let start = self.peek().span.start;
|
|
let base = self.parse_postfix()?;
|
|
if matches!(self.peek().kind, TokenKind::StarStar) {
|
|
self.bump();
|
|
let exponent = self.nested(|parser| parser.parse_unary())?;
|
|
let end = self.previous_end();
|
|
return Ok(binary(BinaryOp::Pow, base, exponent, Span::new(start, end)));
|
|
}
|
|
Ok(base)
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Postfix chain
|
|
// ------------------------------------------------------------------
|
|
|
|
fn parse_postfix(&mut self) -> Result<Expr, ParseError> {
|
|
let start = self.peek().span.start;
|
|
let mut expr = self.parse_primary()?;
|
|
loop {
|
|
if matches!(self.peek().kind, TokenKind::LParen) {
|
|
let args = self.call_arguments()?;
|
|
let end = self.previous_end();
|
|
expr = Expr::Call {
|
|
callee: Box::new(expr),
|
|
args,
|
|
span: Span::new(start, end),
|
|
};
|
|
} else if matches!(self.peek().kind, TokenKind::Dot) {
|
|
self.bump();
|
|
let member = self.expect_identifier("a member name after `.`")?;
|
|
let end = member.span.end;
|
|
expr = Expr::Member {
|
|
base: Box::new(expr),
|
|
name: member.name,
|
|
span: Span::new(start, end),
|
|
};
|
|
} else if matches!(self.peek().kind, TokenKind::LBracket) {
|
|
self.bump();
|
|
let index = self.nested(|parser| parser.expression())?;
|
|
let close = self.expect(TokenKind::RBracket, "`]`")?;
|
|
expr = Expr::Index {
|
|
base: Box::new(expr),
|
|
index: Box::new(index),
|
|
span: Span::new(start, close.span.end),
|
|
};
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
Ok(expr)
|
|
}
|
|
|
|
/// Parse `( [argument {"," argument} [","]] )`.
|
|
///
|
|
/// Shared by call postfixes and by [`crate::parse_arguments`].
|
|
fn call_arguments(&mut self) -> Result<Vec<Arg>, ParseError> {
|
|
self.expect(TokenKind::LParen, "`(`")?;
|
|
let mut args = Vec::new();
|
|
if matches!(self.peek().kind, TokenKind::RParen) {
|
|
self.bump();
|
|
return Ok(args);
|
|
}
|
|
let mut seen_named = false;
|
|
let mut names: Vec<String> = Vec::new();
|
|
loop {
|
|
let start = self.peek().span.start;
|
|
let mut name = None;
|
|
if let Some((candidate, name_span)) = self.peek_named_argument() {
|
|
seen_named = true;
|
|
if names.iter().any(|existing| existing == &candidate) {
|
|
return Err(ParseError::new(
|
|
name_span,
|
|
format!("duplicate named argument `{candidate}`"),
|
|
));
|
|
}
|
|
self.bump(); // identifier
|
|
self.bump(); // `=`
|
|
names.push(candidate.clone());
|
|
name = Some(candidate);
|
|
} else if seen_named {
|
|
return Err(
|
|
self.error_here("positional arguments must all precede named arguments")
|
|
);
|
|
}
|
|
if matches!(self.peek().kind, TokenKind::Comma | TokenKind::RParen) {
|
|
return Err(self.error_here(format!(
|
|
"expected an argument expression, found {}",
|
|
self.describe_current()
|
|
)));
|
|
}
|
|
let value = self.nested(|parser| parser.expression())?;
|
|
let end = self.previous_end();
|
|
args.push(Arg {
|
|
name,
|
|
value,
|
|
span: Span::new(start, end),
|
|
});
|
|
if matches!(self.peek().kind, TokenKind::Comma) {
|
|
self.bump();
|
|
if matches!(self.peek().kind, TokenKind::RParen) {
|
|
self.bump();
|
|
break;
|
|
}
|
|
} else if matches!(self.peek().kind, TokenKind::RParen) {
|
|
self.bump();
|
|
break;
|
|
} else {
|
|
return Err(self.error_here(format!(
|
|
"expected `,` or `)` in argument list, found {}",
|
|
self.describe_current()
|
|
)));
|
|
}
|
|
}
|
|
Ok(args)
|
|
}
|
|
|
|
fn peek_named_argument(&self) -> Option<(String, Span)> {
|
|
let name = match &self.peek().kind {
|
|
TokenKind::Ident(name) => name.clone(),
|
|
_ => return None,
|
|
};
|
|
match self.peek_n(1).map(|token| &token.kind) {
|
|
Some(TokenKind::Assign) => Some((name, self.peek().span)),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Primaries
|
|
// ------------------------------------------------------------------
|
|
|
|
fn parse_primary(&mut self) -> Result<Expr, ParseError> {
|
|
let token = self.peek().clone();
|
|
match token.kind {
|
|
TokenKind::Int(raw) => {
|
|
self.bump();
|
|
let value = BigInt::parse_bytes(raw.as_bytes(), 10)
|
|
.ok_or_else(|| ParseError::new(token.span, "invalid integer literal"))?;
|
|
Ok(Expr::Int(value))
|
|
}
|
|
TokenKind::Float(raw) => {
|
|
self.bump();
|
|
self.float_literal(&raw, token.span)
|
|
}
|
|
TokenKind::Text(text) => {
|
|
self.bump();
|
|
Ok(Expr::Text(text))
|
|
}
|
|
TokenKind::True => {
|
|
self.bump();
|
|
Ok(Expr::Bool(true))
|
|
}
|
|
TokenKind::False => {
|
|
self.bump();
|
|
Ok(Expr::Bool(false))
|
|
}
|
|
TokenKind::NoneKw => {
|
|
self.bump();
|
|
Ok(Expr::None)
|
|
}
|
|
TokenKind::Ident(name) => {
|
|
self.bump();
|
|
Ok(Expr::Ident(Ident {
|
|
name,
|
|
span: token.span,
|
|
}))
|
|
}
|
|
TokenKind::LParen => {
|
|
self.bump();
|
|
let inner = self.nested(|parser| parser.expression())?;
|
|
self.expect(TokenKind::RParen, "`)`")?;
|
|
Ok(inner)
|
|
}
|
|
TokenKind::LBracket => self.list_literal(),
|
|
TokenKind::LBrace => self.record_literal(),
|
|
_ => Err(self.error_here(format!(
|
|
"expected an expression, found {}",
|
|
self.describe_current()
|
|
))),
|
|
}
|
|
}
|
|
|
|
fn float_literal(&self, raw: &str, span: Span) -> Result<Expr, ParseError> {
|
|
let value = raw
|
|
.parse::<f64>()
|
|
.map_err(|_| ParseError::new(span, format!("invalid float literal `{raw}`")))?;
|
|
if !value.is_finite() {
|
|
return Err(ParseError::new(
|
|
span,
|
|
format!(
|
|
"float literal `{raw}` is out of range for IEEE-754 binary64; infinities are \
|
|
only allowed as operation results"
|
|
),
|
|
));
|
|
}
|
|
Ok(Expr::Float(value))
|
|
}
|
|
|
|
fn list_literal(&mut self) -> Result<Expr, ParseError> {
|
|
self.bump(); // `[`
|
|
let mut items = Vec::new();
|
|
if matches!(self.peek().kind, TokenKind::RBracket) {
|
|
self.bump();
|
|
return Ok(Expr::List(items));
|
|
}
|
|
loop {
|
|
let item = self.nested(|parser| parser.expression())?;
|
|
items.push(item);
|
|
if matches!(self.peek().kind, TokenKind::Comma) {
|
|
self.bump();
|
|
if matches!(self.peek().kind, TokenKind::RBracket) {
|
|
self.bump();
|
|
break;
|
|
}
|
|
} else if matches!(self.peek().kind, TokenKind::RBracket) {
|
|
self.bump();
|
|
break;
|
|
} else {
|
|
return Err(self.error_here(format!(
|
|
"expected `,` or `]` in list literal, found {}",
|
|
self.describe_current()
|
|
)));
|
|
}
|
|
}
|
|
Ok(Expr::List(items))
|
|
}
|
|
|
|
fn record_literal(&mut self) -> Result<Expr, ParseError> {
|
|
self.bump(); // `{`
|
|
let mut entries = Vec::new();
|
|
if matches!(self.peek().kind, TokenKind::RBrace) {
|
|
self.bump();
|
|
return Ok(Expr::Record(entries));
|
|
}
|
|
loop {
|
|
let (key, key_span) = self.record_key()?;
|
|
self.expect(TokenKind::Colon, "`:` after a record key")?;
|
|
if matches!(self.peek().kind, TokenKind::RBrace | TokenKind::Comma) {
|
|
return Err(self.error_here(format!(
|
|
"expected a record value expression, found {}",
|
|
self.describe_current()
|
|
)));
|
|
}
|
|
let value = self.nested(|parser| parser.expression())?;
|
|
entries.push(RecordEntry {
|
|
key,
|
|
key_span,
|
|
value,
|
|
});
|
|
if matches!(self.peek().kind, TokenKind::Comma) {
|
|
self.bump();
|
|
if matches!(self.peek().kind, TokenKind::RBrace) {
|
|
self.bump();
|
|
break;
|
|
}
|
|
} else if matches!(self.peek().kind, TokenKind::RBrace) {
|
|
self.bump();
|
|
break;
|
|
} else {
|
|
return Err(self.error_here(format!(
|
|
"expected `,` or `}}` in record literal, found {}",
|
|
self.describe_current()
|
|
)));
|
|
}
|
|
}
|
|
Ok(Expr::Record(entries))
|
|
}
|
|
|
|
fn record_key(&mut self) -> Result<(String, Span), ParseError> {
|
|
match self.peek().kind.clone() {
|
|
TokenKind::Ident(name) => {
|
|
let token = self.bump();
|
|
Ok((name, token.span))
|
|
}
|
|
TokenKind::Text(text) => {
|
|
let token = self.bump();
|
|
Ok((text, token.span))
|
|
}
|
|
_ if is_keyword(&self.peek().kind) => Err(self.error_here(format!(
|
|
"record keys must be identifiers or text literals; {} is a reserved keyword \
|
|
(quote it as a text literal)",
|
|
self.describe_current()
|
|
))),
|
|
_ => Err(self.error_here(format!(
|
|
"expected a record key (identifier or text literal), found {}",
|
|
self.describe_current()
|
|
))),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn binary(op: BinaryOp, left: Expr, right: Expr, span: Span) -> Expr {
|
|
Expr::Binary {
|
|
op,
|
|
left: Box::new(left),
|
|
right: Box::new(right),
|
|
span,
|
|
}
|
|
}
|
|
|
|
fn compare_op(kind: &TokenKind) -> Option<BinaryOp> {
|
|
match kind {
|
|
TokenKind::EqEq => Some(BinaryOp::Eq),
|
|
TokenKind::Ne => Some(BinaryOp::Ne),
|
|
TokenKind::Lt => Some(BinaryOp::Lt),
|
|
TokenKind::Le => Some(BinaryOp::Le),
|
|
TokenKind::Gt => Some(BinaryOp::Gt),
|
|
TokenKind::Ge => Some(BinaryOp::Ge),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn is_keyword(kind: &TokenKind) -> bool {
|
|
matches!(
|
|
kind,
|
|
TokenKind::True
|
|
| TokenKind::False
|
|
| TokenKind::NoneKw
|
|
| TokenKind::And
|
|
| TokenKind::Or
|
|
| TokenKind::Not
|
|
)
|
|
}
|
|
|
|
fn describe_expr(expr: &Expr) -> &'static str {
|
|
match expr {
|
|
Expr::Int(_) => "an integer literal",
|
|
Expr::Float(_) => "a float literal",
|
|
Expr::Text(_) => "a text literal",
|
|
Expr::Bool(_) => "a boolean literal",
|
|
Expr::None => "`None`",
|
|
Expr::Ident(_) => "an identifier",
|
|
Expr::List(_) => "a list literal",
|
|
Expr::Record(_) => "a record literal",
|
|
Expr::Unary { .. } => "a unary expression",
|
|
Expr::Binary { .. } => "a binary expression",
|
|
Expr::Call { .. } => "a call expression",
|
|
Expr::Member { .. } => "a member access",
|
|
Expr::Index { .. } => "an index expression",
|
|
}
|
|
}
|
|
|
|
fn describe_token(kind: &TokenKind) -> String {
|
|
match kind {
|
|
TokenKind::Int(raw) => format!("integer literal `{raw}`"),
|
|
TokenKind::Float(raw) => format!("float literal `{raw}`"),
|
|
TokenKind::Text(_) => "text literal".to_owned(),
|
|
TokenKind::Ident(name) => format!("identifier `{name}`"),
|
|
TokenKind::True => "`true`".to_owned(),
|
|
TokenKind::False => "`false`".to_owned(),
|
|
TokenKind::NoneKw => "`None`".to_owned(),
|
|
TokenKind::And => "`and`".to_owned(),
|
|
TokenKind::Or => "`or`".to_owned(),
|
|
TokenKind::Not => "`not`".to_owned(),
|
|
TokenKind::LParen => "`(`".to_owned(),
|
|
TokenKind::RParen => "`)`".to_owned(),
|
|
TokenKind::LBracket => "`[`".to_owned(),
|
|
TokenKind::RBracket => "`]`".to_owned(),
|
|
TokenKind::LBrace => "`{`".to_owned(),
|
|
TokenKind::RBrace => "`}`".to_owned(),
|
|
TokenKind::Comma => "`,`".to_owned(),
|
|
TokenKind::Colon => "`:`".to_owned(),
|
|
TokenKind::Dot => "`.`".to_owned(),
|
|
TokenKind::Assign => "`=`".to_owned(),
|
|
TokenKind::EqEq => "`==`".to_owned(),
|
|
TokenKind::Ne => "`!=`".to_owned(),
|
|
TokenKind::Lt => "`<`".to_owned(),
|
|
TokenKind::Le => "`<=`".to_owned(),
|
|
TokenKind::Gt => "`>`".to_owned(),
|
|
TokenKind::Ge => "`>=`".to_owned(),
|
|
TokenKind::Plus => "`+`".to_owned(),
|
|
TokenKind::Minus => "`-`".to_owned(),
|
|
TokenKind::Star => "`*`".to_owned(),
|
|
TokenKind::StarStar => "`**`".to_owned(),
|
|
TokenKind::Slash => "`/`".to_owned(),
|
|
TokenKind::SlashSlash => "`//`".to_owned(),
|
|
TokenKind::Percent => "`%`".to_owned(),
|
|
TokenKind::Eof => "end of input".to_owned(),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::{parse_arguments, parse_expression, parse_statement, ParseError};
|
|
use std::str::FromStr;
|
|
|
|
// ------------------------------------------------------------------
|
|
// Helpers
|
|
// ------------------------------------------------------------------
|
|
|
|
fn ex(source: &str) -> Expr {
|
|
parse_expression(source)
|
|
.unwrap_or_else(|err| panic!("{source:?} should parse but failed: {err}"))
|
|
}
|
|
|
|
fn ex_err(source: &str) -> ParseError {
|
|
match parse_expression(source) {
|
|
Ok(expr) => panic!("{source:?} should fail but parsed as {expr:?}"),
|
|
Err(err) => err,
|
|
}
|
|
}
|
|
|
|
fn st(source: &str) -> Statement {
|
|
parse_statement(source)
|
|
.unwrap_or_else(|err| panic!("{source:?} should parse as a statement: {err}"))
|
|
}
|
|
|
|
fn st_err(source: &str) -> ParseError {
|
|
match parse_statement(source) {
|
|
Ok(stmt) => panic!("{source:?} should fail but parsed as {stmt:?}"),
|
|
Err(err) => err,
|
|
}
|
|
}
|
|
|
|
/// Compact span-insensitive rendering, used to assert tree shape.
|
|
fn sx(source: &str) -> String {
|
|
sexpr(&ex(source))
|
|
}
|
|
|
|
fn sexpr(expr: &Expr) -> String {
|
|
match expr {
|
|
Expr::Int(value) => value.to_string(),
|
|
Expr::Float(value) => format!("{value:?}"),
|
|
Expr::Text(value) => format!("{value:?}"),
|
|
Expr::Bool(value) => value.to_string(),
|
|
Expr::None => "None".to_owned(),
|
|
Expr::Ident(ident) => ident.name.clone(),
|
|
Expr::List(items) => {
|
|
let mut out = String::from("(list");
|
|
for item in items {
|
|
out.push(' ');
|
|
out.push_str(&sexpr(item));
|
|
}
|
|
out.push(')');
|
|
out
|
|
}
|
|
Expr::Record(entries) => {
|
|
let mut out = String::from("(record");
|
|
for entry in entries {
|
|
out.push_str(&format!(" {}: {}", entry.key, sexpr(&entry.value)));
|
|
}
|
|
out.push(')');
|
|
out
|
|
}
|
|
Expr::Unary { op, operand, .. } => {
|
|
format!("({} {})", unary_symbol(*op), sexpr(operand))
|
|
}
|
|
Expr::Binary {
|
|
op, left, right, ..
|
|
} => format!("({} {} {})", binary_symbol(*op), sexpr(left), sexpr(right)),
|
|
Expr::Call { callee, args, .. } => {
|
|
let mut out = format!("(call {}", sexpr(callee));
|
|
for arg in args {
|
|
match &arg.name {
|
|
Some(name) => out.push_str(&format!(" {name}={}", sexpr(&arg.value))),
|
|
None => {
|
|
out.push(' ');
|
|
out.push_str(&sexpr(&arg.value));
|
|
}
|
|
}
|
|
}
|
|
out.push(')');
|
|
out
|
|
}
|
|
Expr::Member { base, name, .. } => format!("(. {} {name})", sexpr(base)),
|
|
Expr::Index { base, index, .. } => {
|
|
format!("(index {} {})", sexpr(base), sexpr(index))
|
|
}
|
|
}
|
|
}
|
|
|
|
fn binary_symbol(op: BinaryOp) -> &'static str {
|
|
match op {
|
|
BinaryOp::Or => "or",
|
|
BinaryOp::And => "and",
|
|
BinaryOp::Eq => "==",
|
|
BinaryOp::Ne => "!=",
|
|
BinaryOp::Lt => "<",
|
|
BinaryOp::Le => "<=",
|
|
BinaryOp::Gt => ">",
|
|
BinaryOp::Ge => ">=",
|
|
BinaryOp::Add => "+",
|
|
BinaryOp::Sub => "-",
|
|
BinaryOp::Mul => "*",
|
|
BinaryOp::Div => "/",
|
|
BinaryOp::FloorDiv => "//",
|
|
BinaryOp::Mod => "%",
|
|
BinaryOp::Pow => "**",
|
|
}
|
|
}
|
|
|
|
fn unary_symbol(op: UnaryOp) -> &'static str {
|
|
match op {
|
|
UnaryOp::Not => "not",
|
|
UnaryOp::Neg => "neg",
|
|
UnaryOp::Pos => "pos",
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Literals
|
|
// ------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn integer_literals_are_arbitrary_precision() {
|
|
assert_eq!(ex("0"), Expr::Int(BigInt::from(0)));
|
|
assert_eq!(ex("007"), Expr::Int(BigInt::from(7)));
|
|
assert_eq!(
|
|
ex("9223372036854775808"),
|
|
Expr::Int(BigInt::from(9_223_372_036_854_775_808u64))
|
|
);
|
|
let huge = "123456789012345678901234567890123456789012345678901234567890";
|
|
assert_eq!(ex(huge), Expr::Int(BigInt::from_str(huge).unwrap()));
|
|
assert_eq!(sx("-9223372036854775809"), "(neg 9223372036854775809)");
|
|
}
|
|
|
|
#[test]
|
|
fn float_literals_cover_the_grammar() {
|
|
assert_eq!(ex("1.5"), Expr::Float(1.5));
|
|
assert_eq!(ex("0.0"), Expr::Float(0.0));
|
|
assert_eq!(ex("1.5e3"), Expr::Float(1500.0));
|
|
assert_eq!(ex("1.25E-2"), Expr::Float(0.0125));
|
|
assert_eq!(ex("0.5e+2"), Expr::Float(50.0));
|
|
assert_eq!(
|
|
ex("-2.5"),
|
|
Expr::Unary {
|
|
op: UnaryOp::Neg,
|
|
operand: Box::new(Expr::Float(2.5)),
|
|
span: Span::new(0, 4),
|
|
}
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_float_literals_are_rejected() {
|
|
for bad in [
|
|
".5", "1.", "1e5", "1.2.3", "1.5e", "1.5e+", "1_000", "1.0e999",
|
|
] {
|
|
let err = ex_err(bad);
|
|
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
|
assert!(!err.message.is_empty(), "input {bad:?}");
|
|
}
|
|
// `.5` gets the specific "must be preceded" diagnostic from the lexer.
|
|
assert!(ex_err(".5").message.contains("preceded"));
|
|
assert!(ex_err("1.0e999").message.contains("out of range"));
|
|
}
|
|
|
|
#[test]
|
|
fn text_literals_decode_escapes() {
|
|
assert_eq!(ex(r#""hello""#), Expr::Text("hello".to_owned()));
|
|
assert_eq!(ex(r#""""#), Expr::Text(String::new()));
|
|
assert_eq!(
|
|
ex(r#""a\\b\"c\n\t\r\u{4e2d}""#),
|
|
Expr::Text("a\\b\"c\n\t\r中".to_owned())
|
|
);
|
|
assert_eq!(ex(r#""\u{4E2D}\u{1F600}""#), Expr::Text("中😀".to_owned()));
|
|
// A raw newline is a valid text character inside a literal.
|
|
assert_eq!(ex("\"a\nb\""), Expr::Text("a\nb".to_owned()));
|
|
assert_eq!(ex(r#""{}()[],:.""#), Expr::Text("{}()[],:.".to_owned()));
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_text_literals_are_rejected() {
|
|
for bad in [
|
|
r#""unterminated"#,
|
|
r#""bad \q escape""#,
|
|
r#""bad \u{} escape""#,
|
|
r#""bad \u{D800} escape""#,
|
|
r#""bad \u{110000} escape""#,
|
|
r#""bad \u12 escape""#,
|
|
] {
|
|
let err = ex_err(bad);
|
|
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
|
}
|
|
// `\\` at the end of the input is an unterminated literal.
|
|
let err = parse_expression("\"abc\\").unwrap_err();
|
|
assert!(err.message.contains("unterminated"));
|
|
}
|
|
|
|
#[test]
|
|
fn boolean_and_none_literals_parse() {
|
|
assert_eq!(ex("true"), Expr::Bool(true));
|
|
assert_eq!(ex("false"), Expr::Bool(false));
|
|
assert_eq!(ex("None"), Expr::None);
|
|
// `truex` is an identifier, not the keyword `true` plus `x`.
|
|
assert_eq!(sx("truex"), "truex");
|
|
assert_eq!(sx("None_"), "None_");
|
|
}
|
|
|
|
#[test]
|
|
fn unicode_identifiers_are_accepted() {
|
|
assert_eq!(sx("阶乘"), "阶乘");
|
|
assert_eq!(sx("计算_面积2"), "计算_面积2");
|
|
assert_eq!(sx("café"), "café");
|
|
assert!(matches!(ex("π"), Expr::Ident(_)));
|
|
// U+005F `_` is not XID_Start, so it cannot start an identifier.
|
|
let _ = st("变量 = 1").kind;
|
|
for bad in ["_x", "__", "1abc"] {
|
|
assert_eq!(ex_err(bad).code, "expr.parse", "input {bad:?}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn reserved_keywords_cannot_be_identifiers() {
|
|
for bad in ["and", "or", "not", "true = 1"] {
|
|
let err = ex_err(bad);
|
|
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
|
}
|
|
assert!(ex_err("a.and").message.contains("reserved"));
|
|
assert!(ex_err("a.true").message.contains("reserved"));
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Operator precedence and associativity
|
|
// ------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn additive_and_multiplicative_precedence() {
|
|
assert_eq!(sx("1 + 2 * 3"), "(+ 1 (* 2 3))");
|
|
assert_eq!(sx("(1 + 2) * 3"), "(* (+ 1 2) 3)");
|
|
assert_eq!(sx("1 * 2 + 3"), "(+ (* 1 2) 3)");
|
|
assert_eq!(sx("1 - 2 - 3"), "(- (- 1 2) 3)");
|
|
assert_eq!(sx("8 / 4 // 2 % 3"), "(% (// (/ 8 4) 2) 3)");
|
|
assert_eq!(sx("1 + 2 + 3"), "(+ (+ 1 2) 3)");
|
|
}
|
|
|
|
#[test]
|
|
fn power_is_right_associative_and_binds_tighter_than_unary_minus() {
|
|
assert_eq!(sx("2 ** 3 ** 2"), "(** 2 (** 3 2))");
|
|
assert_eq!(sx("-x ** 2"), "(neg (** x 2))");
|
|
assert_eq!(sx("(-x) ** 2"), "(** (neg x) 2)");
|
|
assert_eq!(sx("2 ** -1"), "(** 2 (neg 1))");
|
|
assert_eq!(sx("2 ** -1 ** 2"), "(** 2 (neg (** 1 2)))");
|
|
assert_eq!(sx("+x ** 2"), "(pos (** x 2))");
|
|
assert_eq!(sx("a * b ** c * d"), "(* (* a (** b c)) d)");
|
|
}
|
|
|
|
#[test]
|
|
fn unary_operators_stack() {
|
|
assert_eq!(sx("--1"), "(neg (neg 1))");
|
|
assert_eq!(sx("not not true"), "(not (not true))");
|
|
assert_eq!(sx("+1"), "(pos 1)");
|
|
assert_eq!(sx("-+1"), "(neg (pos 1))");
|
|
assert_eq!(sx("not a"), "(not a)");
|
|
assert!(ex_err("not").message.contains("expected an expression"));
|
|
assert!(ex_err("!a").message.contains("`not`"));
|
|
}
|
|
|
|
#[test]
|
|
fn comparisons_bind_looser_than_arithmetic() {
|
|
assert_eq!(sx("a < b + c"), "(< a (+ b c))");
|
|
assert_eq!(sx("a + b < c * d"), "(< (+ a b) (* c d))");
|
|
assert_eq!(sx("a == b"), "(== a b)");
|
|
assert_eq!(sx("a != b"), "(!= a b)");
|
|
assert_eq!(sx("a <= b"), "(<= a b)");
|
|
assert_eq!(sx("a > b"), "(> a b)");
|
|
assert_eq!(sx("a >= b"), "(>= a b)");
|
|
// `not` binds tighter than comparison.
|
|
assert_eq!(sx("not a == b"), "(== (not a) b)");
|
|
assert_eq!(sx("-a < b"), "(< (neg a) b)");
|
|
}
|
|
|
|
#[test]
|
|
fn comparisons_are_non_associative() {
|
|
let err = ex_err("a < b < c");
|
|
assert!(err.message.contains("non-associative"));
|
|
assert_eq!(err.span, Span::new(6, 7));
|
|
assert!(ex_err("a == b == c").message.contains("non-associative"));
|
|
assert!(ex_err("a <= b > c").message.contains("non-associative"));
|
|
// The documented workaround parses.
|
|
assert_eq!(sx("a < b and b < c"), "(and (< a b) (< b c))");
|
|
}
|
|
|
|
#[test]
|
|
fn logical_operators_chain_left_associatively() {
|
|
assert_eq!(sx("a and b or c"), "(or (and a b) c)");
|
|
assert_eq!(sx("a or b and c"), "(or a (and b c))");
|
|
assert_eq!(sx("a or b or c"), "(or (or a b) c)");
|
|
assert_eq!(sx("a and b and c"), "(and (and a b) c)");
|
|
assert_eq!(sx("not a and not b"), "(and (not a) (not b))");
|
|
assert_eq!(sx("a and b == c"), "(and a (== b c))");
|
|
assert_eq!(sx("a == b and c != d"), "(and (== a b) (!= c d))");
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Calls, members, indexes
|
|
// ------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn calls_parse_positional_and_named_arguments() {
|
|
assert_eq!(sx("f()"), "(call f)");
|
|
assert_eq!(sx("f(1, 2)"), "(call f 1 2)");
|
|
assert_eq!(sx("f(1,)"), "(call f 1)");
|
|
assert_eq!(sx("f(x=1)"), "(call f x=1)");
|
|
assert_eq!(sx("f(1, y=2,)"), "(call f 1 y=2)");
|
|
assert_eq!(sx("f(g(1), h(x=2))"), "(call f (call g 1) (call h x=2))");
|
|
assert_eq!(sx(r#"打印("你好")"#), r#"(call 打印 "你好")"#);
|
|
}
|
|
|
|
#[test]
|
|
fn call_argument_errors_are_rejected() {
|
|
let err = ex_err("f(a=1, 2)");
|
|
assert!(err.message.contains("positional arguments"));
|
|
let err = ex_err("f(a=1, a=2)");
|
|
assert!(err.message.contains("duplicate named argument"));
|
|
for bad in ["f(,)", "f(1,,2)", "f(1 2)", "f(", "f(x=)", "f(a=1, b)"] {
|
|
let err = ex_err(bad);
|
|
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn member_and_index_access_chain() {
|
|
assert_eq!(sx("a.b"), "(. a b)");
|
|
assert_eq!(sx("a.b.c"), "(. (. a b) c)");
|
|
assert_eq!(sx("a.b[0].c(d)"), "(call (. (index (. a b) 0) c) d)");
|
|
assert_eq!(sx("a[0][1]"), "(index (index a 0) 1)");
|
|
assert_eq!(sx("a[1 + 2]"), "(index a (+ 1 2))");
|
|
assert_eq!(sx("f(1).name"), "(. (call f 1) name)");
|
|
}
|
|
|
|
#[test]
|
|
fn member_and_index_errors_are_rejected() {
|
|
for bad in ["a.", "a[]", "a[1", "a[1,2]", "a.1", "a[1]]"] {
|
|
let err = ex_err(bad);
|
|
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
|
}
|
|
assert!(ex_err("a.true").message.contains("reserved"));
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Collection literals
|
|
// ------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn list_literals_parse() {
|
|
assert_eq!(sx("[]"), "(list)");
|
|
assert_eq!(sx("[1]"), "(list 1)");
|
|
assert_eq!(sx("[1, 2, 3]"), "(list 1 2 3)");
|
|
assert_eq!(sx("[1,]"), "(list 1)");
|
|
assert_eq!(sx("[a, [b, c]]"), "(list a (list b c))");
|
|
assert_eq!(sx(r#"["文本", true, None]"#), r#"(list "文本" true None)"#);
|
|
}
|
|
|
|
#[test]
|
|
fn list_literal_errors_are_rejected() {
|
|
for bad in ["[1 2]", "[,]", "[1,,2]", "[1", "[1, ", "[1; 2]"] {
|
|
let err = ex_err(bad);
|
|
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn record_literals_parse() {
|
|
assert_eq!(sx("{}"), "(record)");
|
|
assert_eq!(sx("{a: 1}"), "(record a: 1)");
|
|
assert_eq!(sx("{a: 1, b: 2,}"), "(record a: 1 b: 2)");
|
|
assert_eq!(sx(r#"{"键": "值"}"#), r#"(record 键: "值")"#);
|
|
assert_eq!(sx("{a: {b: [1]}}"), "(record a: (record b: (list 1)))");
|
|
let expr = ex("{答案: 42}");
|
|
match expr {
|
|
Expr::Record(entries) => {
|
|
assert_eq!(entries.len(), 1);
|
|
assert_eq!(entries[0].key, "答案");
|
|
assert_eq!(entries[0].key_span, Span::new(1, 7));
|
|
}
|
|
other => panic!("expected a record, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn record_literal_errors_are_rejected() {
|
|
for bad in ["{a 1}", "{true: 1}", "{a: }", "{1: 2}", "{a: 1,,}", "{"] {
|
|
let err = ex_err(bad);
|
|
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
|
}
|
|
let err = ex_err("{true: 1}");
|
|
assert!(err.message.contains("reserved keyword"));
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Statements
|
|
// ------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn assignment_statements_parse() {
|
|
let stmt = st("x = 1");
|
|
assert_eq!(
|
|
stmt.kind,
|
|
StatementKind::Assign {
|
|
target: AssignTarget::Local {
|
|
name: "x".to_owned(),
|
|
name_span: Span::new(0, 1),
|
|
},
|
|
value: Expr::Int(BigInt::from(1)),
|
|
}
|
|
);
|
|
let stmt = st("结果 = 阶乘(n=3)");
|
|
match &stmt.kind {
|
|
StatementKind::Assign {
|
|
target: AssignTarget::Local { name, .. },
|
|
value,
|
|
} => {
|
|
assert_eq!(name, "结果");
|
|
assert_eq!(sexpr(value), "(call 阶乘 n=3)");
|
|
}
|
|
other => panic!("expected assignment, got {other:?}"),
|
|
}
|
|
assert_eq!(st("x = y + 1").span(), Span::new(0, 9));
|
|
}
|
|
|
|
#[test]
|
|
fn memory_index_assignment_parses_as_a_place() {
|
|
let stmt = st("m[8] = 42");
|
|
let StatementKind::Assign { target, value } = &stmt.kind else {
|
|
panic!("expected assignment, got {:?}", stmt.kind);
|
|
};
|
|
let AssignTarget::Index { base, index, span } = target else {
|
|
panic!("expected index target, got {target:?}");
|
|
};
|
|
assert_eq!(
|
|
**base,
|
|
Expr::Ident(Ident {
|
|
name: "m".to_owned(),
|
|
span: Span::new(0, 1),
|
|
})
|
|
);
|
|
assert_eq!(**index, Expr::Int(BigInt::from(8)));
|
|
assert_eq!(*span, Span::new(0, 4));
|
|
assert_eq!(*value, Expr::Int(BigInt::from(42)));
|
|
assert_eq!(stmt.span, Span::new(0, 9));
|
|
|
|
// Nested targets are syntactically places; the runtime decides whether
|
|
// the base value is a writable Memory region.
|
|
let stmt = st("outer[0][1] = 2");
|
|
let StatementKind::Assign {
|
|
target: AssignTarget::Index { base, .. },
|
|
..
|
|
} = &stmt.kind
|
|
else {
|
|
panic!("expected nested index target, got {:?}", stmt.kind);
|
|
};
|
|
assert!(matches!(base.as_ref(), Expr::Index { .. }));
|
|
}
|
|
|
|
#[test]
|
|
fn expression_statements_parse() {
|
|
assert_eq!(st("打印(1)").kind, StatementKind::Expr(ex("打印(1)")));
|
|
assert_eq!(st("x").kind, StatementKind::Expr(ex("x")));
|
|
assert_eq!(st("x == 1").kind, StatementKind::Expr(ex("x == 1")));
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_statements_are_rejected() {
|
|
for bad in [
|
|
"",
|
|
" ",
|
|
"x =",
|
|
"x = 1 = 2",
|
|
"a.b = 1",
|
|
"1 = 2",
|
|
"f() = 1",
|
|
"= 1",
|
|
"x = 1 2",
|
|
] {
|
|
let err = st_err(bad);
|
|
assert_eq!(err.code, "expr.parse", "input {bad:?}");
|
|
}
|
|
assert!(st_err("x = 1 = 2").message.contains("chained assignment"));
|
|
assert!(st_err("a.b = 1").message.contains("member assignment"));
|
|
assert!(st_err("1 = 2").message.contains("writable place"));
|
|
assert!(st_err("x =").message.contains("after `=`"));
|
|
}
|
|
|
|
#[test]
|
|
fn expressions_reject_assignment_syntax() {
|
|
assert!(ex_err("x = 1")
|
|
.message
|
|
.contains("assignment is only supported as a statement"));
|
|
}
|
|
|
|
#[test]
|
|
fn comments_are_not_supported() {
|
|
for bad in ["# comment", "1 # 2", "f() # tail"] {
|
|
let err = ex_err(bad);
|
|
assert!(err.message.contains("comments"), "input {bad:?}");
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Spans
|
|
// ------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn spans_are_byte_offsets_into_the_payload() {
|
|
let expr = ex("1 + 2");
|
|
let Expr::Binary { span, .. } = expr else {
|
|
panic!("expected binary");
|
|
};
|
|
assert_eq!(span, Span::new(0, 5));
|
|
|
|
let stmt = st(" x = 1 ");
|
|
assert_eq!(stmt.span(), Span::new(2, 7));
|
|
match stmt.kind {
|
|
StatementKind::Assign {
|
|
target: AssignTarget::Local { name_span, .. },
|
|
..
|
|
} => assert_eq!(name_span, Span::new(2, 3)),
|
|
other => panic!("expected assignment, got {other:?}"),
|
|
}
|
|
|
|
let Expr::Call {
|
|
callee, span, args, ..
|
|
} = ex("f(1)")
|
|
else {
|
|
panic!("expected call");
|
|
};
|
|
assert_eq!(span, Span::new(0, 4));
|
|
assert_eq!(callee.span(), Span::new(0, 1));
|
|
assert_eq!(args[0].span, Span::new(2, 3));
|
|
|
|
let Expr::Member { span, .. } = ex("a.b") else {
|
|
panic!("expected member");
|
|
};
|
|
assert_eq!(span, Span::new(0, 3));
|
|
|
|
let Expr::Index { span, .. } = ex("a[0]") else {
|
|
panic!("expected index");
|
|
};
|
|
assert_eq!(span, Span::new(0, 4));
|
|
|
|
let Expr::Unary { span, .. } = ex("-x") else {
|
|
panic!("expected unary");
|
|
};
|
|
assert_eq!(span, Span::new(0, 2));
|
|
|
|
let Expr::Binary { span, .. } = ex("2 ** 3") else {
|
|
panic!("expected power");
|
|
};
|
|
assert_eq!(span, Span::new(0, 6));
|
|
|
|
let Expr::Binary { span, .. } = ex("(1) + 2") else {
|
|
panic!("expected binary");
|
|
};
|
|
assert_eq!(span, Span::new(0, 7));
|
|
|
|
// Multibyte payloads use byte offsets (张三 is three bytes per scalar).
|
|
let Expr::Binary { span, .. } = ex("张三 + 1") else {
|
|
panic!("expected binary");
|
|
};
|
|
assert_eq!(span, Span::new(0, 10));
|
|
|
|
// Literal variants have no stored span in the frozen AST shape.
|
|
assert_eq!(ex("42").span(), Span::default());
|
|
assert_eq!(ex("[1]").span(), Span::default());
|
|
assert_eq!(ex("{a: 1}").span(), Span::default());
|
|
assert_eq!(ex(r#""s""#).span(), Span::default());
|
|
}
|
|
|
|
#[test]
|
|
fn error_spans_point_at_the_offending_token() {
|
|
assert_eq!(ex_err("1 2").span, Span::new(2, 3));
|
|
assert_eq!(ex_err("f(1 2)").span, Span::new(4, 5));
|
|
assert_eq!(ex_err("1 +").span, Span::empty(3));
|
|
assert_eq!(ex_err("f(").span, Span::empty(2));
|
|
assert_eq!(ex_err("a < b < c").span, Span::new(6, 7));
|
|
let err = ex_err("f(a=1, a=2)");
|
|
assert_eq!(err.span, Span::new(7, 8)); // span of the duplicate name token
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Nesting limit
|
|
// ------------------------------------------------------------------
|
|
|
|
fn nested_parens(depth: usize) -> String {
|
|
format!("{}1{}", "(".repeat(depth), ")".repeat(depth))
|
|
}
|
|
|
|
#[test]
|
|
fn nesting_depth_limit_is_enforced() {
|
|
assert!(parse_expression(&nested_parens(MAX_NESTING_DEPTH)).is_ok());
|
|
let err = parse_expression(&nested_parens(MAX_NESTING_DEPTH + 1)).unwrap_err();
|
|
assert_eq!(err.code, "expr.parse");
|
|
assert!(err.message.contains("nesting"));
|
|
assert_eq!(
|
|
err.span,
|
|
Span::new(MAX_NESTING_DEPTH + 1, MAX_NESTING_DEPTH + 2)
|
|
);
|
|
|
|
// Each prefix operator nests its operand, so exactly
|
|
// MAX_NESTING_DEPTH prefix operators are accepted.
|
|
let ok_unary = format!("{}1", "-".repeat(MAX_NESTING_DEPTH));
|
|
assert!(parse_expression(&ok_unary).is_ok());
|
|
let bad_unary = format!("{}1", "-".repeat(MAX_NESTING_DEPTH + 1));
|
|
assert!(parse_expression(&bad_unary)
|
|
.unwrap_err()
|
|
.message
|
|
.contains("nesting"));
|
|
|
|
let deep_list = format!("{}1{}", "[".repeat(1_000), "]".repeat(1_000));
|
|
let err = parse_expression(&deep_list).unwrap_err();
|
|
assert!(err.message.contains("nesting"));
|
|
|
|
let deep_record = format!("{}1{}", "{a: ".repeat(1_000), "}".repeat(1_000));
|
|
assert!(parse_expression(&deep_record)
|
|
.unwrap_err()
|
|
.message
|
|
.contains("nesting"));
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// parse_arguments()
|
|
// ------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn parse_arguments_parses_a_leading_argument_list() {
|
|
assert!(parse_arguments("()").unwrap().is_empty());
|
|
|
|
let args = parse_arguments("(r=3, precision=2)").unwrap();
|
|
assert_eq!(args.len(), 2);
|
|
assert_eq!(args[0].name.as_deref(), Some("r"));
|
|
assert_eq!(sexpr(&args[0].value), "3");
|
|
assert_eq!(args[0].span, Span::new(1, 4));
|
|
assert_eq!(args[1].name.as_deref(), Some("precision"));
|
|
assert_eq!(args[1].span, Span::new(6, 17));
|
|
|
|
let mixed = parse_arguments("(1, r=2,)").unwrap();
|
|
assert_eq!(mixed.len(), 2);
|
|
assert_eq!(mixed[0].name, None);
|
|
assert_eq!(sexpr(&mixed[0].value), "1");
|
|
assert_eq!(mixed[1].name.as_deref(), Some("r"));
|
|
assert_eq!(mixed[1].value, Expr::Int(BigInt::from(2)));
|
|
}
|
|
|
|
#[test]
|
|
fn parse_arguments_matches_call_argument_semantics() {
|
|
let standalone = parse_arguments("(1, x=2)").unwrap();
|
|
let via_call = match ex("f(1, x=2)") {
|
|
Expr::Call { args, .. } => args,
|
|
other => panic!("expected call, got {other:?}"),
|
|
};
|
|
assert_eq!(standalone.len(), via_call.len());
|
|
assert_eq!(standalone[0].name, via_call[0].name);
|
|
assert_eq!(sexpr(&standalone[0].value), sexpr(&via_call[0].value));
|
|
assert_eq!(standalone[1].name, via_call[1].name);
|
|
assert_eq!(sexpr(&standalone[1].value), sexpr(&via_call[1].value));
|
|
}
|
|
|
|
#[test]
|
|
fn parse_arguments_rejects_invalid_lists() {
|
|
assert!(parse_arguments("").is_err());
|
|
assert!(parse_arguments("r=3").is_err());
|
|
assert!(parse_arguments("(r=3) junk").is_err());
|
|
assert!(parse_arguments("(1, r=2, 3)").is_err());
|
|
assert!(parse_arguments("(").is_err());
|
|
assert!(parse_arguments("()extra").is_err());
|
|
assert!(parse_arguments("(1 2)").is_err());
|
|
let err = parse_arguments("(a=1, a=2)").unwrap_err();
|
|
assert!(err.message.contains("duplicate named argument"));
|
|
let err = parse_arguments("(1, r=2, 3)").unwrap_err();
|
|
assert!(err.message.contains("positional arguments"));
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Adversarial inputs must never panic
|
|
// ------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn adversarial_inputs_never_panic() {
|
|
let inputs = [
|
|
"",
|
|
" ",
|
|
"\t\n",
|
|
"\\",
|
|
"\"",
|
|
"\u{0}",
|
|
"\u{10FFFF}",
|
|
"\u{feff}1",
|
|
"1.2.3.4",
|
|
"((((((((((",
|
|
"))))))",
|
|
"**",
|
|
"//",
|
|
"==",
|
|
"!",
|
|
"not not not",
|
|
"{}[]()",
|
|
"a..b",
|
|
"a[[[",
|
|
"[\\u{",
|
|
"😀",
|
|
"1e",
|
|
"0x10",
|
|
"...",
|
|
"a = = b",
|
|
"f(((((((((",
|
|
"🎉 + 1",
|
|
"{\"a\": }",
|
|
"1234567890123456789012345678901234567890.0e-99999",
|
|
];
|
|
for input in inputs {
|
|
let _ = parse_expression(input);
|
|
let _ = parse_statement(input);
|
|
let _ = parse_arguments(input);
|
|
}
|
|
|
|
// A long flat expression parses without exhausting the stack.
|
|
let long = format!("1{}", " + 1".repeat(2_000));
|
|
assert!(parse_expression(&long).is_ok());
|
|
}
|
|
}
|