refactor(expression): own the v2 payload statement grammar

Move keyword recognition out of the Markdown mapper and into the single
expression/statement parser:

- add parse_payload_statement + PayloadStatement for if/else/while/end/
  return/params/ordinary; keyword matching uses complete tokens, requires
  whitespace before a control expression, rejects trailing tokens/colons
  and reserves the statement words in expression positions
- keep expression spans payload-relative so the frontend can rewrite them
  onto the file; malformed params(...) interiors stay owned by the
  signature parser (one sem.syntax diagnostic, not a duplicate)
- simplify the frontend mapper to control-stack lowering and mapped
  semantic diagnostics only
- emit sem.no_entry_function from run when no Exec H2 main exists
- add else-under-while, params-in-quote, Knowledge-inertness and
  no-entry tests; update the corpus counts
- make the EBNF parameter type annotation optional (params(n) is valid)

Verified: cargo fmt --check, cargo clippy -D warnings and
cargo test --workspace --locked: 501 passed, 0 failed, 2 ignored.
This commit is contained in:
cflmy committed 2026-11-21 12:29:07 +08:00
1 parent e029853e32
commit ffd1cc9862
14 files changed
+334 -145

No files matched your search

+12 -4
View File
@@ -189,10 +189,18 @@ pub fn execute(
let function_name = function.unwrap_or("main").to_string(); let function_name = function.unwrap_or("main").to_string();
let entry_module = &project.program.modules[&project.entry]; let entry_module = &project.program.modules[&project.entry];
if entry_module.function(&function_name).is_none() { if entry_module.function(&function_name).is_none() {
eprintln!( if function.is_none() {
"error: no function named `{function_name}` in {}", eprintln!(
project.entry_file().path.display() "error[sem.no_entry_function]: {} has no callable `main`; declare an Exec H2 \
); (`## @exec main`) or pass an explicit function name",
project.entry_file().path.display()
);
} else {
eprintln!(
"error: no function named `{function_name}` in {}",
project.entry_file().path.display()
);
}
return 1; return 1;
} }
let runtime_args: Vec<EvaluatedArg> = named_args let runtime_args: Vec<EvaluatedArg> = named_args
+16
View File
@@ -1122,3 +1122,19 @@ fn condition_parse_diagnostic_points_inside_the_code_span() {
let column = diagnostic["column"].as_u64().expect("column"); let column = diagnostic["column"].as_u64().expect("column");
assert!((11..=14).contains(&column), "unexpected column {column}"); assert!((11..=14).contains(&column), "unexpected column {column}");
} }
#[test]
fn run_reports_sem_no_entry_function_without_exec_main() {
let dir = TempDir::new("no-entry");
dir.write(
"index.mq.md",
"# @knowledge 文档\n\n这里没有可调用的函数。\n",
);
let output = cli(dir.path(), &["run", "index.mq.md"]);
assert_exit(&output, 1, "missing Exec H2 main is an error");
let stderr = stderr_text(&output);
assert!(
stderr.contains("sem.no_entry_function"),
"stderr must carry the stable code: {stderr}"
);
}
+18
View File
@@ -41,6 +41,18 @@ pub enum TokenKind {
Or, Or,
/// `not` /// `not`
Not, Not,
/// `if` (statement keyword, not an identifier)
If,
/// `else` (statement keyword)
Else,
/// `while` (statement keyword)
While,
/// `end` (statement keyword)
End,
/// `return` (statement keyword)
Return,
/// `params` (signature keyword)
Params,
/// `(` /// `(`
LParen, LParen,
/// `)` /// `)`
@@ -307,6 +319,12 @@ impl<'a> Lexer<'a> {
"and" => TokenKind::And, "and" => TokenKind::And,
"or" => TokenKind::Or, "or" => TokenKind::Or,
"not" => TokenKind::Not, "not" => TokenKind::Not,
"if" => TokenKind::If,
"else" => TokenKind::Else,
"while" => TokenKind::While,
"end" => TokenKind::End,
"return" => TokenKind::Return,
"params" => TokenKind::Params,
_ => TokenKind::Ident(text.to_owned()), _ => TokenKind::Ident(text.to_owned()),
} }
} }
+2
View File
@@ -79,6 +79,7 @@
pub mod ast; pub mod ast;
pub mod lexer; pub mod lexer;
mod parser; mod parser;
pub mod statement;
use std::fmt; use std::fmt;
@@ -89,6 +90,7 @@ pub use ast::{
}; };
pub use marqdo_span::Span; pub use marqdo_span::Span;
pub use parser::MAX_NESTING_DEPTH; pub use parser::MAX_NESTING_DEPTH;
pub use statement::{parse_payload_statement, PayloadStatement};
/// Stable diagnostic code for every expression parse failure. /// Stable diagnostic code for every expression parse failure.
pub const EXPR_PARSE: &str = "expr.parse"; pub const EXPR_PARSE: &str = "expr.parse";
+12
View File
@@ -707,6 +707,12 @@ fn is_keyword(kind: &TokenKind) -> bool {
| TokenKind::And | TokenKind::And
| TokenKind::Or | TokenKind::Or
| TokenKind::Not | TokenKind::Not
| TokenKind::If
| TokenKind::Else
| TokenKind::While
| TokenKind::End
| TokenKind::Return
| TokenKind::Params
) )
} }
@@ -740,6 +746,12 @@ fn describe_token(kind: &TokenKind) -> String {
TokenKind::And => "`and`".to_owned(), TokenKind::And => "`and`".to_owned(),
TokenKind::Or => "`or`".to_owned(), TokenKind::Or => "`or`".to_owned(),
TokenKind::Not => "`not`".to_owned(), TokenKind::Not => "`not`".to_owned(),
TokenKind::If => "`if`".to_owned(),
TokenKind::Else => "`else`".to_owned(),
TokenKind::While => "`while`".to_owned(),
TokenKind::End => "`end`".to_owned(),
TokenKind::Return => "`return`".to_owned(),
TokenKind::Params => "`params`".to_owned(),
TokenKind::LParen => "`(`".to_owned(), TokenKind::LParen => "`(`".to_owned(),
TokenKind::RParen => "`)`".to_owned(), TokenKind::RParen => "`)`".to_owned(),
TokenKind::LBracket => "`[`".to_owned(), TokenKind::LBracket => "`[`".to_owned(),
+229
View File
@@ -0,0 +1,229 @@
//! The v2 payload statement grammar (RFC-LANG-0002).
//!
//! One Markdown carrier holds exactly one payload. This module is the only
//! place that decides whether that payload is a control statement
//! (`if`/`else`/`while`/`end`/`return`), a `params(...)` declaration, or an
//! ordinary assignment/expression. The frontend still owns pairing the control
//! words into blocks; this module owns their lexical recognition and the
//! expression grammar they embed.
//!
//! Keyword matching is by complete token: `ifdef(x)` and `returning()` are
//! ordinary calls, while a bare `if` used as an identifier is a reserved-word
//! error. `if`/`while`/`return` require whitespace before their expression and
//! the whole payload must be consumed.
use crate::ast::{Expr, Statement};
use crate::lexer::{self, TokenKind};
use crate::{parse_expression, parse_statement, ParseError};
use marqdo_span::Span;
/// A payload parsed by the v2 statement grammar.
#[derive(Debug, Clone, PartialEq)]
pub enum PayloadStatement {
/// `if <expr>`
If(Expr),
/// `else` (no expression)
Else,
/// `while <expr>`
While(Expr),
/// `end` (no expression)
End,
/// `return <expr>`
Return(Expr),
/// `params(...)`; the parameter list itself is validated by the frontend
/// signature parser, which owns IR parameter types and defaults.
Params,
/// An ordinary assignment or expression statement.
Ordinary(Statement),
}
/// Parse one complete executable payload under the v2 statement grammar.
///
/// The input is the concatenated carrier text, already trimmed or not; leading
/// and trailing whitespace is ignored. Expressions keep byte spans relative to
/// the original payload, including the keyword prefix, so diagnostics and IR
/// source maps point inside the file after the frontend rewrites them.
pub fn parse_payload_statement(source: &str) -> Result<PayloadStatement, ParseError> {
let trimmed = source.trim();
if trimmed.is_empty() {
return Err(ParseError::new(Span::empty(0), "empty executable payload"));
}
// `params(...)` is a declaration whose interior is validated by
// `signature::parse`; recognize the prefix before tokenizing so malformed
// parameter text (for example `params(1x: Int)`) is reported once by the
// signature parser instead of as a lexer failure.
if trimmed.starts_with("params(") {
return Ok(PayloadStatement::Params);
}
if trimmed == "params" || trimmed.starts_with("params ") || trimmed.starts_with("params\t") {
return Err(ParseError::new(
Span::new(0, "params".len()),
"malformed params declaration; write `params(name: Type, ...)`",
));
}
let tokens = lexer::tokenize(trimmed)?;
let first = tokens.first().expect("tokenize always returns EOF");
match first.kind {
TokenKind::If => {
let expr = control_expression(trimmed, first.span, "if")?;
Ok(PayloadStatement::If(expr))
}
TokenKind::While => {
let expr = control_expression(trimmed, first.span, "while")?;
Ok(PayloadStatement::While(expr))
}
TokenKind::Return => {
let expr = control_expression(trimmed, first.span, "return")?;
Ok(PayloadStatement::Return(expr))
}
TokenKind::Else => {
require_only_keyword(&tokens, "else")?;
Ok(PayloadStatement::Else)
}
TokenKind::End => {
require_only_keyword(&tokens, "end")?;
Ok(PayloadStatement::End)
}
TokenKind::Params => Err(ParseError::new(
first.span,
"malformed params declaration; write `params(name: Type, ...)`",
)),
_ => parse_statement(trimmed).map(PayloadStatement::Ordinary),
}
}
fn control_expression(source: &str, keyword: Span, name: &str) -> Result<Expr, ParseError> {
let rest = &source[keyword.end..];
if rest.is_empty() || !rest.starts_with(char::is_whitespace) {
return Err(ParseError::new(
keyword,
format!("`{name}` must be followed by whitespace and an expression"),
));
}
// Pad the expression so its AST spans stay payload-relative.
let padded = format!("{}{}", " ".repeat(keyword.end), rest);
parse_expression(&padded)
}
fn require_only_keyword(tokens: &[crate::lexer::Token], name: &str) -> Result<(), ParseError> {
let mut non_eof = tokens
.iter()
.filter(|token| !matches!(token.kind, TokenKind::Eof));
let _keyword = non_eof.next();
if let Some(extra) = non_eof.next() {
return Err(ParseError::new(
extra.span,
format!("`{name}` takes no expression"),
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn expr_of(source: &str) -> Expr {
match parse_payload_statement(source).expect("valid payload") {
PayloadStatement::If(expr)
| PayloadStatement::While(expr)
| PayloadStatement::Return(expr) => expr,
other => panic!("unexpected payload: {other:?}"),
}
}
#[test]
fn parses_control_statements_with_payload_relative_spans() {
for keyword in ["if", "while", "return"] {
let source = format!("{keyword} x > 0");
let expr = expr_of(&source);
assert_eq!(expr.span().start, keyword.len() + 1);
}
}
#[test]
fn else_and_end_are_exact_payloads() {
assert_eq!(
parse_payload_statement("else").unwrap(),
PayloadStatement::Else
);
assert_eq!(
parse_payload_statement("end").unwrap(),
PayloadStatement::End
);
assert!(parse_payload_statement("else x").is_err());
assert!(parse_payload_statement("end if").is_err());
}
#[test]
fn keywords_require_whitespace_and_an_expression() {
for source in ["if", "while", "return", "if(x)", "while(1)", "return(x)"] {
assert!(
parse_payload_statement(source).is_err(),
"{source} must be rejected"
);
}
}
#[test]
fn control_expressions_must_consume_the_payload() {
assert!(parse_payload_statement("if x > 0:").is_err());
assert!(parse_payload_statement("if x y").is_err());
}
#[test]
fn keyword_prefixes_are_ordinary_identifiers() {
for source in ["ifdef(x)", "returning(x)", "endpoint(x)", "while_count(x)"] {
match parse_payload_statement(source).unwrap() {
PayloadStatement::Ordinary(statement) => {
assert!(matches!(statement.kind, crate::StatementKind::Expr(_)));
}
other => panic!("{source} parsed as {other:?}"),
}
}
}
#[test]
fn bare_keywords_are_reserved_in_expressions() {
for source in ["x = if", "x = return", "[else]", "{while: 1}"] {
assert!(parse_payload_statement(source).is_err(), "{source}");
}
}
#[test]
fn params_are_recognized_before_signature_validation() {
assert_eq!(
parse_payload_statement("params(n: Int)").unwrap(),
PayloadStatement::Params
);
// Interior validation belongs to `signature::parse`; the statement
// grammar only needs to recognize the declaration payload.
assert_eq!(
parse_payload_statement("params(1x: Int)").unwrap(),
PayloadStatement::Params
);
assert!(parse_payload_statement("params").is_err());
assert!(parse_payload_statement("params (x)").is_err());
assert!(parse_payload_statement("params\t(x)").is_err());
}
#[test]
fn ordinary_statements_stay_ordinary() {
for source in ["x = 1", "print(x)", "m[0] = 1", "x = 1; return x"] {
let parsed = parse_payload_statement(source);
if source.contains(';') {
assert!(parsed.is_err(), "{source}");
} else {
assert!(matches!(parsed.unwrap(), PayloadStatement::Ordinary(_)));
}
}
}
#[test]
fn expressions_cannot_use_statement_keywords_as_identifiers() {
assert!(crate::parse_expression("if").is_err());
assert!(crate::parse_expression("return").is_err());
}
}
+10 -136
View File
@@ -5,7 +5,9 @@
//! parsed only after the role is known. Invalid executable shapes produce hard //! parsed only after the role is known. Invalid executable shapes produce hard
//! diagnostics and never fall back to prose. //! diagnostics and never fall back to prose.
use marqdo_expression::{parse_expression, parse_statement, AssignTarget, Expr, StatementKind}; use marqdo_expression::{
parse_payload_statement, AssignTarget, Expr, PayloadStatement as Payload, StatementKind,
};
use marqdo_ir::{PlaceTarget, Stmt}; use marqdo_ir::{PlaceTarget, Stmt};
use marqdo_markdown::{normalize_name, MarkdownDocument, MdKind, MdNode}; use marqdo_markdown::{normalize_name, MarkdownDocument, MdKind, MdNode};
use marqdo_span::Span; use marqdo_span::Span;
@@ -75,16 +77,6 @@ enum Stop {
Eof, Eof,
} }
enum Payload {
If(Expr),
Else,
While(Expr),
End,
Return(Expr),
Params,
Statement(marqdo_expression::Statement),
}
struct RawPayload { struct RawPayload {
payload: Payload, payload: Payload,
span: Span, span: Span,
@@ -189,7 +181,7 @@ impl Mapper {
); );
return; return;
} }
if let Some(mut payload) = self.parse_payload(&text, span, &segments) { if let Some(mut payload) = self.parse_payload(&text, &segments) {
if let Payload::If(expr) | Payload::While(expr) | Payload::Return(expr) = &mut payload { if let Payload::If(expr) | Payload::While(expr) | Payload::Return(expr) = &mut payload {
segments.rewrite_expr(expr); segments.rewrite_expr(expr);
} }
@@ -202,133 +194,15 @@ impl Mapper {
} }
} }
fn parse_payload( fn parse_payload(&mut self, text: &str, segments: &PayloadSegments) -> Option<Payload> {
&mut self, match parse_payload_statement(text) {
text: &str, Ok(payload) => Some(payload),
span: Span,
segments: &PayloadSegments,
) -> Option<Payload> {
let map = |span: Span| segments.map_span(span);
let text = text.trim();
if text.starts_with("params(") {
// Signature syntax and diagnostics are owned by `signature::parse`;
// the mapper only recognizes the declaration payload.
return Some(Payload::Params);
}
if text == "params" || text.starts_with("params ") {
self.error(
codes::SEM_SYNTAX,
Some(span),
"malformed params declaration; write `params(name: Type, ...)`",
);
return None;
}
let (word, rest) = match text.split_once(char::is_whitespace) {
Some((word, rest)) => (word, rest.trim()),
None => (text, ""),
};
// Parse control expressions with the keyword replaced by spaces so the
// expression AST keeps payload-relative (not rest-relative) spans.
let expression_at = |rest: &str| {
let offset = text.len().saturating_sub(rest.len());
parse_expression(&format!("{}{}", " ".repeat(offset), rest))
};
match word {
"if" => {
if rest.is_empty() {
self.error(codes::SEM_SYNTAX, Some(span), "`if` requires a condition");
return None;
}
match expression_at(rest) {
Ok(expr) => Some(Payload::If(expr)),
Err(error) => {
self.error(
codes::SEM_SYNTAX,
Some(map(error.span)),
format!("invalid if condition: {}", error.message),
);
None
}
}
}
"else" if rest.is_empty() => Some(Payload::Else),
"else" => {
self.error(codes::SEM_SYNTAX, Some(span), "`else` takes no expression");
None
}
"while" => {
if rest.is_empty() {
self.error(
codes::SEM_SYNTAX,
Some(span),
"`while` requires a condition",
);
return None;
}
match expression_at(rest) {
Ok(expr) => Some(Payload::While(expr)),
Err(error) => {
self.error(
codes::SEM_SYNTAX,
Some(map(error.span)),
format!("invalid while condition: {}", error.message),
);
None
}
}
}
"end" if rest.is_empty() => Some(Payload::End),
"end" => {
self.error(codes::SEM_SYNTAX, Some(span), "`end` takes no expression");
None
}
"return" => {
if rest.is_empty() {
self.error(
codes::SEM_SYNTAX,
Some(span),
"`return` requires an expression",
);
return None;
}
match expression_at(rest) {
Ok(expr) => Some(Payload::Return(expr)),
Err(error) => {
self.error(
codes::SEM_SYNTAX,
Some(map(error.span)),
format!("invalid return expression: {}", error.message),
);
None
}
}
}
"params" => {
if text.starts_with("params(") && text.ends_with(')') {
Some(Payload::Params)
} else {
self.error(
codes::SEM_SYNTAX,
Some(span),
"malformed params declaration",
);
None
}
}
"ifdef" | "returning" | "paramsx" => self.statement_payload(text, span),
_ => self.statement_payload(text, span),
}
}
fn statement_payload(&mut self, text: &str, span: Span) -> Option<Payload> {
match parse_statement(text) {
Ok(statement) => Some(Payload::Statement(statement)),
Err(error) => { Err(error) => {
self.error_with_hint( self.error_with_hint(
codes::SEM_SYNTAX, codes::SEM_SYNTAX,
Some(span), Some(segments.map_span(error.span)),
format!("invalid execution payload: {}", error.message), format!("invalid execution payload: {}", error.message),
"one payload is exactly one statement; put control keywords in their own carriers", "one carrier is exactly one statement; control keywords need their own carrier",
); );
None None
} }
@@ -461,7 +335,7 @@ impl Mapper {
} }
*position += 1; *position += 1;
} }
Payload::Statement(statement) => { Payload::Ordinary(statement) => {
let mut statement = statement.clone(); let mut statement = statement.clone();
raw.segments.rewrite_statement(&mut statement); raw.segments.rewrite_statement(&mut statement);
let span = raw.span; let span = raw.span;
+4 -2
View File
@@ -54,10 +54,12 @@ params_stmt = "params" , "(" , [ parameter_list ] , ")" ;
list or block quote. The declaration is compile-time metadata, not a list or block quote. The declaration is compile-time metadata, not a
runtime call. *) runtime call. *)
parameter_list = parameter , { "," , parameter } , [ "," ] ; parameter_list = parameter , { "," , parameter } , [ "," ] ;
parameter = identifier , ":" , type_name , [ "=" , expression ] ; parameter = identifier , [ ":" , type_name ] , [ "=" , expression ] ;
type_name = identifier ; type_name = identifier ;
(* Parameter names are unique; positional parameters precede defaulted ones. (* Parameter names are unique; positional parameters precede defaulted ones.
Types and default expressions reuse the existing signature rules. *) The type annotation is optional to preserve the existing untyped signature
form (`params(n)`); types and default expressions reuse the existing
signature rules. *)
(* Assignment and place writes are unchanged from the previous revision. *) (* Assignment and place writes are unchanged from the previous revision. *)
assignment = assign_target , "=" , expression ; assignment = assign_target , "=" , expression ;
+3 -3
View File
@@ -31,7 +31,7 @@ tests/
│ ├── ast/ 160 × *.mq.md + sibling *.ast.json goldens │ ├── ast/ 160 × *.mq.md + sibling *.ast.json goldens
│ └── semantics/ 73 × *.mq.md + sibling *.expected.json goldens │ └── semantics/ 73 × *.mq.md + sibling *.expected.json goldens
├── negative/ ├── negative/
│ ├── *.mq.md 45 × compile-time hard-error fixtures + *.expected.json │ ├── *.mq.md 47 × compile-time hard-error fixtures + *.expected.json
│ ├── warnings/ 2 × warning-model/control fixtures + *.expected.json │ ├── warnings/ 2 × warning-model/control fixtures + *.expected.json
│ └── runtime/ 21 × runtime-error fixtures + *.expected.json │ └── runtime/ 21 × runtime-error fixtures + *.expected.json
└── README.md this file └── README.md this file
@@ -79,9 +79,9 @@ The power/control-flow slice is pinned by:
loop and inert prose; loop and inert prose;
* negative fixtures 18–23: malformed payload statements (missing condition, * negative fixtures 18–23: malformed payload statements (missing condition,
trailing colon, `end if`, empty `return`, multi-statement payload, bad trailing colon, `end if`, empty `return`, multi-statement payload, bad
`params` syntax); fixtures 24–29 pin control-stack pairing, including the `params` syntax); fixtures 24–29 and 46 pin control-stack pairing, including the
heading boundary that reports `sem.unclosed_control` and `sem.unmatched_end` heading boundary that reports `sem.unclosed_control` and `sem.unmatched_end`
independently; independently and `else` under an open `while`;
* runtime negatives 11–12: negative integer exponent and exponent budget; * runtime negatives 11–12: negative integer exponent and exponent budget;
* `crates/marqdo-frontend/src/payload.rs` span tests and * `crates/marqdo-frontend/src/payload.rs` span tests and
`crates/cli/tests/integration_cli.rs` CLI span/E2E assertions. `crates/cli/tests/integration_cli.rs` CLI span/E2E assertions.
@@ -0,0 +1,5 @@
{
"codes": [
"sem.unmatched_else"
]
}
+9
View File
@@ -0,0 +1,9 @@
## @exec f
`params(flag: Bool)`
`while flag`
`else`
`end`
@@ -0,0 +1,5 @@
{
"codes": [
"sem.signature_position"
]
}
@@ -0,0 +1,5 @@
## @exec f
> `params(n: Int)`
`return n`
@@ -19,3 +19,7 @@
`params(a: Int, b: Int)` `params(a: Int, b: Int)`
`return a + b` `return a + b`
## @knowledge 补充说明
这里的 `if x > 0` 与 **重点** 都只是文档。