Files
marqdo-cli/crates/cli/tests/conformance_semantics.rs
T
cflmy 3a29fce061
CI / fmt + clippy + test (Linux x86_64) (push) Waiting to run
CI / schema instance validation + consumers (push) Waiting to run
CI / runtime contract (macos-latest) (push) Waiting to run
CI / runtime contract (windows-latest) (push) Waiting to run
Linux strict security lane (G-C) / strict isolation escape matrix (push) Waiting to run
ci/woodpecker/push/woodpecker Pipeline was successful
fix(foreign): resolve Windows PATH executables with PATHEXT
The built-in Python provider scanned PATH for the extension-less token
python3; a Windows PATH contains python3.exe, so a working interpreter was
reported as kernel.unavailable and the semantic golden suite failed on
windows-latest. Expand candidates with PATHEXT (conservative default) and use
the same platform-aware resolver in the conformance test's availability probe.

Also make native failures diagnosable: strip ANSI before matching and emit the
panic/build context as public check annotations, and run Linux tests through
the same annotator. This fixes the Windows native lane and gives the macOS
failure the context needed to diagnose it.
2026-11-21 18:22:22 +08:00

398 lines
14 KiB
Rust

//! End-to-end semantic conformance tests.
//!
//! Each `tests/conformance/semantics/*.mq.md` fixture is analysed by
//! [`marqdo_frontend::analyze_source`], assembled into an
//! [`marqdo_ir::Program`] and executed through [`marqdo_runtime::run`] with a
//! [`marqdo_host::StdHost`] in `project` mode. The typed wire representation
//! ([`marqdo_value::to_wire`]) of the returned value and the captured program
//! stdout are compared with the sibling `*.expected.json`:
//!
//! ```json
//! {"function": "<name>", "args": {"name": <wire value>},
//! "value": <wire value>, "stdout": "..."}
//! ```
//!
//! Fixtures that need an explicit grant declare it as
//! `"grants": ["foreign"]`; they are skipped when the kernel is unavailable.
//!
//! Developer note: `UPDATE_SEMANTICS_GOLDENS=1` rewrites `value`/`stdout` from
//! the implementation. Never use it without hand-checking every rewritten
//! expectation; the checked-in goldens were reviewed individually.
use std::fs;
use std::path::{Path, PathBuf};
use marqdo_host::{Capabilities, Host, StdHost};
use marqdo_ir::{ModuleId, Program, ResolvedImport};
use marqdo_runtime::{run, EvaluatedArg};
use marqdo_sandbox::SandboxMode;
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("crates/cli has a parent")
.parent()
.expect("crates has a parent")
.to_path_buf()
}
fn semantics_dir() -> PathBuf {
repo_root()
.join("tests")
.join("conformance")
.join("semantics")
}
fn fixture_paths() -> Vec<PathBuf> {
let dir = semantics_dir();
let mut paths: Vec<PathBuf> = fs::read_dir(&dir)
.unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display()))
.filter_map(Result::ok)
.map(|entry| entry.path())
.filter(|path| path.extension().is_some_and(|extension| extension == "md"))
.collect();
paths.sort();
paths
}
fn expected_path(fixture: &Path) -> PathBuf {
let name = fixture
.file_name()
.expect("fixture has a file name")
.to_string_lossy();
let stem = name
.strip_suffix(".mq.md")
.unwrap_or_else(|| panic!("fixture `{}` must end in .mq.md", fixture.display()));
fixture.with_file_name(format!("{stem}.expected.json"))
}
fn python_available() -> bool {
// Ask the same platform-aware resolver the runtime uses. On Windows a
// PATH entry is `python3.exe`, not the extension-less `python3` token, so a
// raw `Command::new("python3")` probe can disagree with execution.
marqdo_foreign::detect("python").is_some()
}
/// Unique scratch directory for the host project root. Program fixtures are
/// pure, so nothing is read from or written to it; it only satisfies the host
/// constructor contract.
fn scratch_project_root() -> PathBuf {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0);
let dir = std::env::temp_dir().join(format!(
"marqdo-conformance-semantics-{}-{nanos}",
std::process::id()
));
fs::create_dir_all(&dir).unwrap_or_else(|error| {
panic!(
"cannot create scratch project root {}: {error}",
dir.display()
)
});
dir
}
fn render_diagnostics(result: &marqdo_frontend::AnalysisResult) -> String {
let mut out = String::new();
for diagnostic in &result.diagnostics {
out.push_str(&format!(
" {}[{}]: {}\n",
diagnostic.severity, diagnostic.code, diagnostic.message
));
}
out
}
#[test]
fn semantic_goldens_match_end_to_end_execution() {
let fixtures = fixture_paths();
assert!(
fixtures.len() >= 25,
"the P0 corpus must keep at least 25 semantic fixtures, found {}",
fixtures.len()
);
let update = std::env::var_os("UPDATE_SEMANTICS_GOLDENS").is_some_and(|value| value == "1");
let has_python = python_available();
let project_root = scratch_project_root();
let mut skipped = 0usize;
let mut updated = 0usize;
for fixture in &fixtures {
let expected_path = expected_path(fixture);
let expected_text = fs::read_to_string(&expected_path).unwrap_or_else(|error| {
panic!(
"missing semantic golden {}: {error}",
expected_path.display()
)
});
let mut expected: serde_json::Value = serde_json::from_str(&expected_text)
.unwrap_or_else(|error| panic!("invalid JSON in {}: {error}", expected_path.display()));
let object = expected
.as_object()
.unwrap_or_else(|| panic!("{} must be a JSON object", expected_path.display()));
let function = object
.get("function")
.and_then(serde_json::Value::as_str)
.unwrap_or_else(|| panic!("{} has no `function`", expected_path.display()))
.to_string();
let grant_foreign = object
.get("grants")
.and_then(serde_json::Value::as_array)
.is_some_and(|grants| grants.iter().any(|grant| grant == "foreign"));
if grant_foreign && !has_python {
skipped += 1;
println!(
"skipping {}: python3 is unavailable (foreign kernel fixture)",
fixture.display()
);
continue;
}
let source = fs::read_to_string(fixture)
.unwrap_or_else(|error| panic!("cannot read {}: {error}", fixture.display()));
let module_id = ModuleId::from("index.mq.md");
let analysis = marqdo_frontend::analyze_source(&module_id.to_string(), &source);
if analysis.has_errors() {
panic!(
"semantic fixture `{}` must compile without errors:\n{}",
fixture.display(),
render_diagnostics(&analysis)
);
}
let mut program = marqdo_ir::Program::default();
program.insert(analysis.module.clone());
program.main_module = Some(module_id.clone());
install_stdlib_print(&mut program);
install_entry_stdlib_imports(&mut program, &module_id, &analysis);
let mut capabilities = Capabilities::pure();
if grant_foreign {
capabilities = capabilities.allow_foreign();
}
let mut host = StdHost::new(SandboxMode::Project, capabilities, project_root.clone());
let args_value = object
.get("args")
.cloned()
.unwrap_or_else(|| serde_json::json!({}));
let args_object = args_value
.as_object()
.unwrap_or_else(|| panic!("`args` of {} must be an object", expected_path.display()));
let mut runtime_args = Vec::new();
// BTreeMap-backed JSON objects iterate in sorted key order; argument
// evaluation order is irrelevant here because arguments are constants.
for (name, wire) in args_object {
let value = marqdo_value::from_wire(wire).unwrap_or_else(|error| {
panic!(
"invalid wire argument `{name}` in {}: {error}",
expected_path.display()
)
});
runtime_args.push(EvaluatedArg::named(
name.clone(),
value,
marqdo_span::Span::default(),
));
}
let outcome =
run(&program, &mut host, &module_id, &function, runtime_args).unwrap_or_else(|error| {
panic!(
"semantic fixture `{}` failed with {}: {}\nframes: {:?}",
fixture.display(),
error.code(),
error.message,
error.frames
)
});
let stdout = String::from_utf8_lossy(&host.take_output()).to_string();
let value = marqdo_value::to_wire(&outcome.value);
if update {
let mut object = expected.as_object().cloned().expect("object");
object.insert("value".into(), value);
object.insert("stdout".into(), serde_json::Value::String(stdout));
expected = serde_json::Value::Object(object);
let mut text = serde_json::to_string_pretty(&expected).expect("serializable");
text.push('\n');
fs::write(&expected_path, text).unwrap_or_else(|error| {
panic!("cannot write {}: {error}", expected_path.display())
});
updated += 1;
continue;
}
let expected_value = object.get("value").unwrap_or_else(|| {
panic!(
"{} has no `value`; regenerate it with UPDATE_SEMANTICS_GOLDENS=1 \
after reviewing the behaviour",
expected_path.display()
)
});
let expected_stdout = object
.get("stdout")
.and_then(serde_json::Value::as_str)
.unwrap_or_else(|| panic!("{} has no `stdout`", expected_path.display()));
assert_eq!(
&value,
expected_value,
"value mismatch for `{}`\n expected: {}\n actual: {}",
fixture.display(),
expected_value,
value
);
assert_eq!(
stdout,
expected_stdout,
"stdout mismatch for `{}`",
fixture.display()
);
}
if update {
println!("UPDATE_SEMANTICS_GOLDENS=1: rewrote {updated} semantic golden(s)");
return;
}
let _ = fs::remove_dir_all(&project_root);
println!(
"semantic conformance: {} fixture(s) checked, {skipped} skipped",
fixtures.len()
);
}
fn normalise_relative(path: &Path) -> String {
let mut parts: Vec<std::ffi::OsString> = Vec::new();
for component in path.components() {
match component {
std::path::Component::Normal(part) => parts.push(part.to_os_string()),
std::path::Component::ParentDir => {
parts.pop();
}
std::path::Component::CurDir
| std::path::Component::RootDir
| std::path::Component::Prefix(_) => {}
}
}
parts
.iter()
.map(|part| part.to_string_lossy().to_string())
.collect::<Vec<_>>()
.join("/")
}
/// Load the bundled stdlib print modules for direct-runtime fixtures.
///
/// The CLI loader normally injects these for Compat-B. The semantic harness
/// builds a Program directly, so it mirrors the same resolution here; without
/// it, fixtures that call `打印` would hit the Compat-C "no bundle" error.
fn install_stdlib_print(program: &mut Program) {
let root = repo_root().join("stdlib");
if !root.join("stdlib.json").is_file() {
return;
}
let mut loaded = std::collections::BTreeSet::new();
let io = match load_stdlib_module(&root, "io.mq.md", program, &mut loaded) {
Ok(id) => id,
Err(error) => panic!("cannot load bundled stdlib: {error}"),
};
let function = program
.module(&io)
.and_then(|module| module.function("写值"))
.unwrap_or_else(|| panic!("stdlib io has no 写值 function"));
program.print_override = Some(function.id.clone());
}
fn load_stdlib_module(
root: &Path,
relative: &str,
program: &mut Program,
loaded: &mut std::collections::BTreeSet<String>,
) -> Result<ModuleId, String> {
let root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
let relative = normalise_relative(Path::new(relative));
if loaded.contains(&relative) {
if let Some((id, _)) = program
.modules
.iter()
.find(|(id, _)| id.relative_path == relative)
{
return Ok(id.clone());
}
return Err(format!("stdlib import cycle at `{relative}`"));
}
loaded.insert(relative.clone());
let path = root.join(&relative);
let source = fs::read_to_string(&path)
.map_err(|error| format!("cannot read `{}`: {error}", path.display()))?;
let module_id = ModuleId::new("std:test", relative.clone());
let analysis = marqdo_frontend::analyze_source(&module_id.to_string(), &source);
if analysis.has_errors() {
return Err(format!("stdlib module `{relative}` has diagnostics"));
}
let imports: Vec<(String, String)> = analysis
.document
.frontmatter()
.and_then(|frontmatter| frontmatter.imports().ok())
.map(|mapping| mapping.into_iter().collect())
.unwrap_or_default();
for (alias, target) in imports {
let target = target.strip_prefix("std:").unwrap_or(&target);
let child = if target.starts_with("./") || target.starts_with("../") {
let base = Path::new(&relative)
.parent()
.unwrap_or_else(|| Path::new(""));
normalise_relative(&base.join(target))
} else {
target.to_string()
};
let to = load_stdlib_module(&root, &child, program, loaded)?;
program.resolved_imports.push(ResolvedImport {
from: module_id.clone(),
alias,
to,
});
}
program.insert(analysis.module);
Ok(module_id)
}
/// Resolve `std:` imports declared by a direct-runtime fixture entry.
fn install_entry_stdlib_imports(
program: &mut Program,
module_id: &ModuleId,
analysis: &marqdo_frontend::AnalysisResult,
) {
let root = repo_root().join("stdlib");
let Some(imports) = analysis
.document
.frontmatter()
.and_then(|frontmatter| frontmatter.imports().ok())
else {
return;
};
let mut loaded = std::collections::BTreeSet::new();
for (alias, target) in imports {
let Some(relative) = target.strip_prefix("std:") else {
continue;
};
let relative = if relative.ends_with(".mq.md") {
relative.to_string()
} else {
format!("{relative}.mq.md")
};
let to = match load_stdlib_module(&root, &relative, program, &mut loaded) {
Ok(id) => id,
Err(error) => panic!("cannot load stdlib import `{target}`: {error}"),
};
program.resolved_imports.push(ResolvedImport {
from: module_id.clone(),
alias,
to,
});
}
}