Findings from the conformance corpus workstream, all fixed at the source: - value/wire: enable serde_json's float_roundtrip so typed wire floats keep their exact digits across foreign/plugin boundaries (Python pi*9 now stays 28.274333882308138). - host: default interpreter stack budget lowered to 128 frames so deeply recursive documents hit a structured resource.exhausted instead of exhausting the native debug-build Rust stack; --stack remains available. - frontend/map: calling an external HTTP(S) link is reported as ref.unresolved (consistent with other unresolvable call targets) instead of math.unsupported. - frontend/analyze: invalid front matter emits a single frontmatter.invalid diagnostic (the YAML error is not repeated by the imports pass). - spec/mapping: clarify that would-be execution nodes outside any function body are inert with a markdown.inert_executable warning (no "return outside a function" hard-error claim). - tests: add the conformance corpus and harness (150 AST goldens, 36 semantic goldens, 48 hard-error + 6 warning + 10 runtime-error negatives, CLI/module/ plugin integration tests, prose-invariance properties) and update the three reported observations to the fixed behaviour.
266 lines
9.3 KiB
Rust
266 lines
9.3 KiB
Rust
//! End-to-end semantic conformance tests.
|
|
//!
|
|
//! Each `tests/conformance/semantics/*.mq.md` fixture is analysed by
|
|
//! [`marqdo_frontend::analyze_source`], assembled into an
|
|
//! [`marqdo_ir::Program`] and executed through [`marqdo_runtime::run`] with a
|
|
//! [`marqdo_host::StdHost`] in `project` mode. The typed wire representation
|
|
//! ([`marqdo_value::to_wire`]) of the returned value and the captured program
|
|
//! stdout are compared with the sibling `*.expected.json`:
|
|
//!
|
|
//! ```json
|
|
//! {"function": "<name>", "args": {"name": <wire value>},
|
|
//! "value": <wire value>, "stdout": "..."}
|
|
//! ```
|
|
//!
|
|
//! Fixtures that need an explicit grant declare it as
|
|
//! `"grants": ["foreign"]`; they are skipped when the kernel is unavailable.
|
|
//!
|
|
//! Developer note: `UPDATE_SEMANTICS_GOLDENS=1` rewrites `value`/`stdout` from
|
|
//! the implementation. Never use it without hand-checking every rewritten
|
|
//! expectation; the checked-in goldens were reviewed individually.
|
|
|
|
use std::fs;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
|
|
use marqdo_host::{Capabilities, Host, StdHost};
|
|
use marqdo_ir::ModuleId;
|
|
use marqdo_runtime::{run, EvaluatedArg};
|
|
use marqdo_sandbox::SandboxMode;
|
|
|
|
fn repo_root() -> PathBuf {
|
|
Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.parent()
|
|
.expect("crates/cli has a parent")
|
|
.parent()
|
|
.expect("crates has a parent")
|
|
.to_path_buf()
|
|
}
|
|
|
|
fn semantics_dir() -> PathBuf {
|
|
repo_root()
|
|
.join("tests")
|
|
.join("conformance")
|
|
.join("semantics")
|
|
}
|
|
|
|
fn fixture_paths() -> Vec<PathBuf> {
|
|
let dir = semantics_dir();
|
|
let mut paths: Vec<PathBuf> = fs::read_dir(&dir)
|
|
.unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display()))
|
|
.filter_map(Result::ok)
|
|
.map(|entry| entry.path())
|
|
.filter(|path| path.extension().is_some_and(|extension| extension == "md"))
|
|
.collect();
|
|
paths.sort();
|
|
paths
|
|
}
|
|
|
|
fn expected_path(fixture: &Path) -> PathBuf {
|
|
let name = fixture
|
|
.file_name()
|
|
.expect("fixture has a file name")
|
|
.to_string_lossy();
|
|
let stem = name
|
|
.strip_suffix(".mq.md")
|
|
.unwrap_or_else(|| panic!("fixture `{}` must end in .mq.md", fixture.display()));
|
|
fixture.with_file_name(format!("{stem}.expected.json"))
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new("python3")
|
|
.arg("--version")
|
|
.output()
|
|
.map(|output| output.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Unique scratch directory for the host project root. Program fixtures are
|
|
/// pure, so nothing is read from or written to it; it only satisfies the host
|
|
/// constructor contract.
|
|
fn scratch_project_root() -> PathBuf {
|
|
let nanos = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map(|duration| duration.as_nanos())
|
|
.unwrap_or(0);
|
|
let dir = std::env::temp_dir().join(format!(
|
|
"marqdo-conformance-semantics-{}-{nanos}",
|
|
std::process::id()
|
|
));
|
|
fs::create_dir_all(&dir).unwrap_or_else(|error| {
|
|
panic!(
|
|
"cannot create scratch project root {}: {error}",
|
|
dir.display()
|
|
)
|
|
});
|
|
dir
|
|
}
|
|
|
|
fn render_diagnostics(result: &marqdo_frontend::AnalysisResult) -> String {
|
|
let mut out = String::new();
|
|
for diagnostic in &result.diagnostics {
|
|
out.push_str(&format!(
|
|
" {}[{}]: {}\n",
|
|
diagnostic.severity, diagnostic.code, diagnostic.message
|
|
));
|
|
}
|
|
out
|
|
}
|
|
|
|
#[test]
|
|
fn semantic_goldens_match_end_to_end_execution() {
|
|
let fixtures = fixture_paths();
|
|
assert!(
|
|
fixtures.len() >= 25,
|
|
"the P0 corpus must keep at least 25 semantic fixtures, found {}",
|
|
fixtures.len()
|
|
);
|
|
let update = std::env::var_os("UPDATE_SEMANTICS_GOLDENS").is_some_and(|value| value == "1");
|
|
let has_python = python_available();
|
|
let project_root = scratch_project_root();
|
|
let mut skipped = 0usize;
|
|
let mut updated = 0usize;
|
|
|
|
for fixture in &fixtures {
|
|
let expected_path = expected_path(fixture);
|
|
let expected_text = fs::read_to_string(&expected_path).unwrap_or_else(|error| {
|
|
panic!(
|
|
"missing semantic golden {}: {error}",
|
|
expected_path.display()
|
|
)
|
|
});
|
|
let mut expected: serde_json::Value = serde_json::from_str(&expected_text)
|
|
.unwrap_or_else(|error| panic!("invalid JSON in {}: {error}", expected_path.display()));
|
|
let object = expected
|
|
.as_object()
|
|
.unwrap_or_else(|| panic!("{} must be a JSON object", expected_path.display()));
|
|
|
|
let function = object
|
|
.get("function")
|
|
.and_then(serde_json::Value::as_str)
|
|
.unwrap_or_else(|| panic!("{} has no `function`", expected_path.display()))
|
|
.to_string();
|
|
let grant_foreign = object
|
|
.get("grants")
|
|
.and_then(serde_json::Value::as_array)
|
|
.is_some_and(|grants| grants.iter().any(|grant| grant == "foreign"));
|
|
if grant_foreign && !has_python {
|
|
skipped += 1;
|
|
println!(
|
|
"skipping {}: python3 is unavailable (foreign kernel fixture)",
|
|
fixture.display()
|
|
);
|
|
continue;
|
|
}
|
|
|
|
let source = fs::read_to_string(fixture)
|
|
.unwrap_or_else(|error| panic!("cannot read {}: {error}", fixture.display()));
|
|
let module_id = ModuleId::from("index.mq.md");
|
|
let analysis = marqdo_frontend::analyze_source(&module_id.0, &source);
|
|
if analysis.has_errors() {
|
|
panic!(
|
|
"semantic fixture `{}` must compile without errors:\n{}",
|
|
fixture.display(),
|
|
render_diagnostics(&analysis)
|
|
);
|
|
}
|
|
|
|
let mut program = marqdo_ir::Program::default();
|
|
program.insert(analysis.module.clone());
|
|
program.main_module = Some(module_id.clone());
|
|
|
|
let mut capabilities = Capabilities::pure();
|
|
if grant_foreign {
|
|
capabilities = capabilities.allow_foreign();
|
|
}
|
|
let mut host = StdHost::new(SandboxMode::Project, capabilities, project_root.clone());
|
|
|
|
let args_value = object
|
|
.get("args")
|
|
.cloned()
|
|
.unwrap_or_else(|| serde_json::json!({}));
|
|
let args_object = args_value
|
|
.as_object()
|
|
.unwrap_or_else(|| panic!("`args` of {} must be an object", expected_path.display()));
|
|
let mut runtime_args = Vec::new();
|
|
// BTreeMap-backed JSON objects iterate in sorted key order; argument
|
|
// evaluation order is irrelevant here because arguments are constants.
|
|
for (name, wire) in args_object {
|
|
let value = marqdo_value::from_wire(wire).unwrap_or_else(|error| {
|
|
panic!(
|
|
"invalid wire argument `{name}` in {}: {error}",
|
|
expected_path.display()
|
|
)
|
|
});
|
|
runtime_args.push(EvaluatedArg::named(
|
|
name.clone(),
|
|
value,
|
|
marqdo_span::Span::default(),
|
|
));
|
|
}
|
|
|
|
let outcome =
|
|
run(&program, &mut host, &module_id, &function, runtime_args).unwrap_or_else(|error| {
|
|
panic!(
|
|
"semantic fixture `{}` failed with {}: {}\nframes: {:?}",
|
|
fixture.display(),
|
|
error.code(),
|
|
error.message,
|
|
error.frames
|
|
)
|
|
});
|
|
let stdout = String::from_utf8_lossy(&host.take_output()).to_string();
|
|
let value = marqdo_value::to_wire(&outcome.value);
|
|
|
|
if update {
|
|
let mut object = expected.as_object().cloned().expect("object");
|
|
object.insert("value".into(), value);
|
|
object.insert("stdout".into(), serde_json::Value::String(stdout));
|
|
expected = serde_json::Value::Object(object);
|
|
let mut text = serde_json::to_string_pretty(&expected).expect("serializable");
|
|
text.push('\n');
|
|
fs::write(&expected_path, text).unwrap_or_else(|error| {
|
|
panic!("cannot write {}: {error}", expected_path.display())
|
|
});
|
|
updated += 1;
|
|
continue;
|
|
}
|
|
|
|
let expected_value = object.get("value").unwrap_or_else(|| {
|
|
panic!(
|
|
"{} has no `value`; regenerate it with UPDATE_SEMANTICS_GOLDENS=1 \
|
|
after reviewing the behaviour",
|
|
expected_path.display()
|
|
)
|
|
});
|
|
let expected_stdout = object
|
|
.get("stdout")
|
|
.and_then(serde_json::Value::as_str)
|
|
.unwrap_or_else(|| panic!("{} has no `stdout`", expected_path.display()));
|
|
assert_eq!(
|
|
&value,
|
|
expected_value,
|
|
"value mismatch for `{}`\n expected: {}\n actual: {}",
|
|
fixture.display(),
|
|
expected_value,
|
|
value
|
|
);
|
|
assert_eq!(
|
|
stdout,
|
|
expected_stdout,
|
|
"stdout mismatch for `{}`",
|
|
fixture.display()
|
|
);
|
|
}
|
|
|
|
if update {
|
|
println!("UPDATE_SEMANTICS_GOLDENS=1: rewrote {updated} semantic golden(s)");
|
|
return;
|
|
}
|
|
let _ = fs::remove_dir_all(&project_root);
|
|
println!(
|
|
"semantic conformance: {} fixture(s) checked, {skipped} skipped",
|
|
fixtures.len()
|
|
);
|
|
}
|