//! End-to-end semantic conformance tests. //! //! Each `tests/conformance/semantics/*.mq.md` fixture is analysed by //! [`marqdo_frontend::analyze_source`], assembled into an //! [`marqdo_ir::Program`] and executed through [`marqdo_runtime::run`] with a //! [`marqdo_host::StdHost`] in `project` mode. The typed wire representation //! ([`marqdo_value::to_wire`]) of the returned value and the captured program //! stdout are compared with the sibling `*.expected.json`: //! //! ```json //! {"function": "", "args": {"name": }, //! "value": , "stdout": "..."} //! ``` //! //! Fixtures that need an explicit grant declare it as //! `"grants": ["foreign"]`; they are skipped when the kernel is unavailable. //! //! Developer note: `UPDATE_SEMANTICS_GOLDENS=1` rewrites `value`/`stdout` from //! the implementation. Never use it without hand-checking every rewritten //! expectation; the checked-in goldens were reviewed individually. use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use marqdo_host::{Capabilities, Host, StdHost}; use marqdo_ir::ModuleId; use marqdo_runtime::{run, EvaluatedArg}; use marqdo_sandbox::SandboxMode; fn repo_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .expect("crates/cli has a parent") .parent() .expect("crates has a parent") .to_path_buf() } fn semantics_dir() -> PathBuf { repo_root() .join("tests") .join("conformance") .join("semantics") } fn fixture_paths() -> Vec { let dir = semantics_dir(); let mut paths: Vec = fs::read_dir(&dir) .unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display())) .filter_map(Result::ok) .map(|entry| entry.path()) .filter(|path| path.extension().is_some_and(|extension| extension == "md")) .collect(); paths.sort(); paths } fn expected_path(fixture: &Path) -> PathBuf { let name = fixture .file_name() .expect("fixture has a file name") .to_string_lossy(); let stem = name .strip_suffix(".mq.md") .unwrap_or_else(|| panic!("fixture `{}` must end in .mq.md", fixture.display())); fixture.with_file_name(format!("{stem}.expected.json")) } fn python_available() -> bool { Command::new("python3") .arg("--version") .output() .map(|output| output.status.success()) .unwrap_or(false) } /// Unique scratch directory for the host project root. Program fixtures are /// pure, so nothing is read from or written to it; it only satisfies the host /// constructor contract. fn scratch_project_root() -> PathBuf { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|duration| duration.as_nanos()) .unwrap_or(0); let dir = std::env::temp_dir().join(format!( "marqdo-conformance-semantics-{}-{nanos}", std::process::id() )); fs::create_dir_all(&dir).unwrap_or_else(|error| { panic!( "cannot create scratch project root {}: {error}", dir.display() ) }); dir } fn render_diagnostics(result: &marqdo_frontend::AnalysisResult) -> String { let mut out = String::new(); for diagnostic in &result.diagnostics { out.push_str(&format!( " {}[{}]: {}\n", diagnostic.severity, diagnostic.code, diagnostic.message )); } out } #[test] fn semantic_goldens_match_end_to_end_execution() { let fixtures = fixture_paths(); assert!( fixtures.len() >= 25, "the P0 corpus must keep at least 25 semantic fixtures, found {}", fixtures.len() ); let update = std::env::var_os("UPDATE_SEMANTICS_GOLDENS").is_some_and(|value| value == "1"); let has_python = python_available(); let project_root = scratch_project_root(); let mut skipped = 0usize; let mut updated = 0usize; for fixture in &fixtures { let expected_path = expected_path(fixture); let expected_text = fs::read_to_string(&expected_path).unwrap_or_else(|error| { panic!( "missing semantic golden {}: {error}", expected_path.display() ) }); let mut expected: serde_json::Value = serde_json::from_str(&expected_text) .unwrap_or_else(|error| panic!("invalid JSON in {}: {error}", expected_path.display())); let object = expected .as_object() .unwrap_or_else(|| panic!("{} must be a JSON object", expected_path.display())); let function = object .get("function") .and_then(serde_json::Value::as_str) .unwrap_or_else(|| panic!("{} has no `function`", expected_path.display())) .to_string(); let grant_foreign = object .get("grants") .and_then(serde_json::Value::as_array) .is_some_and(|grants| grants.iter().any(|grant| grant == "foreign")); if grant_foreign && !has_python { skipped += 1; println!( "skipping {}: python3 is unavailable (foreign kernel fixture)", fixture.display() ); continue; } let source = fs::read_to_string(fixture) .unwrap_or_else(|error| panic!("cannot read {}: {error}", fixture.display())); let module_id = ModuleId::from("index.mq.md"); let analysis = marqdo_frontend::analyze_source(&module_id.0, &source); if analysis.has_errors() { panic!( "semantic fixture `{}` must compile without errors:\n{}", fixture.display(), render_diagnostics(&analysis) ); } let mut program = marqdo_ir::Program::default(); program.insert(analysis.module.clone()); program.main_module = Some(module_id.clone()); let mut capabilities = Capabilities::pure(); if grant_foreign { capabilities = capabilities.allow_foreign(); } let mut host = StdHost::new(SandboxMode::Project, capabilities, project_root.clone()); let args_value = object .get("args") .cloned() .unwrap_or_else(|| serde_json::json!({})); let args_object = args_value .as_object() .unwrap_or_else(|| panic!("`args` of {} must be an object", expected_path.display())); let mut runtime_args = Vec::new(); // BTreeMap-backed JSON objects iterate in sorted key order; argument // evaluation order is irrelevant here because arguments are constants. for (name, wire) in args_object { let value = marqdo_value::from_wire(wire).unwrap_or_else(|error| { panic!( "invalid wire argument `{name}` in {}: {error}", expected_path.display() ) }); runtime_args.push(EvaluatedArg::named( name.clone(), value, marqdo_span::Span::default(), )); } let outcome = run(&program, &mut host, &module_id, &function, runtime_args).unwrap_or_else(|error| { panic!( "semantic fixture `{}` failed with {}: {}\nframes: {:?}", fixture.display(), error.code(), error.message, error.frames ) }); let stdout = String::from_utf8_lossy(&host.take_output()).to_string(); let value = marqdo_value::to_wire(&outcome.value); if update { let mut object = expected.as_object().cloned().expect("object"); object.insert("value".into(), value); object.insert("stdout".into(), serde_json::Value::String(stdout)); expected = serde_json::Value::Object(object); let mut text = serde_json::to_string_pretty(&expected).expect("serializable"); text.push('\n'); fs::write(&expected_path, text).unwrap_or_else(|error| { panic!("cannot write {}: {error}", expected_path.display()) }); updated += 1; continue; } let expected_value = object.get("value").unwrap_or_else(|| { panic!( "{} has no `value`; regenerate it with UPDATE_SEMANTICS_GOLDENS=1 \ after reviewing the behaviour", expected_path.display() ) }); let expected_stdout = object .get("stdout") .and_then(serde_json::Value::as_str) .unwrap_or_else(|| panic!("{} has no `stdout`", expected_path.display())); assert_eq!( &value, expected_value, "value mismatch for `{}`\n expected: {}\n actual: {}", fixture.display(), expected_value, value ); assert_eq!( stdout, expected_stdout, "stdout mismatch for `{}`", fixture.display() ); } if update { println!("UPDATE_SEMANTICS_GOLDENS=1: rewrote {updated} semantic golden(s)"); return; } let _ = fs::remove_dir_all(&project_root); println!( "semantic conformance: {} fixture(s) checked, {skipped} skipped", fixtures.len() ); }