//! Black-box Execution Events v1 contract tests (RFC-OBS-0001). //! //! These drive the real `marqdo` binary (`CARGO_BIN_EXE_marqdo`) and parse the //! stdout line stream exactly like an external consumer. They cover the P0 //! envelope/lifecycle/byte-fidelity contract and the P1 frame/value policy, //! plus trace coexistence, truncation, live delivery and a broken pipe. use std::fs; use std::io::{BufRead, BufReader, Read}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; use serde_json::Value; const BASIC: &str = "## @exec main\n\n`打印(\"START\")`\n\n`n = 10`\n\n`打印(\"DONE\")`\n\n`return n`\n"; struct TempDir(PathBuf); impl TempDir { fn new(tag: &str) -> Self { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|duration| duration.as_nanos()) .unwrap_or(0); let path = std::env::temp_dir().join(format!( "marqdo-execution-events-{tag}-{}-{nanos}", std::process::id() )); let _ = fs::remove_dir_all(&path); fs::create_dir_all(&path).unwrap(); Self(path) } fn path(&self) -> &Path { &self.0 } fn write(&self, name: &str, source: &str) -> PathBuf { let path = self.0.join(name); fs::write(&path, source).unwrap(); path } } impl Drop for TempDir { fn drop(&mut self) { let _ = fs::remove_dir_all(&self.0); } } fn cli(dir: &Path, args: &[&str]) -> std::process::Output { Command::new(env!("CARGO_BIN_EXE_marqdo")) .args(args) .current_dir(dir) .output() .expect("run marqdo") } fn parse_events(stdout: &[u8]) -> Vec { String::from_utf8_lossy(stdout) .lines() .map(|line| { serde_json::from_str(line) .unwrap_or_else(|error| panic!("invalid JSONL line {line:?}: {error}")) }) .collect() } fn events_of<'a>(events: &'a [Value], kind: &str) -> Vec<&'a Value> { events .iter() .filter(|event| event["kind"] == kind) .collect() } fn chunk_bytes(events: &[Value], kind: &str) -> Vec { let mut out = Vec::new(); for event in events_of(events, kind) { let payload = &event["payload"]; assert_eq!(payload["encoding"], "base64"); let data = base64_decode(payload["data_base64"].as_str().unwrap()); assert_eq!( payload["byte_count"].as_u64().unwrap() as usize, data.len(), "byte_count must equal decoded length" ); out.extend(data); } out } fn base64_decode(text: &str) -> Vec { const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; let mut out = Vec::new(); let bytes: Vec = text .bytes() .filter(|byte| *byte != b'=') .map(|byte| { ALPHABET .iter() .position(|item| *item == byte) .unwrap_or_else(|| panic!("invalid base64 byte {byte}")) as u8 }) .collect(); for chunk in bytes.chunks(4) { let mut triple = 0u32; for (index, byte) in chunk.iter().enumerate() { triple |= u32::from(*byte) << (18 - 6 * index); } out.push((triple >> 16) as u8); if chunk.len() > 2 { out.push((triple >> 8) as u8); } if chunk.len() > 3 { out.push(triple as u8); } } out } fn assert_envelope(events: &[Value]) { assert!(!events.is_empty()); let first = &events[0]; assert_eq!(first["kind"], "stream.opened"); assert_eq!(first["schema"], "marqdo.execution-events"); assert_eq!(first["schema_version"], 1); assert_eq!(first["seq"], 0); let run_id = first["run_id"].as_str().unwrap().to_string(); for (index, event) in events.iter().enumerate() { assert_eq!(event["seq"].as_u64().unwrap(), index as u64); assert_eq!(event["run_id"], run_id); assert_eq!(event["schema_version"], 1); assert!(event["payload"].is_object()); } let terminals: Vec<&str> = events .iter() .filter_map(|event| match event["kind"].as_str().unwrap() { "run.completed" | "run.failed" | "run.rejected" => { Some(event["kind"].as_str().unwrap()) } _ => None, }) .collect(); assert_eq!(terminals.len(), 1, "exactly one terminal event"); assert_eq!(events.last().unwrap()["kind"], terminals[0]); assert_eq!(events_of(events, "stream.opened").len(), 1); assert!(events_of(events, "run.started").len() <= 1); } fn repo_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .unwrap() .parent() .unwrap() .to_path_buf() } fn python_consumer_available() -> bool { Command::new("python3") .args(["-c", "import jsonschema"]) .stdout(Stdio::null()) .stderr(Stdio::null()) .status() .map(|status| status.success()) .unwrap_or(false) } #[test] fn checked_in_fixtures_validate_against_the_public_schema() { if !python_consumer_available() { eprintln!("python3 + jsonschema unavailable; schema consumer skipped"); return; } let fixtures = repo_root().join("tests/consumers/fixtures/execution-events"); let consumer = repo_root().join("tests/consumers/execution_events_consumer.py"); for name in ["minimal.jsonl", "full.jsonl"] { let path = fixtures.join("valid").join(name); let output = Command::new("python3") .arg(&consumer) .arg(&path) .output() .expect("run execution events consumer"); assert!( output.status.success(), "valid fixture {name} rejected: {}", String::from_utf8_lossy(&output.stdout) ); } for entry in fs::read_dir(fixtures.join("invalid")).unwrap() { let path = entry.unwrap().path(); let output = Command::new("python3") .arg(&consumer) .arg(&path) .output() .expect("run execution events consumer"); assert!( !output.status.success(), "invalid fixture {} was accepted", path.display() ); } } #[test] fn structure_fixture_attributes_repeated_recursive_frames() { let dir = TempDir::new("structure"); let source = fs::read_to_string( repo_root().join("tests/consumers/fixtures/execution-events/structure.mq.md"), ) .unwrap(); dir.write("index.mq.md", &source); let output = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--observe-max-events", "1000", ], ); assert_eq!( output.status.code(), Some(0), "{}", String::from_utf8_lossy(&output.stderr) ); let events = parse_events(&output.stdout); assert_envelope(&events); assert_eq!(events.last().unwrap()["payload"]["value"]["decimal"], "3"); let enters = events_of(&events, "call.enter"); assert_eq!(enters.len(), 4, "main + f(2) + g(2) + f(0)"); let frames: Vec<&str> = enters .iter() .map(|event| event["payload"]["frame_id"].as_str().unwrap()) .collect(); let unique: std::collections::BTreeSet<&str> = frames.iter().copied().collect(); assert_eq!(unique.len(), 4, "frame identities are run-unique"); let main = enters[0]["payload"]["frame_id"].as_str().unwrap(); assert!(enters[0]["payload"]["parent_frame_id"].is_null()); let f2 = enters[1]["payload"]["frame_id"].as_str().unwrap(); let g2 = enters[2]["payload"]["frame_id"].as_str().unwrap(); let f0 = enters[3]["payload"]["frame_id"].as_str().unwrap(); assert_eq!(enters[1]["payload"]["parent_frame_id"], main); assert_eq!(enters[2]["payload"]["parent_frame_id"], f2); assert_eq!(enters[3]["payload"]["parent_frame_id"], main); let exit_frames: Vec<&str> = events_of(&events, "call.exit") .iter() .map(|event| event["payload"]["frame_id"].as_str().unwrap()) .collect(); for frame in [main, f2, g2, f0] { assert!(exit_frames.contains(&frame), "frame {frame} must close"); } assert_eq!(events_of(&events, "branch.eval").len(), 2); assert_eq!( events_of(&events, "loop.iteration") .iter() .map(|event| event["payload"]["decision"].as_str().unwrap()) .collect::>(), vec!["continue", "continue", "exit"] ); } #[test] fn native_plugin_is_fail_closed_in_event_mode() { use sha2::Digest; let dir = TempDir::new("native-plugin"); dir.write("index.mq.md", "## @exec main\n\n`return 1`\n"); let plugin_dir = dir.path().join("plugin"); fs::create_dir_all(&plugin_dir).unwrap(); let artifact = plugin_dir.join("solver.so"); fs::write(&artifact, b"not-a-real-library").unwrap(); let digest = hex::encode(sha2::Sha256::digest(fs::read(&artifact).unwrap())); let manifest = format!( "plugin:\n name: demo.solver\n version: 0.1.0\n abi: \"^1.0\"\n kind: native\n exports:\n - name: solver.add\n signature: \"($a: Int, $b: Int) -> Int\"\n requires:\n - math.compute\n artifacts:\n - target: {}\n file: solver.so\n sha256: \"{}\"\n", marqdo_plugin_abi::HostTarget::current().triple(), digest ); fs::write(plugin_dir.join("plugin.yaml"), manifest).unwrap(); let output = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--plugin", plugin_dir.join("plugin.yaml").to_str().unwrap(), "--allow-plugin", "demo.solver", ], ); assert_eq!(output.status.code(), Some(2)); let events = parse_events(&output.stdout); assert_envelope(&events); let terminal = events.last().unwrap(); assert_eq!(terminal["kind"], "run.rejected"); assert_eq!( terminal["payload"]["error"]["code"], "ext.event_stream_unsupported_native_plugin" ); assert!( events_of(&events, "run.started").is_empty(), "the native plugin is rejected before execution" ); } #[test] fn observed_values_never_enter_trace_v2() { let dir = TempDir::new("trace-redaction"); dir.write( "index.mq.md", "## @exec main\n\n`secret_token = \"DO_NOT_LEAK_SENTINEL_1\"`\n\n`visible_count = 42`\n\n`return visible_count`\n", ); let trace_path = dir.path().join("trace.json"); let output = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--observe-variable", "secret_token", "--observe-text-values", "--trace-file", trace_path.to_str().unwrap(), ], ); assert_eq!(output.status.code(), Some(0)); assert!( String::from_utf8_lossy(&output.stdout).contains("DO_NOT_LEAK_SENTINEL_1"), "the explicit event consent publishes the selected value in the stream" ); let trace = fs::read(&trace_path).unwrap(); assert!( !String::from_utf8_lossy(&trace).contains("DO_NOT_LEAK_SENTINEL_1"), "Trace v2 must not gain observed values" ); let trace: Value = serde_json::from_slice(&trace).unwrap(); assert_eq!(trace["schema"], "marqdo.trace"); assert_eq!(trace["schema_version"], 2); } #[test] fn basic_stream_is_pure_jsonl_and_reconstructs_program_output() { let dir = TempDir::new("basic"); dir.write("index.mq.md", BASIC); let output = cli( dir.path(), &["run", "index.mq.md", "--event-stream", "jsonl"], ); assert_eq!( output.status.code(), Some(0), "{}", String::from_utf8_lossy(&output.stderr) ); let events = parse_events(&output.stdout); assert_envelope(&events); assert_eq!(events_of(&events, "run.started").len(), 1); assert_eq!(chunk_bytes(&events, "stdout.chunk"), b"START\nDONE\n"); let terminal = events.last().unwrap(); assert_eq!(terminal["kind"], "run.completed"); assert_eq!(terminal["payload"]["status"], "ok"); assert_eq!(terminal["payload"]["value"]["decimal"], "10"); // Terminal sequence is after all chunk sequences. let terminal_seq = terminal["seq"].as_u64().unwrap(); for event in events_of(&events, "stdout.chunk") { assert!(event["seq"].as_u64().unwrap() < terminal_seq); } // The normal (non-event) run writes the same bytes and return value. let normal = cli(dir.path(), &["run", "index.mq.md"]); assert_eq!(normal.status.code(), Some(0)); assert_eq!(normal.stdout, b"START\nDONE\n"); } #[test] fn event_stream_rejects_mixed_json_and_static_commands() { let dir = TempDir::new("reject"); dir.write("index.mq.md", BASIC); let mixed = cli( dir.path(), &["run", "index.mq.md", "--event-stream", "jsonl", "--json"], ); assert_eq!(mixed.status.code(), Some(2)); assert!(mixed.stdout.is_empty(), "no mixed stdout"); assert!(String::from_utf8_lossy(&mixed.stderr).contains("--json")); for command in ["check", "inspect"] { let rejected = cli( dir.path(), &[command, "index.mq.md", "--event-stream", "jsonl"], ); assert_eq!( rejected.status.code(), Some(2), "{command} must reject the option" ); assert!(rejected.stdout.is_empty(), "{command} must not execute"); } } #[test] fn prepare_and_runtime_failures_have_defined_terminals() { let dir = TempDir::new("failures"); dir.write("broken.mq.md", "## @exec main\n\n`if true`\n\n`return 1`\n"); let rejected = cli( dir.path(), &["run", "broken.mq.md", "--event-stream", "jsonl"], ); assert_eq!(rejected.status.code(), Some(1)); let events = parse_events(&rejected.stdout); assert_envelope(&events); assert_eq!(events.last().unwrap()["kind"], "run.rejected"); assert_eq!(events.last().unwrap()["payload"]["phase"], "compile"); assert!( events_of(&events, "run.started").is_empty(), "a rejected run never starts" ); dir.write( "runtime.mq.md", "## @exec main\n\n`x = 1 / 0`\n\n`return x`\n", ); let failed = cli( dir.path(), &["run", "runtime.mq.md", "--event-stream", "jsonl"], ); assert_eq!(failed.status.code(), Some(1)); let events = parse_events(&failed.stdout); assert_envelope(&events); assert_eq!(events.last().unwrap()["kind"], "run.failed"); assert_eq!( events.last().unwrap()["payload"]["error"]["code"], "runtime.division_by_zero" ); assert!(events_of(&events, "run.completed").is_empty()); let exit = events_of(&events, "call.exit"); assert_eq!(exit.len(), 1, "failure still closes the entered frame"); assert_eq!(exit[0]["payload"]["status"], "runtime.division_by_zero"); } #[test] fn value_observation_is_exact_and_requires_text_consent() { let dir = TempDir::new("values"); dir.write( "index.mq.md", "## @exec main\n\n`secret_token = \"DO_NOT_LEAK_SENTINEL_1\"`\n\n`visible_count = 42`\n\n`return visible_count`\n", ); // Default: metadata only; no value, no sentinel anywhere. let default = cli( dir.path(), &["run", "index.mq.md", "--event-stream", "jsonl"], ); assert_eq!(default.status.code(), Some(0)); assert!(!String::from_utf8_lossy(&default.stdout).contains("DO_NOT_LEAK_SENTINEL_1")); for event in events_of(&parse_events(&default.stdout), "variable.assigned") { assert_eq!(event["payload"]["value_state"], "not_requested"); assert!(event["payload"].get("value").is_none()); } let selected = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--observe-variable", "visible_count", ], ); let events = parse_events(&selected.stdout); let visible = events_of(&events, "variable.assigned") .into_iter() .find(|event| event["payload"]["name"] == "visible_count") .expect("visible_count event"); assert_eq!(visible["payload"]["value_state"], "observed"); assert_eq!(visible["payload"]["value"]["decimal"], "42"); // The unselected variable must never carry a value. for event in events_of(&events, "variable.assigned") { if event["payload"]["name"] == "secret_token" { assert_eq!(event["payload"]["value_state"], "not_requested"); assert!(event["payload"].get("value").is_none()); } } // Selecting the text variable is not enough: the bytes stay redacted. let redacted = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--observe-variable", "secret_token", ], ); assert!(!String::from_utf8_lossy(&redacted.stdout).contains("DO_NOT_LEAK_SENTINEL_1")); let events = parse_events(&redacted.stdout); let secret = events_of(&events, "variable.assigned") .into_iter() .find(|event| event["payload"]["name"] == "secret_token") .expect("secret_token event"); assert_eq!(secret["payload"]["value_state"], "redacted"); assert!(secret["payload"].get("value").is_none()); // A second explicit consent publishes only the selected value. let consented = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--observe-variable", "secret_token", "--observe-text-values", ], ); let stdout = String::from_utf8_lossy(&consented.stdout); assert!(stdout.contains("DO_NOT_LEAK_SENTINEL_1")); let events = parse_events(&consented.stdout); let secret = events_of(&events, "variable.assigned") .into_iter() .find(|event| event["payload"]["name"] == "secret_token") .unwrap(); assert_eq!(secret["payload"]["value_state"], "observed"); assert_eq!( secret["payload"]["value"]["value"], "DO_NOT_LEAK_SENTINEL_1" ); } #[test] fn binary_and_multibyte_output_is_byte_exact() { let dir = TempDir::new("bytes"); dir.write( "index.mq.md", "## @exec main\n\n`打印(\"开始文本\")`\n\n`流 = resource.open(\"stdio\", \"stdout\", [\"write\"])`\n\n`内存 = memory.new(1, 1)`\n\n`内存[0] = 255`\n\n`内存[1] = 0`\n\n`内存[2] = 128`\n\n`写入 = resource.write(流, 内存, 0, 3)`\n\n`关闭 = resource.close(流)`\n\n`return 写入`\n", ); let normal = cli(dir.path(), &["run", "index.mq.md"]); assert_eq!( normal.status.code(), Some(0), "{}", String::from_utf8_lossy(&normal.stderr) ); let streamed = cli( dir.path(), &["run", "index.mq.md", "--event-stream", "jsonl"], ); assert_eq!(streamed.status.code(), Some(0)); let events = parse_events(&streamed.stdout); let chunks = chunk_bytes(&events, "stdout.chunk"); assert_eq!( chunks, normal.stdout, "error text may not go to stdout; concatenated chunks must equal the program bytes" ); assert!(chunks.starts_with("开始文本".as_bytes())); assert_eq!(&chunks[chunks.len() - 3..], &[0xff, 0x00, 0x80]); } #[test] fn trace_file_coexists_and_shares_the_run_identity() { let dir = TempDir::new("trace"); dir.write("index.mq.md", BASIC); let trace_path = dir.path().join("trace.json"); let output = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--trace-file", trace_path.to_str().unwrap(), ], ); assert_eq!(output.status.code(), Some(0)); let events = parse_events(&output.stdout); assert_envelope(&events); let trace: Value = serde_json::from_slice(&fs::read(&trace_path).unwrap()).unwrap(); assert_eq!(trace["schema"], "marqdo.trace"); assert_eq!(trace["schema_version"], 2); assert_eq!(trace["run"]["id"], events[0]["run_id"]); let started = events_of(&events, "run.started")[0]; assert_eq!( started["payload"]["source"]["analysis_digest"], trace["source"]["analysis_digest"] ); assert!( events_of(&events, "run.completed").len() == 1, "trace writing does not suppress the terminal event" ); } #[test] fn structural_budget_truncates_without_losing_critical_events() { let dir = TempDir::new("budget"); dir.write("index.mq.md", BASIC); let output = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--observe-max-events", "5", ], ); assert_eq!(output.status.code(), Some(0)); let events = parse_events(&output.stdout); assert_envelope(&events); let markers = events_of(&events, "observation.truncated"); assert_eq!(markers.len(), 1); assert_eq!(markers[0]["payload"]["reason"], "event_budget"); assert_eq!(chunk_bytes(&events, "stdout.chunk"), b"START\nDONE\n"); assert_eq!(events.last().unwrap()["kind"], "run.completed"); } #[test] fn output_budget_still_fails_the_run_and_emits_no_chunk() { let dir = TempDir::new("output-budget"); dir.write("index.mq.md", BASIC); let output = cli( dir.path(), &[ "run", "index.mq.md", "--event-stream", "jsonl", "--max-output", "3", ], ); assert_eq!(output.status.code(), Some(1)); let events = parse_events(&output.stdout); assert_envelope(&events); assert!(chunk_bytes(&events, "stdout.chunk").is_empty()); assert_eq!(events.last().unwrap()["kind"], "run.failed"); assert_eq!( events.last().unwrap()["payload"]["error"]["code"], "resource.exhausted" ); } #[test] fn live_chunks_arrive_while_the_program_is_still_running() { let dir = TempDir::new("live"); let source = "## @exec main\n\n`打印(\"READY\")`\n\n`i = 0`\n\n`while i < 2000000`\n\n`i = i + 1`\n\n`end`\n\n`打印(\"FINISH\")`\n\n`return i`\n"; dir.write("index.mq.md", source); let mut child = Command::new(env!("CARGO_BIN_EXE_marqdo")) .args([ "run", "index.mq.md", "--event-stream", "jsonl", "--observe-max-events", "20", "--fuel", "100000000", ]) .current_dir(dir.path()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .expect("spawn marqdo"); let mut reader = BufReader::new(child.stdout.take().unwrap()); let mut events = Vec::new(); let mut ready_seen_alive = false; loop { let mut line = String::new(); let read = reader.read_line(&mut line).expect("read event line"); if read == 0 { break; } let event: Value = serde_json::from_str(line.trim_end()).expect("event JSON"); if event["kind"] == "stdout.chunk" { let data = base64_decode(event["payload"]["data_base64"].as_str().unwrap()); if data == b"READY\n" || data == b"READY" { assert!( child.try_wait().expect("try_wait").is_none(), "READY was only readable after the process exited; this is not live output" ); ready_seen_alive = true; } } events.push(event); } let status = child.wait().expect("wait"); assert_eq!(status.code(), Some(0)); assert!( ready_seen_alive, "the READY chunk must be readable before the process exits" ); assert_eq!(chunk_bytes(&events, "stdout.chunk"), b"READY\nFINISH\n"); assert_eq!(events.last().unwrap()["kind"], "run.completed"); } #[test] fn broken_pipe_makes_the_run_fail_instead_of_running_forever() { let dir = TempDir::new("pipe"); let source = "## @exec main\n\n`打印(\"START\")`\n\n`i = 0`\n\n`while i < 5000000`\n\n`i = i + 1`\n\n`end`\n\n`return i`\n"; dir.write("index.mq.md", source); let mut child = Command::new(env!("CARGO_BIN_EXE_marqdo")) .args([ "run", "index.mq.md", "--event-stream", "jsonl", "--observe-max-events", "10", "--fuel", "100000000", ]) .current_dir(dir.path()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .expect("spawn marqdo"); { let mut reader = BufReader::new(child.stdout.take().unwrap()); let mut line = String::new(); reader.read_line(&mut line).expect("read one line"); assert!(line.contains("stream.opened")); // Dropping the reader closes the pipe; the next write must fail. } let deadline = Instant::now() + Duration::from_secs(20); loop { match child.try_wait().expect("try_wait") { Some(status) => { assert_ne!( status.code(), Some(0), "a broken event pipe must fail the run" ); break; } None if Instant::now() > deadline => { let _ = child.kill(); panic!("the run kept executing after its event pipe was closed"); } None => std::thread::sleep(Duration::from_millis(20)), } } let mut stderr = String::new(); child .stderr .take() .unwrap() .read_to_string(&mut stderr) .unwrap(); assert!( stderr.contains("observe.transport_failed"), "expected a transport diagnostic on stderr: {stderr}" ); }