From d890686a2bef7b57f2b7aca84db83f49c9ca20fe Mon Sep 17 00:00:00 2001 From: cflmy Date: Fri, 20 Nov 2026 13:01:22 +0800 Subject: [PATCH] feat(trace): Trace v2 identity, module-owned events and capacity bounds (B-16) - bind traces to the logical project identity and the exact emitted Analysis digest; every span-bearing event carries module_id - add host TraceState budgets (100k events / 32 MiB defaults), CLI --trace-max-events/--trace-max-bytes, terminal trace.truncated events, and a final serialized byte cap; budgets never change run.status - disable semantic recording when no trace is requested - cap worker frames at 8 MiB, align request/result/capability wire caps, make clean EOF distinct from protocol errors, and mark/kill a worker after a transport failure instead of reusing it - add trace-v2 spec/schema, independent trace consumer, CI check and tests - record the S2/C state in STATUS --- .github/workflows/ci.yml | 6 +- README.md | 2 +- crates/cli/src/commands.rs | 17 +++- crates/cli/src/main.rs | 10 +- crates/cli/src/trace.rs | 66 ++++++++++--- crates/cli/tests/analysis_trace.rs | 53 ++++++++++- crates/marqdo-host/src/host.rs | 48 ++++++++-- crates/marqdo-host/src/lib.rs | 2 +- crates/marqdo-host/src/plugin_worker.rs | 61 +++++++++--- crates/marqdo-host/src/trace.rs | 96 ++++++++++++++++++- crates/marqdo-plugin-abi/src/abi.rs | 12 +-- crates/marqdo-plugin-abi/src/worker.rs | 108 ++++++++++++++++++---- crates/marqdo-runtime/src/interpreter.rs | 25 ++++- docs/STATUS.md | 28 +++++- docs/contributor-guide.md | 1 + include/marqdo_plugin_abi_v1.h | 12 +-- spec/plugin-abi.md | 15 +-- spec/schemas/trace-v2.schema.json | 86 +++++++++++++++++ spec/trace-v1.md | 3 +- spec/trace-v2.md | 112 +++++++++++++++++++++++ tests/consumers/trace_consumer.py | 53 +++++++++++ 21 files changed, 732 insertions(+), 84 deletions(-) create mode 100644 spec/schemas/trace-v2.schema.json create mode 100644 spec/trace-v2.md create mode 100644 tests/consumers/trace_consumer.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd48450..4ef8322 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,13 @@ jobs: ./target/debug/marqdo inspect examples/references.mq.md --refs ./target/debug/marqdo check examples/multi-module/index.mq.md --refs \ --emit /tmp/marqdo-analysis.json + ./target/debug/marqdo run examples/multi-module/index.mq.md \ + --trace-file /tmp/marqdo-analysis-trace.json + python3 tests/consumers/analysis_consumer.py /tmp/marqdo-analysis.json + python3 tests/consumers/trace_consumer.py \ + /tmp/marqdo-analysis.json /tmp/marqdo-analysis-trace.json ./target/debug/marqdo run examples/hello.mq.md \ --trace-file /tmp/marqdo-trace.json - python3 tests/consumers/analysis_consumer.py /tmp/marqdo-analysis.json python3 - <<'PY' import json json.load(open("/tmp/marqdo-trace.json")) diff --git a/README.md b/README.md index ce249de..31fa7d5 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ tests/ conformance/negative corpus (see tests/README.md) [`package-format.md`](spec/package-format.md), [`plugin-abi.md`](spec/plugin-abi.md), [`math.md`](spec/math.md), [`foreign.md`](spec/foreign.md), [`tool-v1.md`](spec/tool-v1.md), - [`analysis-v2.md`](spec/analysis-v2.md), [`trace-v1.md`](spec/trace-v1.md); + [`analysis-v2.md`](spec/analysis-v2.md), [`trace-v2.md`](spec/trace-v2.md); JSON Schemas in [`spec/schemas/`](spec/schemas/). * [`tests/README.md`](tests/README.md) — conformance corpus and `UPDATE_GOLDENS` workflow. diff --git a/crates/cli/src/commands.rs b/crates/cli/src/commands.rs index 9113f0d..61a3a42 100644 --- a/crates/cli/src/commands.rs +++ b/crates/cli/src/commands.rs @@ -38,8 +38,12 @@ pub struct SandboxSettings { pub plugins: Vec, /// Explicit trust decision for in-process native plugin loading. pub trust_native_plugin: bool, - /// Optional Trace v1 output path. + /// Optional Trace v2 output path. pub trace_file: Option, + /// Trace event budget override (0 = unlimited). + pub trace_max_events: Option, + /// Trace approximate byte budget override (0 = unlimited). + pub trace_max_bytes: Option, /// External foreign-kernel descriptor paths. pub kernels: Vec, } @@ -690,7 +694,16 @@ pub fn execute( }; let mut host = StdHost::new(mode, capabilities, project.root.clone()); if trace || json || settings.trace_file.is_some() { - host = host.with_trace(); + host = host + .with_trace() + .with_trace_budget(marqdo_host::TraceBudget { + max_events: settings + .trace_max_events + .unwrap_or(marqdo_host::trace::DEFAULT_MAX_TRACE_EVENTS), + max_bytes: settings + .trace_max_bytes + .unwrap_or(marqdo_host::trace::DEFAULT_MAX_TRACE_BYTES), + }); } if let Some(fuel) = settings.fuel { host.budget.fuel = fuel; diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index ea715e9..b1c1706 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -109,9 +109,15 @@ struct SandboxArgs { /// Explicitly trusted plugin manifest/directory to load (repeatable). #[arg(long = "plugin", value_name = "PATH")] plugin: Vec, - /// Write a versioned Trace v1 file for this run. + /// Write a versioned Trace v2 file for this run. #[arg(long = "trace-file", value_name = "PATH")] trace_file: Option, + /// Override the trace event budget (0 = unlimited). + #[arg(long = "trace-max-events")] + trace_max_events: Option, + /// Override the approximate trace byte budget (0 = unlimited). + #[arg(long = "trace-max-bytes")] + trace_max_bytes: Option, /// Register an external foreign-kernel provider (repeatable). #[arg(long = "kernel", value_name = "PATH")] kernel: Vec, @@ -135,6 +141,8 @@ impl SandboxArgs { plugins: self.plugin.clone(), trust_native_plugin: self.trust_native_plugin, trace_file: self.trace_file.clone(), + trace_max_events: self.trace_max_events, + trace_max_bytes: self.trace_max_bytes, kernels: self.kernel.clone(), } } diff --git a/crates/cli/src/trace.rs b/crates/cli/src/trace.rs index cac0dff..15844c3 100644 --- a/crates/cli/src/trace.rs +++ b/crates/cli/src/trace.rs @@ -1,4 +1,4 @@ -//! Public Trace v1 writer (spec/trace-v1.md). +//! Public Trace v2 writer (spec/trace-v2.md). //! //! Events come from the interpreter (semantic) and the capability host //! (effects). No argument values, secrets or handles are recorded. @@ -12,7 +12,7 @@ use crate::loader::LoadedProject; /// Schema name and version. pub const SCHEMA: &str = "marqdo.trace"; -pub const SCHEMA_VERSION: u32 = 1; +pub const SCHEMA_VERSION: u32 = 2; /// Write a trace atomically for one run. pub fn write( @@ -32,6 +32,7 @@ pub fn write( ); let analysis_digest = crate::analysis::digest(project); let root_id = project.identity.root_id(); + let state = host.trace_state(); let mut events: Vec = host .semantic_events() .iter() @@ -39,15 +40,55 @@ pub fn write( .chain(host.trace().iter().map(effect_json)) .collect(); events.sort_by_key(|event| event.get("seq").and_then(Value::as_u64).unwrap_or(u64::MAX)); - let trace = json!({ - "schema": SCHEMA, - "schema_version": SCHEMA_VERSION, - "run": {"id": run_id, "entry": entry, "status": status}, - "source": {"root_id": root_id, "analysis_digest": format!("sha256:{analysis_digest}")}, - "events": events, - }); - let bytes = serde_json::to_vec_pretty(&trace) - .map_err(|error| format!("cannot encode trace: {error}"))?; + + let mut truncation = state.truncation.clone(); + let max_bytes = state.budget.max_bytes; + let bytes = loop { + let mut all = events.clone(); + if let Some(truncation) = &truncation { + let seq = all + .last() + .and_then(|event| event.get("seq")) + .and_then(Value::as_u64) + .unwrap_or(0) + + 1; + all.push(json!({ + "seq": seq, + "kind": "trace.truncated", + "module_id": null, + "span": null, + "detail": truncation.reason, + })); + } + let trace = json!({ + "schema": SCHEMA, + "schema_version": SCHEMA_VERSION, + "run": {"id": run_id, "entry": entry, "status": status}, + "source": { + "root_id": root_id, + "analysis_digest": format!("sha256:{analysis_digest}"), + }, + "limits": { + "max_events": state.budget.max_events, + "max_bytes": state.budget.max_bytes, + }, + "truncated": truncation, + "events": all, + }); + let bytes = serde_json::to_vec_pretty(&trace) + .map_err(|error| format!("cannot encode trace: {error}"))?; + if max_bytes == 0 || bytes.len() <= max_bytes || events.is_empty() { + break bytes; + } + events.pop(); + if truncation.is_none() { + truncation = Some(marqdo_host::TraceTruncation { + reason: "byte_budget".to_string(), + limit: max_bytes as u64, + written: events.len() as u64, + }); + } + }; atomic_write(path, &bytes) } @@ -56,6 +97,7 @@ fn semantic_json(event: &SemanticEvent) -> Value { "seq": event.seq, "kind": event.kind, "function_id": event.function_id, + "module_id": event.module_id, "span": event.span.map(|span| json!({"start": span.start, "end": span.end})), "detail": event.detail, }) @@ -65,7 +107,7 @@ fn effect_json(entry: &TraceEntry) -> Value { json!({ "seq": entry.seq, "kind": "effect", - "module": entry.module, + "module_id": entry.module, "span": entry.span.map(|span| json!({"start": span.start, "end": span.end})), "detail": entry.result, "operation": entry.op, diff --git a/crates/cli/tests/analysis_trace.rs b/crates/cli/tests/analysis_trace.rs index ca667d7..d0e4602 100644 --- a/crates/cli/tests/analysis_trace.rs +++ b/crates/cli/tests/analysis_trace.rs @@ -241,6 +241,52 @@ print(1) ); } +#[test] +fn trace_v2_budget_truncates_long_loops_without_failing_the_run() { + let dir = TempDir::new("trace-budget"); + write( + dir.path(), + "index.mq.md", + "## main\n\n**i = 0**\n\n- `i < 500`\n\n **i = i + 1**\n\n*i*\n", + ); + let trace_path = dir.path().join("trace.json"); + let output = cli( + dir.path(), + &[ + "run", + "index.mq.md", + "--trace-file", + trace_path.to_str().unwrap(), + "--trace-max-events", + "20", + ], + ); + assert_eq!( + output.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let trace: serde_json::Value = + serde_json::from_slice(&std::fs::read(&trace_path).unwrap()).unwrap(); + assert_eq!(trace["schema_version"], 2); + assert_eq!(trace["run"]["status"], "ok"); + assert_eq!(trace["limits"]["max_events"], 20); + assert_eq!(trace["truncated"]["reason"], "event_budget"); + assert_eq!(trace["truncated"]["limit"], 20); + let events = trace["events"].as_array().unwrap(); + assert!(events.len() <= 21, "events: {}", events.len()); + assert_eq!(events.last().unwrap()["kind"], "trace.truncated"); + let mut previous = None; + for event in events { + let seq = event["seq"].as_u64().unwrap(); + if let Some(previous) = previous { + assert!(seq > previous); + } + previous = Some(seq); + } +} + #[test] fn failed_check_removes_stale_analysis() { let dir = TempDir::new("analysis-error"); @@ -282,11 +328,14 @@ fn trace_records_calls_effects_and_status_without_secrets() { ); let trace: serde_json::Value = serde_json::from_str(&text).unwrap(); assert_eq!(trace["schema"], "marqdo.trace"); - assert_eq!(trace["schema_version"], 1); + assert_eq!(trace["schema_version"], 2); assert_eq!(trace["run"]["status"], "ok"); assert_eq!(trace["run"]["entry"], "root:default::index.mq.md::main"); let events = trace["events"].as_array().unwrap(); - assert!(events.iter().any(|event| event["kind"] == "call.enter")); + assert!(events + .iter() + .any(|event| event["kind"] == "call.enter" + && event["module_id"] == "root:default::index.mq.md")); assert!(events.iter().any(|event| event["kind"] == "effect")); assert!(events.iter().any(|event| event["kind"] == "call.exit")); let mut previous = None; diff --git a/crates/marqdo-host/src/host.rs b/crates/marqdo-host/src/host.rs index 24c1c24..31ebfbb 100644 --- a/crates/marqdo-host/src/host.rs +++ b/crates/marqdo-host/src/host.rs @@ -20,7 +20,7 @@ use crate::context::ExecutionContext; use crate::fs::ProjectFs; use crate::handles::{contains_plugin_handle, PluginHandleTable}; use crate::plugin_worker::WorkerPlugin; -use crate::trace::{EffectTrace, SemanticEvent, TraceEntry}; +use crate::trace::{EffectTrace, SemanticEvent, TraceBudget, TraceEntry, TraceState}; use marqdo_sandbox::{PlatformCapabilities, SandboxMode}; /// Host operations reachable from the language surface. @@ -99,6 +99,7 @@ pub trait Host { &mut self, _kind: &str, _function_id: Option<&str>, + _module_id: Option<&str>, _span: Option, _detail: Option<&str>, ) { @@ -144,6 +145,7 @@ pub struct StdHost { pub handles: PluginHandleTable, semantic_events: Vec, next_event_seq: u64, + trace_state: TraceState, } struct PluginSlot { @@ -203,6 +205,7 @@ impl StdHost { handles: PluginHandleTable::default(), semantic_events: Vec::new(), next_event_seq: 0, + trace_state: TraceState::default(), } } @@ -338,6 +341,17 @@ impl StdHost { seq } + /// Trace capacity state for the writer. + pub fn trace_state(&self) -> &TraceState { + &self.trace_state + } + + /// Override the trace event/byte budget. + pub fn with_trace_budget(mut self, budget: TraceBudget) -> Self { + self.trace_state.budget = budget; + self + } + /// Audited plugin capability decisions. pub fn capability_audit(&self) -> Vec { self.broker_audit @@ -359,6 +373,7 @@ impl StdHost { /// Enable effect tracing. pub fn with_trace(mut self) -> Self { self.trace = EffectTrace::enabled(); + self.trace_state.enabled = true; self } @@ -504,20 +519,29 @@ impl Host for StdHost { ctx: &ExecutionContext, ) -> Result { let started = Instant::now(); - let seq = self.next_seq(); let result = self.dispatch(op, args, ctx); let summary = match &result { Ok(_) => "ok".to_string(), Err(error) => error.code().to_string(), }; - self.trace.record( - seq, - op.name(), - ctx.module_id.clone(), - ctx.span, - started.elapsed(), - summary, + let estimate = TraceState::estimate( + "effect", + None, + Some(&ctx.module_id), + Some(&summary), + Some(op.name()), ); + if self.trace_state.accept(estimate) { + let seq = self.next_seq(); + self.trace.record( + seq, + op.name(), + ctx.module_id.clone(), + ctx.span, + started.elapsed(), + summary, + ); + } result } @@ -545,14 +569,20 @@ impl Host for StdHost { &mut self, kind: &str, function_id: Option<&str>, + module_id: Option<&str>, span: Option, detail: Option<&str>, ) { + let estimate = TraceState::estimate(kind, function_id, module_id, detail, None); + if !self.trace_state.accept(estimate) { + return; + } let seq = self.next_seq(); self.semantic_events.push(SemanticEvent { seq, kind: kind.to_string(), function_id: function_id.map(str::to_string), + module_id: module_id.map(str::to_string), span, detail: detail.map(str::to_string), }); diff --git a/crates/marqdo-host/src/lib.rs b/crates/marqdo-host/src/lib.rs index 41eb3ed..7269f76 100644 --- a/crates/marqdo-host/src/lib.rs +++ b/crates/marqdo-host/src/lib.rs @@ -21,4 +21,4 @@ pub use fs::ProjectFs; pub use handles::{contains_plugin_handle, PluginHandleTable}; pub use host::{arity, expect_text, Host, HostOp, StdHost}; pub use marqdo_sandbox::{PlatformCapabilities, SandboxMode, StrictIsolation}; -pub use trace::{EffectTrace, SemanticEvent, TraceEntry}; +pub use trace::{EffectTrace, SemanticEvent, TraceBudget, TraceEntry, TraceState, TraceTruncation}; diff --git a/crates/marqdo-host/src/plugin_worker.rs b/crates/marqdo-host/src/plugin_worker.rs index 43e86d8..940f40a 100644 --- a/crates/marqdo-host/src/plugin_worker.rs +++ b/crates/marqdo-host/src/plugin_worker.rs @@ -22,6 +22,7 @@ pub struct WorkerPlugin { stdin: ChildStdin, frames: Receiver>, exports: BTreeMap, + dead: bool, } const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); @@ -94,6 +95,7 @@ impl WorkerPlugin { }; let Some(mut stdout) = child.stdout.take() else { let _ = child.kill(); + let _ = std::fs::remove_file(&manifest_path); return Err(PluginError::load( "worker_spawn_failed", "worker stdout unavailable", @@ -103,10 +105,23 @@ impl WorkerPlugin { thread::Builder::new() .name("marqdo-plugin-worker-reader".to_string()) .spawn(move || loop { - let frame = read_frame(&mut stdout); - let failed = frame.is_err(); - if sender.send(frame).is_err() || failed { - break; + match read_frame(&mut stdout) { + Ok(Some(frame)) => { + if sender.send(Ok(frame)).is_err() { + break; + } + } + Ok(None) => { + let _ = sender.send(Err(PluginError::invoke( + "plugin.crashed", + "plugin worker closed its output stream", + ))); + break; + } + Err(error) => { + let _ = sender.send(Err(error)); + break; + } } }) .map_err(|error| { @@ -169,21 +184,35 @@ impl WorkerPlugin { stdin, frames, exports, + dead: false, }) } /// Invoke one export in the worker. pub fn invoke(&mut self, request: &WireRequest) -> Result { + if self.dead { + return Err(PluginError::invoke( + "plugin.crashed", + "plugin worker is no longer usable after a transport failure", + )); + } let request_value: serde_json::Value = serde_json::from_slice(&request.to_json_bytes()?) .map_err(|error| PluginError::wire("worker_encode_failed", error.to_string()))?; - write_frame( + if let Err(error) = write_frame( &mut self.stdin, &json!({"kind": "invoke", "request": request_value}), - )?; + ) { + self.mark_dead(); + return Err(error); + } let frame = match self.frames.recv_timeout(CALL_TIMEOUT) { - Ok(frame) => frame?, + Ok(Ok(frame)) => frame, + Ok(Err(error)) => { + self.mark_dead(); + return Err(error); + } Err(_) => { - let _ = self.child.kill(); + self.mark_dead(); return Err(PluginError::invoke( "plugin.timeout", "plugin worker exceeded its invocation timeout", @@ -202,12 +231,20 @@ impl WorkerPlugin { .and_then(serde_json::Value::as_str) .unwrap_or("worker reported an error"), )), - _ => Err(PluginError::invoke( - "plugin.crashed", - "plugin worker closed or violated the frame protocol", - )), + _ => { + self.mark_dead(); + Err(PluginError::invoke( + "plugin.crashed", + "plugin worker closed or violated the frame protocol", + )) + } } } + + fn mark_dead(&mut self) { + self.dead = true; + let _ = self.child.kill(); + } } impl Drop for WorkerPlugin { diff --git a/crates/marqdo-host/src/trace.rs b/crates/marqdo-host/src/trace.rs index f7385dd..2ec7cf9 100644 --- a/crates/marqdo-host/src/trace.rs +++ b/crates/marqdo-host/src/trace.rs @@ -70,7 +70,99 @@ impl EffectTrace { } } -/// One semantic event emitted by the interpreter (Trace v1). +/// Default maximum number of trace events. +pub const DEFAULT_MAX_TRACE_EVENTS: usize = 100_000; +/// Default approximate maximum serialized trace bytes. +pub const DEFAULT_MAX_TRACE_BYTES: usize = 32 * 1024 * 1024; + +/// Trace capacity budget. A zero value means unlimited. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TraceBudget { + /// Maximum events (including the terminal truncation marker). + pub max_events: usize, + /// Approximate maximum serialized bytes. + pub max_bytes: usize, +} + +impl Default for TraceBudget { + fn default() -> Self { + Self { + max_events: DEFAULT_MAX_TRACE_EVENTS, + max_bytes: DEFAULT_MAX_TRACE_BYTES, + } + } +} + +/// Why and where a trace stopped recording. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TraceTruncation { + /// `event_budget` or `byte_budget`. + pub reason: String, + /// The configured limit that was reached. + pub limit: u64, + /// Events written before truncation. + pub written: u64, +} + +/// Mutable trace capacity state for one run. +#[derive(Debug, Clone, Default)] +pub struct TraceState { + /// Whether trace recording is enabled. + pub enabled: bool, + /// Configured budget. + pub budget: TraceBudget, + /// Events accepted so far. + pub written: usize, + /// Estimated accepted bytes. + pub bytes: usize, + /// Truncation marker, set once. + pub truncation: Option, +} + +impl TraceState { + /// Accept one event if it fits the budget. + pub fn accept(&mut self, estimate: usize) -> bool { + if !self.enabled || self.truncation.is_some() { + return false; + } + if self.budget.max_events > 0 && self.written >= self.budget.max_events { + self.truncation = Some(TraceTruncation { + reason: "event_budget".to_string(), + limit: self.budget.max_events as u64, + written: self.written as u64, + }); + return false; + } + if self.budget.max_bytes > 0 && self.bytes + estimate > self.budget.max_bytes { + self.truncation = Some(TraceTruncation { + reason: "byte_budget".to_string(), + limit: self.budget.max_bytes as u64, + written: self.written as u64, + }); + return false; + } + self.written += 1; + self.bytes += estimate; + true + } + + /// Estimate one semantic/effect event envelope. + pub fn estimate( + kind: &str, + function_id: Option<&str>, + module_id: Option<&str>, + detail: Option<&str>, + operation: Option<&str>, + ) -> usize { + 96 + kind.len() + + function_id.map(str::len).unwrap_or(0) + + module_id.map(str::len).unwrap_or(0) + + detail.map(str::len).unwrap_or(0) + + operation.map(str::len).unwrap_or(0) + } +} + +/// One semantic event emitted by the interpreter (Trace v2). #[derive(Debug, Clone, PartialEq, Eq)] pub struct SemanticEvent { /// Global event sequence within the run. @@ -79,6 +171,8 @@ pub struct SemanticEvent { pub kind: String, /// Canonical function id, when inside a call. pub function_id: Option, + /// Logical module owning `span`. + pub module_id: Option, /// Source span, when known. pub span: Option, /// Non-sensitive detail (`ok`, `true`, `int`, an error code). diff --git a/crates/marqdo-plugin-abi/src/abi.rs b/crates/marqdo-plugin-abi/src/abi.rs index 9988aa3..ae4d87b 100644 --- a/crates/marqdo-plugin-abi/src/abi.rs +++ b/crates/marqdo-plugin-abi/src/abi.rs @@ -73,14 +73,14 @@ pub const MAX_EXPORT_NAME_BYTES: usize = 256; pub const MAX_SIGNATURE_WIRE_BYTES: usize = 4096; /// Maximum plugin metadata wire length in bytes. pub const MAX_METADATA_WIRE_BYTES: usize = 65_536; -/// Maximum request wire length in bytes (16 MiB). -pub const MAX_REQUEST_WIRE_BYTES: usize = 16 * 1024 * 1024; -/// Maximum result wire length in bytes (64 MiB). -pub const MAX_RESULT_WIRE_BYTES: usize = 64 * 1024 * 1024; +/// Maximum request wire length in bytes (8 MiB frame minus envelope reserve). +pub const MAX_REQUEST_WIRE_BYTES: usize = 8 * 1024 * 1024 - 4096; +/// Maximum result wire length in bytes (8 MiB frame minus envelope reserve). +pub const MAX_RESULT_WIRE_BYTES: usize = 8 * 1024 * 1024 - 4096; /// Maximum error wire length in bytes (1 MiB). pub const MAX_ERROR_WIRE_BYTES: usize = 1024 * 1024; -/// Maximum capability reply length in bytes (16 MiB). -pub const MAX_CAPABILITY_REPLY_BYTES: usize = 16 * 1024 * 1024; +/// Maximum capability reply length in bytes (8 MiB frame minus envelope reserve). +pub const MAX_CAPABILITY_REPLY_BYTES: usize = 8 * 1024 * 1024 - 4096; /// Borrowed, immutable byte view (`MqBytes`). /// diff --git a/crates/marqdo-plugin-abi/src/worker.rs b/crates/marqdo-plugin-abi/src/worker.rs index 099ddaf..9925aad 100644 --- a/crates/marqdo-plugin-abi/src/worker.rs +++ b/crates/marqdo-plugin-abi/src/worker.rs @@ -19,17 +19,34 @@ use crate::wire::{WireRequest, WireResponse}; /// Hidden CLI command name used to start a worker. pub const WORKER_COMMAND: &str = "__plugin-worker"; /// Maximum frame size accepted by either side. -pub const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024; +pub const MAX_FRAME_BYTES: usize = 8 * 1024 * 1024; /// Read one length-prefixed frame (4-byte big-endian length + JSON). -pub fn read_frame(reader: &mut R) -> Result { +/// +/// `Ok(None)` means a clean EOF at a frame boundary. A partial header or +/// truncated payload is a protocol error, never EOF. +pub fn read_frame(reader: &mut R) -> Result, PluginError> { let mut length = [0_u8; 4]; - reader.read_exact(&mut length).map_err(|error| { - PluginError::invoke( - "plugin.crashed", - format!("worker frame read failed: {error}"), - ) - })?; + let mut read = 0usize; + while read < length.len() { + match reader.read(&mut length[read..]) { + Ok(0) if read == 0 => return Ok(None), + Ok(0) => { + return Err(PluginError::wire( + "worker_frame_invalid", + "worker stream ended inside a frame header", + )) + } + Ok(count) => read += count, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => { + return Err(PluginError::invoke( + "plugin.crashed", + format!("worker frame read failed: {error}"), + )) + } + } + } let length = u32::from_be_bytes(length) as usize; if length == 0 || length > MAX_FRAME_BYTES { return Err(PluginError::wire( @@ -37,14 +54,23 @@ pub fn read_frame(reader: &mut R) -> Result { format!("worker frame length {length} is invalid"), )); } - let mut bytes = vec![0_u8; length]; - reader.read_exact(&mut bytes).map_err(|error| { - PluginError::invoke( - "plugin.crashed", - format!("worker frame read failed: {error}"), - ) - })?; - serde_json::from_slice(&bytes).map_err(|error| { + let mut bytes = Vec::with_capacity(length.min(64 * 1024)); + let read = reader + .take(length as u64) + .read_to_end(&mut bytes) + .map_err(|error| { + PluginError::invoke( + "plugin.crashed", + format!("worker frame read failed: {error}"), + ) + })?; + if read != length { + return Err(PluginError::wire( + "worker_frame_invalid", + format!("worker frame declared {length} bytes but only {read} arrived"), + )); + } + serde_json::from_slice(&bytes).map(Some).map_err(|error| { PluginError::wire( "worker_frame_invalid", format!("worker frame is not valid JSON: {error}"), @@ -110,8 +136,12 @@ pub fn serve(manifest_path: &Path, base_dir: &Path) -> i32 { } loop { let frame = match read_frame(&mut stdin) { - Ok(frame) => frame, - Err(_) => return 0, + Ok(Some(frame)) => frame, + Ok(None) => return 0, + Err(error) => { + eprintln!("worker: {}: {}", error.code, error.message); + return 1; + } }; match frame.get("kind").and_then(Value::as_str) { Some("shutdown") => return 0, @@ -189,3 +219,45 @@ pub fn response_from_frame(frame: &Value) -> Result { .map_err(|error| PluginError::wire("worker_response_invalid", error.to_string()))?; WireResponse::from_json_bytes(&bytes) } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn frame_roundtrip_and_clean_eof() { + let mut bytes = Vec::new(); + write_frame(&mut bytes, &json!({"kind": "shutdown"})).unwrap(); + let mut cursor = Cursor::new(bytes.clone()); + assert_eq!( + read_frame(&mut cursor).unwrap(), + Some(json!({"kind": "shutdown"})) + ); + assert_eq!(read_frame(&mut cursor).unwrap(), None); + } + + #[test] + fn partial_header_is_an_error_not_eof() { + let mut cursor = Cursor::new(vec![0_u8, 0]); + let error = read_frame(&mut cursor).unwrap_err(); + assert_eq!(error.code, "worker_frame_invalid"); + } + + #[test] + fn oversized_and_zero_length_frames_are_rejected() { + for length in [0_u32, (MAX_FRAME_BYTES as u32) + 1] { + let mut cursor = Cursor::new(length.to_be_bytes().to_vec()); + let error = read_frame(&mut cursor).unwrap_err(); + assert_eq!(error.code, "worker_frame_invalid"); + } + } + + #[test] + fn truncated_payload_is_rejected() { + let mut bytes = 10_u32.to_be_bytes().to_vec(); + bytes.extend_from_slice(b"{}"); + let error = read_frame(&mut Cursor::new(bytes)).unwrap_err(); + assert_eq!(error.code, "worker_frame_invalid"); + } +} diff --git a/crates/marqdo-runtime/src/interpreter.rs b/crates/marqdo-runtime/src/interpreter.rs index 6d42173..ba4fc11 100644 --- a/crates/marqdo-runtime/src/interpreter.rs +++ b/crates/marqdo-runtime/src/interpreter.rs @@ -135,11 +135,17 @@ impl<'a> Interpreter<'a> { ) -> Result { let display = function.id.name.clone(); let trace_id = function.id.to_string(); + let trace_module = caller_module.unwrap_or(&function.id.module).canonical(); if self.depth + 1 > 0 { self.host.check_stack(self.depth + 1, call_span)?; } - self.host - .record_semantic("call.enter", Some(&trace_id), call_span, None); + self.host.record_semantic( + "call.enter", + Some(&trace_id), + Some(&trace_module), + call_span, + None, + ); let locals = match self.bind_args(function, args, call_span) { Ok(locals) => locals, Err(mut error) => { @@ -162,8 +168,13 @@ impl<'a> Interpreter<'a> { Ok(Flow::Return(_)) => "return", Err(error) => error.code(), }; - self.host - .record_semantic("call.exit", Some(&trace_id), call_span, Some(status)); + self.host.record_semantic( + "call.exit", + Some(&trace_id), + Some(&trace_module), + call_span, + Some(status), + ); match result { Ok(Flow::Next) => Ok(Value::Unit), Ok(Flow::Return(value)) => Ok(value), @@ -301,9 +312,11 @@ impl<'a> Interpreter<'a> { } Stmt::Return { value, span } => { let value = self.eval_expr(value, frame)?; + let module = frame.function.module.canonical(); self.host.record_semantic( "return", Some(&frame.function.to_string()), + Some(&module), Some(*span), Some(value.type_name()), ); @@ -322,9 +335,11 @@ impl<'a> Interpreter<'a> { ), )); }; + let module = frame.function.module.canonical(); self.host.record_semantic( "branch.eval", Some(&frame.function.to_string()), + Some(&module), Some(arm.condition_span), Some(if condition { "true" } else { "false" }), ); @@ -351,9 +366,11 @@ impl<'a> Interpreter<'a> { format!("loop condition must be Bool, got {}", value.type_name()), )); }; + let module = frame.function.module.canonical(); self.host.record_semantic( "loop.iteration", Some(&frame.function.to_string()), + Some(&module), Some(*condition_span), Some(if value { "continue" } else { "exit" }), ); diff --git a/docs/STATUS.md b/docs/STATUS.md index be1e3bf..edeee83 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -125,6 +125,31 @@ Analysis v2 (`spec/analysis-v2.md`, `spec/schemas/analysis-v2.schema.json`): * Tests cover byte-identical output after moving a project and resource addressing for formulas/code fences. +### S2-C/H2-B record (current) + +Trace v2 (`spec/trace-v2.md`, `spec/schemas/trace-v2.schema.json`): + +* `source.root_id` uses the same logical Analysis identity; `analysis_digest` + is the digest of the exact emitted Analysis bytes and is independently + verifiable. +* Every span-bearing event carries `module_id`; `call.enter`/`call.exit` are + owned by the caller module, other semantic events by the current function, + effects by their `ExecutionContext`. The effect key is now `module_id`. +* Trace budgets: 100 000 events / 32 MiB by default, configurable with + `--trace-max-events` and `--trace-max-bytes`; `0` means unlimited. Semantic + recording is disabled when no trace is requested. +* On capacity exhaustion the trace ends with `trace.truncated` and + `truncated.reason`; the run status and execution result are unchanged. The + writer also enforces the byte cap on the final document. +* Worker frames are capped at 8 MiB, with aligned request/result/capability + wire caps; partial headers, oversized lengths, truncated payloads and invalid + JSON are protocol errors, clean EOF is distinct, and the host kills and marks + a worker dead after a transport failure. +* New independent `tests/consumers/trace_consumer.py` verifies identity, + digest, module references and sequence ordering in CI. +* Tests: event module ownership, 20-event truncation of a 500-iteration loop, + worker frame round-trip/EOF/partial/oversized/truncated cases. + ### M0.6 implementation progress | Step | State | Evidence | @@ -133,11 +158,12 @@ Analysis v2 (`spec/analysis-v2.md`, `spec/schemas/analysis-v2.schema.json`): | S1/H1-A lock v3 graph | **Done** | ADR-0016; full-closure `marqdo.lock` v3; fail-closed edge lookup; strict non-rewriting `sync`; verified Git cache; unified file-set profile (see §S1/H1-A) | | S2-A/H1-B source locations | **Done** | ADR-0017; `RuntimeError`/`CallFrame` carry `module_id`; callsite origin; CLI module-aware resolution with no entry fallback (see §S2-A/H1-B) | | S2-B/H2-A Analysis v2 | **Done** | logical project identity, resources array, machine-path-free diagnostics, emitted-bytes digest, v2 schema + independent consumer (see §S2-B/H2-A) | +| S2-C/H2-B Trace v2 | **Done** | logical root/digest, module-owned events, bounded/truncated trace, independent consumer, hardened worker frames (see §S2-C/H2-B) | | R0 baseline + ADRs | **Done** | this section; `docs/adr/0008`–`0012`; full workspace gate | | R1 package-aware identity + callable registry | **Done for statics; superseded by S1 lock v3** | `ModuleId { instance, relative_path }`; `PackageResolver` path sources; lock v3 transitive graph (ADR-0016) replaces the flat v2 lock; `CallableRegistry` for builtins/marqdo/declared ABI exports; external manifest wiring is R2 | | R2 ABI/Kernel/Broker | **Done (worker = crash isolation)** | `KernelRegistry` external providers; complete-name export routing; `CapabilityBroker`; static `check --plugin`; multi-export C test; isolated worker with timeout/crash tests; host handle table. Native/worker are not an OS sandbox | | R3 Tool Protocol | **Done** | `tool install/list/remove`; unknown-command fallback; atomic local install; SHA/tamper checks; black-box tests for arbitrary names, Unicode args, conflicts, traversal and PATH hijacking | -| R4 Analysis v2 / Trace v1 | **Partial** | `check --emit` Analysis v2 (logical identity, resources, path-free diagnostics, emitted-bytes digest); `run/call --trace-file` still Trace v1 pending H2-B; independent Python consumer; stale-artifact removal; value redaction | +| R4 Analysis/Trace v2 | **Done** | `check --emit` Analysis v2 and `run/call --trace-file` Trace v2 with logical identity, resources, module-owned events, capacity bounds, schemas and independent Python consumers; stale-artifact removal; value redaction | | R5 environment/isolation | **Partial** | Pinned Git fetch with verified cache markers, transitive `@alias` resolution, full-closure atomic `.marqdo/env` materialization, foreign process-group cleanup. Strict OS isolation remains explicitly unavailable (fail-closed) | | R6 release/consumers | **Done for Linux** | examples for unknown-name Tool, multi-export plugin, external kernel and multi-version packages; Analysis/Trace/Tool JSON Schemas; CI Linux gate + Windows/macOS `cargo check`; R6 example tests | diff --git a/docs/contributor-guide.md b/docs/contributor-guide.md index 411b713..a0bb351 100644 --- a/docs/contributor-guide.md +++ b/docs/contributor-guide.md @@ -34,6 +34,7 @@ marqdo-cli/ │ ├── foreign.md Python/Node kernel protocol │ ├── tool-v1.md generic external Tool Protocol v1 │ ├── analysis-v2.md public Analysis v2 projection (v1 historical) +│ ├── trace-v2.md public Trace v2 projection (v1 historical) │ └── trace-v1.md public Trace v1 event stream ├── crates/ │ ├── marqdo-span/ Span, LineCol, LineIndex (leaf) diff --git a/include/marqdo_plugin_abi_v1.h b/include/marqdo_plugin_abi_v1.h index 0dab78a..24f73fe 100644 --- a/include/marqdo_plugin_abi_v1.h +++ b/include/marqdo_plugin_abi_v1.h @@ -99,10 +99,10 @@ * MQ_MAX_EXPORT_NAME_BYTES 256 * MQ_MAX_SIGNATURE_WIRE_BYTES 4096 * MQ_MAX_METADATA_WIRE_BYTES 65536 - * MQ_MAX_REQUEST_WIRE_BYTES 16777216 (16 MiB) - * MQ_MAX_RESULT_WIRE_BYTES 67108864 (64 MiB) + * MQ_MAX_REQUEST_WIRE_BYTES 8384512 (8 MiB - 4096) + * MQ_MAX_RESULT_WIRE_BYTES 8384512 (8 MiB - 4096) * MQ_MAX_ERROR_WIRE_BYTES 1048576 (1 MiB) - * MQ_MAX_CAPABILITY_REPLY_BYTES 16777216 (16 MiB) + * MQ_MAX_CAPABILITY_REPLY_BYTES 8384512 (8 MiB - 4096) * JSON depth is additionally limited to 64 levels. Requests/results that * exceed these limits are rejected with a structured error; they must not * be truncated and reparsed. @@ -245,10 +245,10 @@ extern "C" { #define MQ_MAX_EXPORT_NAME_BYTES 256u #define MQ_MAX_SIGNATURE_WIRE_BYTES 4096u #define MQ_MAX_METADATA_WIRE_BYTES 65536u -#define MQ_MAX_REQUEST_WIRE_BYTES (16u * 1024u * 1024u) -#define MQ_MAX_RESULT_WIRE_BYTES (64u * 1024u * 1024u) +#define MQ_MAX_REQUEST_WIRE_BYTES ((8u * 1024u * 1024u) - 4096u) +#define MQ_MAX_RESULT_WIRE_BYTES ((8u * 1024u * 1024u) - 4096u) #define MQ_MAX_ERROR_WIRE_BYTES (1024u * 1024u) -#define MQ_MAX_CAPABILITY_REPLY_BYTES (16u * 1024u * 1024u) +#define MQ_MAX_CAPABILITY_REPLY_BYTES ((8u * 1024u * 1024u) - 4096u) /* ------------------------------------------------------------------ */ /* Stable-layout buffer types */ diff --git a/spec/plugin-abi.md b/spec/plugin-abi.md index e81e783..2e13d83 100644 --- a/spec/plugin-abi.md +++ b/spec/plugin-abi.md @@ -68,10 +68,10 @@ minted for. Plugins must not dereference, cache or copy it. | `MQ_MAX_EXPORT_NAME_BYTES` | 256 | | `MQ_MAX_SIGNATURE_WIRE_BYTES` | 4096 | | `MQ_MAX_METADATA_WIRE_BYTES` | 65536 | -| `MQ_MAX_REQUEST_WIRE_BYTES` | 16 MiB (16,777,216) | -| `MQ_MAX_RESULT_WIRE_BYTES` | 64 MiB (67,108,864) | +| `MQ_MAX_REQUEST_WIRE_BYTES` | 8 MiB - 4096 (8,384,512) | +| `MQ_MAX_RESULT_WIRE_BYTES` | 8 MiB - 4096 (8,384,512) | | `MQ_MAX_ERROR_WIRE_BYTES` | 1 MiB (1,048,576) | -| `MQ_MAX_CAPABILITY_REPLY_BYTES` | 16 MiB | +| `MQ_MAX_CAPABILITY_REPLY_BYTES` | 8 MiB - 4096 | Wire JSON decoding additionally applies the default `WireLimits` from `spec/value-wire.md` (depth 64, 1,000,000 items, 16 MiB strings). Oversized @@ -270,8 +270,8 @@ with both is rejected (`ambiguous_response`). Error: }} ``` -Envelope limits: at most `MAX_ARGS = 1024` request arguments; request ≤ 16 MiB, -response ≤ 64 MiB, error ≤ 1 MiB. `context` carries no secrets and no +Envelope limits: at most `MAX_ARGS = 1024` request arguments; request and +response ≤ 8 MiB - 4096, error ≤ 1 MiB. `context` carries no secrets and no reconstructable OS handles. `signature_wire` is `{"signature":""}`; `plugin_metadata_wire` is `{"name":"","version":""}` (extra fields ignored). @@ -364,7 +364,10 @@ full plugin capability broker for worker requests remain future work. ### 9.1 Worker frames -Frames are a 4-byte big-endian length followed by one JSON object (max 64 MiB). +Frames are a 4-byte big-endian length followed by one JSON object (max 8 MiB). +A zero/oversized length, partial header, truncated payload or invalid JSON is a +protocol error; clean EOF at a frame boundary is not. The host marks the worker +dead after a transport failure and does not reuse it. Worker → host: `{"kind":"ready","exports":[...]}`. Host → worker: `{"kind":"invoke","request":}` and `{"kind":"shutdown"}`. Worker → host responses are `{"kind":"response","response":}` or diff --git a/spec/schemas/trace-v2.schema.json b/spec/schemas/trace-v2.schema.json new file mode 100644 index 0000000..0d0697c --- /dev/null +++ b/spec/schemas/trace-v2.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://marqdo.dev/schemas/trace-v2.schema.json", + "title": "Marqdo Trace v2", + "type": "object", + "required": ["schema", "schema_version", "run", "source", "limits", "truncated", "events"], + "properties": { + "schema": {"const": "marqdo.trace"}, + "schema_version": {"const": 2}, + "run": { + "type": "object", + "required": ["id", "entry", "status"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "entry": {"type": "string", "minLength": 1}, + "status": {"type": "string", "minLength": 1} + }, + "additionalProperties": true + }, + "source": { + "type": "object", + "required": ["root_id", "analysis_digest"], + "properties": { + "root_id": {"type": "string", "pattern": "^(root:|sha256:[0-9a-f]{64}$)"}, + "analysis_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"} + }, + "additionalProperties": true + }, + "limits": { + "type": "object", + "required": ["max_events", "max_bytes"], + "properties": { + "max_events": {"type": "integer", "minimum": 0}, + "max_bytes": {"type": "integer", "minimum": 0} + }, + "additionalProperties": false + }, + "truncated": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "required": ["reason", "limit", "written"], + "properties": { + "reason": {"enum": ["event_budget", "byte_budget"]}, + "limit": {"type": "integer", "minimum": 0}, + "written": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + } + ] + }, + "events": { + "type": "array", + "items": { + "type": "object", + "required": ["seq", "kind", "module_id", "span", "detail"], + "properties": { + "seq": {"type": "integer", "minimum": 0}, + "kind": {"type": "string", "minLength": 1}, + "function_id": {"type": ["string", "null"]}, + "module_id": {"type": ["string", "null"]}, + "span": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "required": ["start", "end"], + "properties": { + "start": {"type": "integer", "minimum": 0}, + "end": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + } + ] + }, + "detail": {"type": ["string", "null"]}, + "operation": {"type": "string"}, + "duration_micros": {"type": "integer", "minimum": 0} + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/spec/trace-v1.md b/spec/trace-v1.md index d4aff64..5475c48 100644 --- a/spec/trace-v1.md +++ b/spec/trace-v1.md @@ -1,6 +1,7 @@ # Marqdo Trace v1 -Status: **implemented by `marqdo run/call --trace-file ` (M0.6 R4).** +Status: **superseded by [Trace v2](trace-v2.md); never released, do not write +v1.** Kept as history. ## 1. Invariants diff --git a/spec/trace-v2.md b/spec/trace-v2.md new file mode 100644 index 0000000..1888dbc --- /dev/null +++ b/spec/trace-v2.md @@ -0,0 +1,112 @@ +# Marqdo Trace v2 + +Status: **implemented by `marqdo run/call --trace-file ` (S2).** +Supersedes Trace v1; v1 was never released and is kept as history. + +## 1. Invariants + +* Tracing is observation, not execution. Only `run`/`call` produce a trace; + `check`, `inspect` and `tool list` never execute a document. +* Events come from the interpreter (`call.*`, `branch.eval`, + `loop.iteration`, `return`) and the host (`effect`). Both share one sequence. +* The writer is atomic; an error run still records its terminal status. +* No argument values, secrets, handles or file contents are recorded. +* Trace capacity is a bounded observation budget. Reaching it truncates the + trace; it never changes `run.status` or the program’s execution result. + Execution budgets (`--fuel`, `--timeout`, ...) remain separate and still + produce `resource.exhausted`. + +## 2. Shape + +```json +{ + "schema": "marqdo.trace", + "schema_version": 2, + "run": {"id": "pid-nanos", "entry": "root:q@0.1.0::index.mq.md::main", "status": "ok"}, + "source": {"root_id": "root:q@0.1.0", "analysis_digest": "sha256:<64 hex>"}, + "limits": {"max_events": 100000, "max_bytes": 33554432}, + "truncated": null, + "events": [ + {"seq": 0, "kind": "call.enter", "function_id": "root:q@0.1.0::index.mq.md::main", + "module_id": "root:q@0.1.0::index.mq.md", "span": null, "detail": null}, + {"seq": 1, "kind": "effect", "module_id": "dep:p@0.1.0#::index.mq.md", + "span": {"start": 40, "end": 52}, "detail": "ok", "operation": "print", + "duration_micros": 3} + ] +} +``` + +`seq` is globally increasing. `run.status` is `ok` or a stable runtime error +code. Every span-bearing event has a `module_id`; the span is a UTF-8 byte +offset range in that module. `source` only carries project identity and the +Analysis digest; it does not identify the span owner. + +`source.root_id` is the Analysis v2 `project.root_id` +(`root:@` or `sha256:`). +`source.analysis_digest` is `sha256:` plus the SHA-256 of the exact Analysis +v2 bytes produced for the same project with the same projection configuration. +A consumer verifies it by hashing the Analysis file. + +`truncated` is `null` or: + +```json +{"reason": "event_budget", "limit": 20, "written": 20} +``` + +`reason` is `event_budget` or `byte_budget`. When truncation occurs, the final +event is `{"kind":"trace.truncated","detail":,"module_id":null, +"span":null,"seq":}`. + +## 3. Event kinds and span owners + +| kind | detail | `module_id` owner | +|---|---|---| +| `call.enter` | null | caller module (entry call: callee) | +| `call.exit` | `ok` / `return` / error code | caller module (entry call: callee) | +| `return` | value type name | current function’s module | +| `branch.eval` | `true` / `false` | current function’s module | +| `loop.iteration` | `continue` / `exit` | current function’s module | +| `effect` | result summary | module from `ExecutionContext` | +| `trace.truncated` | `event_budget` / `byte_budget` | null | + +`function_id` is always the canonical callee function id. Future kinds are +additive; consumers must ignore unknown kinds. `effect.operation` names the +`HostOp` (`print`, `fs.read`, ...). + +## 4. Capacity + +Defaults: 100 000 events and 32 MiB approximate serialized size. `0` means +unlimited for either limit. `run`/`call` expose `--trace-max-events` and +`--trace-max-bytes`; these are options on existing commands, not new commands. +The host stops accepting events once a limit is reached; the writer also +enforces the byte limit on the final serialized document, dropping trailing +events and recording `byte_budget`. Semantic recording is disabled entirely +when no trace was requested. + +The `trace` array in `run --json` uses the same budget. + +## 5. Staleness + +A trace is stale when its `source.analysis_digest` differs from the digest of +the Analysis emitted for the current document. Consumers must not present a +stale trace as current. Moving a project does not change the manifest identity +or digest; changing executable source does. + +## 6. Worker transport limits + +Plugin worker frames are a 4-byte big-endian length plus JSON, capped at +8 MiB (`8 * 1024 * 1024`). The wire request/result/capability caps are aligned +to `8 MiB - 4096`; error frames remain 1 MiB. A partial header, zero/oversized +length, truncated payload or invalid JSON is a protocol error. A clean EOF is +distinct from a protocol failure. The host marks a worker dead after a transport +failure and never reuses its stdin. + +## 7. Compatibility and tests + +* Consumers must reject unknown `schema_version` and validate `module_id` + against Analysis modules. +* JSON Schema: `spec/schemas/trace-v2.schema.json`. +* `tests/consumers/trace_consumer.py` verifies identity, digest and event + module references. +* CLI tests cover call/effect events, no-secret redaction, error terminal + status, and event-budget truncation without changing the run status. diff --git a/tests/consumers/trace_consumer.py b/tests/consumers/trace_consumer.py new file mode 100644 index 0000000..e57a188 --- /dev/null +++ b/tests/consumers/trace_consumer.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Independent Trace v2 consumer: no Marqdo/Rust imports.""" +import hashlib +import json +import sys + +if len(sys.argv) != 3: + raise SystemExit("usage: trace_consumer.py ANALYSIS.json TRACE.json") + +with open(sys.argv[1], "rb") as handle: + analysis_bytes = handle.read() +analysis = json.loads(analysis_bytes) +with open(sys.argv[2], encoding="utf-8") as handle: + trace = json.load(handle) + +if analysis.get("schema") != "marqdo.analysis" or analysis.get("schema_version") != 2: + raise SystemExit("unsupported analysis") +if trace.get("schema") != "marqdo.trace" or trace.get("schema_version") != 2: + raise SystemExit("unsupported trace") + +modules = {module["id"]: module for module in analysis["modules"]} +if trace["source"]["root_id"] != analysis["project"]["root_id"]: + raise SystemExit("trace/analysis root mismatch") +expected = "sha256:" + hashlib.sha256(analysis_bytes).hexdigest() +if trace["source"]["analysis_digest"] != expected: + raise SystemExit("analysis digest mismatch") + +truncated = trace.get("truncated") +if truncated is not None and truncated.get("reason") not in ("event_budget", "byte_budget"): + raise SystemExit("bad truncation reason") + +previous = None +events = trace["events"] +for event in events: + seq = event["seq"] + if previous is not None and seq <= previous: + raise SystemExit("sequence is not increasing") + previous = seq + span = event.get("span") + if span is not None and (span["start"] > span["end"]): + raise SystemExit("bad span") + module_id = event.get("module_id") + if module_id is not None and module_id not in modules: + raise SystemExit("event references an unknown module") + function_id = event.get("function_id") + if function_id is not None: + owner = function_id.rsplit("::", 1)[0] + if owner not in modules: + raise SystemExit("function id references an unknown module") + +if truncated is not None and events and events[-1].get("kind") != "trace.truncated": + raise SystemExit("truncated trace has no terminal marker") +print("events=" + str(len(events)))