feat(trace): Trace v2 identity, module-owned events and capacity bounds (B-16)

- bind traces to the logical project identity and the exact emitted Analysis
  digest; every span-bearing event carries module_id
- add host TraceState budgets (100k events / 32 MiB defaults), CLI
  --trace-max-events/--trace-max-bytes, terminal trace.truncated events, and a
  final serialized byte cap; budgets never change run.status
- disable semantic recording when no trace is requested
- cap worker frames at 8 MiB, align request/result/capability wire caps, make
  clean EOF distinct from protocol errors, and mark/kill a worker after a
  transport failure instead of reusing it
- add trace-v2 spec/schema, independent trace consumer, CI check and tests
- record the S2/C state in STATUS
This commit is contained in:
cflmy committed 2026-11-20 13:01:22 +08:00
1 parent 55dcb57553
commit d890686a2b
21 files changed
+732 -84

No files matched your search

+5 -1
View File
@@ -41,9 +41,13 @@ jobs:
./target/debug/marqdo inspect examples/references.mq.md --refs
./target/debug/marqdo check examples/multi-module/index.mq.md --refs \
--emit /tmp/marqdo-analysis.json
./target/debug/marqdo run examples/multi-module/index.mq.md \
--trace-file /tmp/marqdo-analysis-trace.json
python3 tests/consumers/analysis_consumer.py /tmp/marqdo-analysis.json
python3 tests/consumers/trace_consumer.py \
/tmp/marqdo-analysis.json /tmp/marqdo-analysis-trace.json
./target/debug/marqdo run examples/hello.mq.md \
--trace-file /tmp/marqdo-trace.json
python3 tests/consumers/analysis_consumer.py /tmp/marqdo-analysis.json
python3 - <<'PY'
import json
json.load(open("/tmp/marqdo-trace.json"))
+1 -1
View File
@@ -192,7 +192,7 @@ tests/ conformance/negative corpus (see tests/README.md)
[`package-format.md`](spec/package-format.md),
[`plugin-abi.md`](spec/plugin-abi.md), [`math.md`](spec/math.md),
[`foreign.md`](spec/foreign.md), [`tool-v1.md`](spec/tool-v1.md),
[`analysis-v2.md`](spec/analysis-v2.md), [`trace-v1.md`](spec/trace-v1.md);
[`analysis-v2.md`](spec/analysis-v2.md), [`trace-v2.md`](spec/trace-v2.md);
JSON Schemas in [`spec/schemas/`](spec/schemas/).
* [`tests/README.md`](tests/README.md) — conformance corpus and `UPDATE_GOLDENS`
workflow.
+15 -2
View File
@@ -38,8 +38,12 @@ pub struct SandboxSettings {
pub plugins: Vec<PathBuf>,
/// Explicit trust decision for in-process native plugin loading.
pub trust_native_plugin: bool,
/// Optional Trace v1 output path.
/// Optional Trace v2 output path.
pub trace_file: Option<PathBuf>,
/// Trace event budget override (0 = unlimited).
pub trace_max_events: Option<usize>,
/// Trace approximate byte budget override (0 = unlimited).
pub trace_max_bytes: Option<usize>,
/// External foreign-kernel descriptor paths.
pub kernels: Vec<PathBuf>,
}
@@ -690,7 +694,16 @@ pub fn execute(
};
let mut host = StdHost::new(mode, capabilities, project.root.clone());
if trace || json || settings.trace_file.is_some() {
host = host.with_trace();
host = host
.with_trace()
.with_trace_budget(marqdo_host::TraceBudget {
max_events: settings
.trace_max_events
.unwrap_or(marqdo_host::trace::DEFAULT_MAX_TRACE_EVENTS),
max_bytes: settings
.trace_max_bytes
.unwrap_or(marqdo_host::trace::DEFAULT_MAX_TRACE_BYTES),
});
}
if let Some(fuel) = settings.fuel {
host.budget.fuel = fuel;
+9 -1
View File
@@ -109,9 +109,15 @@ struct SandboxArgs {
/// Explicitly trusted plugin manifest/directory to load (repeatable).
#[arg(long = "plugin", value_name = "PATH")]
plugin: Vec<PathBuf>,
/// Write a versioned Trace v1 file for this run.
/// Write a versioned Trace v2 file for this run.
#[arg(long = "trace-file", value_name = "PATH")]
trace_file: Option<PathBuf>,
/// Override the trace event budget (0 = unlimited).
#[arg(long = "trace-max-events")]
trace_max_events: Option<usize>,
/// Override the approximate trace byte budget (0 = unlimited).
#[arg(long = "trace-max-bytes")]
trace_max_bytes: Option<usize>,
/// Register an external foreign-kernel provider (repeatable).
#[arg(long = "kernel", value_name = "PATH")]
kernel: Vec<PathBuf>,
@@ -135,6 +141,8 @@ impl SandboxArgs {
plugins: self.plugin.clone(),
trust_native_plugin: self.trust_native_plugin,
trace_file: self.trace_file.clone(),
trace_max_events: self.trace_max_events,
trace_max_bytes: self.trace_max_bytes,
kernels: self.kernel.clone(),
}
}
+54 -12
View File
@@ -1,4 +1,4 @@
//! Public Trace v1 writer (spec/trace-v1.md).
//! Public Trace v2 writer (spec/trace-v2.md).
//!
//! Events come from the interpreter (semantic) and the capability host
//! (effects). No argument values, secrets or handles are recorded.
@@ -12,7 +12,7 @@ use crate::loader::LoadedProject;
/// Schema name and version.
pub const SCHEMA: &str = "marqdo.trace";
pub const SCHEMA_VERSION: u32 = 1;
pub const SCHEMA_VERSION: u32 = 2;
/// Write a trace atomically for one run.
pub fn write(
@@ -32,6 +32,7 @@ pub fn write(
);
let analysis_digest = crate::analysis::digest(project);
let root_id = project.identity.root_id();
let state = host.trace_state();
let mut events: Vec<Value> = host
.semantic_events()
.iter()
@@ -39,15 +40,55 @@ pub fn write(
.chain(host.trace().iter().map(effect_json))
.collect();
events.sort_by_key(|event| event.get("seq").and_then(Value::as_u64).unwrap_or(u64::MAX));
let trace = json!({
"schema": SCHEMA,
"schema_version": SCHEMA_VERSION,
"run": {"id": run_id, "entry": entry, "status": status},
"source": {"root_id": root_id, "analysis_digest": format!("sha256:{analysis_digest}")},
"events": events,
});
let bytes = serde_json::to_vec_pretty(&trace)
.map_err(|error| format!("cannot encode trace: {error}"))?;
let mut truncation = state.truncation.clone();
let max_bytes = state.budget.max_bytes;
let bytes = loop {
let mut all = events.clone();
if let Some(truncation) = &truncation {
let seq = all
.last()
.and_then(|event| event.get("seq"))
.and_then(Value::as_u64)
.unwrap_or(0)
+ 1;
all.push(json!({
"seq": seq,
"kind": "trace.truncated",
"module_id": null,
"span": null,
"detail": truncation.reason,
}));
}
let trace = json!({
"schema": SCHEMA,
"schema_version": SCHEMA_VERSION,
"run": {"id": run_id, "entry": entry, "status": status},
"source": {
"root_id": root_id,
"analysis_digest": format!("sha256:{analysis_digest}"),
},
"limits": {
"max_events": state.budget.max_events,
"max_bytes": state.budget.max_bytes,
},
"truncated": truncation,
"events": all,
});
let bytes = serde_json::to_vec_pretty(&trace)
.map_err(|error| format!("cannot encode trace: {error}"))?;
if max_bytes == 0 || bytes.len() <= max_bytes || events.is_empty() {
break bytes;
}
events.pop();
if truncation.is_none() {
truncation = Some(marqdo_host::TraceTruncation {
reason: "byte_budget".to_string(),
limit: max_bytes as u64,
written: events.len() as u64,
});
}
};
atomic_write(path, &bytes)
}
@@ -56,6 +97,7 @@ fn semantic_json(event: &SemanticEvent) -> Value {
"seq": event.seq,
"kind": event.kind,
"function_id": event.function_id,
"module_id": event.module_id,
"span": event.span.map(|span| json!({"start": span.start, "end": span.end})),
"detail": event.detail,
})
@@ -65,7 +107,7 @@ fn effect_json(entry: &TraceEntry) -> Value {
json!({
"seq": entry.seq,
"kind": "effect",
"module": entry.module,
"module_id": entry.module,
"span": entry.span.map(|span| json!({"start": span.start, "end": span.end})),
"detail": entry.result,
"operation": entry.op,
+51 -2
View File
@@ -241,6 +241,52 @@ print(1)
);
}
#[test]
fn trace_v2_budget_truncates_long_loops_without_failing_the_run() {
let dir = TempDir::new("trace-budget");
write(
dir.path(),
"index.mq.md",
"## main\n\n**i = 0**\n\n- `i < 500`\n\n **i = i + 1**\n\n*i*\n",
);
let trace_path = dir.path().join("trace.json");
let output = cli(
dir.path(),
&[
"run",
"index.mq.md",
"--trace-file",
trace_path.to_str().unwrap(),
"--trace-max-events",
"20",
],
);
assert_eq!(
output.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let trace: serde_json::Value =
serde_json::from_slice(&std::fs::read(&trace_path).unwrap()).unwrap();
assert_eq!(trace["schema_version"], 2);
assert_eq!(trace["run"]["status"], "ok");
assert_eq!(trace["limits"]["max_events"], 20);
assert_eq!(trace["truncated"]["reason"], "event_budget");
assert_eq!(trace["truncated"]["limit"], 20);
let events = trace["events"].as_array().unwrap();
assert!(events.len() <= 21, "events: {}", events.len());
assert_eq!(events.last().unwrap()["kind"], "trace.truncated");
let mut previous = None;
for event in events {
let seq = event["seq"].as_u64().unwrap();
if let Some(previous) = previous {
assert!(seq > previous);
}
previous = Some(seq);
}
}
#[test]
fn failed_check_removes_stale_analysis() {
let dir = TempDir::new("analysis-error");
@@ -282,11 +328,14 @@ fn trace_records_calls_effects_and_status_without_secrets() {
);
let trace: serde_json::Value = serde_json::from_str(&text).unwrap();
assert_eq!(trace["schema"], "marqdo.trace");
assert_eq!(trace["schema_version"], 1);
assert_eq!(trace["schema_version"], 2);
assert_eq!(trace["run"]["status"], "ok");
assert_eq!(trace["run"]["entry"], "root:default::index.mq.md::main");
let events = trace["events"].as_array().unwrap();
assert!(events.iter().any(|event| event["kind"] == "call.enter"));
assert!(events
.iter()
.any(|event| event["kind"] == "call.enter"
&& event["module_id"] == "root:default::index.mq.md"));
assert!(events.iter().any(|event| event["kind"] == "effect"));
assert!(events.iter().any(|event| event["kind"] == "call.exit"));
let mut previous = None;
+39 -9
View File
@@ -20,7 +20,7 @@ use crate::context::ExecutionContext;
use crate::fs::ProjectFs;
use crate::handles::{contains_plugin_handle, PluginHandleTable};
use crate::plugin_worker::WorkerPlugin;
use crate::trace::{EffectTrace, SemanticEvent, TraceEntry};
use crate::trace::{EffectTrace, SemanticEvent, TraceBudget, TraceEntry, TraceState};
use marqdo_sandbox::{PlatformCapabilities, SandboxMode};
/// Host operations reachable from the language surface.
@@ -99,6 +99,7 @@ pub trait Host {
&mut self,
_kind: &str,
_function_id: Option<&str>,
_module_id: Option<&str>,
_span: Option<Span>,
_detail: Option<&str>,
) {
@@ -144,6 +145,7 @@ pub struct StdHost {
pub handles: PluginHandleTable,
semantic_events: Vec<SemanticEvent>,
next_event_seq: u64,
trace_state: TraceState,
}
struct PluginSlot {
@@ -203,6 +205,7 @@ impl StdHost {
handles: PluginHandleTable::default(),
semantic_events: Vec::new(),
next_event_seq: 0,
trace_state: TraceState::default(),
}
}
@@ -338,6 +341,17 @@ impl StdHost {
seq
}
/// Trace capacity state for the writer.
pub fn trace_state(&self) -> &TraceState {
&self.trace_state
}
/// Override the trace event/byte budget.
pub fn with_trace_budget(mut self, budget: TraceBudget) -> Self {
self.trace_state.budget = budget;
self
}
/// Audited plugin capability decisions.
pub fn capability_audit(&self) -> Vec<CapabilityAudit> {
self.broker_audit
@@ -359,6 +373,7 @@ impl StdHost {
/// Enable effect tracing.
pub fn with_trace(mut self) -> Self {
self.trace = EffectTrace::enabled();
self.trace_state.enabled = true;
self
}
@@ -504,20 +519,29 @@ impl Host for StdHost {
ctx: &ExecutionContext,
) -> Result<Value, RuntimeError> {
let started = Instant::now();
let seq = self.next_seq();
let result = self.dispatch(op, args, ctx);
let summary = match &result {
Ok(_) => "ok".to_string(),
Err(error) => error.code().to_string(),
};
self.trace.record(
seq,
op.name(),
ctx.module_id.clone(),
ctx.span,
started.elapsed(),
summary,
let estimate = TraceState::estimate(
"effect",
None,
Some(&ctx.module_id),
Some(&summary),
Some(op.name()),
);
if self.trace_state.accept(estimate) {
let seq = self.next_seq();
self.trace.record(
seq,
op.name(),
ctx.module_id.clone(),
ctx.span,
started.elapsed(),
summary,
);
}
result
}
@@ -545,14 +569,20 @@ impl Host for StdHost {
&mut self,
kind: &str,
function_id: Option<&str>,
module_id: Option<&str>,
span: Option<Span>,
detail: Option<&str>,
) {
let estimate = TraceState::estimate(kind, function_id, module_id, detail, None);
if !self.trace_state.accept(estimate) {
return;
}
let seq = self.next_seq();
self.semantic_events.push(SemanticEvent {
seq,
kind: kind.to_string(),
function_id: function_id.map(str::to_string),
module_id: module_id.map(str::to_string),
span,
detail: detail.map(str::to_string),
});
+1 -1
View File
@@ -21,4 +21,4 @@ pub use fs::ProjectFs;
pub use handles::{contains_plugin_handle, PluginHandleTable};
pub use host::{arity, expect_text, Host, HostOp, StdHost};
pub use marqdo_sandbox::{PlatformCapabilities, SandboxMode, StrictIsolation};
pub use trace::{EffectTrace, SemanticEvent, TraceEntry};
pub use trace::{EffectTrace, SemanticEvent, TraceBudget, TraceEntry, TraceState, TraceTruncation};
+49 -12
View File
@@ -22,6 +22,7 @@ pub struct WorkerPlugin {
stdin: ChildStdin,
frames: Receiver<Result<serde_json::Value, PluginError>>,
exports: BTreeMap<String, String>,
dead: bool,
}
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30);
@@ -94,6 +95,7 @@ impl WorkerPlugin {
};
let Some(mut stdout) = child.stdout.take() else {
let _ = child.kill();
let _ = std::fs::remove_file(&manifest_path);
return Err(PluginError::load(
"worker_spawn_failed",
"worker stdout unavailable",
@@ -103,10 +105,23 @@ impl WorkerPlugin {
thread::Builder::new()
.name("marqdo-plugin-worker-reader".to_string())
.spawn(move || loop {
let frame = read_frame(&mut stdout);
let failed = frame.is_err();
if sender.send(frame).is_err() || failed {
break;
match read_frame(&mut stdout) {
Ok(Some(frame)) => {
if sender.send(Ok(frame)).is_err() {
break;
}
}
Ok(None) => {
let _ = sender.send(Err(PluginError::invoke(
"plugin.crashed",
"plugin worker closed its output stream",
)));
break;
}
Err(error) => {
let _ = sender.send(Err(error));
break;
}
}
})
.map_err(|error| {
@@ -169,21 +184,35 @@ impl WorkerPlugin {
stdin,
frames,
exports,
dead: false,
})
}
/// Invoke one export in the worker.
pub fn invoke(&mut self, request: &WireRequest) -> Result<WireResponse, PluginError> {
if self.dead {
return Err(PluginError::invoke(
"plugin.crashed",
"plugin worker is no longer usable after a transport failure",
));
}
let request_value: serde_json::Value = serde_json::from_slice(&request.to_json_bytes()?)
.map_err(|error| PluginError::wire("worker_encode_failed", error.to_string()))?;
write_frame(
if let Err(error) = write_frame(
&mut self.stdin,
&json!({"kind": "invoke", "request": request_value}),
)?;
) {
self.mark_dead();
return Err(error);
}
let frame = match self.frames.recv_timeout(CALL_TIMEOUT) {
Ok(frame) => frame?,
Ok(Ok(frame)) => frame,
Ok(Err(error)) => {
self.mark_dead();
return Err(error);
}
Err(_) => {
let _ = self.child.kill();
self.mark_dead();
return Err(PluginError::invoke(
"plugin.timeout",
"plugin worker exceeded its invocation timeout",
@@ -202,12 +231,20 @@ impl WorkerPlugin {
.and_then(serde_json::Value::as_str)
.unwrap_or("worker reported an error"),
)),
_ => Err(PluginError::invoke(
"plugin.crashed",
"plugin worker closed or violated the frame protocol",
)),
_ => {
self.mark_dead();
Err(PluginError::invoke(
"plugin.crashed",
"plugin worker closed or violated the frame protocol",
))
}
}
}
fn mark_dead(&mut self) {
self.dead = true;
let _ = self.child.kill();
}
}
impl Drop for WorkerPlugin {
+95 -1
View File
@@ -70,7 +70,99 @@ impl EffectTrace {
}
}
/// One semantic event emitted by the interpreter (Trace v1).
/// Default maximum number of trace events.
pub const DEFAULT_MAX_TRACE_EVENTS: usize = 100_000;
/// Default approximate maximum serialized trace bytes.
pub const DEFAULT_MAX_TRACE_BYTES: usize = 32 * 1024 * 1024;
/// Trace capacity budget. A zero value means unlimited.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TraceBudget {
/// Maximum events (including the terminal truncation marker).
pub max_events: usize,
/// Approximate maximum serialized bytes.
pub max_bytes: usize,
}
impl Default for TraceBudget {
fn default() -> Self {
Self {
max_events: DEFAULT_MAX_TRACE_EVENTS,
max_bytes: DEFAULT_MAX_TRACE_BYTES,
}
}
}
/// Why and where a trace stopped recording.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TraceTruncation {
/// `event_budget` or `byte_budget`.
pub reason: String,
/// The configured limit that was reached.
pub limit: u64,
/// Events written before truncation.
pub written: u64,
}
/// Mutable trace capacity state for one run.
#[derive(Debug, Clone, Default)]
pub struct TraceState {
/// Whether trace recording is enabled.
pub enabled: bool,
/// Configured budget.
pub budget: TraceBudget,
/// Events accepted so far.
pub written: usize,
/// Estimated accepted bytes.
pub bytes: usize,
/// Truncation marker, set once.
pub truncation: Option<TraceTruncation>,
}
impl TraceState {
/// Accept one event if it fits the budget.
pub fn accept(&mut self, estimate: usize) -> bool {
if !self.enabled || self.truncation.is_some() {
return false;
}
if self.budget.max_events > 0 && self.written >= self.budget.max_events {
self.truncation = Some(TraceTruncation {
reason: "event_budget".to_string(),
limit: self.budget.max_events as u64,
written: self.written as u64,
});
return false;
}
if self.budget.max_bytes > 0 && self.bytes + estimate > self.budget.max_bytes {
self.truncation = Some(TraceTruncation {
reason: "byte_budget".to_string(),
limit: self.budget.max_bytes as u64,
written: self.written as u64,
});
return false;
}
self.written += 1;
self.bytes += estimate;
true
}
/// Estimate one semantic/effect event envelope.
pub fn estimate(
kind: &str,
function_id: Option<&str>,
module_id: Option<&str>,
detail: Option<&str>,
operation: Option<&str>,
) -> usize {
96 + kind.len()
+ function_id.map(str::len).unwrap_or(0)
+ module_id.map(str::len).unwrap_or(0)
+ detail.map(str::len).unwrap_or(0)
+ operation.map(str::len).unwrap_or(0)
}
}
/// One semantic event emitted by the interpreter (Trace v2).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SemanticEvent {
/// Global event sequence within the run.
@@ -79,6 +171,8 @@ pub struct SemanticEvent {
pub kind: String,
/// Canonical function id, when inside a call.
pub function_id: Option<String>,
/// Logical module owning `span`.
pub module_id: Option<String>,
/// Source span, when known.
pub span: Option<Span>,
/// Non-sensitive detail (`ok`, `true`, `int`, an error code).
+6 -6
View File
@@ -73,14 +73,14 @@ pub const MAX_EXPORT_NAME_BYTES: usize = 256;
pub const MAX_SIGNATURE_WIRE_BYTES: usize = 4096;
/// Maximum plugin metadata wire length in bytes.
pub const MAX_METADATA_WIRE_BYTES: usize = 65_536;
/// Maximum request wire length in bytes (16 MiB).
pub const MAX_REQUEST_WIRE_BYTES: usize = 16 * 1024 * 1024;
/// Maximum result wire length in bytes (64 MiB).
pub const MAX_RESULT_WIRE_BYTES: usize = 64 * 1024 * 1024;
/// Maximum request wire length in bytes (8 MiB frame minus envelope reserve).
pub const MAX_REQUEST_WIRE_BYTES: usize = 8 * 1024 * 1024 - 4096;
/// Maximum result wire length in bytes (8 MiB frame minus envelope reserve).
pub const MAX_RESULT_WIRE_BYTES: usize = 8 * 1024 * 1024 - 4096;
/// Maximum error wire length in bytes (1 MiB).
pub const MAX_ERROR_WIRE_BYTES: usize = 1024 * 1024;
/// Maximum capability reply length in bytes (16 MiB).
pub const MAX_CAPABILITY_REPLY_BYTES: usize = 16 * 1024 * 1024;
/// Maximum capability reply length in bytes (8 MiB frame minus envelope reserve).
pub const MAX_CAPABILITY_REPLY_BYTES: usize = 8 * 1024 * 1024 - 4096;
/// Borrowed, immutable byte view (`MqBytes`).
///
+90 -18
View File
@@ -19,17 +19,34 @@ use crate::wire::{WireRequest, WireResponse};
/// Hidden CLI command name used to start a worker.
pub const WORKER_COMMAND: &str = "__plugin-worker";
/// Maximum frame size accepted by either side.
pub const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024;
pub const MAX_FRAME_BYTES: usize = 8 * 1024 * 1024;
/// Read one length-prefixed frame (4-byte big-endian length + JSON).
pub fn read_frame<R: Read>(reader: &mut R) -> Result<Value, PluginError> {
///
/// `Ok(None)` means a clean EOF at a frame boundary. A partial header or
/// truncated payload is a protocol error, never EOF.
pub fn read_frame<R: Read>(reader: &mut R) -> Result<Option<Value>, PluginError> {
let mut length = [0_u8; 4];
reader.read_exact(&mut length).map_err(|error| {
PluginError::invoke(
"plugin.crashed",
format!("worker frame read failed: {error}"),
)
})?;
let mut read = 0usize;
while read < length.len() {
match reader.read(&mut length[read..]) {
Ok(0) if read == 0 => return Ok(None),
Ok(0) => {
return Err(PluginError::wire(
"worker_frame_invalid",
"worker stream ended inside a frame header",
))
}
Ok(count) => read += count,
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(error) => {
return Err(PluginError::invoke(
"plugin.crashed",
format!("worker frame read failed: {error}"),
))
}
}
}
let length = u32::from_be_bytes(length) as usize;
if length == 0 || length > MAX_FRAME_BYTES {
return Err(PluginError::wire(
@@ -37,14 +54,23 @@ pub fn read_frame<R: Read>(reader: &mut R) -> Result<Value, PluginError> {
format!("worker frame length {length} is invalid"),
));
}
let mut bytes = vec![0_u8; length];
reader.read_exact(&mut bytes).map_err(|error| {
PluginError::invoke(
"plugin.crashed",
format!("worker frame read failed: {error}"),
)
})?;
serde_json::from_slice(&bytes).map_err(|error| {
let mut bytes = Vec::with_capacity(length.min(64 * 1024));
let read = reader
.take(length as u64)
.read_to_end(&mut bytes)
.map_err(|error| {
PluginError::invoke(
"plugin.crashed",
format!("worker frame read failed: {error}"),
)
})?;
if read != length {
return Err(PluginError::wire(
"worker_frame_invalid",
format!("worker frame declared {length} bytes but only {read} arrived"),
));
}
serde_json::from_slice(&bytes).map(Some).map_err(|error| {
PluginError::wire(
"worker_frame_invalid",
format!("worker frame is not valid JSON: {error}"),
@@ -110,8 +136,12 @@ pub fn serve(manifest_path: &Path, base_dir: &Path) -> i32 {
}
loop {
let frame = match read_frame(&mut stdin) {
Ok(frame) => frame,
Err(_) => return 0,
Ok(Some(frame)) => frame,
Ok(None) => return 0,
Err(error) => {
eprintln!("worker: {}: {}", error.code, error.message);
return 1;
}
};
match frame.get("kind").and_then(Value::as_str) {
Some("shutdown") => return 0,
@@ -189,3 +219,45 @@ pub fn response_from_frame(frame: &Value) -> Result<WireResponse, PluginError> {
.map_err(|error| PluginError::wire("worker_response_invalid", error.to_string()))?;
WireResponse::from_json_bytes(&bytes)
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
#[test]
fn frame_roundtrip_and_clean_eof() {
let mut bytes = Vec::new();
write_frame(&mut bytes, &json!({"kind": "shutdown"})).unwrap();
let mut cursor = Cursor::new(bytes.clone());
assert_eq!(
read_frame(&mut cursor).unwrap(),
Some(json!({"kind": "shutdown"}))
);
assert_eq!(read_frame(&mut cursor).unwrap(), None);
}
#[test]
fn partial_header_is_an_error_not_eof() {
let mut cursor = Cursor::new(vec![0_u8, 0]);
let error = read_frame(&mut cursor).unwrap_err();
assert_eq!(error.code, "worker_frame_invalid");
}
#[test]
fn oversized_and_zero_length_frames_are_rejected() {
for length in [0_u32, (MAX_FRAME_BYTES as u32) + 1] {
let mut cursor = Cursor::new(length.to_be_bytes().to_vec());
let error = read_frame(&mut cursor).unwrap_err();
assert_eq!(error.code, "worker_frame_invalid");
}
}
#[test]
fn truncated_payload_is_rejected() {
let mut bytes = 10_u32.to_be_bytes().to_vec();
bytes.extend_from_slice(b"{}");
let error = read_frame(&mut Cursor::new(bytes)).unwrap_err();
assert_eq!(error.code, "worker_frame_invalid");
}
}
+21 -4
View File
@@ -135,11 +135,17 @@ impl<'a> Interpreter<'a> {
) -> Result<Value, RuntimeError> {
let display = function.id.name.clone();
let trace_id = function.id.to_string();
let trace_module = caller_module.unwrap_or(&function.id.module).canonical();
if self.depth + 1 > 0 {
self.host.check_stack(self.depth + 1, call_span)?;
}
self.host
.record_semantic("call.enter", Some(&trace_id), call_span, None);
self.host.record_semantic(
"call.enter",
Some(&trace_id),
Some(&trace_module),
call_span,
None,
);
let locals = match self.bind_args(function, args, call_span) {
Ok(locals) => locals,
Err(mut error) => {
@@ -162,8 +168,13 @@ impl<'a> Interpreter<'a> {
Ok(Flow::Return(_)) => "return",
Err(error) => error.code(),
};
self.host
.record_semantic("call.exit", Some(&trace_id), call_span, Some(status));
self.host.record_semantic(
"call.exit",
Some(&trace_id),
Some(&trace_module),
call_span,
Some(status),
);
match result {
Ok(Flow::Next) => Ok(Value::Unit),
Ok(Flow::Return(value)) => Ok(value),
@@ -301,9 +312,11 @@ impl<'a> Interpreter<'a> {
}
Stmt::Return { value, span } => {
let value = self.eval_expr(value, frame)?;
let module = frame.function.module.canonical();
self.host.record_semantic(
"return",
Some(&frame.function.to_string()),
Some(&module),
Some(*span),
Some(value.type_name()),
);
@@ -322,9 +335,11 @@ impl<'a> Interpreter<'a> {
),
));
};
let module = frame.function.module.canonical();
self.host.record_semantic(
"branch.eval",
Some(&frame.function.to_string()),
Some(&module),
Some(arm.condition_span),
Some(if condition { "true" } else { "false" }),
);
@@ -351,9 +366,11 @@ impl<'a> Interpreter<'a> {
format!("loop condition must be Bool, got {}", value.type_name()),
));
};
let module = frame.function.module.canonical();
self.host.record_semantic(
"loop.iteration",
Some(&frame.function.to_string()),
Some(&module),
Some(*condition_span),
Some(if value { "continue" } else { "exit" }),
);
+27 -1
View File
@@ -125,6 +125,31 @@ Analysis v2 (`spec/analysis-v2.md`, `spec/schemas/analysis-v2.schema.json`):
* Tests cover byte-identical output after moving a project and resource
addressing for formulas/code fences.
### S2-C/H2-B record (current)
Trace v2 (`spec/trace-v2.md`, `spec/schemas/trace-v2.schema.json`):
* `source.root_id` uses the same logical Analysis identity; `analysis_digest`
is the digest of the exact emitted Analysis bytes and is independently
verifiable.
* Every span-bearing event carries `module_id`; `call.enter`/`call.exit` are
owned by the caller module, other semantic events by the current function,
effects by their `ExecutionContext`. The effect key is now `module_id`.
* Trace budgets: 100 000 events / 32 MiB by default, configurable with
`--trace-max-events` and `--trace-max-bytes`; `0` means unlimited. Semantic
recording is disabled when no trace is requested.
* On capacity exhaustion the trace ends with `trace.truncated` and
`truncated.reason`; the run status and execution result are unchanged. The
writer also enforces the byte cap on the final document.
* Worker frames are capped at 8 MiB, with aligned request/result/capability
wire caps; partial headers, oversized lengths, truncated payloads and invalid
JSON are protocol errors, clean EOF is distinct, and the host kills and marks
a worker dead after a transport failure.
* New independent `tests/consumers/trace_consumer.py` verifies identity,
digest, module references and sequence ordering in CI.
* Tests: event module ownership, 20-event truncation of a 500-iteration loop,
worker frame round-trip/EOF/partial/oversized/truncated cases.
### M0.6 implementation progress
| Step | State | Evidence |
@@ -133,11 +158,12 @@ Analysis v2 (`spec/analysis-v2.md`, `spec/schemas/analysis-v2.schema.json`):
| S1/H1-A lock v3 graph | **Done** | ADR-0016; full-closure `marqdo.lock` v3; fail-closed edge lookup; strict non-rewriting `sync`; verified Git cache; unified file-set profile (see §S1/H1-A) |
| S2-A/H1-B source locations | **Done** | ADR-0017; `RuntimeError`/`CallFrame` carry `module_id`; callsite origin; CLI module-aware resolution with no entry fallback (see §S2-A/H1-B) |
| S2-B/H2-A Analysis v2 | **Done** | logical project identity, resources array, machine-path-free diagnostics, emitted-bytes digest, v2 schema + independent consumer (see §S2-B/H2-A) |
| S2-C/H2-B Trace v2 | **Done** | logical root/digest, module-owned events, bounded/truncated trace, independent consumer, hardened worker frames (see §S2-C/H2-B) |
| R0 baseline + ADRs | **Done** | this section; `docs/adr/0008`–`0012`; full workspace gate |
| R1 package-aware identity + callable registry | **Done for statics; superseded by S1 lock v3** | `ModuleId { instance, relative_path }`; `PackageResolver` path sources; lock v3 transitive graph (ADR-0016) replaces the flat v2 lock; `CallableRegistry` for builtins/marqdo/declared ABI exports; external manifest wiring is R2 |
| R2 ABI/Kernel/Broker | **Done (worker = crash isolation)** | `KernelRegistry` external providers; complete-name export routing; `CapabilityBroker`; static `check --plugin`; multi-export C test; isolated worker with timeout/crash tests; host handle table. Native/worker are not an OS sandbox |
| R3 Tool Protocol | **Done** | `tool install/list/remove`; unknown-command fallback; atomic local install; SHA/tamper checks; black-box tests for arbitrary names, Unicode args, conflicts, traversal and PATH hijacking |
| R4 Analysis v2 / Trace v1 | **Partial** | `check --emit` Analysis v2 (logical identity, resources, path-free diagnostics, emitted-bytes digest); `run/call --trace-file` still Trace v1 pending H2-B; independent Python consumer; stale-artifact removal; value redaction |
| R4 Analysis/Trace v2 | **Done** | `check --emit` Analysis v2 and `run/call --trace-file` Trace v2 with logical identity, resources, module-owned events, capacity bounds, schemas and independent Python consumers; stale-artifact removal; value redaction |
| R5 environment/isolation | **Partial** | Pinned Git fetch with verified cache markers, transitive `@alias` resolution, full-closure atomic `.marqdo/env` materialization, foreign process-group cleanup. Strict OS isolation remains explicitly unavailable (fail-closed) |
| R6 release/consumers | **Done for Linux** | examples for unknown-name Tool, multi-export plugin, external kernel and multi-version packages; Analysis/Trace/Tool JSON Schemas; CI Linux gate + Windows/macOS `cargo check`; R6 example tests |
+1
View File
@@ -34,6 +34,7 @@ marqdo-cli/
│ ├── foreign.md Python/Node kernel protocol
│ ├── tool-v1.md generic external Tool Protocol v1
│ ├── analysis-v2.md public Analysis v2 projection (v1 historical)
│ ├── trace-v2.md public Trace v2 projection (v1 historical)
│ └── trace-v1.md public Trace v1 event stream
├── crates/
│ ├── marqdo-span/ Span, LineCol, LineIndex (leaf)
+6 -6
View File
@@ -99,10 +99,10 @@
* MQ_MAX_EXPORT_NAME_BYTES 256
* MQ_MAX_SIGNATURE_WIRE_BYTES 4096
* MQ_MAX_METADATA_WIRE_BYTES 65536
* MQ_MAX_REQUEST_WIRE_BYTES 16777216 (16 MiB)
* MQ_MAX_RESULT_WIRE_BYTES 67108864 (64 MiB)
* MQ_MAX_REQUEST_WIRE_BYTES 8384512 (8 MiB - 4096)
* MQ_MAX_RESULT_WIRE_BYTES 8384512 (8 MiB - 4096)
* MQ_MAX_ERROR_WIRE_BYTES 1048576 (1 MiB)
* MQ_MAX_CAPABILITY_REPLY_BYTES 16777216 (16 MiB)
* MQ_MAX_CAPABILITY_REPLY_BYTES 8384512 (8 MiB - 4096)
* JSON depth is additionally limited to 64 levels. Requests/results that
* exceed these limits are rejected with a structured error; they must not
* be truncated and reparsed.
@@ -245,10 +245,10 @@ extern "C" {
#define MQ_MAX_EXPORT_NAME_BYTES 256u
#define MQ_MAX_SIGNATURE_WIRE_BYTES 4096u
#define MQ_MAX_METADATA_WIRE_BYTES 65536u
#define MQ_MAX_REQUEST_WIRE_BYTES (16u * 1024u * 1024u)
#define MQ_MAX_RESULT_WIRE_BYTES (64u * 1024u * 1024u)
#define MQ_MAX_REQUEST_WIRE_BYTES ((8u * 1024u * 1024u) - 4096u)
#define MQ_MAX_RESULT_WIRE_BYTES ((8u * 1024u * 1024u) - 4096u)
#define MQ_MAX_ERROR_WIRE_BYTES (1024u * 1024u)
#define MQ_MAX_CAPABILITY_REPLY_BYTES (16u * 1024u * 1024u)
#define MQ_MAX_CAPABILITY_REPLY_BYTES ((8u * 1024u * 1024u) - 4096u)
/* ------------------------------------------------------------------ */
/* Stable-layout buffer types */
+9 -6
View File
@@ -68,10 +68,10 @@ minted for. Plugins must not dereference, cache or copy it.
| `MQ_MAX_EXPORT_NAME_BYTES` | 256 |
| `MQ_MAX_SIGNATURE_WIRE_BYTES` | 4096 |
| `MQ_MAX_METADATA_WIRE_BYTES` | 65536 |
| `MQ_MAX_REQUEST_WIRE_BYTES` | 16 MiB (16,777,216) |
| `MQ_MAX_RESULT_WIRE_BYTES` | 64 MiB (67,108,864) |
| `MQ_MAX_REQUEST_WIRE_BYTES` | 8 MiB - 4096 (8,384,512) |
| `MQ_MAX_RESULT_WIRE_BYTES` | 8 MiB - 4096 (8,384,512) |
| `MQ_MAX_ERROR_WIRE_BYTES` | 1 MiB (1,048,576) |
| `MQ_MAX_CAPABILITY_REPLY_BYTES` | 16 MiB |
| `MQ_MAX_CAPABILITY_REPLY_BYTES` | 8 MiB - 4096 |
Wire JSON decoding additionally applies the default `WireLimits` from
`spec/value-wire.md` (depth 64, 1,000,000 items, 16 MiB strings). Oversized
@@ -270,8 +270,8 @@ with both is rejected (`ambiguous_response`). Error:
}}
```
Envelope limits: at most `MAX_ARGS = 1024` request arguments; request ≤ 16 MiB,
response ≤ 64 MiB, error ≤ 1 MiB. `context` carries no secrets and no
Envelope limits: at most `MAX_ARGS = 1024` request arguments; request and
response ≤ 8 MiB - 4096, error ≤ 1 MiB. `context` carries no secrets and no
reconstructable OS handles. `signature_wire` is
`{"signature":"<signature string>"}`; `plugin_metadata_wire` is
`{"name":"<package name>","version":"<semver>"}` (extra fields ignored).
@@ -364,7 +364,10 @@ full plugin capability broker for worker requests remain future work.
### 9.1 Worker frames
Frames are a 4-byte big-endian length followed by one JSON object (max 64 MiB).
Frames are a 4-byte big-endian length followed by one JSON object (max 8 MiB).
A zero/oversized length, partial header, truncated payload or invalid JSON is a
protocol error; clean EOF at a frame boundary is not. The host marks the worker
dead after a transport failure and does not reuse it.
Worker → host: `{"kind":"ready","exports":[...]}`. Host → worker:
`{"kind":"invoke","request":<wire request>}` and `{"kind":"shutdown"}`.
Worker → host responses are `{"kind":"response","response":<wire response>}` or
+86
View File
@@ -0,0 +1,86 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://marqdo.dev/schemas/trace-v2.schema.json",
"title": "Marqdo Trace v2",
"type": "object",
"required": ["schema", "schema_version", "run", "source", "limits", "truncated", "events"],
"properties": {
"schema": {"const": "marqdo.trace"},
"schema_version": {"const": 2},
"run": {
"type": "object",
"required": ["id", "entry", "status"],
"properties": {
"id": {"type": "string", "minLength": 1},
"entry": {"type": "string", "minLength": 1},
"status": {"type": "string", "minLength": 1}
},
"additionalProperties": true
},
"source": {
"type": "object",
"required": ["root_id", "analysis_digest"],
"properties": {
"root_id": {"type": "string", "pattern": "^(root:|sha256:[0-9a-f]{64}$)"},
"analysis_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}
},
"additionalProperties": true
},
"limits": {
"type": "object",
"required": ["max_events", "max_bytes"],
"properties": {
"max_events": {"type": "integer", "minimum": 0},
"max_bytes": {"type": "integer", "minimum": 0}
},
"additionalProperties": false
},
"truncated": {
"oneOf": [
{"type": "null"},
{
"type": "object",
"required": ["reason", "limit", "written"],
"properties": {
"reason": {"enum": ["event_budget", "byte_budget"]},
"limit": {"type": "integer", "minimum": 0},
"written": {"type": "integer", "minimum": 0}
},
"additionalProperties": true
}
]
},
"events": {
"type": "array",
"items": {
"type": "object",
"required": ["seq", "kind", "module_id", "span", "detail"],
"properties": {
"seq": {"type": "integer", "minimum": 0},
"kind": {"type": "string", "minLength": 1},
"function_id": {"type": ["string", "null"]},
"module_id": {"type": ["string", "null"]},
"span": {
"oneOf": [
{"type": "null"},
{
"type": "object",
"required": ["start", "end"],
"properties": {
"start": {"type": "integer", "minimum": 0},
"end": {"type": "integer", "minimum": 0}
},
"additionalProperties": true
}
]
},
"detail": {"type": ["string", "null"]},
"operation": {"type": "string"},
"duration_micros": {"type": "integer", "minimum": 0}
},
"additionalProperties": true
}
}
},
"additionalProperties": true
}
+2 -1
View File
@@ -1,6 +1,7 @@
# Marqdo Trace v1
Status: **implemented by `marqdo run/call --trace-file <path>` (M0.6 R4).**
Status: **superseded by [Trace v2](trace-v2.md); never released, do not write
v1.** Kept as history.
## 1. Invariants
+112
View File
@@ -0,0 +1,112 @@
# Marqdo Trace v2
Status: **implemented by `marqdo run/call --trace-file <path>` (S2).**
Supersedes Trace v1; v1 was never released and is kept as history.
## 1. Invariants
* Tracing is observation, not execution. Only `run`/`call` produce a trace;
`check`, `inspect` and `tool list` never execute a document.
* Events come from the interpreter (`call.*`, `branch.eval`,
`loop.iteration`, `return`) and the host (`effect`). Both share one sequence.
* The writer is atomic; an error run still records its terminal status.
* No argument values, secrets, handles or file contents are recorded.
* Trace capacity is a bounded observation budget. Reaching it truncates the
trace; it never changes `run.status` or the program’s execution result.
Execution budgets (`--fuel`, `--timeout`, ...) remain separate and still
produce `resource.exhausted`.
## 2. Shape
```json
{
"schema": "marqdo.trace",
"schema_version": 2,
"run": {"id": "pid-nanos", "entry": "root:q@0.1.0::index.mq.md::main", "status": "ok"},
"source": {"root_id": "root:q@0.1.0", "analysis_digest": "sha256:<64 hex>"},
"limits": {"max_events": 100000, "max_bytes": 33554432},
"truncated": null,
"events": [
{"seq": 0, "kind": "call.enter", "function_id": "root:q@0.1.0::index.mq.md::main",
"module_id": "root:q@0.1.0::index.mq.md", "span": null, "detail": null},
{"seq": 1, "kind": "effect", "module_id": "dep:p@0.1.0#<digest>::index.mq.md",
"span": {"start": 40, "end": 52}, "detail": "ok", "operation": "print",
"duration_micros": 3}
]
}
```
`seq` is globally increasing. `run.status` is `ok` or a stable runtime error
code. Every span-bearing event has a `module_id`; the span is a UTF-8 byte
offset range in that module. `source` only carries project identity and the
Analysis digest; it does not identify the span owner.
`source.root_id` is the Analysis v2 `project.root_id`
(`root:<name>@<version>` or `sha256:<workspace snapshot>`).
`source.analysis_digest` is `sha256:` plus the SHA-256 of the exact Analysis
v2 bytes produced for the same project with the same projection configuration.
A consumer verifies it by hashing the Analysis file.
`truncated` is `null` or:
```json
{"reason": "event_budget", "limit": 20, "written": 20}
```
`reason` is `event_budget` or `byte_budget`. When truncation occurs, the final
event is `{"kind":"trace.truncated","detail":<reason>,"module_id":null,
"span":null,"seq":<one past the last written event>}`.
## 3. Event kinds and span owners
| kind | detail | `module_id` owner |
|---|---|---|
| `call.enter` | null | caller module (entry call: callee) |
| `call.exit` | `ok` / `return` / error code | caller module (entry call: callee) |
| `return` | value type name | current function’s module |
| `branch.eval` | `true` / `false` | current function’s module |
| `loop.iteration` | `continue` / `exit` | current function’s module |
| `effect` | result summary | module from `ExecutionContext` |
| `trace.truncated` | `event_budget` / `byte_budget` | null |
`function_id` is always the canonical callee function id. Future kinds are
additive; consumers must ignore unknown kinds. `effect.operation` names the
`HostOp` (`print`, `fs.read`, ...).
## 4. Capacity
Defaults: 100 000 events and 32 MiB approximate serialized size. `0` means
unlimited for either limit. `run`/`call` expose `--trace-max-events` and
`--trace-max-bytes`; these are options on existing commands, not new commands.
The host stops accepting events once a limit is reached; the writer also
enforces the byte limit on the final serialized document, dropping trailing
events and recording `byte_budget`. Semantic recording is disabled entirely
when no trace was requested.
The `trace` array in `run --json` uses the same budget.
## 5. Staleness
A trace is stale when its `source.analysis_digest` differs from the digest of
the Analysis emitted for the current document. Consumers must not present a
stale trace as current. Moving a project does not change the manifest identity
or digest; changing executable source does.
## 6. Worker transport limits
Plugin worker frames are a 4-byte big-endian length plus JSON, capped at
8 MiB (`8 * 1024 * 1024`). The wire request/result/capability caps are aligned
to `8 MiB - 4096`; error frames remain 1 MiB. A partial header, zero/oversized
length, truncated payload or invalid JSON is a protocol error. A clean EOF is
distinct from a protocol failure. The host marks a worker dead after a transport
failure and never reuses its stdin.
## 7. Compatibility and tests
* Consumers must reject unknown `schema_version` and validate `module_id`
against Analysis modules.
* JSON Schema: `spec/schemas/trace-v2.schema.json`.
* `tests/consumers/trace_consumer.py` verifies identity, digest and event
module references.
* CLI tests cover call/effect events, no-secret redaction, error terminal
status, and event-budget truncation without changing the run status.
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env python3
"""Independent Trace v2 consumer: no Marqdo/Rust imports."""
import hashlib
import json
import sys
if len(sys.argv) != 3:
raise SystemExit("usage: trace_consumer.py ANALYSIS.json TRACE.json")
with open(sys.argv[1], "rb") as handle:
analysis_bytes = handle.read()
analysis = json.loads(analysis_bytes)
with open(sys.argv[2], encoding="utf-8") as handle:
trace = json.load(handle)
if analysis.get("schema") != "marqdo.analysis" or analysis.get("schema_version") != 2:
raise SystemExit("unsupported analysis")
if trace.get("schema") != "marqdo.trace" or trace.get("schema_version") != 2:
raise SystemExit("unsupported trace")
modules = {module["id"]: module for module in analysis["modules"]}
if trace["source"]["root_id"] != analysis["project"]["root_id"]:
raise SystemExit("trace/analysis root mismatch")
expected = "sha256:" + hashlib.sha256(analysis_bytes).hexdigest()
if trace["source"]["analysis_digest"] != expected:
raise SystemExit("analysis digest mismatch")
truncated = trace.get("truncated")
if truncated is not None and truncated.get("reason") not in ("event_budget", "byte_budget"):
raise SystemExit("bad truncation reason")
previous = None
events = trace["events"]
for event in events:
seq = event["seq"]
if previous is not None and seq <= previous:
raise SystemExit("sequence is not increasing")
previous = seq
span = event.get("span")
if span is not None and (span["start"] > span["end"]):
raise SystemExit("bad span")
module_id = event.get("module_id")
if module_id is not None and module_id not in modules:
raise SystemExit("event references an unknown module")
function_id = event.get("function_id")
if function_id is not None:
owner = function_id.rsplit("::", 1)[0]
if owner not in modules:
raise SystemExit("function id references an unknown module")
if truncated is not None and events and events[-1].get("kind") != "trace.truncated":
raise SystemExit("truncated trace has no terminal marker")
print("events=" + str(len(events)))