feat(trace): Trace v2 identity, module-owned events and capacity bounds (B-16)
- bind traces to the logical project identity and the exact emitted Analysis digest; every span-bearing event carries module_id - add host TraceState budgets (100k events / 32 MiB defaults), CLI --trace-max-events/--trace-max-bytes, terminal trace.truncated events, and a final serialized byte cap; budgets never change run.status - disable semantic recording when no trace is requested - cap worker frames at 8 MiB, align request/result/capability wire caps, make clean EOF distinct from protocol errors, and mark/kill a worker after a transport failure instead of reusing it - add trace-v2 spec/schema, independent trace consumer, CI check and tests - record the S2/C state in STATUS
This commit is contained in:
1 parent
55dcb57553
commit
d890686a2b
21 files changed
+708
-60
No files matched your search
@@ -41,9 +41,13 @@ jobs:
|
||||
./target/debug/marqdo inspect examples/references.mq.md --refs
|
||||
./target/debug/marqdo check examples/multi-module/index.mq.md --refs \
|
||||
--emit /tmp/marqdo-analysis.json
|
||||
./target/debug/marqdo run examples/multi-module/index.mq.md \
|
||||
--trace-file /tmp/marqdo-analysis-trace.json
|
||||
python3 tests/consumers/analysis_consumer.py /tmp/marqdo-analysis.json
|
||||
python3 tests/consumers/trace_consumer.py \
|
||||
/tmp/marqdo-analysis.json /tmp/marqdo-analysis-trace.json
|
||||
./target/debug/marqdo run examples/hello.mq.md \
|
||||
--trace-file /tmp/marqdo-trace.json
|
||||
python3 tests/consumers/analysis_consumer.py /tmp/marqdo-analysis.json
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
json.load(open("/tmp/marqdo-trace.json"))
|
||||
|
||||
@@ -192,7 +192,7 @@ tests/ conformance/negative corpus (see tests/README.md)
|
||||
[`package-format.md`](spec/package-format.md),
|
||||
[`plugin-abi.md`](spec/plugin-abi.md), [`math.md`](spec/math.md),
|
||||
[`foreign.md`](spec/foreign.md), [`tool-v1.md`](spec/tool-v1.md),
|
||||
[`analysis-v2.md`](spec/analysis-v2.md), [`trace-v1.md`](spec/trace-v1.md);
|
||||
[`analysis-v2.md`](spec/analysis-v2.md), [`trace-v2.md`](spec/trace-v2.md);
|
||||
JSON Schemas in [`spec/schemas/`](spec/schemas/).
|
||||
* [`tests/README.md`](tests/README.md) — conformance corpus and `UPDATE_GOLDENS`
|
||||
workflow.
|
||||
|
||||
@@ -38,8 +38,12 @@ pub struct SandboxSettings {
|
||||
pub plugins: Vec<PathBuf>,
|
||||
/// Explicit trust decision for in-process native plugin loading.
|
||||
pub trust_native_plugin: bool,
|
||||
/// Optional Trace v1 output path.
|
||||
/// Optional Trace v2 output path.
|
||||
pub trace_file: Option<PathBuf>,
|
||||
/// Trace event budget override (0 = unlimited).
|
||||
pub trace_max_events: Option<usize>,
|
||||
/// Trace approximate byte budget override (0 = unlimited).
|
||||
pub trace_max_bytes: Option<usize>,
|
||||
/// External foreign-kernel descriptor paths.
|
||||
pub kernels: Vec<PathBuf>,
|
||||
}
|
||||
@@ -690,7 +694,16 @@ pub fn execute(
|
||||
};
|
||||
let mut host = StdHost::new(mode, capabilities, project.root.clone());
|
||||
if trace || json || settings.trace_file.is_some() {
|
||||
host = host.with_trace();
|
||||
host = host
|
||||
.with_trace()
|
||||
.with_trace_budget(marqdo_host::TraceBudget {
|
||||
max_events: settings
|
||||
.trace_max_events
|
||||
.unwrap_or(marqdo_host::trace::DEFAULT_MAX_TRACE_EVENTS),
|
||||
max_bytes: settings
|
||||
.trace_max_bytes
|
||||
.unwrap_or(marqdo_host::trace::DEFAULT_MAX_TRACE_BYTES),
|
||||
});
|
||||
}
|
||||
if let Some(fuel) = settings.fuel {
|
||||
host.budget.fuel = fuel;
|
||||
|
||||
@@ -109,9 +109,15 @@ struct SandboxArgs {
|
||||
/// Explicitly trusted plugin manifest/directory to load (repeatable).
|
||||
#[arg(long = "plugin", value_name = "PATH")]
|
||||
plugin: Vec<PathBuf>,
|
||||
/// Write a versioned Trace v1 file for this run.
|
||||
/// Write a versioned Trace v2 file for this run.
|
||||
#[arg(long = "trace-file", value_name = "PATH")]
|
||||
trace_file: Option<PathBuf>,
|
||||
/// Override the trace event budget (0 = unlimited).
|
||||
#[arg(long = "trace-max-events")]
|
||||
trace_max_events: Option<usize>,
|
||||
/// Override the approximate trace byte budget (0 = unlimited).
|
||||
#[arg(long = "trace-max-bytes")]
|
||||
trace_max_bytes: Option<usize>,
|
||||
/// Register an external foreign-kernel provider (repeatable).
|
||||
#[arg(long = "kernel", value_name = "PATH")]
|
||||
kernel: Vec<PathBuf>,
|
||||
@@ -135,6 +141,8 @@ impl SandboxArgs {
|
||||
plugins: self.plugin.clone(),
|
||||
trust_native_plugin: self.trust_native_plugin,
|
||||
trace_file: self.trace_file.clone(),
|
||||
trace_max_events: self.trace_max_events,
|
||||
trace_max_bytes: self.trace_max_bytes,
|
||||
kernels: self.kernel.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
+47
-5
@@ -1,4 +1,4 @@
|
||||
//! Public Trace v1 writer (spec/trace-v1.md).
|
||||
//! Public Trace v2 writer (spec/trace-v2.md).
|
||||
//!
|
||||
//! Events come from the interpreter (semantic) and the capability host
|
||||
//! (effects). No argument values, secrets or handles are recorded.
|
||||
@@ -12,7 +12,7 @@ use crate::loader::LoadedProject;
|
||||
|
||||
/// Schema name and version.
|
||||
pub const SCHEMA: &str = "marqdo.trace";
|
||||
pub const SCHEMA_VERSION: u32 = 1;
|
||||
pub const SCHEMA_VERSION: u32 = 2;
|
||||
|
||||
/// Write a trace atomically for one run.
|
||||
pub fn write(
|
||||
@@ -32,6 +32,7 @@ pub fn write(
|
||||
);
|
||||
let analysis_digest = crate::analysis::digest(project);
|
||||
let root_id = project.identity.root_id();
|
||||
let state = host.trace_state();
|
||||
let mut events: Vec<Value> = host
|
||||
.semantic_events()
|
||||
.iter()
|
||||
@@ -39,15 +40,55 @@ pub fn write(
|
||||
.chain(host.trace().iter().map(effect_json))
|
||||
.collect();
|
||||
events.sort_by_key(|event| event.get("seq").and_then(Value::as_u64).unwrap_or(u64::MAX));
|
||||
|
||||
let mut truncation = state.truncation.clone();
|
||||
let max_bytes = state.budget.max_bytes;
|
||||
let bytes = loop {
|
||||
let mut all = events.clone();
|
||||
if let Some(truncation) = &truncation {
|
||||
let seq = all
|
||||
.last()
|
||||
.and_then(|event| event.get("seq"))
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0)
|
||||
+ 1;
|
||||
all.push(json!({
|
||||
"seq": seq,
|
||||
"kind": "trace.truncated",
|
||||
"module_id": null,
|
||||
"span": null,
|
||||
"detail": truncation.reason,
|
||||
}));
|
||||
}
|
||||
let trace = json!({
|
||||
"schema": SCHEMA,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"run": {"id": run_id, "entry": entry, "status": status},
|
||||
"source": {"root_id": root_id, "analysis_digest": format!("sha256:{analysis_digest}")},
|
||||
"events": events,
|
||||
"source": {
|
||||
"root_id": root_id,
|
||||
"analysis_digest": format!("sha256:{analysis_digest}"),
|
||||
},
|
||||
"limits": {
|
||||
"max_events": state.budget.max_events,
|
||||
"max_bytes": state.budget.max_bytes,
|
||||
},
|
||||
"truncated": truncation,
|
||||
"events": all,
|
||||
});
|
||||
let bytes = serde_json::to_vec_pretty(&trace)
|
||||
.map_err(|error| format!("cannot encode trace: {error}"))?;
|
||||
if max_bytes == 0 || bytes.len() <= max_bytes || events.is_empty() {
|
||||
break bytes;
|
||||
}
|
||||
events.pop();
|
||||
if truncation.is_none() {
|
||||
truncation = Some(marqdo_host::TraceTruncation {
|
||||
reason: "byte_budget".to_string(),
|
||||
limit: max_bytes as u64,
|
||||
written: events.len() as u64,
|
||||
});
|
||||
}
|
||||
};
|
||||
atomic_write(path, &bytes)
|
||||
}
|
||||
|
||||
@@ -56,6 +97,7 @@ fn semantic_json(event: &SemanticEvent) -> Value {
|
||||
"seq": event.seq,
|
||||
"kind": event.kind,
|
||||
"function_id": event.function_id,
|
||||
"module_id": event.module_id,
|
||||
"span": event.span.map(|span| json!({"start": span.start, "end": span.end})),
|
||||
"detail": event.detail,
|
||||
})
|
||||
@@ -65,7 +107,7 @@ fn effect_json(entry: &TraceEntry) -> Value {
|
||||
json!({
|
||||
"seq": entry.seq,
|
||||
"kind": "effect",
|
||||
"module": entry.module,
|
||||
"module_id": entry.module,
|
||||
"span": entry.span.map(|span| json!({"start": span.start, "end": span.end})),
|
||||
"detail": entry.result,
|
||||
"operation": entry.op,
|
||||
|
||||
@@ -241,6 +241,52 @@ print(1)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_v2_budget_truncates_long_loops_without_failing_the_run() {
|
||||
let dir = TempDir::new("trace-budget");
|
||||
write(
|
||||
dir.path(),
|
||||
"index.mq.md",
|
||||
"## main\n\n**i = 0**\n\n- `i < 500`\n\n **i = i + 1**\n\n*i*\n",
|
||||
);
|
||||
let trace_path = dir.path().join("trace.json");
|
||||
let output = cli(
|
||||
dir.path(),
|
||||
&[
|
||||
"run",
|
||||
"index.mq.md",
|
||||
"--trace-file",
|
||||
trace_path.to_str().unwrap(),
|
||||
"--trace-max-events",
|
||||
"20",
|
||||
],
|
||||
);
|
||||
assert_eq!(
|
||||
output.status.code(),
|
||||
Some(0),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let trace: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&trace_path).unwrap()).unwrap();
|
||||
assert_eq!(trace["schema_version"], 2);
|
||||
assert_eq!(trace["run"]["status"], "ok");
|
||||
assert_eq!(trace["limits"]["max_events"], 20);
|
||||
assert_eq!(trace["truncated"]["reason"], "event_budget");
|
||||
assert_eq!(trace["truncated"]["limit"], 20);
|
||||
let events = trace["events"].as_array().unwrap();
|
||||
assert!(events.len() <= 21, "events: {}", events.len());
|
||||
assert_eq!(events.last().unwrap()["kind"], "trace.truncated");
|
||||
let mut previous = None;
|
||||
for event in events {
|
||||
let seq = event["seq"].as_u64().unwrap();
|
||||
if let Some(previous) = previous {
|
||||
assert!(seq > previous);
|
||||
}
|
||||
previous = Some(seq);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_check_removes_stale_analysis() {
|
||||
let dir = TempDir::new("analysis-error");
|
||||
@@ -282,11 +328,14 @@ fn trace_records_calls_effects_and_status_without_secrets() {
|
||||
);
|
||||
let trace: serde_json::Value = serde_json::from_str(&text).unwrap();
|
||||
assert_eq!(trace["schema"], "marqdo.trace");
|
||||
assert_eq!(trace["schema_version"], 1);
|
||||
assert_eq!(trace["schema_version"], 2);
|
||||
assert_eq!(trace["run"]["status"], "ok");
|
||||
assert_eq!(trace["run"]["entry"], "root:default::index.mq.md::main");
|
||||
let events = trace["events"].as_array().unwrap();
|
||||
assert!(events.iter().any(|event| event["kind"] == "call.enter"));
|
||||
assert!(events
|
||||
.iter()
|
||||
.any(|event| event["kind"] == "call.enter"
|
||||
&& event["module_id"] == "root:default::index.mq.md"));
|
||||
assert!(events.iter().any(|event| event["kind"] == "effect"));
|
||||
assert!(events.iter().any(|event| event["kind"] == "call.exit"));
|
||||
let mut previous = None;
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::context::ExecutionContext;
|
||||
use crate::fs::ProjectFs;
|
||||
use crate::handles::{contains_plugin_handle, PluginHandleTable};
|
||||
use crate::plugin_worker::WorkerPlugin;
|
||||
use crate::trace::{EffectTrace, SemanticEvent, TraceEntry};
|
||||
use crate::trace::{EffectTrace, SemanticEvent, TraceBudget, TraceEntry, TraceState};
|
||||
use marqdo_sandbox::{PlatformCapabilities, SandboxMode};
|
||||
|
||||
/// Host operations reachable from the language surface.
|
||||
@@ -99,6 +99,7 @@ pub trait Host {
|
||||
&mut self,
|
||||
_kind: &str,
|
||||
_function_id: Option<&str>,
|
||||
_module_id: Option<&str>,
|
||||
_span: Option<Span>,
|
||||
_detail: Option<&str>,
|
||||
) {
|
||||
@@ -144,6 +145,7 @@ pub struct StdHost {
|
||||
pub handles: PluginHandleTable,
|
||||
semantic_events: Vec<SemanticEvent>,
|
||||
next_event_seq: u64,
|
||||
trace_state: TraceState,
|
||||
}
|
||||
|
||||
struct PluginSlot {
|
||||
@@ -203,6 +205,7 @@ impl StdHost {
|
||||
handles: PluginHandleTable::default(),
|
||||
semantic_events: Vec::new(),
|
||||
next_event_seq: 0,
|
||||
trace_state: TraceState::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -338,6 +341,17 @@ impl StdHost {
|
||||
seq
|
||||
}
|
||||
|
||||
/// Trace capacity state for the writer.
|
||||
pub fn trace_state(&self) -> &TraceState {
|
||||
&self.trace_state
|
||||
}
|
||||
|
||||
/// Override the trace event/byte budget.
|
||||
pub fn with_trace_budget(mut self, budget: TraceBudget) -> Self {
|
||||
self.trace_state.budget = budget;
|
||||
self
|
||||
}
|
||||
|
||||
/// Audited plugin capability decisions.
|
||||
pub fn capability_audit(&self) -> Vec<CapabilityAudit> {
|
||||
self.broker_audit
|
||||
@@ -359,6 +373,7 @@ impl StdHost {
|
||||
/// Enable effect tracing.
|
||||
pub fn with_trace(mut self) -> Self {
|
||||
self.trace = EffectTrace::enabled();
|
||||
self.trace_state.enabled = true;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -504,12 +519,20 @@ impl Host for StdHost {
|
||||
ctx: &ExecutionContext,
|
||||
) -> Result<Value, RuntimeError> {
|
||||
let started = Instant::now();
|
||||
let seq = self.next_seq();
|
||||
let result = self.dispatch(op, args, ctx);
|
||||
let summary = match &result {
|
||||
Ok(_) => "ok".to_string(),
|
||||
Err(error) => error.code().to_string(),
|
||||
};
|
||||
let estimate = TraceState::estimate(
|
||||
"effect",
|
||||
None,
|
||||
Some(&ctx.module_id),
|
||||
Some(&summary),
|
||||
Some(op.name()),
|
||||
);
|
||||
if self.trace_state.accept(estimate) {
|
||||
let seq = self.next_seq();
|
||||
self.trace.record(
|
||||
seq,
|
||||
op.name(),
|
||||
@@ -518,6 +541,7 @@ impl Host for StdHost {
|
||||
started.elapsed(),
|
||||
summary,
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
@@ -545,14 +569,20 @@ impl Host for StdHost {
|
||||
&mut self,
|
||||
kind: &str,
|
||||
function_id: Option<&str>,
|
||||
module_id: Option<&str>,
|
||||
span: Option<Span>,
|
||||
detail: Option<&str>,
|
||||
) {
|
||||
let estimate = TraceState::estimate(kind, function_id, module_id, detail, None);
|
||||
if !self.trace_state.accept(estimate) {
|
||||
return;
|
||||
}
|
||||
let seq = self.next_seq();
|
||||
self.semantic_events.push(SemanticEvent {
|
||||
seq,
|
||||
kind: kind.to_string(),
|
||||
function_id: function_id.map(str::to_string),
|
||||
module_id: module_id.map(str::to_string),
|
||||
span,
|
||||
detail: detail.map(str::to_string),
|
||||
});
|
||||
|
||||
@@ -21,4 +21,4 @@ pub use fs::ProjectFs;
|
||||
pub use handles::{contains_plugin_handle, PluginHandleTable};
|
||||
pub use host::{arity, expect_text, Host, HostOp, StdHost};
|
||||
pub use marqdo_sandbox::{PlatformCapabilities, SandboxMode, StrictIsolation};
|
||||
pub use trace::{EffectTrace, SemanticEvent, TraceEntry};
|
||||
pub use trace::{EffectTrace, SemanticEvent, TraceBudget, TraceEntry, TraceState, TraceTruncation};
|
||||
@@ -22,6 +22,7 @@ pub struct WorkerPlugin {
|
||||
stdin: ChildStdin,
|
||||
frames: Receiver<Result<serde_json::Value, PluginError>>,
|
||||
exports: BTreeMap<String, String>,
|
||||
dead: bool,
|
||||
}
|
||||
|
||||
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
@@ -94,6 +95,7 @@ impl WorkerPlugin {
|
||||
};
|
||||
let Some(mut stdout) = child.stdout.take() else {
|
||||
let _ = child.kill();
|
||||
let _ = std::fs::remove_file(&manifest_path);
|
||||
return Err(PluginError::load(
|
||||
"worker_spawn_failed",
|
||||
"worker stdout unavailable",
|
||||
@@ -103,11 +105,24 @@ impl WorkerPlugin {
|
||||
thread::Builder::new()
|
||||
.name("marqdo-plugin-worker-reader".to_string())
|
||||
.spawn(move || loop {
|
||||
let frame = read_frame(&mut stdout);
|
||||
let failed = frame.is_err();
|
||||
if sender.send(frame).is_err() || failed {
|
||||
match read_frame(&mut stdout) {
|
||||
Ok(Some(frame)) => {
|
||||
if sender.send(Ok(frame)).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
let _ = sender.send(Err(PluginError::invoke(
|
||||
"plugin.crashed",
|
||||
"plugin worker closed its output stream",
|
||||
)));
|
||||
break;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = sender.send(Err(error));
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(|error| {
|
||||
let _ = child.kill();
|
||||
@@ -169,21 +184,35 @@ impl WorkerPlugin {
|
||||
stdin,
|
||||
frames,
|
||||
exports,
|
||||
dead: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Invoke one export in the worker.
|
||||
pub fn invoke(&mut self, request: &WireRequest) -> Result<WireResponse, PluginError> {
|
||||
if self.dead {
|
||||
return Err(PluginError::invoke(
|
||||
"plugin.crashed",
|
||||
"plugin worker is no longer usable after a transport failure",
|
||||
));
|
||||
}
|
||||
let request_value: serde_json::Value = serde_json::from_slice(&request.to_json_bytes()?)
|
||||
.map_err(|error| PluginError::wire("worker_encode_failed", error.to_string()))?;
|
||||
write_frame(
|
||||
if let Err(error) = write_frame(
|
||||
&mut self.stdin,
|
||||
&json!({"kind": "invoke", "request": request_value}),
|
||||
)?;
|
||||
) {
|
||||
self.mark_dead();
|
||||
return Err(error);
|
||||
}
|
||||
let frame = match self.frames.recv_timeout(CALL_TIMEOUT) {
|
||||
Ok(frame) => frame?,
|
||||
Ok(Ok(frame)) => frame,
|
||||
Ok(Err(error)) => {
|
||||
self.mark_dead();
|
||||
return Err(error);
|
||||
}
|
||||
Err(_) => {
|
||||
let _ = self.child.kill();
|
||||
self.mark_dead();
|
||||
return Err(PluginError::invoke(
|
||||
"plugin.timeout",
|
||||
"plugin worker exceeded its invocation timeout",
|
||||
@@ -202,14 +231,22 @@ impl WorkerPlugin {
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("worker reported an error"),
|
||||
)),
|
||||
_ => Err(PluginError::invoke(
|
||||
_ => {
|
||||
self.mark_dead();
|
||||
Err(PluginError::invoke(
|
||||
"plugin.crashed",
|
||||
"plugin worker closed or violated the frame protocol",
|
||||
)),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_dead(&mut self) {
|
||||
self.dead = true;
|
||||
let _ = self.child.kill();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WorkerPlugin {
|
||||
fn drop(&mut self) {
|
||||
let _ = write_frame(&mut self.stdin, &json!({"kind": "shutdown"}));
|
||||
|
||||
@@ -70,7 +70,99 @@ impl EffectTrace {
|
||||
}
|
||||
}
|
||||
|
||||
/// One semantic event emitted by the interpreter (Trace v1).
|
||||
/// Default maximum number of trace events.
|
||||
pub const DEFAULT_MAX_TRACE_EVENTS: usize = 100_000;
|
||||
/// Default approximate maximum serialized trace bytes.
|
||||
pub const DEFAULT_MAX_TRACE_BYTES: usize = 32 * 1024 * 1024;
|
||||
|
||||
/// Trace capacity budget. A zero value means unlimited.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TraceBudget {
|
||||
/// Maximum events (including the terminal truncation marker).
|
||||
pub max_events: usize,
|
||||
/// Approximate maximum serialized bytes.
|
||||
pub max_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for TraceBudget {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_events: DEFAULT_MAX_TRACE_EVENTS,
|
||||
max_bytes: DEFAULT_MAX_TRACE_BYTES,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why and where a trace stopped recording.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TraceTruncation {
|
||||
/// `event_budget` or `byte_budget`.
|
||||
pub reason: String,
|
||||
/// The configured limit that was reached.
|
||||
pub limit: u64,
|
||||
/// Events written before truncation.
|
||||
pub written: u64,
|
||||
}
|
||||
|
||||
/// Mutable trace capacity state for one run.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct TraceState {
|
||||
/// Whether trace recording is enabled.
|
||||
pub enabled: bool,
|
||||
/// Configured budget.
|
||||
pub budget: TraceBudget,
|
||||
/// Events accepted so far.
|
||||
pub written: usize,
|
||||
/// Estimated accepted bytes.
|
||||
pub bytes: usize,
|
||||
/// Truncation marker, set once.
|
||||
pub truncation: Option<TraceTruncation>,
|
||||
}
|
||||
|
||||
impl TraceState {
|
||||
/// Accept one event if it fits the budget.
|
||||
pub fn accept(&mut self, estimate: usize) -> bool {
|
||||
if !self.enabled || self.truncation.is_some() {
|
||||
return false;
|
||||
}
|
||||
if self.budget.max_events > 0 && self.written >= self.budget.max_events {
|
||||
self.truncation = Some(TraceTruncation {
|
||||
reason: "event_budget".to_string(),
|
||||
limit: self.budget.max_events as u64,
|
||||
written: self.written as u64,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if self.budget.max_bytes > 0 && self.bytes + estimate > self.budget.max_bytes {
|
||||
self.truncation = Some(TraceTruncation {
|
||||
reason: "byte_budget".to_string(),
|
||||
limit: self.budget.max_bytes as u64,
|
||||
written: self.written as u64,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
self.written += 1;
|
||||
self.bytes += estimate;
|
||||
true
|
||||
}
|
||||
|
||||
/// Estimate one semantic/effect event envelope.
|
||||
pub fn estimate(
|
||||
kind: &str,
|
||||
function_id: Option<&str>,
|
||||
module_id: Option<&str>,
|
||||
detail: Option<&str>,
|
||||
operation: Option<&str>,
|
||||
) -> usize {
|
||||
96 + kind.len()
|
||||
+ function_id.map(str::len).unwrap_or(0)
|
||||
+ module_id.map(str::len).unwrap_or(0)
|
||||
+ detail.map(str::len).unwrap_or(0)
|
||||
+ operation.map(str::len).unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
/// One semantic event emitted by the interpreter (Trace v2).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SemanticEvent {
|
||||
/// Global event sequence within the run.
|
||||
@@ -79,6 +171,8 @@ pub struct SemanticEvent {
|
||||
pub kind: String,
|
||||
/// Canonical function id, when inside a call.
|
||||
pub function_id: Option<String>,
|
||||
/// Logical module owning `span`.
|
||||
pub module_id: Option<String>,
|
||||
/// Source span, when known.
|
||||
pub span: Option<Span>,
|
||||
/// Non-sensitive detail (`ok`, `true`, `int`, an error code).
|
||||
|
||||
@@ -73,14 +73,14 @@ pub const MAX_EXPORT_NAME_BYTES: usize = 256;
|
||||
pub const MAX_SIGNATURE_WIRE_BYTES: usize = 4096;
|
||||
/// Maximum plugin metadata wire length in bytes.
|
||||
pub const MAX_METADATA_WIRE_BYTES: usize = 65_536;
|
||||
/// Maximum request wire length in bytes (16 MiB).
|
||||
pub const MAX_REQUEST_WIRE_BYTES: usize = 16 * 1024 * 1024;
|
||||
/// Maximum result wire length in bytes (64 MiB).
|
||||
pub const MAX_RESULT_WIRE_BYTES: usize = 64 * 1024 * 1024;
|
||||
/// Maximum request wire length in bytes (8 MiB frame minus envelope reserve).
|
||||
pub const MAX_REQUEST_WIRE_BYTES: usize = 8 * 1024 * 1024 - 4096;
|
||||
/// Maximum result wire length in bytes (8 MiB frame minus envelope reserve).
|
||||
pub const MAX_RESULT_WIRE_BYTES: usize = 8 * 1024 * 1024 - 4096;
|
||||
/// Maximum error wire length in bytes (1 MiB).
|
||||
pub const MAX_ERROR_WIRE_BYTES: usize = 1024 * 1024;
|
||||
/// Maximum capability reply length in bytes (16 MiB).
|
||||
pub const MAX_CAPABILITY_REPLY_BYTES: usize = 16 * 1024 * 1024;
|
||||
/// Maximum capability reply length in bytes (8 MiB frame minus envelope reserve).
|
||||
pub const MAX_CAPABILITY_REPLY_BYTES: usize = 8 * 1024 * 1024 - 4096;
|
||||
|
||||
/// Borrowed, immutable byte view (`MqBytes`).
|
||||
///
|
||||
|
||||
@@ -19,17 +19,34 @@ use crate::wire::{WireRequest, WireResponse};
|
||||
/// Hidden CLI command name used to start a worker.
|
||||
pub const WORKER_COMMAND: &str = "__plugin-worker";
|
||||
/// Maximum frame size accepted by either side.
|
||||
pub const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024;
|
||||
pub const MAX_FRAME_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Read one length-prefixed frame (4-byte big-endian length + JSON).
|
||||
pub fn read_frame<R: Read>(reader: &mut R) -> Result<Value, PluginError> {
|
||||
///
|
||||
/// `Ok(None)` means a clean EOF at a frame boundary. A partial header or
|
||||
/// truncated payload is a protocol error, never EOF.
|
||||
pub fn read_frame<R: Read>(reader: &mut R) -> Result<Option<Value>, PluginError> {
|
||||
let mut length = [0_u8; 4];
|
||||
reader.read_exact(&mut length).map_err(|error| {
|
||||
PluginError::invoke(
|
||||
let mut read = 0usize;
|
||||
while read < length.len() {
|
||||
match reader.read(&mut length[read..]) {
|
||||
Ok(0) if read == 0 => return Ok(None),
|
||||
Ok(0) => {
|
||||
return Err(PluginError::wire(
|
||||
"worker_frame_invalid",
|
||||
"worker stream ended inside a frame header",
|
||||
))
|
||||
}
|
||||
Ok(count) => read += count,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(error) => {
|
||||
return Err(PluginError::invoke(
|
||||
"plugin.crashed",
|
||||
format!("worker frame read failed: {error}"),
|
||||
)
|
||||
})?;
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
let length = u32::from_be_bytes(length) as usize;
|
||||
if length == 0 || length > MAX_FRAME_BYTES {
|
||||
return Err(PluginError::wire(
|
||||
@@ -37,14 +54,23 @@ pub fn read_frame<R: Read>(reader: &mut R) -> Result<Value, PluginError> {
|
||||
format!("worker frame length {length} is invalid"),
|
||||
));
|
||||
}
|
||||
let mut bytes = vec![0_u8; length];
|
||||
reader.read_exact(&mut bytes).map_err(|error| {
|
||||
let mut bytes = Vec::with_capacity(length.min(64 * 1024));
|
||||
let read = reader
|
||||
.take(length as u64)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| {
|
||||
PluginError::invoke(
|
||||
"plugin.crashed",
|
||||
format!("worker frame read failed: {error}"),
|
||||
)
|
||||
})?;
|
||||
serde_json::from_slice(&bytes).map_err(|error| {
|
||||
if read != length {
|
||||
return Err(PluginError::wire(
|
||||
"worker_frame_invalid",
|
||||
format!("worker frame declared {length} bytes but only {read} arrived"),
|
||||
));
|
||||
}
|
||||
serde_json::from_slice(&bytes).map(Some).map_err(|error| {
|
||||
PluginError::wire(
|
||||
"worker_frame_invalid",
|
||||
format!("worker frame is not valid JSON: {error}"),
|
||||
@@ -110,8 +136,12 @@ pub fn serve(manifest_path: &Path, base_dir: &Path) -> i32 {
|
||||
}
|
||||
loop {
|
||||
let frame = match read_frame(&mut stdin) {
|
||||
Ok(frame) => frame,
|
||||
Err(_) => return 0,
|
||||
Ok(Some(frame)) => frame,
|
||||
Ok(None) => return 0,
|
||||
Err(error) => {
|
||||
eprintln!("worker: {}: {}", error.code, error.message);
|
||||
return 1;
|
||||
}
|
||||
};
|
||||
match frame.get("kind").and_then(Value::as_str) {
|
||||
Some("shutdown") => return 0,
|
||||
@@ -189,3 +219,45 @@ pub fn response_from_frame(frame: &Value) -> Result<WireResponse, PluginError> {
|
||||
.map_err(|error| PluginError::wire("worker_response_invalid", error.to_string()))?;
|
||||
WireResponse::from_json_bytes(&bytes)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
|
||||
#[test]
|
||||
fn frame_roundtrip_and_clean_eof() {
|
||||
let mut bytes = Vec::new();
|
||||
write_frame(&mut bytes, &json!({"kind": "shutdown"})).unwrap();
|
||||
let mut cursor = Cursor::new(bytes.clone());
|
||||
assert_eq!(
|
||||
read_frame(&mut cursor).unwrap(),
|
||||
Some(json!({"kind": "shutdown"}))
|
||||
);
|
||||
assert_eq!(read_frame(&mut cursor).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_header_is_an_error_not_eof() {
|
||||
let mut cursor = Cursor::new(vec![0_u8, 0]);
|
||||
let error = read_frame(&mut cursor).unwrap_err();
|
||||
assert_eq!(error.code, "worker_frame_invalid");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_and_zero_length_frames_are_rejected() {
|
||||
for length in [0_u32, (MAX_FRAME_BYTES as u32) + 1] {
|
||||
let mut cursor = Cursor::new(length.to_be_bytes().to_vec());
|
||||
let error = read_frame(&mut cursor).unwrap_err();
|
||||
assert_eq!(error.code, "worker_frame_invalid");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_payload_is_rejected() {
|
||||
let mut bytes = 10_u32.to_be_bytes().to_vec();
|
||||
bytes.extend_from_slice(b"{}");
|
||||
let error = read_frame(&mut Cursor::new(bytes)).unwrap_err();
|
||||
assert_eq!(error.code, "worker_frame_invalid");
|
||||
}
|
||||
}
|
||||
@@ -135,11 +135,17 @@ impl<'a> Interpreter<'a> {
|
||||
) -> Result<Value, RuntimeError> {
|
||||
let display = function.id.name.clone();
|
||||
let trace_id = function.id.to_string();
|
||||
let trace_module = caller_module.unwrap_or(&function.id.module).canonical();
|
||||
if self.depth + 1 > 0 {
|
||||
self.host.check_stack(self.depth + 1, call_span)?;
|
||||
}
|
||||
self.host
|
||||
.record_semantic("call.enter", Some(&trace_id), call_span, None);
|
||||
self.host.record_semantic(
|
||||
"call.enter",
|
||||
Some(&trace_id),
|
||||
Some(&trace_module),
|
||||
call_span,
|
||||
None,
|
||||
);
|
||||
let locals = match self.bind_args(function, args, call_span) {
|
||||
Ok(locals) => locals,
|
||||
Err(mut error) => {
|
||||
@@ -162,8 +168,13 @@ impl<'a> Interpreter<'a> {
|
||||
Ok(Flow::Return(_)) => "return",
|
||||
Err(error) => error.code(),
|
||||
};
|
||||
self.host
|
||||
.record_semantic("call.exit", Some(&trace_id), call_span, Some(status));
|
||||
self.host.record_semantic(
|
||||
"call.exit",
|
||||
Some(&trace_id),
|
||||
Some(&trace_module),
|
||||
call_span,
|
||||
Some(status),
|
||||
);
|
||||
match result {
|
||||
Ok(Flow::Next) => Ok(Value::Unit),
|
||||
Ok(Flow::Return(value)) => Ok(value),
|
||||
@@ -301,9 +312,11 @@ impl<'a> Interpreter<'a> {
|
||||
}
|
||||
Stmt::Return { value, span } => {
|
||||
let value = self.eval_expr(value, frame)?;
|
||||
let module = frame.function.module.canonical();
|
||||
self.host.record_semantic(
|
||||
"return",
|
||||
Some(&frame.function.to_string()),
|
||||
Some(&module),
|
||||
Some(*span),
|
||||
Some(value.type_name()),
|
||||
);
|
||||
@@ -322,9 +335,11 @@ impl<'a> Interpreter<'a> {
|
||||
),
|
||||
));
|
||||
};
|
||||
let module = frame.function.module.canonical();
|
||||
self.host.record_semantic(
|
||||
"branch.eval",
|
||||
Some(&frame.function.to_string()),
|
||||
Some(&module),
|
||||
Some(arm.condition_span),
|
||||
Some(if condition { "true" } else { "false" }),
|
||||
);
|
||||
@@ -351,9 +366,11 @@ impl<'a> Interpreter<'a> {
|
||||
format!("loop condition must be Bool, got {}", value.type_name()),
|
||||
));
|
||||
};
|
||||
let module = frame.function.module.canonical();
|
||||
self.host.record_semantic(
|
||||
"loop.iteration",
|
||||
Some(&frame.function.to_string()),
|
||||
Some(&module),
|
||||
Some(*condition_span),
|
||||
Some(if value { "continue" } else { "exit" }),
|
||||
);
|
||||
|
||||
+27
-1
@@ -125,6 +125,31 @@ Analysis v2 (`spec/analysis-v2.md`, `spec/schemas/analysis-v2.schema.json`):
|
||||
* Tests cover byte-identical output after moving a project and resource
|
||||
addressing for formulas/code fences.
|
||||
|
||||
### S2-C/H2-B record (current)
|
||||
|
||||
Trace v2 (`spec/trace-v2.md`, `spec/schemas/trace-v2.schema.json`):
|
||||
|
||||
* `source.root_id` uses the same logical Analysis identity; `analysis_digest`
|
||||
is the digest of the exact emitted Analysis bytes and is independently
|
||||
verifiable.
|
||||
* Every span-bearing event carries `module_id`; `call.enter`/`call.exit` are
|
||||
owned by the caller module, other semantic events by the current function,
|
||||
effects by their `ExecutionContext`. The effect key is now `module_id`.
|
||||
* Trace budgets: 100 000 events / 32 MiB by default, configurable with
|
||||
`--trace-max-events` and `--trace-max-bytes`; `0` means unlimited. Semantic
|
||||
recording is disabled when no trace is requested.
|
||||
* On capacity exhaustion the trace ends with `trace.truncated` and
|
||||
`truncated.reason`; the run status and execution result are unchanged. The
|
||||
writer also enforces the byte cap on the final document.
|
||||
* Worker frames are capped at 8 MiB, with aligned request/result/capability
|
||||
wire caps; partial headers, oversized lengths, truncated payloads and invalid
|
||||
JSON are protocol errors, clean EOF is distinct, and the host kills and marks
|
||||
a worker dead after a transport failure.
|
||||
* New independent `tests/consumers/trace_consumer.py` verifies identity,
|
||||
digest, module references and sequence ordering in CI.
|
||||
* Tests: event module ownership, 20-event truncation of a 500-iteration loop,
|
||||
worker frame round-trip/EOF/partial/oversized/truncated cases.
|
||||
|
||||
### M0.6 implementation progress
|
||||
|
||||
| Step | State | Evidence |
|
||||
@@ -133,11 +158,12 @@ Analysis v2 (`spec/analysis-v2.md`, `spec/schemas/analysis-v2.schema.json`):
|
||||
| S1/H1-A lock v3 graph | **Done** | ADR-0016; full-closure `marqdo.lock` v3; fail-closed edge lookup; strict non-rewriting `sync`; verified Git cache; unified file-set profile (see §S1/H1-A) |
|
||||
| S2-A/H1-B source locations | **Done** | ADR-0017; `RuntimeError`/`CallFrame` carry `module_id`; callsite origin; CLI module-aware resolution with no entry fallback (see §S2-A/H1-B) |
|
||||
| S2-B/H2-A Analysis v2 | **Done** | logical project identity, resources array, machine-path-free diagnostics, emitted-bytes digest, v2 schema + independent consumer (see §S2-B/H2-A) |
|
||||
| S2-C/H2-B Trace v2 | **Done** | logical root/digest, module-owned events, bounded/truncated trace, independent consumer, hardened worker frames (see §S2-C/H2-B) |
|
||||
| R0 baseline + ADRs | **Done** | this section; `docs/adr/0008`–`0012`; full workspace gate |
|
||||
| R1 package-aware identity + callable registry | **Done for statics; superseded by S1 lock v3** | `ModuleId { instance, relative_path }`; `PackageResolver` path sources; lock v3 transitive graph (ADR-0016) replaces the flat v2 lock; `CallableRegistry` for builtins/marqdo/declared ABI exports; external manifest wiring is R2 |
|
||||
| R2 ABI/Kernel/Broker | **Done (worker = crash isolation)** | `KernelRegistry` external providers; complete-name export routing; `CapabilityBroker`; static `check --plugin`; multi-export C test; isolated worker with timeout/crash tests; host handle table. Native/worker are not an OS sandbox |
|
||||
| R3 Tool Protocol | **Done** | `tool install/list/remove`; unknown-command fallback; atomic local install; SHA/tamper checks; black-box tests for arbitrary names, Unicode args, conflicts, traversal and PATH hijacking |
|
||||
| R4 Analysis v2 / Trace v1 | **Partial** | `check --emit` Analysis v2 (logical identity, resources, path-free diagnostics, emitted-bytes digest); `run/call --trace-file` still Trace v1 pending H2-B; independent Python consumer; stale-artifact removal; value redaction |
|
||||
| R4 Analysis/Trace v2 | **Done** | `check --emit` Analysis v2 and `run/call --trace-file` Trace v2 with logical identity, resources, module-owned events, capacity bounds, schemas and independent Python consumers; stale-artifact removal; value redaction |
|
||||
| R5 environment/isolation | **Partial** | Pinned Git fetch with verified cache markers, transitive `@alias` resolution, full-closure atomic `.marqdo/env` materialization, foreign process-group cleanup. Strict OS isolation remains explicitly unavailable (fail-closed) |
|
||||
| R6 release/consumers | **Done for Linux** | examples for unknown-name Tool, multi-export plugin, external kernel and multi-version packages; Analysis/Trace/Tool JSON Schemas; CI Linux gate + Windows/macOS `cargo check`; R6 example tests |
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ marqdo-cli/
|
||||
│ ├── foreign.md Python/Node kernel protocol
|
||||
│ ├── tool-v1.md generic external Tool Protocol v1
|
||||
│ ├── analysis-v2.md public Analysis v2 projection (v1 historical)
|
||||
│ ├── trace-v2.md public Trace v2 projection (v1 historical)
|
||||
│ └── trace-v1.md public Trace v1 event stream
|
||||
├── crates/
|
||||
│ ├── marqdo-span/ Span, LineCol, LineIndex (leaf)
|
||||
|
||||
@@ -99,10 +99,10 @@
|
||||
* MQ_MAX_EXPORT_NAME_BYTES 256
|
||||
* MQ_MAX_SIGNATURE_WIRE_BYTES 4096
|
||||
* MQ_MAX_METADATA_WIRE_BYTES 65536
|
||||
* MQ_MAX_REQUEST_WIRE_BYTES 16777216 (16 MiB)
|
||||
* MQ_MAX_RESULT_WIRE_BYTES 67108864 (64 MiB)
|
||||
* MQ_MAX_REQUEST_WIRE_BYTES 8384512 (8 MiB - 4096)
|
||||
* MQ_MAX_RESULT_WIRE_BYTES 8384512 (8 MiB - 4096)
|
||||
* MQ_MAX_ERROR_WIRE_BYTES 1048576 (1 MiB)
|
||||
* MQ_MAX_CAPABILITY_REPLY_BYTES 16777216 (16 MiB)
|
||||
* MQ_MAX_CAPABILITY_REPLY_BYTES 8384512 (8 MiB - 4096)
|
||||
* JSON depth is additionally limited to 64 levels. Requests/results that
|
||||
* exceed these limits are rejected with a structured error; they must not
|
||||
* be truncated and reparsed.
|
||||
@@ -245,10 +245,10 @@ extern "C" {
|
||||
#define MQ_MAX_EXPORT_NAME_BYTES 256u
|
||||
#define MQ_MAX_SIGNATURE_WIRE_BYTES 4096u
|
||||
#define MQ_MAX_METADATA_WIRE_BYTES 65536u
|
||||
#define MQ_MAX_REQUEST_WIRE_BYTES (16u * 1024u * 1024u)
|
||||
#define MQ_MAX_RESULT_WIRE_BYTES (64u * 1024u * 1024u)
|
||||
#define MQ_MAX_REQUEST_WIRE_BYTES ((8u * 1024u * 1024u) - 4096u)
|
||||
#define MQ_MAX_RESULT_WIRE_BYTES ((8u * 1024u * 1024u) - 4096u)
|
||||
#define MQ_MAX_ERROR_WIRE_BYTES (1024u * 1024u)
|
||||
#define MQ_MAX_CAPABILITY_REPLY_BYTES (16u * 1024u * 1024u)
|
||||
#define MQ_MAX_CAPABILITY_REPLY_BYTES ((8u * 1024u * 1024u) - 4096u)
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Stable-layout buffer types */
|
||||
|
||||
+9
-6
@@ -68,10 +68,10 @@ minted for. Plugins must not dereference, cache or copy it.
|
||||
| `MQ_MAX_EXPORT_NAME_BYTES` | 256 |
|
||||
| `MQ_MAX_SIGNATURE_WIRE_BYTES` | 4096 |
|
||||
| `MQ_MAX_METADATA_WIRE_BYTES` | 65536 |
|
||||
| `MQ_MAX_REQUEST_WIRE_BYTES` | 16 MiB (16,777,216) |
|
||||
| `MQ_MAX_RESULT_WIRE_BYTES` | 64 MiB (67,108,864) |
|
||||
| `MQ_MAX_REQUEST_WIRE_BYTES` | 8 MiB - 4096 (8,384,512) |
|
||||
| `MQ_MAX_RESULT_WIRE_BYTES` | 8 MiB - 4096 (8,384,512) |
|
||||
| `MQ_MAX_ERROR_WIRE_BYTES` | 1 MiB (1,048,576) |
|
||||
| `MQ_MAX_CAPABILITY_REPLY_BYTES` | 16 MiB |
|
||||
| `MQ_MAX_CAPABILITY_REPLY_BYTES` | 8 MiB - 4096 |
|
||||
|
||||
Wire JSON decoding additionally applies the default `WireLimits` from
|
||||
`spec/value-wire.md` (depth 64, 1,000,000 items, 16 MiB strings). Oversized
|
||||
@@ -270,8 +270,8 @@ with both is rejected (`ambiguous_response`). Error:
|
||||
}}
|
||||
```
|
||||
|
||||
Envelope limits: at most `MAX_ARGS = 1024` request arguments; request ≤ 16 MiB,
|
||||
response ≤ 64 MiB, error ≤ 1 MiB. `context` carries no secrets and no
|
||||
Envelope limits: at most `MAX_ARGS = 1024` request arguments; request and
|
||||
response ≤ 8 MiB - 4096, error ≤ 1 MiB. `context` carries no secrets and no
|
||||
reconstructable OS handles. `signature_wire` is
|
||||
`{"signature":"<signature string>"}`; `plugin_metadata_wire` is
|
||||
`{"name":"<package name>","version":"<semver>"}` (extra fields ignored).
|
||||
@@ -364,7 +364,10 @@ full plugin capability broker for worker requests remain future work.
|
||||
|
||||
### 9.1 Worker frames
|
||||
|
||||
Frames are a 4-byte big-endian length followed by one JSON object (max 64 MiB).
|
||||
Frames are a 4-byte big-endian length followed by one JSON object (max 8 MiB).
|
||||
A zero/oversized length, partial header, truncated payload or invalid JSON is a
|
||||
protocol error; clean EOF at a frame boundary is not. The host marks the worker
|
||||
dead after a transport failure and does not reuse it.
|
||||
Worker → host: `{"kind":"ready","exports":[...]}`. Host → worker:
|
||||
`{"kind":"invoke","request":<wire request>}` and `{"kind":"shutdown"}`.
|
||||
Worker → host responses are `{"kind":"response","response":<wire response>}` or
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://marqdo.dev/schemas/trace-v2.schema.json",
|
||||
"title": "Marqdo Trace v2",
|
||||
"type": "object",
|
||||
"required": ["schema", "schema_version", "run", "source", "limits", "truncated", "events"],
|
||||
"properties": {
|
||||
"schema": {"const": "marqdo.trace"},
|
||||
"schema_version": {"const": 2},
|
||||
"run": {
|
||||
"type": "object",
|
||||
"required": ["id", "entry", "status"],
|
||||
"properties": {
|
||||
"id": {"type": "string", "minLength": 1},
|
||||
"entry": {"type": "string", "minLength": 1},
|
||||
"status": {"type": "string", "minLength": 1}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"source": {
|
||||
"type": "object",
|
||||
"required": ["root_id", "analysis_digest"],
|
||||
"properties": {
|
||||
"root_id": {"type": "string", "pattern": "^(root:|sha256:[0-9a-f]{64}$)"},
|
||||
"analysis_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"limits": {
|
||||
"type": "object",
|
||||
"required": ["max_events", "max_bytes"],
|
||||
"properties": {
|
||||
"max_events": {"type": "integer", "minimum": 0},
|
||||
"max_bytes": {"type": "integer", "minimum": 0}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"truncated": {
|
||||
"oneOf": [
|
||||
{"type": "null"},
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["reason", "limit", "written"],
|
||||
"properties": {
|
||||
"reason": {"enum": ["event_budget", "byte_budget"]},
|
||||
"limit": {"type": "integer", "minimum": 0},
|
||||
"written": {"type": "integer", "minimum": 0}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"events": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["seq", "kind", "module_id", "span", "detail"],
|
||||
"properties": {
|
||||
"seq": {"type": "integer", "minimum": 0},
|
||||
"kind": {"type": "string", "minLength": 1},
|
||||
"function_id": {"type": ["string", "null"]},
|
||||
"module_id": {"type": ["string", "null"]},
|
||||
"span": {
|
||||
"oneOf": [
|
||||
{"type": "null"},
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["start", "end"],
|
||||
"properties": {
|
||||
"start": {"type": "integer", "minimum": 0},
|
||||
"end": {"type": "integer", "minimum": 0}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"detail": {"type": ["string", "null"]},
|
||||
"operation": {"type": "string"},
|
||||
"duration_micros": {"type": "integer", "minimum": 0}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
# Marqdo Trace v1
|
||||
|
||||
Status: **implemented by `marqdo run/call --trace-file <path>` (M0.6 R4).**
|
||||
Status: **superseded by [Trace v2](trace-v2.md); never released, do not write
|
||||
v1.** Kept as history.
|
||||
|
||||
## 1. Invariants
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# Marqdo Trace v2
|
||||
|
||||
Status: **implemented by `marqdo run/call --trace-file <path>` (S2).**
|
||||
Supersedes Trace v1; v1 was never released and is kept as history.
|
||||
|
||||
## 1. Invariants
|
||||
|
||||
* Tracing is observation, not execution. Only `run`/`call` produce a trace;
|
||||
`check`, `inspect` and `tool list` never execute a document.
|
||||
* Events come from the interpreter (`call.*`, `branch.eval`,
|
||||
`loop.iteration`, `return`) and the host (`effect`). Both share one sequence.
|
||||
* The writer is atomic; an error run still records its terminal status.
|
||||
* No argument values, secrets, handles or file contents are recorded.
|
||||
* Trace capacity is a bounded observation budget. Reaching it truncates the
|
||||
trace; it never changes `run.status` or the program’s execution result.
|
||||
Execution budgets (`--fuel`, `--timeout`, ...) remain separate and still
|
||||
produce `resource.exhausted`.
|
||||
|
||||
## 2. Shape
|
||||
|
||||
```json
|
||||
{
|
||||
"schema": "marqdo.trace",
|
||||
"schema_version": 2,
|
||||
"run": {"id": "pid-nanos", "entry": "root:q@0.1.0::index.mq.md::main", "status": "ok"},
|
||||
"source": {"root_id": "root:q@0.1.0", "analysis_digest": "sha256:<64 hex>"},
|
||||
"limits": {"max_events": 100000, "max_bytes": 33554432},
|
||||
"truncated": null,
|
||||
"events": [
|
||||
{"seq": 0, "kind": "call.enter", "function_id": "root:q@0.1.0::index.mq.md::main",
|
||||
"module_id": "root:q@0.1.0::index.mq.md", "span": null, "detail": null},
|
||||
{"seq": 1, "kind": "effect", "module_id": "dep:p@0.1.0#<digest>::index.mq.md",
|
||||
"span": {"start": 40, "end": 52}, "detail": "ok", "operation": "print",
|
||||
"duration_micros": 3}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`seq` is globally increasing. `run.status` is `ok` or a stable runtime error
|
||||
code. Every span-bearing event has a `module_id`; the span is a UTF-8 byte
|
||||
offset range in that module. `source` only carries project identity and the
|
||||
Analysis digest; it does not identify the span owner.
|
||||
|
||||
`source.root_id` is the Analysis v2 `project.root_id`
|
||||
(`root:<name>@<version>` or `sha256:<workspace snapshot>`).
|
||||
`source.analysis_digest` is `sha256:` plus the SHA-256 of the exact Analysis
|
||||
v2 bytes produced for the same project with the same projection configuration.
|
||||
A consumer verifies it by hashing the Analysis file.
|
||||
|
||||
`truncated` is `null` or:
|
||||
|
||||
```json
|
||||
{"reason": "event_budget", "limit": 20, "written": 20}
|
||||
```
|
||||
|
||||
`reason` is `event_budget` or `byte_budget`. When truncation occurs, the final
|
||||
event is `{"kind":"trace.truncated","detail":<reason>,"module_id":null,
|
||||
"span":null,"seq":<one past the last written event>}`.
|
||||
|
||||
## 3. Event kinds and span owners
|
||||
|
||||
| kind | detail | `module_id` owner |
|
||||
|---|---|---|
|
||||
| `call.enter` | null | caller module (entry call: callee) |
|
||||
| `call.exit` | `ok` / `return` / error code | caller module (entry call: callee) |
|
||||
| `return` | value type name | current function’s module |
|
||||
| `branch.eval` | `true` / `false` | current function’s module |
|
||||
| `loop.iteration` | `continue` / `exit` | current function’s module |
|
||||
| `effect` | result summary | module from `ExecutionContext` |
|
||||
| `trace.truncated` | `event_budget` / `byte_budget` | null |
|
||||
|
||||
`function_id` is always the canonical callee function id. Future kinds are
|
||||
additive; consumers must ignore unknown kinds. `effect.operation` names the
|
||||
`HostOp` (`print`, `fs.read`, ...).
|
||||
|
||||
## 4. Capacity
|
||||
|
||||
Defaults: 100 000 events and 32 MiB approximate serialized size. `0` means
|
||||
unlimited for either limit. `run`/`call` expose `--trace-max-events` and
|
||||
`--trace-max-bytes`; these are options on existing commands, not new commands.
|
||||
The host stops accepting events once a limit is reached; the writer also
|
||||
enforces the byte limit on the final serialized document, dropping trailing
|
||||
events and recording `byte_budget`. Semantic recording is disabled entirely
|
||||
when no trace was requested.
|
||||
|
||||
The `trace` array in `run --json` uses the same budget.
|
||||
|
||||
## 5. Staleness
|
||||
|
||||
A trace is stale when its `source.analysis_digest` differs from the digest of
|
||||
the Analysis emitted for the current document. Consumers must not present a
|
||||
stale trace as current. Moving a project does not change the manifest identity
|
||||
or digest; changing executable source does.
|
||||
|
||||
## 6. Worker transport limits
|
||||
|
||||
Plugin worker frames are a 4-byte big-endian length plus JSON, capped at
|
||||
8 MiB (`8 * 1024 * 1024`). The wire request/result/capability caps are aligned
|
||||
to `8 MiB - 4096`; error frames remain 1 MiB. A partial header, zero/oversized
|
||||
length, truncated payload or invalid JSON is a protocol error. A clean EOF is
|
||||
distinct from a protocol failure. The host marks a worker dead after a transport
|
||||
failure and never reuses its stdin.
|
||||
|
||||
## 7. Compatibility and tests
|
||||
|
||||
* Consumers must reject unknown `schema_version` and validate `module_id`
|
||||
against Analysis modules.
|
||||
* JSON Schema: `spec/schemas/trace-v2.schema.json`.
|
||||
* `tests/consumers/trace_consumer.py` verifies identity, digest and event
|
||||
module references.
|
||||
* CLI tests cover call/effect events, no-secret redaction, error terminal
|
||||
status, and event-budget truncation without changing the run status.
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent Trace v2 consumer: no Marqdo/Rust imports."""
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
raise SystemExit("usage: trace_consumer.py ANALYSIS.json TRACE.json")
|
||||
|
||||
with open(sys.argv[1], "rb") as handle:
|
||||
analysis_bytes = handle.read()
|
||||
analysis = json.loads(analysis_bytes)
|
||||
with open(sys.argv[2], encoding="utf-8") as handle:
|
||||
trace = json.load(handle)
|
||||
|
||||
if analysis.get("schema") != "marqdo.analysis" or analysis.get("schema_version") != 2:
|
||||
raise SystemExit("unsupported analysis")
|
||||
if trace.get("schema") != "marqdo.trace" or trace.get("schema_version") != 2:
|
||||
raise SystemExit("unsupported trace")
|
||||
|
||||
modules = {module["id"]: module for module in analysis["modules"]}
|
||||
if trace["source"]["root_id"] != analysis["project"]["root_id"]:
|
||||
raise SystemExit("trace/analysis root mismatch")
|
||||
expected = "sha256:" + hashlib.sha256(analysis_bytes).hexdigest()
|
||||
if trace["source"]["analysis_digest"] != expected:
|
||||
raise SystemExit("analysis digest mismatch")
|
||||
|
||||
truncated = trace.get("truncated")
|
||||
if truncated is not None and truncated.get("reason") not in ("event_budget", "byte_budget"):
|
||||
raise SystemExit("bad truncation reason")
|
||||
|
||||
previous = None
|
||||
events = trace["events"]
|
||||
for event in events:
|
||||
seq = event["seq"]
|
||||
if previous is not None and seq <= previous:
|
||||
raise SystemExit("sequence is not increasing")
|
||||
previous = seq
|
||||
span = event.get("span")
|
||||
if span is not None and (span["start"] > span["end"]):
|
||||
raise SystemExit("bad span")
|
||||
module_id = event.get("module_id")
|
||||
if module_id is not None and module_id not in modules:
|
||||
raise SystemExit("event references an unknown module")
|
||||
function_id = event.get("function_id")
|
||||
if function_id is not None:
|
||||
owner = function_id.rsplit("::", 1)[0]
|
||||
if owner not in modules:
|
||||
raise SystemExit("function id references an unknown module")
|
||||
|
||||
if truncated is not None and events and events[-1].get("kind") != "trace.truncated":
|
||||
raise SystemExit("truncated trace has no terminal marker")
|
||||
print("events=" + str(len(events)))
|
||||
Reference in new issue
Block a user