From 01f6fe46a728d4212a2dd1afe97ad832f3b2421b Mon Sep 17 00:00:00 2001 From: cflmy Date: Fri, 20 Nov 2026 23:25:09 +0800 Subject: [PATCH] feat(host): add worker resource-provider transport v1 - Add a host-managed helper process speaking framed JSON (4-byte BE length) with ping/open/read/write/close/shutdown, max 1 MiB frames and structured error mapping (ADR-0035). - Add StdHost::register_worker_resource_provider using the same protocol, kind-conflict and name-grant checks as the in-process provider; pure/strict refuse provider kinds. - Tests spawn a Python helper and cover round-trip, grant/mode denial and missing-program failure; no parser/Value/runtime/HostOp change. - Update ADR/spec/status/README/changelog and evidence; C ABI binding remains open. Gate: fmt, clippy -D warnings, 433 tests passed. --- CHANGELOG.md | 3 + README.md | 3 + crates/marqdo-host/src/host.rs | 33 ++- crates/marqdo-host/src/lib.rs | 1 + crates/marqdo-host/src/provider_worker.rs | 210 ++++++++++++++++++ crates/marqdo-host/src/resource.rs | 9 + crates/marqdo-host/tests/worker_provider.rs | 162 ++++++++++++++ docs/STATUS.md | 10 +- ...0035-worker-resource-provider-transport.md | 92 ++++++++ docs/evidence/2026-11-20-worker-provider.md | 62 ++++++ spec/security.md | 2 +- tests/README.md | 6 + 12 files changed, 585 insertions(+), 8 deletions(-) create mode 100644 crates/marqdo-host/src/provider_worker.rs create mode 100644 crates/marqdo-host/tests/worker_provider.rs create mode 100644 docs/adr/0035-worker-resource-provider-transport.md create mode 100644 docs/evidence/2026-11-20-worker-provider.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 48bc2e6..6ea8553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,6 +136,9 @@ currently `0.1.0`; no release tag has been cut. Dates use ISO 8601. JSON report; no bytecode/JIT/native backend is introduced. - P4 cross-storage table equivalence: fixture 72 compares row-store Table sort/filter output with a `.mq.md` columnar store cell-by-cell. +- CM-A09 worker provider transport (ADR-0035): host-managed helper process + speaking framed JSON for resource open/read/write/close; trusted-only, + project-mode + name grant; C ABI binding still open. - P5 HTTP framing/client: `stdlib/net/http.mq.md` parses status/headers/chunked responses and builds requests; `http_over_tcp.rs` runs a loopback TCP HTTP exchange through `resource.open("tcp")`. No host HTTP/TLS implementation. diff --git a/README.md b/README.md index ad82e19..42d6a59 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,9 @@ executes anything. * **Blocking TCP endpoint (RFC-CM P5 first slice)** — `resource.open("tcp", ":", ["read","write"])` with exact IP grants, project mode and 5-second timeouts; DNS/TLS/UDP/server sockets excluded (ADR-0034). +* **Worker ResourceProvider transport v1 (RFC-CM CM-A09)** — trusted helper + process with framed JSON resource operations; project + name grant; C ABI + binding remains open (ADR-0035). * **In-process ResourceProvider v1 (RFC-CM CM-A09 first slice)** — trusted providers register extra resource kinds, project-mode + name-grant gated, using the existing `resource.*`/`Result` ABI; no parser/runtime/`HostOp` diff --git a/crates/marqdo-host/src/host.rs b/crates/marqdo-host/src/host.rs index 0a2a878..e11caa3 100644 --- a/crates/marqdo-host/src/host.rs +++ b/crates/marqdo-host/src/host.rs @@ -21,6 +21,7 @@ use crate::context::ExecutionContext; use crate::fs::ProjectFs; use crate::handles::{contains_plugin_handle, PluginHandleTable}; use crate::plugin_worker::WorkerPlugin; +use crate::provider_worker::WorkerResourceProvider; use crate::resource::{ResourceProvider, ResourceRead, ResourceTable, SharedResourceProvider}; use crate::trace::{EffectTrace, SemanticEvent, TraceBudget, TraceEntry, TraceState}; use marqdo_sandbox::{PlatformCapabilities, SandboxMode}; @@ -577,6 +578,32 @@ impl StdHost { pub fn register_resource_provider( &mut self, provider: Box, + ) -> Result<(), RuntimeError> { + self.register_provider_checked(provider) + } + + /// Register a trusted worker-process resource provider (ADR-0035). + /// + /// The helper speaks framed JSON over stdin/stdout and is **not** an OS + /// sandbox; only explicitly trusted helpers may be spawned. + pub fn register_worker_resource_provider( + &mut self, + name: &str, + kinds: &[&str], + program: &std::path::Path, + args: &[String], + ) -> Result<(), RuntimeError> { + let kinds = kinds + .iter() + .map(|kind| (*kind).to_string()) + .collect::>(); + let provider = WorkerResourceProvider::spawn(name, &kinds, program, args)?; + self.register_provider_checked(Box::new(provider)) + } + + fn register_provider_checked( + &mut self, + provider: Box, ) -> Result<(), RuntimeError> { if provider.protocol_version() != 1 { return Err(RuntimeError::new( @@ -597,11 +624,7 @@ impl StdHost { format!("resource provider name `{name}` is reserved or empty"), )); } - let kinds = provider - .kinds() - .iter() - .map(|kind| kind.to_string()) - .collect::>(); + let kinds = provider.kind_strings(); if kinds.is_empty() { return Err(RuntimeError::new( ErrorCategory::ResourceUnsupported, diff --git a/crates/marqdo-host/src/lib.rs b/crates/marqdo-host/src/lib.rs index b32b0a9..e2cf518 100644 --- a/crates/marqdo-host/src/lib.rs +++ b/crates/marqdo-host/src/lib.rs @@ -12,6 +12,7 @@ pub mod handles; pub mod host; pub mod kernels; mod plugin_worker; +mod provider_worker; pub mod resource; pub mod trace; diff --git a/crates/marqdo-host/src/provider_worker.rs b/crates/marqdo-host/src/provider_worker.rs new file mode 100644 index 0000000..9ab4980 --- /dev/null +++ b/crates/marqdo-host/src/provider_worker.rs @@ -0,0 +1,210 @@ +//! Host-managed worker resource provider (ADR-0035). +//! +//! This is a small, versioned framed-JSON transport for **trusted helper +//! processes**. It is not a C ABI binding and not an OS sandbox: the helper +//! runs as the host user. Registration is a host embedding API; language code +//! only sees the normal `resource.*` operations. + +use std::io::{BufReader, Read, Write}; +use std::path::Path; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; + +use marqdo_span::Span; +use serde_json::{json, Value as Json}; + +use crate::resource::{ResourceProvider, ResourceRead}; +use marqdo_value::{ErrorCategory, RuntimeError}; + +const MAX_FRAME_BYTES: usize = 1024 * 1024; + +/// A `ResourceProvider` backed by a child process speaking framed JSON. +pub(crate) struct WorkerResourceProvider { + name: String, + kinds: Vec, + child: Child, + stdin: ChildStdin, + stdout: BufReader, +} + +impl WorkerResourceProvider { + pub(crate) fn spawn( + name: &str, + kinds: &[String], + program: &Path, + args: &[String], + ) -> Result { + let mut child = Command::new(program) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| { + RuntimeError::new( + ErrorCategory::KernelUnavailable, + None, + format!( + "cannot spawn worker resource provider `{name}` ({}): {error}", + program.display() + ), + ) + })?; + let stdin = child + .stdin + .take() + .ok_or_else(|| worker_error("missing stdin pipe"))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| worker_error("missing stdout pipe"))?; + let mut provider = Self { + name: name.to_string(), + kinds: kinds.to_vec(), + child, + stdin, + stdout: BufReader::new(stdout), + }; + // Handshake: the helper must answer a ping before it is trusted. + provider.request(json!({"op": "ping"}))?; + Ok(provider) + } + + fn request(&mut self, request: Json) -> Result { + let bytes = serde_json::to_vec(&request) + .map_err(|error| worker_error(&format!("cannot encode request: {error}")))?; + if bytes.len() > MAX_FRAME_BYTES { + return Err(worker_error("request frame is too large")); + } + self.stdin + .write_all(&(bytes.len() as u32).to_be_bytes()) + .and_then(|()| self.stdin.write_all(&bytes)) + .and_then(|()| self.stdin.flush()) + .map_err(|error| worker_error(&format!("cannot write request frame: {error}")))?; + let mut header = [0u8; 4]; + self.stdout + .read_exact(&mut header) + .map_err(|error| worker_error(&format!("cannot read response header: {error}")))?; + let length = u32::from_be_bytes(header) as usize; + if length == 0 || length > MAX_FRAME_BYTES { + return Err(worker_error("invalid response frame length")); + } + let mut payload = vec![0u8; length]; + self.stdout + .read_exact(&mut payload) + .map_err(|error| worker_error(&format!("cannot read response frame: {error}")))?; + let response: Json = serde_json::from_slice(&payload) + .map_err(|error| worker_error(&format!("invalid response JSON: {error}")))?; + if response.get("ok").and_then(Json::as_bool) != Some(true) { + let code = response + .get("code") + .and_then(Json::as_str) + .unwrap_or("host.error"); + let message = response + .get("message") + .and_then(Json::as_str) + .unwrap_or("worker resource provider failed"); + let category = match code { + "permission.denied" => ErrorCategory::PermissionDenied, + "resource.closed" => ErrorCategory::ResourceClosed, + "resource.exhausted" => ErrorCategory::ResourceExhausted, + "resource.unsupported" => ErrorCategory::ResourceUnsupported, + _ => ErrorCategory::HostError, + }; + return Err(RuntimeError::new(category, None, message.to_string())); + } + Ok(response) + } +} + +impl ResourceProvider for WorkerResourceProvider { + fn name(&self) -> &str { + &self.name + } + fn kinds(&self) -> &[&str] { + // Owned kinds are exposed through `kind_strings`; registration never + // calls this for the worker transport. + &[] + } + fn kind_strings(&self) -> Vec { + self.kinds.clone() + } + fn open( + &mut self, + kind: &str, + descriptor: &str, + rights: &[String], + span: Option, + ) -> Result { + let response = self + .request(json!({ + "op": "open", + "kind": kind, + "descriptor": descriptor, + "rights": rights, + })) + .map_err(|error| with_span(error, span))?; + response + .get("token") + .and_then(Json::as_u64) + .ok_or_else(|| worker_error("open response has no token")) + } + fn read( + &mut self, + token: u64, + max_bytes: usize, + span: Option, + ) -> Result { + let response = self + .request(json!({"op": "read", "token": token, "max_bytes": max_bytes})) + .map_err(|error| with_span(error, span))?; + let bytes = response + .get("bytes") + .and_then(Json::as_array) + .ok_or_else(|| worker_error("read response has no bytes"))? + .iter() + .map(|value| value.as_u64().and_then(|v| u8::try_from(v).ok())) + .collect::>>() + .ok_or_else(|| worker_error("read response contains a non-byte value"))?; + let eof = response.get("eof").and_then(Json::as_bool).unwrap_or(false); + Ok(ResourceRead { bytes, eof }) + } + fn write( + &mut self, + token: u64, + bytes: &[u8], + span: Option, + ) -> Result { + let response = self + .request(json!({"op": "write", "token": token, "bytes": bytes})) + .map_err(|error| with_span(error, span))?; + response + .get("count") + .and_then(Json::as_u64) + .and_then(|count| usize::try_from(count).ok()) + .ok_or_else(|| worker_error("write response has no count")) + } + fn close(&mut self, token: u64, span: Option) -> Result<(), RuntimeError> { + self.request(json!({"op": "close", "token": token})) + .map_err(|error| with_span(error, span))?; + Ok(()) + } +} + +impl Drop for WorkerResourceProvider { + fn drop(&mut self) { + let _ = self.request(json!({"op": "shutdown"})); + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn with_span(mut error: RuntimeError, span: Option) -> RuntimeError { + if error.span.is_none() { + error.span = span; + } + error +} + +fn worker_error(message: &str) -> RuntimeError { + RuntimeError::new(ErrorCategory::HostError, None, message.to_string()) +} diff --git a/crates/marqdo-host/src/resource.rs b/crates/marqdo-host/src/resource.rs index 578088b..edfe411 100644 --- a/crates/marqdo-host/src/resource.rs +++ b/crates/marqdo-host/src/resource.rs @@ -28,6 +28,15 @@ pub trait ResourceProvider: Send { } /// Resource kinds served by this provider. fn kinds(&self) -> &[&str]; + + /// Owned kind list used by registration. Providers with dynamically owned + /// kind strings override this. + fn kind_strings(&self) -> Vec { + self.kinds() + .iter() + .map(|kind| (*kind).to_string()) + .collect() + } /// Open one descriptor, returning a provider-private token. fn open( &mut self, diff --git a/crates/marqdo-host/tests/worker_provider.rs b/crates/marqdo-host/tests/worker_provider.rs new file mode 100644 index 0000000..895ad9f --- /dev/null +++ b/crates/marqdo-host/tests/worker_provider.rs @@ -0,0 +1,162 @@ +use marqdo_host::{Capabilities, Host, StdHost}; +use marqdo_sandbox::SandboxMode; +use marqdo_value::ErrorCategory; + +fn context() -> marqdo_host::ExecutionContext { + marqdo_host::ExecutionContext::new("test".to_string(), None) +} + +fn handle(value: marqdo_value::Value) -> marqdo_value::ResourceHandle { + match value { + marqdo_value::Value::Resource(handle) => handle, + other => panic!("expected resource, got {other:?}"), + } +} + +fn helper() -> (tempfile::TempDir, std::path::PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("provider.py"); + std::fs::write( + &script, + r#" +import sys, json, struct + +def read_frame(): + header = sys.stdin.buffer.read(4) + if len(header) < 4: + return None + size = struct.unpack(">I", header)[0] + return json.loads(sys.stdin.buffer.read(size)) + +def write_frame(value): + payload = json.dumps(value).encode() + sys.stdout.buffer.write(struct.pack(">I", len(payload)) + payload) + sys.stdout.buffer.flush() + +data = b"hello world" +position = 0 +while True: + request = read_frame() + if request is None: + break + op = request["op"] + if op == "ping": + write_frame({"ok": True}) + elif op == "open": + write_frame({"ok": True, "token": 7}) + elif op == "read": + count = request["max_bytes"] + chunk = data[position:position + count] + position += len(chunk) + write_frame({"ok": True, "bytes": list(chunk), "eof": position >= len(data)}) + elif op == "write": + write_frame({"ok": True, "count": len(request["bytes"])}) + elif op == "close": + write_frame({"ok": True}) + elif op == "shutdown": + break + else: + write_frame({"ok": False, "code": "resource.unsupported", "message": op}) +"#, + ) + .unwrap(); + (dir, script) +} + +#[test] +fn worker_provider_round_trips_through_framed_json() { + let (dir, script) = helper(); + let project = tempfile::tempdir().unwrap(); + let mut host = StdHost::new( + SandboxMode::Project, + Capabilities::project().allow_resource_provider("worker"), + project.path(), + ); + host.register_worker_resource_provider( + "worker", + &["worker.sensor"], + &std::path::PathBuf::from("python3"), + &[script.to_string_lossy().to_string()], + ) + .unwrap(); + let ctx = context(); + let handle = handle( + host.resource_open("worker.sensor", "sensor-1", &["read".to_string()], &ctx) + .unwrap(), + ); + let first = host.resource_read(handle, 5, &ctx).unwrap(); + assert_eq!(first.bytes, b"hello"); + assert!(!first.eof); + let rest = host.resource_read(handle, 64, &ctx).unwrap(); + assert_eq!(rest.bytes, b" world"); + assert!(rest.eof); + assert_eq!(host.resource_write(handle, b"abc", &ctx).unwrap(), 3); + host.resource_close(handle, &ctx).unwrap(); + assert_eq!( + host.resource_close(handle, &ctx).unwrap_err().code(), + "resource.closed" + ); + drop(dir); +} + +#[test] +fn worker_provider_registration_requires_a_grant_and_project_mode() { + let (_dir, script) = helper(); + let project = tempfile::tempdir().unwrap(); + let ctx = context(); + let mut denied = StdHost::new( + SandboxMode::Project, + Capabilities::project(), + project.path(), + ); + denied + .register_worker_resource_provider( + "worker", + &["worker.sensor"], + &std::path::PathBuf::from("python3"), + &[script.to_string_lossy().to_string()], + ) + .unwrap(); + let error = denied + .resource_open("worker.sensor", "s", &["read".to_string()], &ctx) + .unwrap_err(); + assert_eq!(error.category, ErrorCategory::PermissionDenied); + + for mode in [SandboxMode::Pure, SandboxMode::Strict] { + let mut host = StdHost::new( + mode, + Capabilities::project().allow_resource_provider("worker"), + project.path(), + ); + host.register_worker_resource_provider( + "worker", + &["worker.sensor"], + &std::path::PathBuf::from("python3"), + &[script.to_string_lossy().to_string()], + ) + .unwrap(); + let error = host + .resource_open("worker.sensor", "s", &["read".to_string()], &ctx) + .unwrap_err(); + assert_eq!(error.category, ErrorCategory::PermissionDenied); + } +} + +#[test] +fn missing_worker_program_is_a_structured_error() { + let project = tempfile::tempdir().unwrap(); + let mut host = StdHost::new( + SandboxMode::Project, + Capabilities::project().allow_resource_provider("worker"), + project.path(), + ); + let error = host + .register_worker_resource_provider( + "worker", + &["worker.sensor"], + &std::path::PathBuf::from("/definitely/missing/marqdo-provider"), + &[], + ) + .unwrap_err(); + assert_eq!(error.category, ErrorCategory::KernelUnavailable); +} diff --git a/docs/STATUS.md b/docs/STATUS.md index 56cc888..95836ff 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -136,6 +136,9 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL PluginHandle `plugin/export` are read-only observation members (host id private); `stdlib/format.mq.md` reproduces builtin `打印`/`repr` for all 15 value types, including Formula/Code/FunctionRef and Code UTF-8 byte length. +- Worker resource provider transport (CM-A09, ADR-0035): host-managed helper + process with framed JSON, ping/open/read/write/close; project-mode + name + grant; trusted, not a sandbox; C ABI binding still open. - HTTP framing and client (P5/C10): `stdlib/net/http.mq.md` parses status, headers and `Transfer-Encoding: chunked`, builds requests, and an end-to-end test parses a loopback TCP response; fixture 71 pins the same logic. No host @@ -210,9 +213,9 @@ behavior. This table records only what is in this tree now. | P0 freeze (ADRs, spec deltas, baseline) | **P1 + partial P2 slice done** | ADR-0022 freezes CM-A01/CM-A02/CM-A03; ADR-0023 freezes CM-A04 (minimal List/Map/Text/Table observation and construction); ADR-0024 freezes CM-A05 (recoverable `Result` map convention); `spec/expression.ebnf`, `spec/evaluation.md`, `spec/value-wire.md`, `spec/analysis-v2.md`, `spec/security.md` updated; existing AST/value goldens unchanged | | P1 Memory + place assignment | **Implemented locally** | `Value::Memory`, run-scoped `MemoryStore` (64 KiB pages, 64 MiB total / 16 MiB region), `memory.new/size/grow`, `m[i]` read, `m[i] = byte`; semantic fixtures 45–47, runtime negatives 13–18, `examples/memory-buffer.mq.md`; evidence [`evidence/2026-11-20-memory-p1.md`](evidence/2026-11-20-memory-p1.md) | | P2 Marqdo-written allocator/data structures + minimal structure observation | **PASS (local pure lane); external stdlib repo publication pending** | `.mq.md`-only evidence: bump allocator with u32 codec, linked list, failed-allocation atomicity (48–50); CM-A04 structure primitives and a table filter (51–54, ADR-0023); a first-fit free-list allocator with block reuse, splitting, address-adjacent coalescing and `Result` maps (55, ADR-0024), a `Memory`-backed binary tree with recursive insert/in-order traversal (56), a `.mq.md` CSV row splitter (57), an open-addressing hash table with linear probing, update and `hash.full` (58), a `.mq.md` UTF-8 encoder over `Memory` (59), a validating UTF-8 decoder rejecting truncated, bad-continuation and surrogate/overlong input (60), a `Memory`-backed graph adjacency list (61), a recursive JSON parser for nested arrays/objects, strings with escapes, integers, booleans, null and error Results (62), a `Memory`-backed stack bytecode VM with PUSH/ADD/MUL/JMP/JZ/HALT (63), big-endian 16/32/64-bit plus LEB128 integer codecs (64), a `.mq.md` table sort rebuilding a new table from `table.row/columns/cell` (65), a table join (67), a quoted/escaped CSV row parser (68), table group (69), an Int matrix multiply (70), and a sparse-matrix multiply (73); the first-fit allocator also detects double free (`alloc.double_free`). Evidence: [`memory-p2-closure`](evidence/2026-11-20-memory-p2-closure.md), [`structure-api-p2`](evidence/2026-11-20-structure-api-p2.md), [`result-and-free-list-p2`](evidence/2026-11-20-result-and-free-list-p2.md). Local gate PASS: `crates/cli/tests/p2_closure.rs` runs 21 C01–C06/C11/C12 fixtures plus the stdlib table self-test in `--mode pure` (no foreign/plugin/import grants), scans sources for foreign/plugin/import shortcuts and compares value/stdout to goldens; evidence [`p2-closure-gate`](evidence/2026-11-20-p2-closure-gate.md). The bundled `stdlib/` tree, `std:` imports, digest verification, Analysis identity and frozen-CLI closure test are implemented (ADR-0028). Remaining outside C01–C12: publication as a separate external stdlib repository, and no remote CI URL. | -| P3 Resource/Stream ABI | **PARTIAL** | ADR-0025/0026 CM-A06 slices: opaque `Resource`, `resource.open/read/write/close`, stdout write in pure/project, granted project-file read and buffered project-file write in project, EOF/closed/handle errors, wire/host rejection; strict permits only stdio/stdout and refuses file/network resources. Fixture 66 + CLI/host tests. CM-A07 **Compat-C** is implemented: when a program calls `打印/print`, the loader injects the bundled `stdlib/io.mq.md` and the runtime routes to `io.写值` (all 15 value types, UTF-8 via `Memory`, `resource.write(stdout)`); `HostOp::Print` and the legacy `Value::display()` print path are removed, and a missing bundle fails with `kernel.unavailable`. Strict permits only the existing stdio console channel. Still missing: directories, network, generic provider registry, Table storage, strict file/network resources. | +| P3 Resource/Stream ABI | **PARTIAL** | ADR-0025/0026 CM-A06 slices: opaque `Resource`, `resource.open/read/write/close`, stdout write in pure/project, granted project-file read and buffered project-file write in project, EOF/closed/handle errors, wire/host rejection; strict permits only stdio/stdout and refuses file/network resources. Fixture 66 + CLI/host tests. CM-A07 **Compat-C** is implemented: when a program calls `打印/print`, the loader injects the bundled `stdlib/io.mq.md` and the runtime routes to `io.写值` (all 15 value types, UTF-8 via `Memory`, `resource.write(stdout)`); `HostOp::Print` and the legacy `Value::display()` print path are removed, and a missing bundle fails with `kernel.unavailable`. Strict permits only the existing stdio console channel. Providers are registered through ADR-0033 (in-process) and ADR-0035 (worker helper). Still missing: directories, C ABI provider binding, provider packages, strict provider support. | | P4 Table logical/physical migration | **PASS (local gate); T1/T2/T3 future** | ADR-0032/CM-A08: `Value::Table` semantics and GFM AST laziness unchanged; `stdlib/data/table.mq.md` provides select/filter/sort/append/set/join/group/`转GFM` as value-returning `.mq.md` functions with input immutability; zero Rust table algorithms were added. Cross-storage equivalence is pinned by fixture 72 (row Table vs `.mq.md` columnar store, identical headers/rows/cells). The optional T1/T2/T3 physical-storage/column-store migration was not required by the gate and is not implemented. | -| P5 third-party Provider / TCP / HTTP | **PASS (local gate); external network/CI and C-Worker transport open** | C10: `stdlib/net/http.mq.md` + fixture 71 parse HTTP/1.1 status/headers/chunked bodies and build requests; `http_over_tcp.rs` runs a loopback HTTP exchange wholly in `.mq.md` over `resource.open("tcp",...)`. ADR-0034 IP-literal blocking TCP endpoint (exact IP grant, project-only, 5s timeouts) and ADR-0033 in-process `ResourceProvider` v1 (registration, protocol/kind checks, name grant, mock read/write provider) are tested. External DNS/TLS/UDP, host HTTP, provider packages and the C/Worker provider transport remain out of scope. | +| P5 third-party Provider / TCP / HTTP | **PASS (local gate); external network/CI and C-Worker transport open** | C10: `stdlib/net/http.mq.md` + fixture 71 parse HTTP/1.1 status/headers/chunked bodies and build requests; `http_over_tcp.rs` runs a loopback HTTP exchange wholly in `.mq.md` over `resource.open("tcp",...)`. ADR-0034 IP-literal blocking TCP endpoint (exact IP grant, project-only, 5s timeouts) and ADR-0033 in-process `ResourceProvider` v1 (registration, protocol/kind checks, name grant, mock read/write provider) and ADR-0035 host-managed worker provider transport (framed JSON helper, ping/open/read/write/close, trusted-only) are tested. External DNS/TLS/UDP, host HTTP, provider packages and the C ABI provider binding remain out of scope. | | P6 performance / native backends | **BASELINE ONLY** | reproducible profile harness `tools/perf/bench.py` and [`p6-profile`](evidence/2026-11-20-p6-profile.md) for 7 workloads; no bytecode/JIT/columnar numeric backend or equivalence/optimization gate yet | The P1 memory domain is pure computation: it needs no grant and is available in @@ -258,6 +261,9 @@ user-written allocator is explicitly **not** claimed. echo tests. - [`evidence/2026-11-20-table-cross-storage.md`](evidence/2026-11-20-table-cross-storage.md) — P4 row-store Table vs `.mq.md` columnar store equivalence for sort/filter. +- [`evidence/2026-11-20-worker-provider.md`](evidence/2026-11-20-worker-provider.md) + — ADR-0035 trusted worker resource-provider transport, framed-JSON helper + round-trip and grant/mode/error tests. - [`evidence/2026-11-20-http-over-tcp.md`](evidence/2026-11-20-http-over-tcp.md) — P5/C10 `.mq.md` HTTP framing over a loopback TCP stream, stdlib module and end-to-end client test. diff --git a/docs/adr/0035-worker-resource-provider-transport.md b/docs/adr/0035-worker-resource-provider-transport.md new file mode 100644 index 0000000..1c82952 --- /dev/null +++ b/docs/adr/0035-worker-resource-provider-transport.md @@ -0,0 +1,92 @@ +# ADR-0035: Worker resource-provider transport v1 + +Status: **Accepted (trusted helper transport; C ABI binding still open)** +Date: 2026-11-20 +Relates to: `docs/MARQDO_COMPUTE_CORE_STDLIB_MIGRATION_RFC_0002.md` +(CM-A09, CM-A11, P5), ADR-0033, ADR-0034 +Specs: `spec/evaluation.md`, `spec/security.md` +Evidence: `docs/evidence/2026-11-20-worker-provider.md` + +## Context + +ADR-0033 provides an in-process `ResourceProvider` v1. CM-A09 also asks for a +process boundary so an adapter can crash or hang without taking down the +interpreter, and for a transport that can later be implemented by the C/Worker +plugin ecosystem. This ADR adds the first process-boundary transport: a +host-managed helper process speaking framed JSON. + +It deliberately does **not** claim the C ABI or OS sandbox: the helper runs as +the host user and is trusted. The existing plugin Worker transport remains the +untrusted-plugin path; this is a resource-provider transport for explicitly +trusted adapters. + +## Decision + +### 1. Framing and protocol v1 + +* 4-byte big-endian length + JSON, maximum 1 MiB per frame. +* A `ping` handshake must succeed before the provider is registered. +* Operations: + +```text +{"op":"open","kind","descriptor","rights"} -> {"ok":true,"token":u64} +{"op":"read","token","max_bytes"} -> {"ok":true,"bytes":[u8...],"eof":bool} +{"op":"write","token","bytes":[u8...]} -> {"ok":true,"count":u64} +{"op":"close","token"} -> {"ok":true} +{"op":"shutdown"} -> process exit +{"ok":false,"code","message"} -> structured failure +``` + +* Provider tokens stay in the helper; the host still issues the opaque language + `ResourceHandle`. Bytes are copied per call. +* The host maps helper error codes to the existing categories + (`permission.denied`, `resource.closed`, `resource.exhausted`, + `resource.unsupported`, otherwise `host.error`) and attaches the callsite + span. +* The provider is killed on drop; a protocol/IO failure is `host.error`, never a + fallback to another transport. + +### 2. Registration and authorization + +* `StdHost::register_worker_resource_provider(name, kinds, program, args)` is a + host embedding API. It validates name/kinds with the same rules as ADR-0033 + (non-empty, not `stdio`/`file`, no duplicate kind), then registers the + provider under its kinds. +* `resource.open` still requires the provider name grant and project mode. + Pure/strict refuse provider kinds; strict remains fail-closed. +* Language code cannot spawn a helper; registration is not reachable from + `.mq.md` or a package import. + +### 3. Security boundary + +* The helper runs as the host user with full host privileges. It is **trusted**; + this transport is not a sandbox and does not make arbitrary native code safe. +* The project-mode + name-grant checks are the same as ADR-0033. The helper's + own descriptor/rights validation is additional, not a substitute. +* No network, filesystem or process rights are delegated implicitly by + registration; the helper is responsible for its own behavior, and the host + grant only authorizes the resource kind. + +### 4. What remains for CM-A09 + +* A C ABI / plugin-Worker binding for `ResourceProvider` (buffer handoff, + ownership, version negotiation through the plugin ABI). +* Provider package installation, signatures and discovery. +* Untrusted-provider isolation and strict-mode provider support. +* The transport is versioned (`protocol_version() == 1`); future versions need a + new ADR and negotiation. + +## Consequences + +* A trusted adapter can live in its own process and be reached through the same + `resource.*`/`Result` language surface without touching the parser, `Value`, + runtime dispatch or `HostOp`. +* A helper crash/exit is a structured host error; the interpreter never falls + back to a different provider. +* The helper protocol is small enough to reimplement in C/Worker adapters later. + +## Verification + +`docs/evidence/2026-11-20-worker-provider.md` records a Python helper exercising +ping/open/read/write/close, grant/pure/strict denial, missing-program error and +the full workspace gate. diff --git a/docs/evidence/2026-11-20-worker-provider.md b/docs/evidence/2026-11-20-worker-provider.md new file mode 100644 index 0000000..5cf6bdf --- /dev/null +++ b/docs/evidence/2026-11-20-worker-provider.md @@ -0,0 +1,62 @@ +# RFC-CM CM-A09 evidence: worker resource-provider transport v1 + +Date: 2026-11-20 (Asia/Shanghai) +Repository: `marqdo-cli` +Baseline: `12be02e` plus this step's changes. +Decision record: [ADR-0035](../adr/0035-worker-resource-provider-transport.md) +Related: [in-process provider](2026-11-20-provider-abi-cm-a09.md), +[STATUS §5](../STATUS.md). +Scope: trusted host-managed helper process; not a sandbox and not the C ABI binding. + +## Transport + +`crates/marqdo-host/src/provider_worker.rs` implements the ADR-0035 framed-JSON +transport: + +```text +4-byte BE length + JSON, max 1 MiB +ping handshake +open -> token +read -> bytes+eof +write -> count +close +shutdown on drop +``` + +`StdHost::register_worker_resource_provider(name, kinds, program, args)` spawns +the helper and registers it through the same validation/grant path as ADR-0033. +The helper token never enters a language `Value`; the host still issues the +opaque `ResourceHandle`. + +## Tests + +`crates/marqdo-host/tests/worker_provider.rs` (3 tests): + +```text +worker_provider_round_trips_through_framed_json + Python helper: ping/open/read 5 "hello"/read rest " world" eof/write 3/close; + second close resource.closed. + +worker_provider_registration_requires_a_grant_and_project_mode + no name grant -> permission.denied; pure/strict -> permission.denied. + +missing_worker_program_is_a_structured_error + non-existent helper -> kernel.unavailable (no fallback). +``` + +No runtime, `HostOp`, `Value` or parser change was needed; language code still +uses `resource.*`. + +## Security boundary + +The helper runs as the host user with host privileges. Registration is an +embedding API, project-mode only, and requires a provider-name grant. It is +trusted infrastructure, **not** an OS sandbox; the untrusted-plugin path remains +the separate plugin Worker transport. + +## Not done + +* C ABI / plugin-Worker binding for `ResourceProvider`. +* Provider package installation/signatures/discovery. +* Untrusted-provider isolation; strict-mode provider support. +* External DNS/TLS/network endpoint work beyond the IP-literal TCP slice. diff --git a/spec/security.md b/spec/security.md index e8338ab..689a05f 100644 --- a/spec/security.md +++ b/spec/security.md @@ -63,7 +63,7 @@ set for one run: | `process: bool` | `false` | required together with `foreign` for `foreign.run`/`foreign.call` | | `foreign: bool` | `false` | foreign kernels | | `plugins: Vec` | empty | `plugin.invoke` and `StdHost::load_plugin` | -| `resource_providers: Vec` | empty | registered in-process `ResourceProvider` kinds; project mode only (ADR-0033) | +| `resource_providers: Vec` | empty | registered in-process/worker `ResourceProvider` kinds; project mode only; helpers are trusted, not sandboxed (ADR-0033/0035) | Requests versus grants: diff --git a/tests/README.md b/tests/README.md index e2537ff..a8d0bb2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -177,6 +177,12 @@ grant/pure/strict/DNS refusal) and `crates/cli/tests/resource_streams.rs::tcp_stream_round_trips_through_mqmd_with_a_network_grant` (`--allow-net 127.0.0.1`). +RFC-CM CM-A09 worker provider transport (ADR-0035) is pinned by +`crates/marqdo-host/tests/worker_provider.rs`: a Python helper is spawned, +pinged, opened/read/written/closed through framed JSON; missing grant, pure, +strict and missing-program cases are rejected. The helper is trusted, not a +sandbox. + RFC-CM CM-A09 in-process provider v1 (ADR-0033) is pinned by the three `registered_provider_streams_*` / `provider_*` tests in `crates/marqdo-host/tests/resource_streams.rs`: a mock provider is registered