From 974102dfc34bd7302fbebba4ea91983c970db6a7 Mon Sep 17 00:00:00 2001 From: cflmy Date: Fri, 20 Nov 2026 23:28:30 +0800 Subject: [PATCH] feat(host): add provider package manifest v1 - Add a marqdo.provider schema v1 loader (schema/version/ABI/platform library) and StdHost::register_provider_package, reusing the C ABI adapter and shared kind/name checks (ADR-0037). - Add a compiled-provider + provider.json installation test; language use still requires project mode and allow_resource_provider. - Update ADR/spec/status/README/changelog and evidence; no registry, signature or CLI install is claimed. Gate: fmt, clippy -D warnings, 437 tests passed. --- CHANGELOG.md | 2 + README.md | 3 + crates/marqdo-host/src/host.rs | 13 +++- crates/marqdo-host/src/provider_c.rs | 80 +++++++++++++++++++- crates/marqdo-host/tests/c_provider.rs | 45 +++++++++++ docs/STATUS.md | 8 +- docs/adr/0037-provider-package-manifest.md | 77 +++++++++++++++++++ docs/architecture/README.md | 1 + docs/evidence/2026-11-20-provider-package.md | 45 +++++++++++ spec/security.md | 2 +- tests/README.md | 3 +- 11 files changed, 274 insertions(+), 5 deletions(-) create mode 100644 docs/adr/0037-provider-package-manifest.md create mode 100644 docs/evidence/2026-11-20-provider-package.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5de47ba..7709ee9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,6 +136,8 @@ currently `0.1.0`; no release tag has been cut. Dates use ISO 8601. JSON report; no bytecode/JIT/native backend is introduced. - P4 cross-storage table equivalence: fixture 72 compares row-store Table sort/filter output with a `.mq.md` columnar store cell-by-cell. +- CM-A09 provider package manifest v1 (ADR-0037): local manifest resolves a + trusted C ABI library and registers it; no registry/signatures/CLI install. - CM-A09 C ABI provider v1 (ADR-0036): trusted shared libraries implement open/read/write/close/version/last_error; host loads them with version and symbol checks, project-mode + name grant only. Plugin-Worker binding remains diff --git a/README.md b/README.md index 69d3cbc..f6655f3 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,9 @@ executes anything. * **Blocking TCP endpoint (RFC-CM P5 first slice)** — `resource.open("tcp", ":", ["read","write"])` with exact IP grants, project mode and 5-second timeouts; DNS/TLS/UDP/server sockets excluded (ADR-0034). +* **Provider package manifest v1 (RFC-CM CM-A09)** — local `marqdo.provider` + manifest installs a trusted C ABI provider; no registry/signature/CLI install + (ADR-0037). * **C ABI ResourceProvider v1 (RFC-CM CM-A09)** — trusted native shared libraries expose resource open/read/write/close through a frozen header; project + name grant; plugin-Worker binding remains open (ADR-0036). diff --git a/crates/marqdo-host/src/host.rs b/crates/marqdo-host/src/host.rs index 9616854..baaf21e 100644 --- a/crates/marqdo-host/src/host.rs +++ b/crates/marqdo-host/src/host.rs @@ -21,7 +21,7 @@ use crate::context::ExecutionContext; use crate::fs::ProjectFs; use crate::handles::{contains_plugin_handle, PluginHandleTable}; use crate::plugin_worker::WorkerPlugin; -use crate::provider_c::CResourceProvider; +use crate::provider_c::{load_package, CResourceProvider}; use crate::provider_worker::WorkerResourceProvider; use crate::resource::{ResourceProvider, ResourceRead, ResourceTable, SharedResourceProvider}; use crate::trace::{EffectTrace, SemanticEvent, TraceBudget, TraceEntry, TraceState}; @@ -602,6 +602,17 @@ impl StdHost { self.register_provider_checked(Box::new(provider)) } + /// Register a trusted C ABI provider from a `marqdo.provider` manifest + /// (ADR-0037). The manifest is host-side installation metadata; language + /// imports cannot install or authorize a provider. + pub fn register_provider_package( + &mut self, + manifest_path: &std::path::Path, + ) -> Result<(), RuntimeError> { + let provider = load_package(manifest_path)?; + self.register_provider_checked(Box::new(provider)) + } + /// Register a trusted worker-process resource provider (ADR-0035). /// /// The helper speaks framed JSON over stdin/stdout and is **not** an OS diff --git a/crates/marqdo-host/src/provider_c.rs b/crates/marqdo-host/src/provider_c.rs index 1a1eaa3..2b1dd8a 100644 --- a/crates/marqdo-host/src/provider_c.rs +++ b/crates/marqdo-host/src/provider_c.rs @@ -4,8 +4,9 @@ //! and is explicitly not a sandbox; only explicitly trusted adapters may be //! registered. +use std::collections::BTreeMap; use std::ffi::{c_char, CString}; -use std::path::Path; +use std::path::{Path, PathBuf}; use libloading::Library; use marqdo_span::Span; @@ -23,6 +24,83 @@ type WriteFn = unsafe extern "C" fn(u64, *const u8, u32, *mut u32) -> i32; type CloseFn = unsafe extern "C" fn(u64) -> i32; type LastErrorFn = unsafe extern "C" fn(*mut c_char, u32) -> i32; +#[derive(Debug, serde::Deserialize)] +struct ProviderPackage { + schema: String, + schema_version: u32, + name: String, + kinds: Vec, + abi: String, + #[serde(default)] + library: BTreeMap, +} + +/// Load a C ABI provider from its package manifest (ADR-0037). +pub(crate) fn load_package(manifest_path: &Path) -> Result { + let bytes = std::fs::read(manifest_path).map_err(|error| { + RuntimeError::new( + ErrorCategory::ResourceUnsupported, + None, + format!( + "cannot read provider manifest {}: {error}", + manifest_path.display() + ), + ) + })?; + let package: ProviderPackage = serde_json::from_slice(&bytes).map_err(|error| { + RuntimeError::new( + ErrorCategory::ResourceUnsupported, + None, + format!( + "invalid provider manifest {}: {error}", + manifest_path.display() + ), + ) + })?; + if package.schema != "marqdo.provider" || package.schema_version != 1 { + return Err(provider_error( + &package.name, + "manifest schema must be marqdo.provider version 1", + )); + } + if package.abi != "c-abi-1" { + return Err(provider_error( + &package.name, + &format!( + "unsupported provider ABI `{}` (expected c-abi-1)", + package.abi + ), + )); + } + let platform = if cfg!(target_os = "linux") { + "linux" + } else if cfg!(target_os = "macos") { + "macos" + } else if cfg!(target_os = "windows") { + "windows" + } else { + "default" + }; + let library_name = package + .library + .get(platform) + .or_else(|| package.library.get("default")) + .ok_or_else(|| { + provider_error( + &package.name, + &format!("manifest has no `{platform}` or `default` library entry"), + ) + })?; + let base = manifest_path.parent().unwrap_or_else(|| Path::new(".")); + let library = PathBuf::from(library_name); + let library = if library.is_absolute() { + library + } else { + base.join(library) + }; + CResourceProvider::open(&package.name, &package.kinds, &library) +} + pub(crate) struct CResourceProvider { name: String, kinds: Vec, diff --git a/crates/marqdo-host/tests/c_provider.rs b/crates/marqdo-host/tests/c_provider.rs index 203d7a9..f4e5f02 100644 --- a/crates/marqdo-host/tests/c_provider.rs +++ b/crates/marqdo-host/tests/c_provider.rs @@ -194,3 +194,48 @@ fn c_abi_version_mismatch_and_modes_fail_closed() { assert_eq!(error.category, ErrorCategory::PermissionDenied); } } + +#[test] +fn provider_package_manifest_installs_a_c_abi_library() { + if !cc_available() { + eprintln!("skipping C provider test: cc is unavailable"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let library = build_provider(dir.path()); + let manifest = dir.path().join("provider.json"); + let platform = if cfg!(target_os = "linux") { + "linux" + } else if cfg!(target_os = "macos") { + "macos" + } else { + "default" + }; + std::fs::write( + &manifest, + serde_json::json!({ + "schema": "marqdo.provider", + "schema_version": 1, + "name": "cabi", + "kinds": ["cabi.sensor"], + "abi": "c-abi-1", + "library": { platform: library.file_name().unwrap().to_string_lossy() } + }) + .to_string(), + ) + .unwrap(); + let project = tempfile::tempdir().unwrap(); + let mut host = StdHost::new( + SandboxMode::Project, + Capabilities::project().allow_resource_provider("cabi"), + project.path(), + ); + host.register_provider_package(&manifest).unwrap(); + let ctx = context(); + let handle = handle( + host.resource_open("cabi.sensor", "s", &["read".to_string()], &ctx) + .unwrap(), + ); + let read = host.resource_read(handle, 2, &ctx).unwrap(); + assert_eq!(read.bytes, b"c-"); +} diff --git a/docs/STATUS.md b/docs/STATUS.md index 07c72a3..73ba615 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -136,6 +136,9 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL PluginHandle `plugin/export` are read-only observation members (host id private); `stdlib/format.mq.md` reproduces builtin `打印`/`repr` for all 15 value types, including Formula/Code/FunctionRef and Code UTF-8 byte length. +- Provider package manifest v1 (CM-A09, ADR-0037): a local `marqdo.provider` + manifest resolves a trusted C ABI library and registers it; no registry, + signatures or CLI install yet. - C ABI ResourceProvider v1 (CM-A09, ADR-0036): trusted shared libraries expose open/read/write/close through the frozen header; version/load failures are fail-closed; project-mode + name grant only. Plugin-Worker binding remains open. @@ -218,7 +221,7 @@ behavior. This table records only what is in this tree now. | P2 Marqdo-written allocator/data structures + minimal structure observation | **PASS (local pure lane); external stdlib repo publication pending** | `.mq.md`-only evidence: bump allocator with u32 codec, linked list, failed-allocation atomicity (48–50); CM-A04 structure primitives and a table filter (51–54, ADR-0023); a first-fit free-list allocator with block reuse, splitting, address-adjacent coalescing and `Result` maps (55, ADR-0024), a `Memory`-backed binary tree with recursive insert/in-order traversal (56), a `.mq.md` CSV row splitter (57), an open-addressing hash table with linear probing, update and `hash.full` (58), a `.mq.md` UTF-8 encoder over `Memory` (59), a validating UTF-8 decoder rejecting truncated, bad-continuation and surrogate/overlong input (60), a `Memory`-backed graph adjacency list (61), a recursive JSON parser for nested arrays/objects, strings with escapes, integers, booleans, null and error Results (62), a `Memory`-backed stack bytecode VM with PUSH/ADD/MUL/JMP/JZ/HALT (63), big-endian 16/32/64-bit plus LEB128 integer codecs (64), a `.mq.md` table sort rebuilding a new table from `table.row/columns/cell` (65), a table join (67), a quoted/escaped CSV row parser (68), table group (69), an Int matrix multiply (70), and a sparse-matrix multiply (73); the first-fit allocator also detects double free (`alloc.double_free`). Evidence: [`memory-p2-closure`](evidence/2026-11-20-memory-p2-closure.md), [`structure-api-p2`](evidence/2026-11-20-structure-api-p2.md), [`result-and-free-list-p2`](evidence/2026-11-20-result-and-free-list-p2.md). Local gate PASS: `crates/cli/tests/p2_closure.rs` runs 21 C01–C06/C11/C12 fixtures plus the stdlib table self-test in `--mode pure` (no foreign/plugin/import grants), scans sources for foreign/plugin/import shortcuts and compares value/stdout to goldens; evidence [`p2-closure-gate`](evidence/2026-11-20-p2-closure-gate.md). The bundled `stdlib/` tree, `std:` imports, digest verification, Analysis identity and frozen-CLI closure test are implemented (ADR-0028). Remaining outside C01–C12: publication as a separate external stdlib repository, and no remote CI URL. | | P3 Resource/Stream ABI | **PARTIAL** | ADR-0025/0026 CM-A06 slices: opaque `Resource`, `resource.open/read/write/close`, stdout write in pure/project, granted project-file read and buffered project-file write in project, EOF/closed/handle errors, wire/host rejection; strict permits only stdio/stdout and refuses file/network resources. Fixture 66 + CLI/host tests. CM-A07 **Compat-C** is implemented: when a program calls `打印/print`, the loader injects the bundled `stdlib/io.mq.md` and the runtime routes to `io.写值` (all 15 value types, UTF-8 via `Memory`, `resource.write(stdout)`); `HostOp::Print` and the legacy `Value::display()` print path are removed, and a missing bundle fails with `kernel.unavailable`. Strict permits only the existing stdio console channel. Providers are registered through ADR-0033 (in-process) and ADR-0035 (worker helper). Still missing: directories, C ABI provider binding, provider packages, strict provider support. | | P4 Table logical/physical migration | **PASS (local gate); T1/T2/T3 future** | ADR-0032/CM-A08: `Value::Table` semantics and GFM AST laziness unchanged; `stdlib/data/table.mq.md` provides select/filter/sort/append/set/join/group/`转GFM` as value-returning `.mq.md` functions with input immutability; zero Rust table algorithms were added. Cross-storage equivalence is pinned by fixture 72 (row Table vs `.mq.md` columnar store, identical headers/rows/cells). The optional T1/T2/T3 physical-storage/column-store migration was not required by the gate and is not implemented. | -| P5 third-party Provider / TCP / HTTP | **PASS (local gate); external network/CI and plugin-Worker resource binding open** | C10: `stdlib/net/http.mq.md` + fixture 71 parse HTTP/1.1 status/headers/chunked bodies and build requests; `http_over_tcp.rs` runs a loopback HTTP exchange wholly in `.mq.md` over `resource.open("tcp",...)`. ADR-0034 IP-literal blocking TCP endpoint (exact IP grant, project-only, 5s timeouts) and ADR-0033 in-process `ResourceProvider` v1 (registration, protocol/kind checks, name grant, mock read/write provider) ADR-0035 host-managed worker provider transport (framed JSON helper, ping/open/read/write/close, trusted-only) and ADR-0036 trusted C ABI provider (shared-library symbols, version/load checks) are tested. External DNS/TLS/UDP, host HTTP, provider packages/signatures and the plugin-Worker resource binding remain out of scope. | +| P5 third-party Provider / TCP / HTTP | **PASS (local gate); external network/CI and plugin-Worker resource binding open** | C10: `stdlib/net/http.mq.md` + fixture 71 parse HTTP/1.1 status/headers/chunked bodies and build requests; `http_over_tcp.rs` runs a loopback HTTP exchange wholly in `.mq.md` over `resource.open("tcp",...)`. ADR-0034 IP-literal blocking TCP endpoint (exact IP grant, project-only, 5s timeouts) and ADR-0033 in-process `ResourceProvider` v1 (registration, protocol/kind checks, name grant, mock read/write provider) ADR-0035 host-managed worker provider transport (framed JSON helper, ping/open/read/write/close, trusted-only) and ADR-0036 trusted C ABI provider (shared-library symbols, version/load checks) and ADR-0037 local provider-package manifest installation are tested. External DNS/TLS/UDP, host HTTP, provider packages/signatures and the plugin-Worker resource binding remain out of scope. | | P6 performance / native backends | **BASELINE ONLY** | reproducible profile harness `tools/perf/bench.py` and [`p6-profile`](evidence/2026-11-20-p6-profile.md) for 7 workloads; no bytecode/JIT/columnar numeric backend or equivalence/optimization gate yet | The P1 memory domain is pure computation: it needs no grant and is available in @@ -264,6 +267,9 @@ user-written allocator is explicitly **not** claimed. echo tests. - [`evidence/2026-11-20-table-cross-storage.md`](evidence/2026-11-20-table-cross-storage.md) — P4 row-store Table vs `.mq.md` columnar store equivalence for sort/filter. +- [`evidence/2026-11-20-provider-package.md`](evidence/2026-11-20-provider-package.md) + — ADR-0037 local provider package manifest, C library resolution and + registration test. - [`evidence/2026-11-20-c-provider.md`](evidence/2026-11-20-c-provider.md) — ADR-0036 C ABI ResourceProvider v1, compiled C adapter round-trip and fail-closed load/version checks. diff --git a/docs/adr/0037-provider-package-manifest.md b/docs/adr/0037-provider-package-manifest.md new file mode 100644 index 0000000..49d5008 --- /dev/null +++ b/docs/adr/0037-provider-package-manifest.md @@ -0,0 +1,77 @@ +# ADR-0037: Provider package manifest v1 + +Status: **Accepted (host-side installation metadata)** +Date: 2026-11-20 +Relates to: `docs/MARQDO_COMPUTE_CORE_STDLIB_MIGRATION_RFC_0002.md` +(CM-A09, CM-A10, P5), ADR-0036 +Specs: `spec/evaluation.md`, `spec/security.md` +Evidence: `docs/evidence/2026-11-20-provider-package.md` + +## Context + +ADR-0036 lets a trusted C shared library implement resource operations, but the +host still needed a code-level path to the library. CM-A09 asks for provider +installation/discovery without changing the parser/runtime. This ADR defines a +small, versioned **package manifest** so a provider distribution can be +installed by a host embedding and reused by future external provider +repositories. + +## Decision + +### 1. Manifest format + +```json +{ + "schema": "marqdo.provider", + "schema_version": 1, + "name": "cabi", + "kinds": ["cabi.sensor"], + "abi": "c-abi-1", + "library": { + "linux": "libcabi_provider.so", + "macos": "libcabi_provider.dylib", + "default": "libcabi_provider.so" + } +} +``` + +* file name: `provider.json` by convention; the host API takes an explicit + manifest path. +* `schema`/`schema_version` must be exactly `marqdo.provider`/`1`. +* `abi` must be `c-abi-1` (ADR-0036); future ABIs need a new ADR. +* library paths are relative to the manifest directory (absolute paths are + accepted only for explicit local tests); platform key is `linux`, `macos`, + `windows`, or `default`. +* unknown fields are tolerated by serde, but unknown schema/ABI are rejected. + +### 2. Host API and authorization + +* `StdHost::register_provider_package(manifest_path)` reads/validates the + manifest, loads the C ABI library and registers it through the same + kind/name validation as ADR-0033/0036. +* Registration does not authorize language use: `resource.open` still requires + project mode and `Capabilities.allow_resource_provider(name)`. Pure/strict + refuse provider kinds. +* Language imports cannot read or install a provider manifest. Installation is + a host/package-manager action (INV-06). + +### 3. Trust boundary + +* The manifest points at trusted native code loaded in-process; it is not a + sandbox and does not add signatures. A future provider repository must add + signatures/verification and a Worker binding. +* No network/package registry fetch is implemented here; the host passes a + local path. + +## Non-goals + +* Provider registry download, signatures, dependency resolution or semver. +* CLI install/uninstall commands (host embedding only). +* Untrusted-provider isolation or strict-mode provider support. +* Automatic platform selection beyond the `library` map. + +## Verification + +`docs/evidence/2026-11-20-provider-package.md` records the manifest round-trip +test: compile a C provider, write `provider.json`, `register_provider_package`, +then `resource.open/read`. diff --git a/docs/architecture/README.md b/docs/architecture/README.md index c6d8a60..e09b9de 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -33,6 +33,7 @@ implementation status lives in `docs/STATUS.md`; frozen behavior is in | P5 blocking TCP resource endpoint v1 | [0034](../adr/0034-tcp-resource-endpoint.md) | Accepted (IP-literal, no DNS/TLS) | | CM-A09 worker resource-provider transport v1 | [0035](../adr/0035-worker-resource-provider-transport.md) | Accepted (trusted helper) | | CM-A09 C ABI ResourceProvider v1 | [0036](../adr/0036-c-abi-resource-provider.md) | Accepted (trusted native adapter) | +| CM-A09 provider package manifest v1 | [0037](../adr/0037-provider-package-manifest.md) | Accepted (host-side install) | | Table physical storage (T1/T2/T3), C/Worker provider transport, strict file/network resources | — | Not decided; do not implement by implication | ## What a future phase must not do diff --git a/docs/evidence/2026-11-20-provider-package.md b/docs/evidence/2026-11-20-provider-package.md new file mode 100644 index 0000000..df62544 --- /dev/null +++ b/docs/evidence/2026-11-20-provider-package.md @@ -0,0 +1,45 @@ +# RFC-CM CM-A09 evidence: provider package manifest v1 + +Date: 2026-11-20 (Asia/Shanghai) +Repository: `marqdo-cli` +Baseline: `5e5c3a8` plus this step's changes. +Decision record: [ADR-0037](../adr/0037-provider-package-manifest.md) +Related: [C ABI provider](2026-11-20-c-provider.md), [STATUS §5](../STATUS.md). +Scope: local manifest installation; no registry/signature/Worker binding. + +## Manifest + +```json +{ + "schema": "marqdo.provider", + "schema_version": 1, + "name": "cabi", + "kinds": ["cabi.sensor"], + "abi": "c-abi-1", + "library": {"linux": "libecho_provider.so", "default": "libecho_provider.so"} +} +``` + +`StdHost::register_provider_package` validates schema/version/ABI, resolves the +platform library relative to the manifest, loads it through the ADR-0036 C ABI +adapter and registers it through the shared kind/name checks. Language use still +requires project mode and `allow_resource_provider("cabi")`. + +## Test + +`crates/marqdo-host/tests/c_provider.rs::provider_package_manifest_installs_a_c_abi_library`: + +1. compile the C provider with `cc -shared`; +2. write `provider.json` next to it; +3. `register_provider_package(&manifest)` succeeds; +4. `resource.open("cabi.sensor", ...)` and `resource.read` return `c-`. + +Existing C tests cover version mismatch, missing library and pure/strict refusal. +No parser/`Value`/runtime/`HostOp` change was needed. + +## Not done + +* Provider registry/download, signatures, dependency resolution, semver. +* CLI install/uninstall commands; installation is host embedding API only. +* Untrusted-provider isolation, strict-mode providers, plugin-Worker binding. +* Windows library build lane. diff --git a/spec/security.md b/spec/security.md index 194d975..76cd476 100644 --- a/spec/security.md +++ b/spec/security.md @@ -63,7 +63,7 @@ set for one run: | `process: bool` | `false` | required together with `foreign` for `foreign.run`/`foreign.call` | | `foreign: bool` | `false` | foreign kernels | | `plugins: Vec` | empty | `plugin.invoke` and `StdHost::load_plugin` | -| `resource_providers: Vec` | empty | registered in-process/worker/C-ABI `ResourceProvider` kinds; project mode only; helper processes and native libraries are trusted, not sandboxed (ADR-0033/0035/0036) | +| `resource_providers: Vec` | empty | registered in-process/worker/C-ABI `ResourceProvider` kinds; project mode only; helper processes and native libraries are trusted, not sandboxed (ADR-0033/0035/0036); local manifests are host-installed, unsigned (ADR-0037) | Requests versus grants: diff --git a/tests/README.md b/tests/README.md index e401f2c..76a9471 100644 --- a/tests/README.md +++ b/tests/README.md @@ -180,7 +180,8 @@ grant/pure/strict/DNS refusal) and RFC-CM CM-A09 C ABI provider (ADR-0036) is pinned by `crates/marqdo-host/tests/c_provider.rs`, which compiles a small C shared library, registers it, round-trips `resource.*`, checks missing-library, -wrong-ABI-version and pure/strict fail-closed behavior. +wrong-ABI-version and pure/strict fail-closed behavior; the package-manifest +test installs a C provider from `provider.json`. RFC-CM CM-A09 worker provider transport (ADR-0035) is pinned by `crates/marqdo-host/tests/worker_provider.rs`: a Python helper is spawned,