feat(foreign): verify provider digests and record environment metadata (B-12)
- add environment_id/environment_digest/interpreter_version to KernelDescriptor - require external CLI descriptors to pin an absolute executable and a full provider digest - verify the digest at registration and immediately before every spawn; tampering after registration fails with ForeignError::Integrity before spawn - validate digest formats and reject relative external entrypoints - pin the example provider digest and add an integrity test - document that uv/npm environment materialization and lock integration remain open; strict isolation is unchanged and still refused
This commit is contained in:
1 parent
20d15fd5a2
commit
b37026b79d
9 files changed
+214
-8
No files matched your search
Generated
+2
@@ -361,11 +361,13 @@ dependencies = [
|
||||
name = "marqdo-foreign"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"hex",
|
||||
"libc",
|
||||
"marqdo-span",
|
||||
"marqdo-value",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
@@ -863,6 +863,12 @@ fn load_kernel_descriptor(host: &mut StdHost, path: &Path) -> Result<(), String>
|
||||
if descriptor.builtin {
|
||||
return Err("external kernel descriptors must not set `builtin: true`".to_string());
|
||||
}
|
||||
if descriptor.digest.is_none() {
|
||||
return Err(
|
||||
"external kernel descriptors must declare a `digest` of the provider program"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if descriptor.entrypoint.is_relative() {
|
||||
let base = path.parent().unwrap_or_else(|| Path::new("."));
|
||||
descriptor.entrypoint = base.join(&descriptor.entrypoint);
|
||||
|
||||
@@ -10,6 +10,8 @@ authors.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
|
||||
[dependencies]
|
||||
marqdo-span = { workspace = true }
|
||||
@@ -17,6 +19,8 @@ marqdo-value = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = { workspace = true }
|
||||
|
||||
@@ -95,6 +95,15 @@ pub struct KernelDescriptor {
|
||||
/// descriptors must leave this false.
|
||||
#[serde(default)]
|
||||
pub builtin: bool,
|
||||
/// Optional opaque environment id (for example a uv/npm lock id).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub environment_id: Option<String>,
|
||||
/// Optional digest of the environment descriptor/closure.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub environment_digest: Option<String>,
|
||||
/// Optional expected interpreter version string.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub interpreter_version: Option<String>,
|
||||
}
|
||||
|
||||
impl KernelDescriptor {
|
||||
@@ -111,6 +120,9 @@ impl KernelDescriptor {
|
||||
digest: None,
|
||||
permissions_requested: Vec::new(),
|
||||
builtin: true,
|
||||
environment_id: None,
|
||||
environment_digest: None,
|
||||
interpreter_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,6 +143,9 @@ impl KernelDescriptor {
|
||||
digest: None,
|
||||
permissions_requested: Vec::new(),
|
||||
builtin: false,
|
||||
environment_id: None,
|
||||
environment_digest: None,
|
||||
interpreter_version: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -189,6 +204,47 @@ impl KernelRegistry {
|
||||
),
|
||||
});
|
||||
}
|
||||
if let Some(environment) = &descriptor.environment_id {
|
||||
if environment.trim().is_empty() {
|
||||
return Err(ForeignError::Protocol {
|
||||
message: format!(
|
||||
"kernel provider `{}` has an empty environment_id",
|
||||
descriptor.provider_id
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
if let Some(digest) = &descriptor.environment_digest {
|
||||
if !is_sha256_hex(digest) {
|
||||
return Err(ForeignError::Protocol {
|
||||
message: format!(
|
||||
"kernel provider `{}` environment_digest must be 64 lowercase hex characters",
|
||||
descriptor.provider_id
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
if let Some(digest) = &descriptor.digest {
|
||||
if !is_sha256_hex(digest) {
|
||||
return Err(ForeignError::Protocol {
|
||||
message: format!(
|
||||
"kernel provider `{}` digest must be 64 lowercase hex characters",
|
||||
descriptor.provider_id
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
if !descriptor.builtin {
|
||||
if !descriptor.entrypoint.is_absolute() {
|
||||
return Err(ForeignError::Protocol {
|
||||
message: format!(
|
||||
"external kernel provider `{}` entrypoint must be absolute",
|
||||
descriptor.provider_id
|
||||
),
|
||||
});
|
||||
}
|
||||
verify_provider(&descriptor)?;
|
||||
}
|
||||
for language in &mut descriptor.languages {
|
||||
*language = language.trim().to_ascii_lowercase();
|
||||
if language.is_empty() {
|
||||
@@ -366,6 +422,16 @@ pub enum ForeignError {
|
||||
/// What went wrong.
|
||||
message: String,
|
||||
},
|
||||
/// The external provider binary no longer matches its declared digest.
|
||||
#[error("foreign kernel `{provider}` failed integrity verification: expected {expected}, got {actual}")]
|
||||
Integrity {
|
||||
/// Provider id.
|
||||
provider: String,
|
||||
/// Declared digest.
|
||||
expected: String,
|
||||
/// Actual digest.
|
||||
actual: String,
|
||||
},
|
||||
/// Captured stdout/stderr exceeded `max_output_bytes`; the child was
|
||||
/// killed and the partial output dropped.
|
||||
#[error("foreign kernel output exceeded the {limit} byte limit")]
|
||||
@@ -375,6 +441,41 @@ pub enum ForeignError {
|
||||
},
|
||||
}
|
||||
|
||||
fn is_sha256_hex(value: &str) -> bool {
|
||||
value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
fn sha256_file(path: &Path) -> Result<String, ForeignError> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let bytes = fs::read(path).map_err(|error| ForeignError::Io {
|
||||
message: format!("cannot read kernel provider `{}`: {error}", path.display()),
|
||||
})?;
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&bytes);
|
||||
Ok(hex::encode(hasher.finalize()))
|
||||
}
|
||||
|
||||
fn verify_provider(descriptor: &KernelDescriptor) -> Result<(), ForeignError> {
|
||||
if descriptor.builtin {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(expected) = &descriptor.digest else {
|
||||
return Ok(());
|
||||
};
|
||||
let actual = sha256_file(&descriptor.entrypoint)?;
|
||||
if &actual != expected {
|
||||
return Err(ForeignError::Integrity {
|
||||
provider: descriptor.provider_id.clone(),
|
||||
expected: expected.clone(),
|
||||
actual,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Detect a kernel for `language` by scanning `PATH`.
|
||||
///
|
||||
/// Candidates are `python3`, then `python` for Python, and `node` for
|
||||
@@ -494,6 +595,9 @@ fn execute(request: &RunRequest, registry: &KernelRegistry) -> Result<RawRun, Fo
|
||||
request.language
|
||||
),
|
||||
})?;
|
||||
if !descriptor.builtin {
|
||||
verify_provider(descriptor)?;
|
||||
}
|
||||
let program = resolve_program(descriptor).ok_or_else(|| ForeignError::KernelUnavailable {
|
||||
language: descriptor.provider_id.clone(),
|
||||
hint: if descriptor.builtin {
|
||||
@@ -593,6 +697,9 @@ fn is_executable_file(path: &Path) -> bool {
|
||||
|
||||
fn resolve_program(descriptor: &KernelDescriptor) -> Option<PathBuf> {
|
||||
if !descriptor.builtin {
|
||||
if !descriptor.entrypoint.is_absolute() {
|
||||
return None;
|
||||
}
|
||||
return descriptor
|
||||
.entrypoint
|
||||
.is_file()
|
||||
|
||||
@@ -98,3 +98,42 @@ fn external_provider_timeout_is_enforced() {
|
||||
assert!(matches!(error, ForeignError::Timeout { .. }));
|
||||
// The built-in path remains unchanged.
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn provider_digest_is_verified_at_registration_and_execution() {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let script = mock_provider(dir.path());
|
||||
let digest = {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(std::fs::read(&script).unwrap());
|
||||
hex::encode(hasher.finalize())
|
||||
};
|
||||
let mut registry = KernelRegistry::builtin();
|
||||
let mut descriptor = KernelDescriptor::external("mocklang", &["mocklang", "mock"], &script);
|
||||
descriptor.digest = Some(digest.clone());
|
||||
descriptor.environment_id = Some("env-test".to_string());
|
||||
registry.register(descriptor).unwrap();
|
||||
assert_eq!(
|
||||
registry
|
||||
.descriptor("mocklang")
|
||||
.unwrap()
|
||||
.environment_id
|
||||
.as_deref(),
|
||||
Some("env-test")
|
||||
);
|
||||
let result = call_with_registry(&request("mocklang", "ignored"), ®istry).unwrap();
|
||||
assert_eq!(result.status, RunStatus::Success);
|
||||
|
||||
// Tampering after registration must be caught before spawn.
|
||||
std::fs::write(&script, "#!/bin/sh\nexit 0\n").unwrap();
|
||||
let error = call_with_registry(&request("mocklang", "ignored"), ®istry).unwrap_err();
|
||||
assert!(matches!(error, ForeignError::Integrity { .. }), "{error:?}");
|
||||
|
||||
// Malformed digests and relative external entrypoints are rejected.
|
||||
let mut bad = KernelDescriptor::external("bad", &["bad"], "relative.sh");
|
||||
bad.digest = Some("not-a-digest".to_string());
|
||||
assert!(registry.register(bad).is_err());
|
||||
}
|
||||
@@ -268,6 +268,16 @@ fn foreign_error(
|
||||
format!("foreign kernel exceeded its {seconds}s timeout"),
|
||||
),
|
||||
ForeignError::Protocol { message } => (ErrorCategory::ForeignFailure, message),
|
||||
ForeignError::Integrity {
|
||||
provider,
|
||||
expected,
|
||||
actual,
|
||||
} => (
|
||||
ErrorCategory::KernelUnavailable,
|
||||
format!(
|
||||
"foreign kernel `{provider}` failed integrity verification: expected {expected}, got {actual}"
|
||||
),
|
||||
),
|
||||
ForeignError::OutputTooLarge { limit } => (
|
||||
ErrorCategory::ResourceExhausted,
|
||||
format!("foreign kernel exceeded the {limit} byte output limit"),
|
||||
|
||||
+22
-1
@@ -188,6 +188,26 @@ Tool Protocol hardening (`spec/tool-v1.md`):
|
||||
not counted as cross-platform evidence; `docs/STATUS.md` records this rather
|
||||
than treating `cargo test` skips as passes.
|
||||
|
||||
### S5-A/B-12 record (current)
|
||||
|
||||
Foreign provider integrity and environment metadata:
|
||||
|
||||
* `KernelDescriptor` gains `environment_id`, `environment_digest` and
|
||||
`interpreter_version`; external descriptors loaded by the CLI must declare a
|
||||
full provider `digest` and an absolute executable entrypoint.
|
||||
* Descriptors are verified at registration and again before every spawn; a
|
||||
post-registration binary change returns `ForeignError::Integrity` without
|
||||
starting a process. Malformed digests and relative external paths fail closed.
|
||||
* The example provider pins its digest; tests cover success, post-registration
|
||||
tampering and malformed descriptors.
|
||||
* The environment fields are metadata only. uv/npm environment materialization,
|
||||
environment-directory hashing and lock-v3 integration are **not**
|
||||
implemented.
|
||||
* H3-A (capability broker executing real operations) and H3-B (OS isolation)
|
||||
remain open. `CapabilityBroker::authorize` is still policy/audit-only, Worker
|
||||
capability requests are still denied, and `strict` is refused on every
|
||||
platform. No G-C claim is made.
|
||||
|
||||
### M0.6 implementation progress
|
||||
|
||||
| Step | State | Evidence |
|
||||
@@ -199,12 +219,13 @@ Tool Protocol hardening (`spec/tool-v1.md`):
|
||||
| S2-C/H2-B Trace v2 | **Done** | logical root/digest, module-owned events, bounded/truncated trace, independent consumer, hardened worker frames (see §S2-C/H2-B) |
|
||||
| S3-A/H2-C Tool protocol | **Done** | SemVer negotiation at install+dispatch, whole-bundle manifest/digest verification, atomic install/remove, reserved record file (see §S3-A/H2-C) |
|
||||
| S4-A/B-13 B-14 quality gates | **Partial** | real schema instance validation + negative mutations in CI; Windows/macOS runtime tests for core/Analysis/Trace and strict refusal. Tool/C-ABI/worker runtime evidence remains Unix-only (see §S4-A) |
|
||||
| S5-A/B-12 foreign integrity | **Done for metadata/integrity** | provider digest verified at registration and before spawn; environment descriptor fields; full uv/npm materialization and H3-A/B remain open (see §S5-A/B-12) |
|
||||
| R0 baseline + ADRs | **Done** | this section; `docs/adr/0008`–`0012`; full workspace gate |
|
||||
| R1 package-aware identity + callable registry | **Done for statics; superseded by S1 lock v3** | `ModuleId { instance, relative_path }`; `PackageResolver` path sources; lock v3 transitive graph (ADR-0016) replaces the flat v2 lock; `CallableRegistry` for builtins/marqdo/declared ABI exports; external manifest wiring is R2 |
|
||||
| R2 ABI/Kernel/Broker | **Done (worker = crash isolation)** | `KernelRegistry` external providers; complete-name export routing; `CapabilityBroker`; static `check --plugin`; multi-export C test; isolated worker with timeout/crash tests; host handle table. Native/worker are not an OS sandbox |
|
||||
| R3 Tool Protocol | **Done (S3 hardened)** | `tool install/list/remove`; unknown-command fallback; SemVer + Analysis/Trace negotiation; whole-bundle digest verification; staged install and rename-based removal; black-box tests for arbitrary names, Unicode args, conflicts, traversal, PATH hijacking and bundle tampering |
|
||||
| R4 Analysis/Trace v2 | **Done** | `check --emit` Analysis v2 and `run/call --trace-file` Trace v2 with logical identity, resources, module-owned events, capacity bounds, schemas and independent Python consumers; stale-artifact removal; value redaction |
|
||||
| R5 environment/isolation | **Partial** | Pinned Git fetch with verified cache markers, transitive `@alias` resolution, full-closure atomic `.marqdo/env` materialization, foreign process-group cleanup. Strict OS isolation remains explicitly unavailable (fail-closed) |
|
||||
| R5 environment/isolation | **Partial** | Pinned Git fetch with verified cache markers, transitive `@alias` resolution, full-closure atomic `.marqdo/env` materialization, foreign process-group cleanup, verified foreign provider digests and environment descriptor metadata. Strict OS isolation and uv/npm environment materialization remain explicitly unavailable (fail-closed) |
|
||||
| R6 release/consumers | **Partial** | examples for unknown-name Tool, multi-export plugin, external kernel and multi-version packages; Analysis/Trace/Tool JSON Schemas with instance validation; Linux full gate + Windows/macOS workspace tests and portable analysis/trace smoke; Unix-only Tool/ABI/worker runtime evidence |
|
||||
|
||||
Reproduce the evidence:
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
{
|
||||
"provider_id": "mocklang",
|
||||
"languages": ["mocklang", "mock"],
|
||||
"languages": [
|
||||
"mocklang",
|
||||
"mock"
|
||||
],
|
||||
"wire_version": 1,
|
||||
"entrypoint": "./mocklang",
|
||||
"permissions_requested": []
|
||||
"permissions_requested": [],
|
||||
"digest": "9083fd340d06a9e3e0356a9f5b202ff8d61bcd22b2aaedd190f7c593bbeab23a",
|
||||
"environment_id": "mocklang-env-1"
|
||||
}
|
||||
+17
-5
@@ -44,7 +44,8 @@ heading text.
|
||||
|
||||
`KernelRegistry` maps case-insensitive language labels to an explicit
|
||||
`KernelDescriptor` (`provider_id`, `languages`, `wire_version`, `entrypoint`,
|
||||
optional digest, advisory permission requests). Built-in providers are:
|
||||
`digest`, optional `environment_id`/`environment_digest`/`interpreter_version`,
|
||||
advisory permission requests). Built-in providers are:
|
||||
|
||||
| Canonical | Accepted aliases | Interpreter lookup |
|
||||
|---|---|---|
|
||||
@@ -53,10 +54,21 @@ optional digest, advisory permission requests). Built-in providers are:
|
||||
|
||||
Additional providers are registered with
|
||||
`KernelDescriptor::external(provider_id, languages, entrypoint)` and an exact
|
||||
wire version (`1`). External entrypoints must be explicit executable files, not
|
||||
PATH names. Duplicate language labels and incompatible wire versions are
|
||||
rejected at registration. The host owns a registry (`StdHost::kernel_registry`,
|
||||
`StdHost::register_kernel`); the interpreter never hard-codes a language list.
|
||||
wire version (`1`). External entrypoints must be absolute executable files, not
|
||||
PATH names, and descriptors loaded by the CLI must declare a 64-character
|
||||
lowercase hex `digest` of the provider program. The digest is verified at
|
||||
registration and again immediately before every execution; a mismatch returns
|
||||
`ForeignError::Integrity` before spawn, so a provider tampered with after
|
||||
registration cannot run. Duplicate language labels and incompatible wire
|
||||
versions are rejected at registration. The host owns a registry
|
||||
(`StdHost::kernel_registry`, `StdHost::register_kernel`); the interpreter never
|
||||
hard-codes a language list.
|
||||
|
||||
`environment_id`, `environment_digest` and `interpreter_version` record the
|
||||
intended reproducible environment. This build validates their shape and echoes
|
||||
`environment_id` through `KernelInfo`; it does **not** materialize uv/npm
|
||||
environments or integrate them into lock v3. A production descriptor should pin
|
||||
all three once a materializer exists.
|
||||
|
||||
A `Code` value for an unregistered label is still valid inert data. Only an
|
||||
explicit `foreign.run`/`foreign.call` lookup can fail with
|
||||
|
||||
Reference in new issue
Block a user