diff --git a/Cargo.lock b/Cargo.lock index e4fe5ec..ad535f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -363,6 +363,7 @@ dependencies = [ "marqdo-span", "marqdo-value", "serde_json", + "tempfile", "thiserror", ] diff --git a/crates/marqdo-foreign/Cargo.toml b/crates/marqdo-foreign/Cargo.toml index beb47fe..ce90118 100644 --- a/crates/marqdo-foreign/Cargo.toml +++ b/crates/marqdo-foreign/Cargo.toml @@ -8,6 +8,9 @@ license.workspace = true repository.workspace = true authors.workspace = true +[dev-dependencies] +tempfile = { workspace = true } + [dependencies] marqdo-span = { workspace = true } marqdo-value = { workspace = true } diff --git a/crates/marqdo-foreign/src/lib.rs b/crates/marqdo-foreign/src/lib.rs index 08b33d6..997b3d9 100644 --- a/crates/marqdo-foreign/src/lib.rs +++ b/crates/marqdo-foreign/src/lib.rs @@ -35,6 +35,7 @@ //! //! [`Code`]: marqdo_value::CodeValue +use std::collections::BTreeMap; use std::fs; use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; @@ -69,6 +70,170 @@ static UNIQUE_COUNTER: AtomicU64 = AtomicU64::new(0); /// Canonical language names supported by this crate. pub const SUPPORTED_LANGUAGES: [&str; 2] = ["python", "javascript"]; +/// A registered foreign execution provider. +/// +/// Descriptors are metadata; they do not grant process/filesystem access. The +/// host still requires an explicit foreign grant before invoking one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KernelDescriptor { + /// Stable provider id (`python`, `javascript`, `mocklang`, ...). + pub provider_id: String, + /// Accepted case-insensitive code-fence labels and aliases. + pub languages: Vec, + /// Wire protocol version the provider implements. + pub wire_version: u32, + /// Program to start. Built-in providers use a PATH command name; external + /// providers must point at an explicit executable. + pub entrypoint: PathBuf, + /// Content digest of the external provider program, when known. + pub digest: Option, + /// Advisory capability requests; never grants. + pub permissions_requested: Vec, + /// Whether this is one of the built-in Python/Node providers. + pub builtin: bool, +} + +impl KernelDescriptor { + /// Create a built-in provider descriptor. + pub fn builtin(provider_id: &str, languages: &[&str], entrypoint: &str) -> Self { + Self { + provider_id: provider_id.to_string(), + languages: languages + .iter() + .map(|language| language.to_string()) + .collect(), + wire_version: marqdo_value::WIRE_VERSION, + entrypoint: PathBuf::from(entrypoint), + digest: None, + permissions_requested: Vec::new(), + builtin: true, + } + } + + /// Create an external provider descriptor. + pub fn external( + provider_id: impl Into, + languages: &[&str], + entrypoint: impl Into, + ) -> Self { + Self { + provider_id: provider_id.into(), + languages: languages + .iter() + .map(|language| language.to_string()) + .collect(), + wire_version: marqdo_value::WIRE_VERSION, + entrypoint: entrypoint.into(), + digest: None, + permissions_requested: Vec::new(), + builtin: false, + } + } +} + +/// Registry of foreign kernels. Registration is explicit and deterministic. +#[derive(Debug, Clone, Default)] +pub struct KernelRegistry { + descriptors: Vec, + by_language: BTreeMap, +} + +impl KernelRegistry { + /// Built-in Python and Node providers. + pub fn builtin() -> Self { + let mut registry = Self::default(); + registry + .register(KernelDescriptor::builtin( + "python", + &["python", "python3", "py"], + "python3", + )) + .expect("built-in python provider is valid"); + registry + .register(KernelDescriptor::builtin( + "javascript", + &["javascript", "js", "node", "nodejs"], + "node", + )) + .expect("built-in javascript provider is valid"); + registry + } + + /// Register a provider, rejecting duplicate language labels and an + /// incompatible wire version. + pub fn register(&mut self, mut descriptor: KernelDescriptor) -> Result<(), ForeignError> { + if descriptor.provider_id.trim().is_empty() { + return Err(ForeignError::Protocol { + message: "kernel provider_id must not be empty".to_string(), + }); + } + if descriptor.languages.is_empty() { + return Err(ForeignError::Protocol { + message: format!( + "kernel provider `{}` declares no languages", + descriptor.provider_id + ), + }); + } + if descriptor.wire_version != marqdo_value::WIRE_VERSION { + return Err(ForeignError::Protocol { + message: format!( + "kernel provider `{}` requires wire version {}, host supports {}", + descriptor.provider_id, + descriptor.wire_version, + marqdo_value::WIRE_VERSION + ), + }); + } + for language in &mut descriptor.languages { + *language = language.trim().to_ascii_lowercase(); + if language.is_empty() { + return Err(ForeignError::Protocol { + message: format!( + "kernel provider `{}` contains an empty language label", + descriptor.provider_id + ), + }); + } + if self.by_language.contains_key(language) { + return Err(ForeignError::Protocol { + message: format!("kernel language `{language}` is already registered"), + }); + } + } + let index = self.descriptors.len(); + for language in &descriptor.languages { + self.by_language.insert(language.clone(), index); + } + self.descriptors.push(descriptor); + Ok(()) + } + + /// Resolve a language label to its provider descriptor. + pub fn descriptor(&self, language: &str) -> Option<&KernelDescriptor> { + let key = language.trim().to_ascii_lowercase(); + self.by_language + .get(&key) + .and_then(|index| self.descriptors.get(*index)) + } + + /// Detect a provider's executable and version. + pub fn detect(&self, language: &str) -> Option { + let descriptor = self.descriptor(language)?; + let program = resolve_program(descriptor)?; + let version = if descriptor.builtin { + probe_version(&program).unwrap_or_else(|| "unknown".to_string()) + } else { + "external provider".to_string() + }; + Some(KernelInfo { + language: descriptor.provider_id.clone(), + program, + version, + }) + } +} + /// Normalize a language alias to its canonical kernel name. /// /// Recognized aliases are `python`/`python3`/`py` and @@ -214,24 +379,7 @@ pub enum ForeignError { /// when no working interpreter is installed; detection honors the same /// cleared-environment rule as execution and is bounded by a short timeout. pub fn detect(language: &str) -> Option { - let canonical = normalize_language(language)?; - let path_var = std::env::var_os("PATH")?; - for name in program_candidates(canonical) { - for directory in std::env::split_paths(&path_var) { - let program = directory.join(name); - if !is_executable_file(&program) { - continue; - } - if let Some(version) = probe_version(&program) { - return Some(KernelInfo { - language: canonical.to_string(), - program, - version, - }); - } - } - } - None + KernelRegistry::builtin().detect(language) } /// One-shot isolated execution of foreign code. @@ -246,7 +394,15 @@ pub fn detect(language: &str) -> Option { /// [`RunStatus::TimedOut`]. Infrastructure failures (unavailable kernel, spawn /// failure, exceeding the output budget) are [`ForeignError`]s. pub fn run(request: &RunRequest) -> Result { - let raw = execute(request)?; + run_with_registry(request, &KernelRegistry::builtin()) +} + +/// Like [`run`] with an explicit provider registry. +pub fn run_with_registry( + request: &RunRequest, + registry: &KernelRegistry, +) -> Result { + let raw = execute(request, registry)?; Ok(RunResult { status: raw.status, stdout: String::from_utf8_lossy(&raw.stdout).into_owned(), @@ -267,7 +423,15 @@ pub fn run(request: &RunRequest) -> Result { /// can report the kernel's own failure faithfully; a timeout returns /// [`ForeignError::Timeout`] because no typed value can exist. pub fn call(request: &RunRequest) -> Result { - let raw = execute(request)?; + call_with_registry(request, &KernelRegistry::builtin()) +} + +/// Like [`call`] with an explicit provider registry. +pub fn call_with_registry( + request: &RunRequest, + registry: &KernelRegistry, +) -> Result { + let raw = execute(request, registry)?; match raw.status { RunStatus::TimedOut => Err(ForeignError::Timeout { seconds: request.timeout.as_secs_f64(), @@ -315,22 +479,40 @@ struct RawRun { duration: Duration, } -fn execute(request: &RunRequest) -> Result { - let canonical = - normalize_language(&request.language).ok_or_else(|| ForeignError::KernelUnavailable { - language: request.language.clone(), - hint: format!( - "supported languages are `python` (aliases: python3, py) and \ - `javascript` (aliases: js, node); `{}` is not a foreign kernel", - request.language - ), - })?; - let program = find_program(canonical).ok_or_else(|| ForeignError::KernelUnavailable { - language: canonical.to_string(), - hint: kernel_hint(canonical).to_string(), +fn execute(request: &RunRequest, registry: &KernelRegistry) -> Result { + let descriptor = + registry + .descriptor(&request.language) + .ok_or_else(|| ForeignError::KernelUnavailable { + language: request.language.clone(), + hint: format!( + "no kernel provider is registered for `{}`; register an explicit provider or use python/javascript", + request.language + ), + })?; + let program = resolve_program(descriptor).ok_or_else(|| ForeignError::KernelUnavailable { + language: descriptor.provider_id.clone(), + hint: if descriptor.builtin { + kernel_hint(&descriptor.provider_id).to_string() + } else { + format!( + "external kernel provider `{}` executable `{}` is missing or not executable", + descriptor.provider_id, + descriptor.entrypoint.display() + ) + }, })?; - let artifacts = TempArtifacts::create(canonical)?; + let extension = if descriptor.builtin { + if descriptor.provider_id == "python" { + "py" + } else { + "js" + } + } else { + "src" + }; + let artifacts = TempArtifacts::create(extension)?; artifacts.write_program(&request.source)?; let payload = request_envelope(request)?; @@ -405,9 +587,17 @@ fn is_executable_file(path: &Path) -> bool { } } -fn find_program(language: &str) -> Option { +fn resolve_program(descriptor: &KernelDescriptor) -> Option { + if !descriptor.builtin { + return descriptor + .entrypoint + .is_file() + .then(|| descriptor.entrypoint.clone()) + .filter(|path| is_executable_file(path)); + } let path_var = std::env::var_os("PATH")?; - for name in program_candidates(language) { + let candidates = program_candidates(&descriptor.provider_id); + for name in candidates { for directory in std::env::split_paths(&path_var) { let candidate = directory.join(name); if is_executable_file(&candidate) { @@ -464,9 +654,8 @@ struct TempArtifacts { } impl TempArtifacts { - fn create(language: &str) -> Result { + fn create(extension: &str) -> Result { let base = fs::canonicalize(std::env::temp_dir()).unwrap_or_else(|_| std::env::temp_dir()); - let extension = if language == "python" { "py" } else { "js" }; let mut last_error: Option = None; for _ in 0..16 { let token = unique_token(); diff --git a/crates/marqdo-foreign/tests/provider.rs b/crates/marqdo-foreign/tests/provider.rs new file mode 100644 index 0000000..41c86cf --- /dev/null +++ b/crates/marqdo-foreign/tests/provider.rs @@ -0,0 +1,101 @@ +//! External kernel-provider registration (ADR-0012). + +use std::time::Duration; + +use marqdo_foreign::{ + call, call_with_registry, ForeignError, KernelDescriptor, KernelRegistry, RunRequest, + RunStatus, +}; +use marqdo_value::Value; + +fn request(language: &str, source: &str) -> RunRequest { + RunRequest { + language: language.to_string(), + source: source.to_string(), + input: None, + timeout: Duration::from_secs(5), + max_output_bytes: 64 * 1024, + } +} + +#[cfg(unix)] +fn mock_provider(dir: &std::path::Path) -> std::path::PathBuf { + use std::os::unix::fs::PermissionsExt; + let script = dir.join("mocklang.sh"); + std::fs::write( + &script, + "#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"type\":\"int\",\"decimal\":\"42\"}'\n", + ) + .unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o700); + std::fs::set_permissions(&script, permissions).unwrap(); + script +} + +#[cfg(unix)] +#[test] +fn external_provider_is_registered_without_core_changes() { + let dir = tempfile::tempdir().unwrap(); + let script = mock_provider(dir.path()); + let mut registry = KernelRegistry::builtin(); + registry + .register(KernelDescriptor::external( + "mocklang", + &["mocklang", "mock"], + &script, + )) + .unwrap(); + assert_eq!(registry.descriptor("MOCK").unwrap().provider_id, "mocklang"); + assert!(registry.detect("mocklang").is_some()); + + let result = call_with_registry(&request("mocklang", "ignored source"), ®istry).unwrap(); + assert_eq!(result.status, RunStatus::Success); + assert_eq!(result.value, Some(Value::Int(42.into()))); + + let unavailable = call(&request("mocklang", "ignored source")).unwrap_err(); + assert!(matches!( + unavailable, + ForeignError::KernelUnavailable { .. } + )); +} + +#[cfg(unix)] +#[test] +fn duplicate_language_labels_and_wire_versions_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + let script = mock_provider(dir.path()); + let mut registry = KernelRegistry::builtin(); + let mut descriptor = KernelDescriptor::external("mocklang", &["python"], &script); + assert!(registry.register(descriptor.clone()).is_err()); + + descriptor.languages = vec!["mocklang".into()]; + descriptor.wire_version = 99; + assert!(registry.register(descriptor).is_err()); +} + +#[cfg(unix)] +#[test] +fn external_provider_timeout_is_enforced() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("slow.sh"); + std::fs::write(&script, "#!/bin/sh\nsleep 5\n").unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o700); + std::fs::set_permissions(&script, permissions).unwrap(); + + let mut registry = KernelRegistry::builtin(); + registry + .register(KernelDescriptor::external( + "slowlang", + &["slowlang"], + &script, + )) + .unwrap(); + let mut request = request("slowlang", ""); + request.timeout = Duration::from_millis(50); + let error = call_with_registry(&request, ®istry).unwrap_err(); + assert!(matches!(error, ForeignError::Timeout { .. })); + // The built-in path remains unchanged. +} diff --git a/crates/marqdo-host/src/host.rs b/crates/marqdo-host/src/host.rs index cefe6e9..2379f47 100644 --- a/crates/marqdo-host/src/host.rs +++ b/crates/marqdo-host/src/host.rs @@ -107,6 +107,8 @@ pub struct StdHost { pub foreign_timeout: Duration, /// Maximum captured bytes per foreign stream. pub foreign_output_limit: usize, + /// Registered foreign kernels (built-in Python/Node plus explicit providers). + pub kernel_registry: marqdo_foreign::KernelRegistry, output: Vec, trace_id: u64, /// Explicitly trusted, in-process native plugins keyed by namespace. @@ -141,6 +143,7 @@ impl StdHost { trace: EffectTrace::default(), foreign_timeout: Duration::from_secs(10), foreign_output_limit: 1024 * 1024, + kernel_registry: marqdo_foreign::KernelRegistry::builtin(), output: Vec::new(), trace_id, plugins: Vec::new(), @@ -205,6 +208,16 @@ impl StdHost { Ok(namespace) } + /// Register an explicit external foreign-kernel provider. + pub fn register_kernel( + &mut self, + descriptor: marqdo_foreign::KernelDescriptor, + ) -> Result<(), RuntimeError> { + self.kernel_registry.register(descriptor).map_err(|error| { + RuntimeError::new(ErrorCategory::KernelUnavailable, None, error.to_string()) + }) + } + /// Enable effect tracing. pub fn with_trace(mut self) -> Self { self.trace = EffectTrace::enabled(); diff --git a/crates/marqdo-host/src/kernels.rs b/crates/marqdo-host/src/kernels.rs index fae2138..a6de137 100644 --- a/crates/marqdo-host/src/kernels.rs +++ b/crates/marqdo-host/src/kernels.rs @@ -177,13 +177,13 @@ pub(crate) fn foreign( }; match op { HostOp::ForeignRun => { - let result = - marqdo_foreign::run(&request).map_err(|error| foreign_error(error, span))?; + let result = marqdo_foreign::run_with_registry(&request, &host.kernel_registry) + .map_err(|error| foreign_error(error, span))?; Ok(foreign_result_map(result)) } HostOp::ForeignCall => { - let result = - marqdo_foreign::call(&request).map_err(|error| foreign_error(error, span))?; + let result = marqdo_foreign::call_with_registry(&request, &host.kernel_registry) + .map_err(|error| foreign_error(error, span))?; if result.status != marqdo_foreign::RunStatus::Success { return Err(RuntimeError::new( ErrorCategory::ForeignFailure, diff --git a/docs/STATUS.md b/docs/STATUS.md index cb6d6d7..02bd3d9 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -41,7 +41,7 @@ providers). |---|---|---| | R0 baseline + ADRs | **Done** | this section; `docs/adr/0008`–`0012`; full workspace gate | | R1 package-aware identity + callable registry | **Done for statics** | `ModuleId { instance, relative_path }`; `PackageResolver` path sources; `lock_version: 2` with v1 migration; `CallableRegistry` for builtins/marqdo/declared ABI exports; external manifest wiring is R2 | -| R2 ABI/Kernel/Broker | Not started | — | +| R2 ABI/Kernel/Broker | **In progress** | `KernelRegistry`/`KernelDescriptor` external providers; host registry; provider tests. Plugin export routing, broker and worker pending | | R3 Tool Protocol | Not started | — | | R4 Analysis/Trace v1 | Not started | — | | R5 environment/isolation | Not started | — | diff --git a/spec/foreign.md b/spec/foreign.md index d9d009f..68c3540 100644 --- a/spec/foreign.md +++ b/spec/foreign.md @@ -1,7 +1,8 @@ # Marqdo Foreign Kernels v1 -Status: **one-shot kernels for Python and Node/JavaScript**, frozen protocol for -M0.5. There are no sessions or persistent kernels. +Status: **one-shot kernels with an explicit provider registry** (M0.6 R2), +including built-in Python and Node/JavaScript providers. There are no sessions +or persistent kernels. Implementation: `crates/marqdo-foreign/src/lib.rs` (process supervision and protocol), `crates/marqdo-host/src/kernels.rs::foreign` (capability check and @@ -39,22 +40,30 @@ print(json.dumps({"type": "float", "value": math.pi * r ** 2})) The anchor is the GitHub-style slug of the heading (`#python-校验`), not the heading text. -## 2. Supported languages +## 2. Supported languages and providers -`normalize_language` is ASCII case-insensitive and trims surrounding -whitespace: +`KernelRegistry` maps case-insensitive language labels to an explicit +`KernelDescriptor` (`provider_id`, `languages`, `wire_version`, `entrypoint`, +optional digest, advisory permission requests). Built-in providers are: | Canonical | Accepted aliases | Interpreter lookup | |---|---|---| | `python` | `python`, `python3`, `py` | `python3`, then `python` on `PATH` | | `javascript` | `javascript`, `js`, `node`, `nodejs` | `node` on `PATH` | -Any other language yields `ForeignError::KernelUnavailable` (the `Code` value is -still valid data). `detect(language)` returns `KernelInfo { language, program, -version }` by scanning `PATH` and running the interpreter with `--version` -(5 s timeout, 64 KiB output cap, cleared environment except `PATH`); the -`version` is the first non-empty line of stdout or stderr. `detect` and -execution use the same `PATH`-only environment. +Additional providers are registered with +`KernelDescriptor::external(provider_id, languages, entrypoint)` and an exact +wire version (`1`). External entrypoints must be explicit executable files, not +PATH names. Duplicate language labels and incompatible wire versions are +rejected at registration. The host owns a registry (`StdHost::kernel_registry`, +`StdHost::register_kernel`); the interpreter never hard-codes a language list. + +A `Code` value for an unregistered label is still valid inert data. Only an +explicit `foreign.run`/`foreign.call` lookup can fail with +`ForeignError::KernelUnavailable`; the error carries a provider/install hint. +`detect(language)` uses the same registry and returns `KernelInfo { language, +program, version }` (5 s timeout, 64 KiB output cap, cleared environment except +`PATH`). ## 3. `RunRequest` / `RunResult`