Compare commits

...
9 Commits
Author SHA1 Message Date
cflmy 24c88bcc78 fix(resource): reject provider read and write over-reports
CI / fmt + clippy + test (Linux x86_64) (push) Waiting to run
CI / schema instance validation + consumers (push) Waiting to run
CI / runtime contract (macos-latest) (push) Waiting to run
CI / runtime contract (windows-latest) (push) Waiting to run
ci/woodpecker/push/woodpecker Pipeline was successful
2026-11-21 01:05:14 +08:00
cflmy 6aa3d605af fix(stdlib): return the validated write-all byte count 2026-11-21 01:02:45 +08:00
cflmy e6b5678350 fix(stdlib): strip CR from console lines 2026-11-21 00:59:58 +08:00
cflmy 4a0caabc03 fix(runtime): keep resource.read destination bounds strict 2026-11-21 00:58:42 +08:00
cflmy 23e68cba5f docs(resource): record stdio stream endpoints and stdlib APIs 2026-11-21 00:58:34 +08:00
cflmy 02aecaa747 test(conformance): load nested stdlib imports without false cycles 2026-11-21 00:58:34 +08:00
cflmy 64a6b48866 fix(resource): share one stdin cursor across handles 2026-11-21 00:49:07 +08:00
cflmy 3b13d8499a feat(stdlib): add streaming IO and console line APIs 2026-11-21 00:48:46 +08:00
cflmy aed6a80ba7 feat(resource): add stdin and stderr stdio endpoints 2026-11-21 00:36:25 +08:00
21 changed files with 1633 additions and 77 deletions

No files matched your search

+31 -1
View File
@@ -815,7 +815,8 @@ pub fn execute(
return 2;
}
};
let mut host = StdHost::new(mode, capabilities, project.root.clone());
let stdin = read_stdin();
let mut host = StdHost::new(mode, capabilities, project.root.clone()).with_stdin(stdin);
if trace || json || settings.trace_file.is_some() {
host = host
.with_trace()
@@ -935,6 +936,8 @@ pub fn execute(
}
let stdout = host.take_output();
let stdout_text = String::from_utf8_lossy(&stdout).to_string();
let stderr = host.take_error();
let stderr_text = String::from_utf8_lossy(&stderr).to_string();
let trace_entries = host.trace().to_vec();
match result {
Ok(outcome) => {
@@ -944,11 +947,15 @@ pub fn execute(
"success": true,
"value": marqdo_value::to_wire(&outcome.value),
"stdout": stdout_text,
"stderr": stderr_text,
"diagnostics": output::diagnostics_json(&project.diagnostics),
"trace": trace_entries,
}));
} else {
use std::io::Write;
if !stderr.is_empty() {
let _ = std::io::stderr().write_all(&stderr);
}
let _ = std::io::stdout().write_all(&stdout);
if function.is_some() && outcome.value != Value::Unit {
println!("{}", outcome.value.display());
@@ -970,11 +977,15 @@ pub fn execute(
"command": command_name,
"success": false,
"stdout": stdout_text,
"stderr": stderr_text,
"error": runtime_error_json(&error, &project),
"trace": trace_entries,
}));
} else {
use std::io::Write;
if !stderr.is_empty() {
let _ = std::io::stderr().write_all(&stderr);
}
let _ = std::io::stdout().write_all(&stdout);
eprint!("{}", render_runtime_error(&error, &project));
}
@@ -983,6 +994,25 @@ pub fn execute(
}
}
/// Read piped stdin once at the composition boundary.
///
/// A terminal stdin is not read, so `marqdo run` never blocks waiting for an
/// interactive EOF. Non-terminal input is a finite pipe/file and is copied into
/// the run-scoped `stdio/stdin` resource.
fn read_stdin() -> Vec<u8> {
use std::io::{IsTerminal, Read};
let mut stdin = std::io::stdin();
if stdin.is_terminal() {
return Vec::new();
}
let mut bytes = Vec::new();
if let Err(error) = stdin.read_to_end(&mut bytes) {
eprintln!("error: cannot read stdin: {error}");
return Vec::new();
}
bytes
}
fn load_kernel_descriptor(host: &mut StdHost, path: &Path) -> Result<(), String> {
let text = std::fs::read_to_string(path).map_err(|error| {
format!(
+36 -10
View File
@@ -266,6 +266,26 @@ fn semantic_goldens_match_end_to_end_execution() {
);
}
fn normalise_relative(path: &Path) -> String {
let mut parts: Vec<std::ffi::OsString> = Vec::new();
for component in path.components() {
match component {
std::path::Component::Normal(part) => parts.push(part.to_os_string()),
std::path::Component::ParentDir => {
parts.pop();
}
std::path::Component::CurDir
| std::path::Component::RootDir
| std::path::Component::Prefix(_) => {}
}
}
parts
.iter()
.map(|part| part.to_string_lossy().to_string())
.collect::<Vec<_>>()
.join("/")
}
/// Load the bundled stdlib print modules for direct-runtime fixtures.
///
/// The CLI loader normally injects these for Compat-B. The semantic harness
@@ -294,13 +314,23 @@ fn load_stdlib_module(
program: &mut Program,
loaded: &mut std::collections::BTreeSet<String>,
) -> Result<ModuleId, String> {
if !loaded.insert(relative.to_string()) {
let root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
let relative = normalise_relative(Path::new(relative));
if loaded.contains(&relative) {
if let Some((id, _)) = program
.modules
.iter()
.find(|(id, _)| id.relative_path == relative)
{
return Ok(id.clone());
}
return Err(format!("stdlib import cycle at `{relative}`"));
}
let path = root.join(relative);
loaded.insert(relative.clone());
let path = root.join(&relative);
let source = fs::read_to_string(&path)
.map_err(|error| format!("cannot read `{}`: {error}", path.display()))?;
let module_id = ModuleId::new("std:test", relative.replace('\\', "/"));
let module_id = ModuleId::new("std:test", relative.clone());
let analysis = marqdo_frontend::analyze_source(&module_id.to_string(), &source);
if analysis.has_errors() {
return Err(format!("stdlib module `{relative}` has diagnostics"));
@@ -314,18 +344,14 @@ fn load_stdlib_module(
for (alias, target) in imports {
let target = target.strip_prefix("std:").unwrap_or(&target);
let child = if target.starts_with("./") || target.starts_with("../") {
let base = Path::new(relative)
let base = Path::new(&relative)
.parent()
.unwrap_or_else(|| Path::new(""));
base.join(target)
.components()
.map(|component| component.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>()
.join("/")
normalise_relative(&base.join(target))
} else {
target.to_string()
};
let to = load_stdlib_module(root, &child, program, loaded)?;
let to = load_stdlib_module(&root, &child, program, loaded)?;
program.resolved_imports.push(ResolvedImport {
from: module_id.clone(),
alias,
+152 -1
View File
@@ -1,7 +1,8 @@
//! End-to-end resource stream tests through the CLI.
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use std::process::{Command, Output, Stdio};
struct TempDir(PathBuf);
@@ -33,6 +34,24 @@ fn cli(dir: &Path, args: &[&str]) -> Output {
.expect("run marqdo")
}
fn cli_with_stdin(dir: &Path, args: &[&str], input: &[u8]) -> Output {
let mut child = Command::new(env!("CARGO_BIN_EXE_marqdo"))
.args(args)
.current_dir(dir)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("spawn marqdo");
child
.stdin
.take()
.expect("stdin pipe")
.write_all(input)
.expect("write stdin");
child.wait_with_output().expect("wait for marqdo")
}
fn write_file(path: &Path, contents: &str) {
std::fs::write(path, contents).unwrap();
}
@@ -295,3 +314,135 @@ fn file_stream_requires_a_read_grant() {
assert_eq!(report["success"], false);
assert_eq!(report["error"]["code"], "permission.denied");
}
const STDIN_READER: &str = r#"## main
**输入 = resource.open("stdio", "stdin", ["read"])**
**内存 = memory.new(1, 1)**
**第一 = resource.read(输入, 内存, 0, 2)**
**第二 = resource.read(输入, 内存, 2, 8)**
**第三 = resource.read(输入, 内存, 0, 1)**
*[第一.value.count, 第一.value.eof, 第二.value.count, 第二.value.eof, 第三.value.eof, 内存[0], 内存[1], 内存[2], 内存[3]]*
"#;
#[test]
fn piped_stdin_is_chunked_and_reaches_eof() {
let dir = TempDir::new("stdin");
write_file(&dir.path().join("index.mq.md"), STDIN_READER);
let output = cli_with_stdin(dir.path(), &["run", "index.mq.md", "--json"], b"abcd");
assert_eq!(
output.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let report: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json output");
assert_eq!(report["success"], true);
let values = report["value"]["items"].as_array().expect("list result");
assert_eq!(values[0]["decimal"], "2");
assert_eq!(values[1]["value"], false);
assert_eq!(values[2]["decimal"], "2");
assert_eq!(values[3]["value"], true);
assert_eq!(values[4]["value"], true);
for (index, expected) in ["97", "98", "99", "100"].into_iter().enumerate() {
assert_eq!(values[5 + index]["decimal"], expected);
}
}
const STDERR_WRITER: &str = r#"## main
**错误 = resource.open("stdio", "stderr", ["write"])**
**输出 = resource.open("stdio", "stdout", ["write"])**
**内存 = memory.new(1, 1)**
**`内存[0] = 101`**
**`内存[1] = 114`**
**`内存[2] = 114`**
**写错误 = resource.write(错误, 内存, 0, 3)**
**`内存[0] = 111`**
**`内存[1] = 117`**
**`内存[2] = 116`**
**写输出 = resource.write(输出, 内存, 0, 3)**
**关闭错误 = resource.close(错误)**
**关闭输出 = resource.close(输出)**
*[写错误.value.count, 写输出.value.count, 关闭错误.ok, 关闭输出.ok]*
"#;
#[test]
fn program_stderr_is_separate_from_stdout_and_diagnostics() {
let dir = TempDir::new("stderr");
write_file(&dir.path().join("index.mq.md"), STDERR_WRITER);
let output = cli(dir.path(), &["run", "index.mq.md", "--json"]);
assert_eq!(
output.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let report: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json output");
assert_eq!(report["success"], true);
assert_eq!(report["stdout"], "out");
assert_eq!(report["stderr"], "err");
let values = report["value"]["items"].as_array().expect("list result");
assert_eq!(values[0]["decimal"], "3");
assert_eq!(values[1]["decimal"], "3");
assert_eq!(values[2]["value"], true);
assert_eq!(values[3]["value"], true);
}
const CONSOLE_READER: &str = r#"---
import:
console: "std:io/console"
---
## main
**第一 = console.读取行()**
**第二 = console.读取行()**
**第三 = console.读取行()**
**输出 = console.打印(第一.value + "|" + 第二.value)**
*[第一.ok, 第二.ok, 第三.ok, 第三.code, 输出.ok]*
"#;
#[test]
fn stdlib_console_reads_lines_and_reports_eof() {
let dir = TempDir::new("console");
write_file(&dir.path().join("index.mq.md"), CONSOLE_READER);
let output = cli_with_stdin(dir.path(), &["run", "index.mq.md", "--json"], b"one\ntwo\n");
assert_eq!(
output.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let report: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json output");
assert_eq!(report["success"], true);
assert_eq!(report["stdout"], "one|two\n");
let values = report["value"]["items"].as_array().expect("list result");
assert_eq!(values[0]["value"], true);
assert_eq!(values[1]["value"], true);
assert_eq!(values[2]["value"], false);
assert_eq!(values[3]["value"], "io.eof");
assert_eq!(values[4]["value"], true);
}
+45 -7
View File
@@ -203,6 +203,8 @@ pub struct StdHost {
broker_audit: Arc<Mutex<Vec<CapabilityAudit>>>,
/// Host-issued opaque plugin handles.
pub handles: PluginHandleTable,
/// Program stderr captured separately from stdout and diagnostics.
stderr: Vec<u8>,
/// Host-owned resource/stream table (ADR-0025).
resources: ResourceTable,
/// Registered in-process providers keyed by resource kind (ADR-0033).
@@ -267,6 +269,7 @@ impl StdHost {
vendor_capabilities: BTreeSet::new(),
broker_audit: Arc::new(Mutex::new(Vec::new())),
handles: PluginHandleTable::default(),
stderr: Vec::new(),
resources: ResourceTable::new(),
resource_providers: BTreeMap::new(),
semantic_events: Vec::new(),
@@ -476,6 +479,25 @@ impl StdHost {
&self.output
}
/// Attach run-scoped stdin bytes at assembly time (composition root).
///
/// This does not start a thread, spawn a process or read a hidden second
/// channel. A program reads these bytes only through `stdio/stdin`.
pub fn with_stdin(mut self, bytes: impl Into<Vec<u8>>) -> Self {
self.resources.set_stdin(bytes.into());
self
}
/// Buffered program stderr without consuming it.
pub fn error(&self) -> &[u8] {
&self.stderr
}
/// Take buffered program stderr.
pub fn take_error(&mut self) -> Vec<u8> {
std::mem::take(&mut self.stderr)
}
fn dispatch(
&mut self,
op: HostOp,
@@ -728,7 +750,8 @@ impl StdHost {
let requested: BTreeSet<&str> = rights.iter().map(String::as_str).collect();
if self.mode == SandboxMode::Strict {
// stdio/stdout is the existing console channel already permitted by
// strict L1 (CM-A11); file/network/process providers are not.
// strict L1 (CM-A11); stdin, stderr, file/network/process providers
// are not, because strict isolation has not been proven for them.
let stdio_console =
kind == "stdio" && descriptor == "stdout" && requested == BTreeSet::from(["write"]);
if !stdio_console {
@@ -749,27 +772,37 @@ impl StdHost {
}
match kind {
"stdio" => {
if descriptor != "stdout" || requested != BTreeSet::from(["write"]) {
let allowed = match descriptor {
"stdout" | "stderr" => requested == BTreeSet::from(["write"]),
"stdin" => requested == BTreeSet::from(["read"]),
_ => false,
};
if !allowed {
return Err(RuntimeError::new(
ErrorCategory::ResourceUnsupported,
ctx.span,
"the first stream slice supports only stdio/stdout with rights [write]",
"stdio supports stdout/stderr with rights [write] and stdin with rights [read]",
));
}
if !self.capabilities.console {
return Err(RuntimeError::new(
ErrorCategory::PermissionDenied,
ctx.span,
"the console capability is required for stdio/stdout",
"the console capability is required for stdio resources",
));
}
let handle = self.resources.open_stdout();
let handle = match descriptor {
"stdin" => self.resources.open_stdin(),
"stderr" => self.resources.open_stderr(),
"stdout" => self.resources.open_stdout(),
_ => unreachable!(),
};
self.record_semantic(
"resource.open",
None,
Some(&ctx.module_id),
ctx.span,
Some("kind=stdio descriptor=stdout"),
Some(&format!("kind=stdio descriptor={descriptor}")),
);
self.record_resource_effect("resource.open", ctx, started, "ok");
Ok(Value::Resource(handle))
@@ -999,10 +1032,15 @@ impl Host for StdHost {
) -> Result<usize, RuntimeError> {
let started = Instant::now();
let kind = self.resources.kind(handle, ctx.span)?;
let stdio = self.resources.stdio_name(handle, ctx.span)?;
let count = self.resources.write(handle, bytes, ctx.span)?;
if kind == "stdio" {
self.budget.consume_output(count, ctx.span)?;
self.output.extend_from_slice(bytes);
if stdio == Some("stderr") {
self.stderr.extend_from_slice(&bytes[..count]);
} else {
self.output.extend_from_slice(&bytes[..count]);
}
}
self.record_semantic(
"resource.write",
+237 -7
View File
@@ -1,8 +1,9 @@
//! Host-owned resource/stream table (ADR-0025).
//!
//! The first slice deliberately supports only stdout writes and bounded
//! project-file reads loaded at open. Handles are run-scoped and opaque; the
//! table validates run identity, slot identity and generation on every call.
//! The stream table owns run-scoped stdio endpoints, bounded project-file
//! read/write streams, explicitly registered provider tokens and IP-literal TCP
//! streams. Handles are opaque and the table validates run identity, slot
//! identity and generation on every call. Bytes move only through `Memory`.
use std::collections::BTreeMap;
use std::io::{Read, Write};
@@ -75,10 +76,22 @@ pub struct ResourceRead {
pub eof: bool,
}
struct StdinCursor {
bytes: Vec<u8>,
position: usize,
}
enum Stream {
Stdin {
cursor: Arc<Mutex<StdinCursor>>,
closed: bool,
},
Stdout {
closed: bool,
},
Stderr {
closed: bool,
},
File {
relative: String,
bytes: Vec<u8>,
@@ -110,7 +123,9 @@ struct Slot {
impl Slot {
fn closed(&self) -> bool {
match &self.stream {
Stream::Stdin { closed, .. } => *closed,
Stream::Stdout { closed } => *closed,
Stream::Stderr { closed } => *closed,
Stream::File { closed, .. } => *closed,
Stream::FileWriter { closed, .. } => *closed,
Stream::Provider { closed, .. } => *closed,
@@ -124,6 +139,7 @@ pub struct ResourceTable {
run_id: u64,
next_resource: u64,
slots: BTreeMap<u64, Slot>,
stdin: Arc<Mutex<StdinCursor>>,
}
impl Default for ResourceTable {
@@ -139,6 +155,10 @@ impl ResourceTable {
run_id: NEXT_RUN_ID.fetch_add(1, Ordering::Relaxed),
next_resource: 1,
slots: BTreeMap::new(),
stdin: Arc::new(Mutex::new(StdinCursor {
bytes: Vec::new(),
position: 0,
})),
}
}
@@ -152,6 +172,43 @@ impl ResourceTable {
self.insert(Stream::Stdout { closed: false })
}
/// Register the pre-opened stderr stream.
pub fn open_stderr(&mut self) -> ResourceHandle {
self.insert(Stream::Stderr { closed: false })
}
/// Set the run-scoped stdin bytes at assembly time.
///
/// All `stdio/stdin` handles share this cursor so repeated console reads
/// continue through the input instead of restarting at byte zero.
pub fn set_stdin(&mut self, bytes: Vec<u8>) {
self.stdin = Arc::new(Mutex::new(StdinCursor { bytes, position: 0 }));
}
/// Register a read-only handle over the run-scoped stdin cursor.
pub fn open_stdin(&mut self) -> ResourceHandle {
self.insert(Stream::Stdin {
cursor: Arc::clone(&self.stdin),
closed: false,
})
}
/// Return the stdio descriptor name for a handle, if it is a stdio stream.
pub fn stdio_name(
&self,
handle: ResourceHandle,
span: Option<Span>,
) -> Result<Option<&'static str>, RuntimeError> {
let slot = self.slot(handle, span)?;
Ok(match &slot.stream {
Stream::Stdin { .. } => Some("stdin"),
Stream::Stdout { .. } => Some("stdout"),
Stream::Stderr { .. } => Some("stderr"),
Stream::File { .. } | Stream::FileWriter { .. } => None,
Stream::Provider { .. } | Stream::Tcp { .. } => None,
})
}
/// Register a project-file read stream opened from already-bounded bytes.
pub fn open_file(&mut self, relative: impl Into<String>, bytes: Vec<u8>) -> ResourceHandle {
self.insert(Stream::File {
@@ -268,6 +325,17 @@ impl ResourceTable {
));
}
match &mut slot.stream {
Stream::Stdin { cursor, .. } => {
let mut cursor = cursor.lock().map_err(|_| stdin_lock_error(span))?;
let start = cursor.position;
let end = start.saturating_add(max_bytes).min(cursor.bytes.len());
let chunk = cursor.bytes[start..end].to_vec();
cursor.position = end;
Ok(ResourceRead {
bytes: chunk,
eof: end >= cursor.bytes.len(),
})
}
Stream::File {
bytes, position, ..
} => {
@@ -285,6 +353,11 @@ impl ResourceTable {
span,
"stdout does not support resource.read",
)),
Stream::Stderr { .. } => Err(RuntimeError::new(
ErrorCategory::ResourceUnsupported,
span,
"stderr does not support resource.read",
)),
Stream::FileWriter { .. } => Err(RuntimeError::new(
ErrorCategory::ResourceUnsupported,
span,
@@ -294,7 +367,18 @@ impl ResourceTable {
provider, token, ..
} => {
let mut provider = provider.lock().map_err(|_| provider_lock_error(span))?;
provider.read(*token, max_bytes, span)
let read = provider.read(*token, max_bytes, span)?;
if read.bytes.len() > max_bytes {
return Err(RuntimeError::new(
ErrorCategory::ResourceFault,
span,
format!(
"resource provider returned {} bytes for a {max_bytes}-byte read",
read.bytes.len()
),
));
}
Ok(read)
}
Stream::Tcp { stream, .. } => {
let mut buffer = vec![0u8; max_bytes];
@@ -333,7 +417,12 @@ impl ResourceTable {
));
}
match &mut slot.stream {
Stream::Stdout { .. } => Ok(bytes.len()),
Stream::Stdout { .. } | Stream::Stderr { .. } => Ok(bytes.len()),
Stream::Stdin { .. } => Err(RuntimeError::new(
ErrorCategory::ResourceUnsupported,
span,
"stdin is a read-only resource stream",
)),
Stream::File { relative, .. } => Err(RuntimeError::new(
ErrorCategory::ResourceUnsupported,
span,
@@ -369,7 +458,18 @@ impl ResourceTable {
provider, token, ..
} => {
let mut provider = provider.lock().map_err(|_| provider_lock_error(span))?;
provider.write(*token, bytes, span)
let count = provider.write(*token, bytes, span)?;
if count > bytes.len() {
return Err(RuntimeError::new(
ErrorCategory::ResourceFault,
span,
format!(
"resource provider reported {count} bytes written for a {}-byte request",
bytes.len()
),
));
}
Ok(count)
}
Stream::Tcp { stream, .. } => stream.write(bytes).map_err(|error| {
RuntimeError::new(
@@ -396,7 +496,9 @@ impl ResourceTable {
));
}
match &mut slot.stream {
Stream::Stdin { closed, .. } => *closed = true,
Stream::Stdout { closed } => *closed = true,
Stream::Stderr { closed } => *closed = true,
Stream::File { closed, .. } => *closed = true,
Stream::FileWriter { closed, .. } => *closed = true,
Stream::Provider {
@@ -447,7 +549,9 @@ impl ResourceTable {
pub fn kind(&self, handle: ResourceHandle, span: Option<Span>) -> Result<String, RuntimeError> {
let slot = self.slot(handle, span)?;
Ok(match &slot.stream {
Stream::Stdout { .. } => "stdio".to_string(),
Stream::Stdin { .. } | Stream::Stdout { .. } | Stream::Stderr { .. } => {
"stdio".to_string()
}
Stream::File { .. } | Stream::FileWriter { .. } => "file".to_string(),
Stream::Provider { kind, .. } => kind.clone(),
Stream::Tcp { .. } => "tcp".to_string(),
@@ -463,6 +567,14 @@ fn provider_lock_error(span: Option<Span>) -> RuntimeError {
)
}
fn stdin_lock_error(span: Option<Span>) -> RuntimeError {
RuntimeError::new(
ErrorCategory::HostError,
span,
"stdin cursor lock is poisoned",
)
}
fn unknown_handle(span: Option<Span>) -> RuntimeError {
RuntimeError::new(
ErrorCategory::ResourceFault,
@@ -536,4 +648,122 @@ mod tests {
"resource.unsupported"
);
}
#[test]
fn stdin_reads_are_chunked_and_end_at_eof() {
let mut table = ResourceTable::new();
table.set_stdin(b"ab".to_vec());
let stdin = table.open_stdin();
assert_eq!(
table.read(stdin, 1, None).unwrap(),
ResourceRead {
bytes: b"a".to_vec(),
eof: false
}
);
assert_eq!(
table.read(stdin, 8, None).unwrap(),
ResourceRead {
bytes: b"b".to_vec(),
eof: true
}
);
assert_eq!(
table.read(stdin, 1, None).unwrap(),
ResourceRead {
bytes: Vec::new(),
eof: true
}
);
assert_eq!(
table.write(stdin, b"x", None).unwrap_err().code(),
"resource.unsupported"
);
table.close(stdin, None).unwrap();
assert_eq!(
table.read(stdin, 1, None).unwrap_err().code(),
"resource.closed"
);
}
#[test]
fn stderr_is_write_only() {
let mut table = ResourceTable::new();
let stderr = table.open_stderr();
assert_eq!(table.write(stderr, b"diagnostic", None).unwrap(), 10);
assert_eq!(
table.read(stderr, 1, None).unwrap_err().code(),
"resource.unsupported"
);
assert_eq!(table.stdio_name(stderr, None).unwrap(), Some("stderr"));
table.close(stderr, None).unwrap();
assert_eq!(
table.write(stderr, b"x", None).unwrap_err().code(),
"resource.closed"
);
}
}
#[cfg(test)]
mod provider_tests {
use super::*;
struct OverReporting;
impl ResourceProvider for OverReporting {
fn name(&self) -> &str {
"over-reporting"
}
fn kinds(&self) -> &[&str] {
&["bad"]
}
fn open(
&mut self,
_kind: &str,
_descriptor: &str,
_rights: &[String],
_span: Option<Span>,
) -> Result<u64, RuntimeError> {
Ok(1)
}
fn read(
&mut self,
_token: u64,
max_bytes: usize,
_span: Option<Span>,
) -> Result<ResourceRead, RuntimeError> {
Ok(ResourceRead {
bytes: vec![0; max_bytes + 1],
eof: false,
})
}
fn write(
&mut self,
_token: u64,
bytes: &[u8],
_span: Option<Span>,
) -> Result<usize, RuntimeError> {
Ok(bytes.len() + 1)
}
fn close(&mut self, _token: u64, _span: Option<Span>) -> Result<(), RuntimeError> {
Ok(())
}
}
#[test]
fn provider_counts_are_bounded_by_the_request() {
let mut table = ResourceTable::new();
let provider: SharedResourceProvider = Arc::new(Mutex::new(Box::new(OverReporting)));
let handle = table
.open_provider(provider, "bad", "descriptor", &["read".to_string()], None)
.unwrap();
assert_eq!(
table.read(handle, 2, None).unwrap_err().code(),
"runtime.resource"
);
assert_eq!(
table.write(handle, b"x", None).unwrap_err().code(),
"runtime.resource"
);
}
}
@@ -53,6 +53,63 @@ fn stdout_write_obeys_the_output_budget_atomically() {
assert!(host.take_output().is_empty());
}
#[test]
fn pure_mode_reads_stdin_through_a_resource() {
let dir = tempfile::tempdir().unwrap();
let mut host = StdHost::new(SandboxMode::Pure, Capabilities::pure(), dir.path())
.with_stdin(b"abcd".to_vec());
let ctx = context();
let handle = handle(
host.resource_open("stdio", "stdin", &["read".to_string()], &ctx)
.unwrap(),
);
let first = host.resource_read(handle, 2, &ctx).unwrap();
assert_eq!(first.bytes, b"ab");
assert!(!first.eof);
let rest = host.resource_read(handle, 8, &ctx).unwrap();
assert_eq!(rest.bytes, b"cd");
assert!(rest.eof);
assert_eq!(host.resource_read(handle, 1, &ctx).unwrap().bytes, b"");
let write = host.resource_write(handle, b"x", &ctx).unwrap_err();
assert_eq!(write.code(), "resource.unsupported");
}
#[test]
fn stdout_and_stderr_are_captured_separately() {
let dir = tempfile::tempdir().unwrap();
let mut host = StdHost::new(SandboxMode::Pure, Capabilities::pure(), dir.path());
let ctx = context();
let stdout = handle(
host.resource_open("stdio", "stdout", &["write".to_string()], &ctx)
.unwrap(),
);
let stderr = handle(
host.resource_open("stdio", "stderr", &["write".to_string()], &ctx)
.unwrap(),
);
assert_eq!(host.resource_write(stdout, b"out", &ctx).unwrap(), 3);
assert_eq!(host.resource_write(stderr, b"err", &ctx).unwrap(), 3);
assert_eq!(host.output(), b"out");
assert_eq!(host.error(), b"err");
assert_eq!(host.take_output(), b"out");
assert_eq!(host.take_error(), b"err");
}
#[test]
fn strict_mode_refuses_stdin_and_stderr() {
let dir = tempfile::tempdir().unwrap();
let mut host = StdHost::new(SandboxMode::Strict, Capabilities::pure(), dir.path())
.with_stdin(b"x".to_vec());
let ctx = context();
for (descriptor, rights) in [("stdin", "read"), ("stderr", "write")] {
let error = host
.resource_open("stdio", descriptor, &[rights.to_string()], &ctx)
.unwrap_err();
assert_eq!(error.category, ErrorCategory::PermissionDenied);
}
}
#[test]
fn strict_mode_allows_only_the_existing_stdio_console_channel() {
let dir = tempfile::tempdir().unwrap();
@@ -0,0 +1,468 @@
# Marqdo CLI 最新提交架构审查与重构 RFC
> 状态:**审查报告 / 待讨论设计,不是已实施代码**
> 仓库:[`cflmy/marqdo-cli`](https://github.com/cflmy/marqdo-cli)
> 冻结审查提交:[`7fbe069196e61d06a00d64fe8d65049c327373eb`](https://github.com/cflmy/marqdo-cli/commit/7fbe069196e61d06a00d64fe8d65049c327373eb)
> 编写日期:2026-11-21;依据 GitHub 仓库树、源码、ADR、spec、stdlib、测试文件和状态记录,**没有在本环境重新构建或执行完整测试**。
> 重要约束:**标准库继续保存在同一仓库**;保持 Markdown 原生语法、`std:` 导入、能力授权及既有稳定语义。
## 0. 审查结论
**不是计算模型需要再推倒重来,而是 `语言核心 → 运行时 → 平台边界 → 标准库 → CLI` 的职责未完成收敛。**
- 已有 `Memory` 和 `resource.*`,意味着 RFC-CM-0001/0002 的重要骨架已落地;不能再把“缺少独立内存”“打印仍由 `HostOp::Print` 实现”当作现状。
- 但标准库目前主要是 `format.mq.md`、`text.mq.md`、`io.mq.md`、`data/table.mq.md`、`net/http.mq.md`;`io` 仅提供 `写文本/写值`,缺乏输入、文件和面向用户的通用流操作。
- 仓库有大量**证明可以写**的 `.mq.md` 测试,但它们不等于已经**作为标准库产品发布**:`JSON`、CSV、分配器、UTF-8 解码、字节码虚拟机等主要见于 `tests/conformance/semantics/`。
- 现有 `resource.*` 与旧 `fs.read/write` 并存;`Host` 仍有数学、Foreign、Plugin 等操作;Provider、Plugin、Foreign Kernel、Tool 具有不同职责,但其注册、策略、诊断、输送方式的实现入口日益分散。
- 物理文件组织尚可维护,不应以“crates 多”为由全面合并。真正要减少的是**同类逻辑的实现入口**、大文件承担多种职责、文档互相矛盾,以及把测试夹具当成用户标准库的错觉。
**本 RFC 的最高优先级:构建一个可靠的 `stdin/stdout/stderr + file + byte streams` 底层端点;基于它用 `.mq.md` 交付完整 `std:io` / `std:fs` / `std:data`;在此之后再拆分超大模块、收敛扩展注册面。**
## 1. 事实依据与边界
### 1.1 已实现、但容易被忽略的能力
| 能力 | 最新代码中的事实 | 限制 |
|---|---|---|
| 自有计算 | Markdown/Expression/Frontend/IR/Interpreter 均已独立实现 | 仍为 tree-walk,未实现 native optimizer/JIT |
| 程序可寻址内存 | `marqdo-runtime/src/memory.rs`:独立区域、64 KiB 页、增长、字节读写、别名共享、运行级预算 | 区域在整次 run 中存在;无语言级释放整个 region;默认每区域16 MiB、每次运行64 MiB |
| 位置更新 | `m[i] = byte` 已实现 | 并非通用的 `list[i]=...` / `map[k]=...` 原地可变引用语义;普通集合仍为值语义 |
| 结构操作 | `list.append/concat`、`map.keys/has/get/set/remove`、`table.columns/row/cell`、`text.codepoints/from_codepoint` | 只是一组必需底层操作,不能替代完整标准库 |
| 标准库载入 | `stdlib/` 同仓库;`std:` 显式导入;`stdlib.json` 摘要校验 | 缺安装包一致性和 API 分层的完整验收;不要拆库 |
| 打印 | `print/打印` 通过加载 `stdlib/io.mq.md` 的 `写值` 执行;原 `HostOp::Print` 已移除 | stdlib 输出路径仍只写一次、固定一页缓冲、未逐项检查 Result |
| 底层文件 | `resource.open("file",..., ["read"]/["write"])`;`fs.read/write` 遗留接口 | 读取时整文件载入、有 8 MiB 限额;写入时缓冲到 close;没有完整 `std:fs` |
| TCP | `resource.open("tcp", IP:port,["read","write"])` | IP 字面量、同步、5 秒超时;缺 DNS、TLS、监听、UDP |
| 资源 Provider | 进程内、worker、C ABI 等实现已经存在 | 与 Plugin、Foreign 的注册和安全边界重叠,不能认为它们都是一种 API |
| 严格隔离 | Linux Bubblewrap 且探测成功时严格执行;本地 VM 有证据 | 不能称 Windows/macOS 或所有 Provider 已获相同保障 |
### 1.2 标准库中哪些是真正的公开 API?
此 HEAD 的正式 `stdlib` 函数源码集中于 `format.mq.md`、`text.mq.md`、`io.mq.md`、`data/table.mq.md`、`net/http.mq.md`,另有 `demo.mq.md` 和三个测试文档。现状**不是空标准库**,但明显没有达到日常 CLI 所需的覆盖范围。
已在 conformance 中证明,但仍需产品化的候选:
- `tests/conformance/semantics/55-first-fit-free-list-allocator.mq.md`:空闲链表分配器;
- `59-utf8-encoder` / `60-utf8-decoder`:编码/校验解码;
- `62-json-array-parser`:JSON 解析相关能力;
- `63-memory-bytecode-vm`:虚拟 CPU/字节码;
- `68-csv-quoted-row`:引号转义 CSV;
- `70-matrix-multiply`、`71-http-parser`、`72-table-columnar`、`73-sparse-matrix`。
**结论:测试证明的是表达能力,标准库证明的是用户能依赖的 API 及错误、性能、版本契约;两者不能相互代替。**
### 1.3 可以明确定位的结构偏移
1. `crates/cli/src/commands.rs` 约 1539 行,同时承担加载调用、运行环境组装、严格隔离启动、工具输出、依赖命令和文件写入;`cli/src/loader.rs` 也掺入特定 `print` 识别/重定向。CLI 过度承担执行策略。
2. `crates/marqdo-host/src/host.rs` 约 1138 行,负责 `HostOp`、资源分发、权限与 Provider 注册;`resource.rs` 约 540 行混合 stdout、文件、TCP、Provider 句柄。资源能力正确,但组织已超出一个文件的合适边界。
3. `marqdo-runtime` 依赖 `marqdo-host`,即语言运行内核直接依赖具体平台边界模块,尚未做到执行引擎与平台实现清楚的依赖方向;是否引入一个薄接口 crate 必须根据下一阶段的独立嵌入需求决定,禁止盲目增 crate。
4. `HostOp` 同时拥有高层数学内核、Foreign、Plugin、遗留 `fs.*`,又存在新的 `resource.*`;**高级操作与基础系统操作在一个分发层并存**。
5. `Plugin ABI` / `Resource Provider` / `Foreign Kernel` / `CLI Tool` 在功能上不同,但基础注册、授权、版本、签名、传输、诊断等通用机制缺少统一的架构地图。删除其中某一种协议会伤害已有消费者;应先定义所有权,后考虑传输层复用。
6. 文档状态漂移:`docs/architecture/README.md` 仍称 P5 Provider 未实现,`docs/evidence/2026-11-20-rfc-cm-completion-audit.md` 却记录 Provider/P5 的本地验收;`docs/STATUS.md` 把较老的 `c9f01aa` 描述为 GitHub master,而此次实际最新为 `7fbe069`。**应把状态“生成与冻结”机制纳入重构,而非靠人手更新数十份计划。**
7. `stdlib/io.mq.md` 的“写文本”总是 `memory.new(1,1)`,把 Unicode 写入缓冲后只 `resource.write` 一次,不检查 `{ok, count}`;输入大于 64 KiB 会越界/失败,潜在部分写失败被吞掉。这个问题属于标准库正确性,不是“再加一个 Rust println”能解决的。
8. `stdlib/net/http.mq.md` 以 Unicode 码点文本为基础解析响应和 chunked 数据,而 HTTP 的长度以**字节**为单位;对非 ASCII 正文、混合编码、大小写不敏感头字段、碎片读取和消息上限不能据此宣称完整 HTTP/1.1 实现。
## 2. 目标架构:五个清晰的责任层
```text
用户 .mq.md 程序 / Agent / 科研工作流
│
stdlib/* (.mq.md,仓库内版本化与签名/摘要)
├─ core/collections/text/encoding/data/math
├─ io/console/stream/fs/net/process
└─ memory/alloc + 可移植纯算法
│
语言语义层:Markdown → AST → Frontend → IR
│
执行层:Value / MemoryStore / evaluator / budgets
(只描述计算,不拥有 stdout 文件句柄)
│
受控平台接口:open/read/write/close + 非流式请求
├─ capability / budget / audit / tracing
├─ stdio / file / tcp / OS 适配器
└─ 显式注册的 Provider(可 in-process / worker / C)
│
操作系统 / 外部服务 / 设备
CLI 只做 Composition Root:参数、工程加载、装配、执行、输出
Plugin callable / Foreign legacy / Tool protocol = 并列的扩展类别,
由独立注册模块负责,不直接污染标准库或语言 IR。
```
### 2.1 宪法性边界(应成为 CI 架构检查规则)
- **语言语义内核**:表达式、控制流、局部变量、List/Map/Table 语义、数值和字节行为、内存区域、值与位置赋值、函数/文档引用、错误源位置。它不应知道 `stdout`、项目目录、TCP 超时或第三方插件路径。
- **平台接口**:负责不可纯计算实现的 I/O、系统时钟、熵源、网络、文件句柄、权限校验;这些接口的宿主实现是必要的,但不能承载 JSON 解析、文本格式化、HTTP 语义或内存分配策略。
- **标准库**:以 `.mq.md` 实现普通 API、协议、算法、容器管理。**普通能力新增时,只改 stdlib + tests + docs,不改 Rust Builtin。**
- **CLI**:只负责选择输入项目、解析命令、加载配置、装配 Host 和 Runtime、路由人类/机器输出;不是语义策略中心。
- **扩展接口**:原生函数扩展和资源扩展都经同一授权审计机制,但函数调用与有状态流仍保留不同能力类型。
### 2.2 引擎与平台的依赖方向
当前 `marqdo-runtime → marqdo-host` 并非立即必须合并或新增接口 crate 的理由。分两步:
**第一步**:把所有 I/O 分发和许可策略严格收敛到 `marqdo-host`,`runtime` 不再增加外部操作分支;建立可以注入 FakeHost 的语义测试。
**第二步(当支持嵌入式 Runtime 或第二个执行后端时)**:引入最薄、无平台依赖的 `ExecutionPort` 契约,并让 `marqdo-host` 实现它。若 Rust crate 会形成环依赖,才考虑一个不足数百行的共享接口 crate;**禁止为每个端口和 Provider 单独设 crate**。
## 3. 目标目录(同一仓库,不拆 stdlib)
只展示重要目录;不代表一次性移动所有 crate:
```text
marqdo-cli/
├── Cargo.toml
├── crates/
│ ├── marqdo-markdown/ # 只保留 Markdown profile、AST、source span
│ ├── marqdo-expression/ # 表达式词法、语法与 AST
│ ├── marqdo-frontend/ # 映射、名称分析、静态诊断
│ ├── marqdo-ir/ # 稳定执行 IR;不积累高层 API
│ ├── marqdo-value/ # 值语义及 wire 合约
│ ├── marqdo-runtime/
│ │ └── src/
│ │ ├── interpreter.rs
│ │ ├── memory.rs
│ │ ├── place.rs # 可赋值位置与统一读取/写入
│ │ ├── primitives/ # 确实不可由 .mq.md 有效组合的计算原语
│ │ └── lib.rs
│ ├── marqdo-host/
│ │ └── src/
│ │ ├── session.rs # 每次执行的 Host 状态、资源预算
│ │ ├── resource/
│ │ │ ├── table.rs # 句柄登记、生命周期、权限
│ │ │ ├── stdio.rs
│ │ │ ├── file.rs
│ │ │ ├── tcp.rs
│ │ │ └── provider.rs
│ │ ├── providers/ # registry/native/worker/C 实现适配
│ │ ├── effects/ # auditing、trace、capability
│ │ ├── legacy/ # 旧 fs / foreign 迁移包装
│ │ └── lib.rs
│ ├── marqdo-package/ # import / lock / package identity
│ ├── marqdo-sandbox/ # OS 隔离证明,绝不移入 stdlib
│ ├── marqdo-foreign/ # 可选兼容执行器,非内核
│ ├── marqdo-math/ # 临时可选原生数学实现,逐项确立归属
│ ├── marqdo-plugin-abi/ # 原生可调用扩展协议
│ ├── marqdo-plugin-sdk/
│ ├── marqdo-provider-worker/
│ └── cli/
│ └── src/
│ ├── main.rs
│ ├── args.rs
│ ├── commands/ # run, call, check, inspect, package, tooling
│ ├── project/ # loader, stdlib resolver, project context
│ ├── assembly.rs # RunSession 的惟一装配位置
│ └── output.rs
├── stdlib/ # 保留在同一个 git 仓库
│ ├── stdlib.json # 继续 digest 校验
│ ├── core/result.mq.md
│ ├── collections/list.mq.md
│ ├── collections/map.mq.md
│ ├── text/utf8.mq.md
│ ├── text/format.mq.md
│ ├── text/scan.mq.md
│ ├── memory/alloc.mq.md
│ ├── io/stream.mq.md
│ ├── io/console.mq.md
│ ├── fs/file.mq.md
│ ├── fs/path.mq.md
│ ├── net/tcp.mq.md
│ ├── net/http1.mq.md
│ ├── data/json.mq.md
│ ├── data/csv.mq.md
│ ├── data/table.mq.md
│ ├── io.mq.md # 兼容门面,保留旧 std:io
│ ├── format.mq.md # 兼容门面,保留旧 std:format
│ ├── text.mq.md # 兼容门面,保留旧 std:text
│ └── tests/ # 对公共 std: API 做产品级测试
├── tests/
│ ├── conformance/ # 语义契约(无需变成文档产品)
│ ├── architecture/ # import graph / primitive-only tests
│ ├── e2e/ # 纯 Marqdo CLI,输入/文件/HTTP/JSON 等
│ └── security/
├── spec/ # 真正冻结的语言和 ABI 规范
├── docs/
│ ├── architecture/ # 唯一当前架构总览 + RFC 决策索引
│ ├── adr/ # 只记录决策,不冒充实时状态
│ ├── evidence/ # SHA 固定的事实快照
│ ├── history/ # 已过时规划
│ └── STATUS.md # 自动采集或引用当前状态来源
└── tools/ # 打包、CI、文档一致性检查和 benchmark
```
**迁移原则:先移动职责,再移动路径。** 不因为希望目录好看就制造公共 API 大面积重命名;已公开的 `std:io`、`std:format`、`std:text`、`std:data/table`、`std:net/http` 必须有兼容导入门面和逐版本弃用协议。
## 4. 系统 I/O 的最小充分基座
### 4.1 必须补上的真正底层能力
当前 `resource.open("stdio","stdout",["write"])` 已有。**缺少 `stdin` 读端点**,这是不能只用已有资源来拼出的新 OS 交互能力。应扩展同一 `stdio` 资源类别,不增加语言关键字、`Input` builtin 或第二套控制台协议。
目标:
```text
resource.open("stdio", "stdin", ["read"]) -> Resource
resource.open("stdio", "stdout", ["write"]) -> Resource
resource.open("stdio", "stderr", ["write"]) -> Resource
resource.read(handle, Memory, offset, max_bytes) -> Result({count,eof})
resource.write(handle, Memory, offset, length) -> Result({count})
resource.close(handle) -> Result
```
stdin 与 stdout 使用同一个 `ResourceTable`、同一个权限检查和同一 Trace 契约。`stderr` 必须区分**应用输出**与 CLI 自身的诊断,以免 `--json` 机器可读输出混合无关数据。所有端点均遵守可取消/超时与资源预算(第一版可采用同步阻塞,但规范明确等待策略与取消路径)。
**不能将 `read_line` 作为新 Rust Builtin。** 它应位于 `stdlib/io/console.mq.md`,由 `resource.read` + 有界缓冲 + 增量 UTF-8 解码 + 换行判定构建。
### 4.2 修复现有 `stdlib/io.mq.md` 的正确性缺陷
现状:固定 `memory.new(1,1)`,单次 `resource.write`,无 `{ok,count}` 校验。目标 `write_all` 必须:
1. 将任意长度文本按**字节**分片编码,或按需增长 Memory;禁止超过区域上限时静默截断。
2. `resource.write` 若 `{ok:false}`,返回带原因的 `Result`;若 `0 < count < requested`,继续写剩余字节;若 `count=0` 且有剩余,返回 `io.write_zero` 防止死循环。
3. 不因 `> resource.write(...)` 语句返回值被忽略而掩盖错误;`打印` 的失败须有明确可观测结果。若要求完全兼容旧 `print` 的致命失败行为,需要规定 `Result` 到致命错误的桥接方式,不能在 `.mq.md` 中擅自吞错。
4. 使用统一缓冲生命周期与容量预算;stdout、文件和 TCP 共享 `write_all`,只在入口处选择资源。
5. 确保多字节 Unicode、嵌入 NUL、超过一页、短写、失效资源、无权限、EOF 等均有测试。
### 4.3 文件:不是没有底层能力,而是接口不完整
目前 `resource.open("file", path, ["read"]/["write"])` 可工作。但实现为:读取时整文件转入缓冲(8 MiB 限制),写入时缓存在 Host 并于 `close` 提交。这是当前阶段的安全简化,不是通用 OS 文件流。
**标准库立即可构造**:`read_all`、`read_text`、`write_text`、`write_bytes`、`copy_file`(限当前文件容量)、统一 close/error 检查。
**需要真实的底层新增**:开放大文件的增量读写;追加和 seek;stat/exists/listdir/mkdir/remove/rename 等非流式路径操作。后二类不能通过 `resource.read/write` 神奇推导出来。
推荐能力合同分两类:
- `Resource Stream`:打开、读、写、关闭,负责字节流和文件位置。
- `Resource Operation`:受版本协议和**逐操作授权**的元信息/目录修改请求,用于不能归结为流的系统命令;候选形态 `resource.request(scope, operation, arguments)`,**不是无授权的万能 syscall**。引入前需 ADR 固定操作命名、路径规范化、能力范围、返回错误、审计与默认拒绝。
第一版不扩充复杂的文件事件监听、mmap 及目录枚举细节;只实现能支撑真实 CLI 文件工具的最小操作组。旧 `fs.read/write` 保持一段时间的兼容 shim,最终统一到 Resource 路线,避免双路径权限规则。
### 4.4 网络:低层缺口与 stdlib 缺口分别处理
- 现在:IP-literal TCP、同步、5 秒超时、HTTP `.mq.md` 请求构造和部分响应解析。
- 标准库:`tcp.connect/read_exact/write_all`、可增量的 HTTP/1.1 状态机、二进制 body、Content-Length 和 chunked 字节协议、错误 Result。
- 新 OS/Provider 能力:DNS 解析、TLS(可通过经过审计的安全传输 Provider)、监听、UDP、系统证书存储和可信熵源。**不应宣称普通裸字节内存自动生成安全 TLS 或 DNS 能力。**
- 当前 `stdlib/net/http.mq.md` 不应以码点长度处理分块字节长度;在面向通用网络发布之前先改为字节流解析。
## 5. 功能归属:内核 / 平台 / 标准库 / 可选加速
| 能力 | 推荐归属 | 是否应新增 HostOp / Rust 算法 |
|---|---|---|
| `Int/Float/Bool` 算术与比较、函数控制流 | 内核 | 核心原语,保留 |
| `Memory` 区域创建、增长、边界检查、字节访问 | 内核 | 保留;由解释器维护抽象存储 |
| `List/Map/Table` 必要结构观察与构造 | 内核 | 保留最小闭包,不增排序等算法 |
| `type/len`、结构访问、值/位置语义 | 内核 | 保留契约,慎重扩展 |
| 动态分配器、free-list、内存池、链表和树 | `stdlib/memory`、`collections` | 不增加 Rust 算法 |
| UTF-8 解码、文本转义、数字格式化 | `stdlib/text` | 原生可作为等价加速,不拥有唯一语义 |
| `打印`、输入行、文件复制、日志格式化 | `stdlib/io` / `fs` | 不增高层 Builtin;底层系统操作必要 |
| JSON/CSV、GFM 表格变换、HTTP 解析 | `stdlib/data` / `net` | 纯 `.mq.md` 必须有参考实现 |
| `std:fs` 目录/元数据/重命名 | 标准库包装 + 平台 `Resource Operation` | 需要**通用能力端点**,不是每个函数单独 HostOp |
| OS 文件句柄、网络 socket、stdio、时间、熵源 | Host / OS adapter | 必须有少量可信系统调用实现 |
| 数学符号求值/微分、科学优化 | 可选 library 或加速层 | 逐项评估;不要放进语言语义基石 |
| Python/Node Foreign | 可选兼容扩展 | 不作为标准库存在性的依赖 |
| 原生插件导出 | Callable Plugin | 不能替代普通自举算法 |
| 资源 Provider | Resource Provider | 不能替代普通 `std:io` |
| `marqdo-<tool>` 工具 | CLI Tool Protocol | 与应用运行时 API 严格分离 |
特别说明:当前的 `Value::Table` 使用 `Vec<Vec<Value>>` 作为存储并不意味着架构失败。先保证 GFM 源码、逻辑 Table、物理存储三层分离、确认普通 Table 的值语义,再按性能证据改存储;优先建设**能直接使用的表格 API**而不是立即上列存引擎。
## 6. 扩展体系如何收敛,而不推倒已有工作
**不要粗暴删除四种看起来类似的扩展方式**:它们的边界不同。
| 当前机制 | 保留的真正职责 | 应停止承担的职责 |
|---|---|---|
| Tool Protocol | `marqdo` CLI 自身的外部工具、inspect/verify/IDE 等命令 | 在用户程序运行过程中替代标准 I/O |
| Callable Plugin ABI | 可调用的原生函数扩展、受控实验加速 | 把普通 JSON/CSV/格式化都做成原生 ABI |
| Resource Provider | 新资源类型和设备/网络/文件端点 | 为每个高级算法创建一个 Provider |
| Foreign Kernel | 明确请求跨语言计算的兼容层 | Marqdo 实现常见算法的常规路径 |
共用基础设施(**先集中到 `marqdo-host` 内部的统一 Registry/Policy 模块,暂不增加 crate**):
- package/adapter identity、descriptor/version validation;
- 精确能力授予与每次请求审计;
- 统一超时、预算、错误分类和 trace metadata;
- 进程内/worker/C 等传输后端的记录方式;
- 严格模式拒绝未经证明的适配器,不把“worker 子进程”误称沙盒。
不应人为强迫函数调用与资源流共享一份万能二进制 ABI。共同的是**注册和权限元模型**,而不是计算语义。
## 7. CLI 模块治理与目录重构步骤
### P0:冻结最新行为、建立一张事实总表
- 建立单一 `docs/architecture/CURRENT.md`,固定审查 SHA、实现/测试/标准库发布/多平台验证四个状态维度。
- 对照 `docs/STATUS.md`、`docs/architecture/README.md`、RFC 完成审计,消除旧 `master` SHA 与 P5 状态冲突;旧状态移动至 `docs/history/` 或加明显“历史快照”标志。
- 建立源依赖图、公开 API/stdio/file/TCP 契约、现有扩展路径分类图;记录 baseline 而非先改文件名。
- CI 加入简单文档一致性检查和标准库 manifest/digest 校验。
**验收**:任何对外状态声明都能指向固定 commit、真实测试或源码位置;不允许“测试有例子”冒充“stdlib 已发布”。
### P1:把标准输入与 I/O 正确性做成第一条闭环
1. Host 增加 `stdio/stdin`(可选 `stderr`)且复用 `resource.*`;保留现有 stdout 安全检查。
2. 实现 `stdlib/io/stream`:`read_chunk`、`read_exact`、`read_to_end`、`write_all`、`close_checked`。
3. 实现 `stdlib/io/console`:`read_line`、`prompt`、`print`、`println`,行缓冲、EOF 与 UTF-8 错误一致。
4. 修复当前 `stdlib/io.mq.md`:复用这些函数,保持 `打印` 外部行为不变。
5. 保持 `--json` 通道纯净,诊断/trace 与程序输出严格分流。
**验收**:通过管道将三行输入送入 `marqdo run`,程序按行响应;单行超过 64 KiB、短读/短写、空输入 EOF、无权限、错误 UTF-8 不崩溃、不丢字节;同一测试必须使用 `.mq.md` 实现 `read_line` 而非 Rust 新内置。
### P2:文件与完整数据标准库
- 发布 `std:fs/file` 基本 API:read/write/copy;迁移旧 `fs.read/write` 到兼容包装而不是形成两个长期安全边界。
- 将 conformance 中成熟的 JSON、CSV、UTF-8 decode、整数编码、free-list allocator 迁为公共 stdlib API;移动逻辑后重写测试,确保测试导入**实际标准库模块**。
- 文件增量读写与目录/元数据行为需要最小的底层 Resource 扩展 ADR;**不能将目标 API 列入已实现直到完成**。
- `std:data/table` 保留可读 API,增加列名选择、空值/类型校验、按键访问、流式 CSV→Table;对稳定输出保留复制/值语义。
**验收**:纯 `.mq.md` 实现并测试文件复制、JSON 读改写回、CSV→Table→GFM、UTF-8 错误报告;追加一个新的数据格式适配器仅改 `.mq.md`。
### P3:分离 CLI/Host 中的多重职责
- 不改公开 API,先将 `cli/commands.rs` 按 run/call/check/inspect/package 分成内部模块;`stdlib.rs`、`loader.rs` 属于 project 加载路径。
- 将 `host.rs` 的资源处理和 Provider 注册拆到 `resource/`、`providers/`,Policy 与 Trace 保留统一入口。
- 将 StdHost 建立为一次性 `RunSession`;配置、权限、stdlib digest、资源句柄生命周期及输出路由由一个装配函数确定。
- 外部 Tool Protocol 不参与 `marqdo run` 的普通标准库函数解析。
- 不改变 `marqdo-runtime` 对外执行结果,不因“美观”合并所有小 crate。
**验收**:Public API/trace/ABI digest 不变;主 CLI 的 run/call/check 与 strict 安全测试全部维持;对 `src/commands.rs`、`src/host.rs` 做职责级拆分,而不是纯机械拆文件。
### P4:系统资源扩展与生产可用性
- 在稳定的授予模型上增加 `fs` 元数据与目录事务、DNS/TLS Provider(安全审计单列)、stderr、时间/熵源等;其适用顺序以真实应用需求和安全边界为依据。
- 把 HTTP 从 Unicode 文本演示升级成二进制增量解析;协议处理仍为 `.mq.md`。
- 标准库可在不修改冻结 CLI 二进制的情况下增加新模块/补丁版本,仍在同仓库发布和统一打包。
**验收**:能用标准库完成交互式 CLI(逐行读写)、大文件增量处理、文件清单与复制、HTTP 请求(在有授权的传输端点上);新增协议解析不引入 Rust 算法。
### P5:可选优化,而不是再次堆原语
- 测量 `format.显示` 文本拼接、表格值复制、List 结构变换、字节循环读写的成本。
- 保留纯 `.mq.md` 的参照语义;仅对经基准证明的瓶颈设计内核优化(例如有界 bulk memory copy、结构共享、字符串 builder、编译后端),不得改变可观测语义。
- 在确实需要长驻 Agent/并行的情况下,再讨论对象与资源生命周期、任务调度和原子语义;不提前推入本轮最小修复。
## 8. 能力缺陷与优先级清单
| 优先级 | 缺陷 | 性质 | 应由谁完成 |
|---|---|---|---|
| P0 | 无 stdio/stdin 可读端点 | 真正缺失的 OS 能力 | Host `Resource` adapter |
| P0 | `io.写文本` 固定一页、未检查短写/错误 | 标准库正确性 | stdlib/io |
| P0 | `stdlib` 输入、read_line/read_exact/write_all 缺失 | 标准库产品缺口 | stdlib/io |
| P0 | stdlib 测试夹具与公共 API 混淆 | 工程产品化 | stdlib + tests |
| P0 | 架构状态文档相互矛盾 | 文档治理 | docs/CI |
| P1 | `std:fs` 文件工具缺失 | 标准库产品缺口 | stdlib/fs |
| P1 | 文件 read 预读、write 关闭提交、8 MiB 上限 | 当前低层契约能力限制 | Host Resource + 安全审查 |
| P1 | 缺文件 stat/list/rename/remove/mkdir/append/seek | 新 OS 操作需要平台端点 | Resource Operation + stdlib |
| P1 | JSON/CSV/UTF-8 decode 主要停留在 tests | API 发布缺口 | stdlib/data,text |
| P1 | 大型 `host.rs`/`commands.rs` 的职责混杂 | 架构债 | Rust 模块重组 |
| P1 | 多协议注册/安全/追踪概念交叉 | 架构边界问题 | Provider/Plugin Registry |
| P2 | HTTP 解析基于字符,非通用字节流 | 语义与正确性限制 | stdlib/net |
| P2 | TCP 缺 DNS/TLS/监听/UDP | 平台端点能力缺失 | Provider/OS adapter |
| P2 | 标准库缺 path、regex、datetime、env、process、安全随机、压缩等 | 依赖具体 OS/算法的混合缺口 | stdlib 优先;必要时端点 |
| P2 | 当前每区域16MiB、run 64MiB 的限制 | 资源预算设置限制 | Runtime/CLI 配额策略 |
| P3 | 无并发任务、原子同步、可取消的长驻应用调度 | 高级执行语义缺口 | Runtime,另作 ADR |
| P3 | 无 native 优化器、表格高效后端 | 性能能力缺口 | 依据 benchmark 决定 |
| P3 | 多平台严格隔离与 CI 证据有限 | 验证/发布缺口 | CI/Sandbox |
## 9. 计算闭包验收:防止又变成 Rust Builtin 项目
必须区分四种状态:
- `S0` 原语可表达:语义核心拥有必需的计算/系统边界;
- `S1` 证明可写:测试文档写出过一个算法;
- `S2` 标准库已发布:用户可通过 `std:` 稳定导入;
- `S3` 工程可用:正确性、边界错误、权限、性能预算、文档、版本契约得到验收。
下表要求只在冻结 CLI 之上改 `.mq.md`(涉及新增 OS 端点的用例须先完成对应底层适配器):
| 能力 | 核心闭包验收 | 标准库验收 | 防作弊条件 |
|---|---|---|---|
| 内存分配 | bump / free-list / splitting / coalescing | `std:memory/alloc` | 不加 Rust allocator builtin |
| 字节编码 | UTF-8 / endian / LEB128 | `std:text/utf8` | 不调用 Foreign |
| 数据格式 | JSON/CSV roundtrip、错误位置 | `std:data/json`/`csv` | 不增加 JSON/CSV HostOp |
| 表格 | filter/sort/join/group/GFM | `std:data/table` | 不加表格算法 Builtin |
| 输入输出 | stdin→read_line→stdout;多行 EOF | `std:io` | 只有底层字节读写由 Host 实现 |
| 文件 | read/copy/write/metadata/rename | `std:fs` | 新增 OS 端点须有精确授权,不加 `copy_file` Rust Builtin |
| HTTP | 解析分块/内容长度/UTF-8 body | `std:net/http1` | 不调用 Python HTTP 包 |
| 独立更新 | 仅替换 stdlib bundle 和摘要 | frozen-binary 测试 | CLI 二进制哈希不变 |
**完成门槛**:不允许只在 `tests/conformance/semantics` 放一个 demo 就关闭 stdlib 的待办事项;测试必须从用户导入路径走完整执行流。
## 10. 重构的安全与兼容性验收
1. **纯模式**:任何文件或网络端点无授予则拒绝;控制台 stdin/stderr 是否允许由明确的新 ADR 决定,不能无意突破既有严格模式契约。
2. **项目模式**:每种文件元数据/写入/删除操作单独授权;绝不能复用普通 read 授权执行目录删除;路径规范化防越界、符号链接竞态需要攻击性测试。
3. **严格模式**:目前只明确支持受限 stdout,stdin、file、network、Provider 必须在隔离证明达到要求后才能逐个开放;拒绝时可诊断,不可静默降级。
4. **资源句柄**:跨 run、伪造、关闭后使用均拒绝;标准库 `close_checked` 保证正常路径不遗漏 close;异常致命退出必须由 Host 在 run 结束时清理资源。
5. **短读短写**:所有 `read/write` 返回实际字节数,标准库循环并在零进度时终止;不得认为一次调用即完整成功。
6. **字节与 Unicode**:二进制 I/O 不隐式调用 `文本()`,协议长度以字节计算;UTF-8 解码错误返回有类型的 `Result`。
7. **兼容**:`打印` 输出文本契约、`std:io` 名称、Result 结构和 `Resource` 权限语义不改变;若要调整已有接口,以兼容门面和重大版本演进。
8. **测试**:最基本验收建议为 `cargo fmt --all -- --check`、`cargo clippy --workspace --all-targets -- -D warnings`、`cargo test --workspace --locked`、stdlib digest tests、frozen CLI tests、对应 strict VM 安全矩阵;**这些是目标命令,不代表本审查执行了它们**。
9. **发布文档**:CI 的成功/失败、commit SHA、环境、精确测试命令由自动报告生成;不能用旧审查快照声称新提交已过关。
## 11. 禁止事项:防止再次架构偏移
- 不创建独立的 `marqdo-stdlib` 仓库,除非用户以后决定改变已接受的 ADR-0039。
- 不为 `input()`、`read_line()`、`file_copy()`、JSON、CSV、表格排序分别新增 Rust Builtin。
- 不把标准库失败伪装成成功;不能忽略 `resource.write(...).ok` 与短写。
- 不为了“统一”把普通 `List/Map/Table` 直接改成可共享可变裸内存,破坏既有值语义。
- 不将 Callable Plugin、Resource Provider、Foreign 和 Tool 合并成一个不可审查的万能入口。
- 不在 `strict` 中默许新 native provider,或将 worker 进程隔离称作操作系统沙盒。
- 不因为小 crate 数量多而一次性合并十几个 crate;先识别依赖方向和职责。
- 不再把一次性测试示例宣称成已经交付的标准库产品。
- 不用单纯文本/Unicode 码点算法假装处理 HTTP 的二进制长度和文件协议。
- 不在第一阶段建设无需求支持的 GPU/QPU、JIT、裸机器码或复杂任务调度框架。
## 12. 按文件的最先修改计划(建议前两个 PR)
### PR-1:Correct IO / stdin First
- `crates/marqdo-host/src/resource.rs`:扩展 Stdin、Stderr stream 变体和读写权限;可拆 `resource/stdio.rs`。
- `crates/marqdo-host/src/host.rs`:使 stdio 资源发现和授权支持新端点;保持严格模式 fail-closed。
- `crates/cli/src/commands.rs`:确认真实 stdin 传入 Runtime Session 的路径与 CLI JSON 输出互不污染。
- `stdlib/io.mq.md`:改为兼容门面,提供正确的 write_all + input line API。
- 新增 `stdlib/io/stream.mq.md`、`stdlib/io/console.mq.md`、`stdlib/text/utf8.mq.md`。
- `crates/cli/tests/resource_streams.rs`:新增 stdin/EOF/partial 语义与 stdout/stderr 区分测试。
- 新增 `tests/e2e/interactive-cli`:冻结 CLI、不使用 foreign/plugin。
- 更新 `spec/evaluation.md` 和 security ADR;更新 stdlib digest。
### PR-2:Stdlib Productization + CLI Module Boundaries
- 新增 `stdlib/fs/file.mq.md`、`stdlib/data/json.mq.md`、`stdlib/data/csv.mq.md` 和公共 API 文档。
- 将已验证的 conformance 示例改造为标准库测试,保留原有语义夹具作为后向回归。
- 将 `crates/cli/src/commands.rs` 分为 `commands/`,`loader.rs/stdlib.rs` 归入 project 模块;不改变命令行接口。
- 将 `crates/marqdo-host/src/host.rs` 的资源操作迁至 `resource/*`,将多 Provider 注册迁至 `providers/*`;对 ABI/Capability 策略固定回归。
- 新增 CI 的 `stdlib-import-only`、`no-new-builtin-for-algorithm` 和文档 SHA 一致性门禁。
文件重排应遵循“先行为保护、后内部抽取、最后删冗余”的顺序。不要让大规模结构性 diff 与系统 I/O 新功能混在同一个 PR 里;若严格实施,应把 PR-2 再拆成 stdlib 产品化与纯 Rust 文件重排两个独立变更。
## 13. 与之前 RFC-CM-0001 / RFC-CM-0002 的关系
- 继续沿用 `Managed Value + Addressable Memory Region + Capability Resource`,**不再重启机器码或门电路讨论**。
- 已有 `Memory`、`Resource`、`print` 迁移和表格算法是真实进展,不需要重复开发。
- RFC-CM-0002 中“标准库独立仓库”的早期预设已经被 ADR-0039 改为**同仓库标准库**;应修改旧 RFC 顶部的状态注释并链接 ADR-0039,而不是保留两种相反的实施指导。
- 以前以“证明计算闭包”为主;本次以“**闭包已证明的能力能否成为可靠的、可发行的用户 API**”为中心。
## 14. 审查来源(冻结到同一 commit)
以下路径均对应 `7fbe069196e61d06a00d64fe8d65049c327373eb`:
- [Cargo workspace](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/Cargo.toml)
- [CLI commands](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/crates/cli/src/commands.rs)
- [CLI project/print loader](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/crates/cli/src/loader.rs)
- [Stdlib resolver](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/crates/cli/src/stdlib.rs)
- [Runtime memory](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/crates/marqdo-runtime/src/memory.rs)
- [Builtins](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/crates/marqdo-ir/src/builtins.rs)
- [Host dispatch](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/crates/marqdo-host/src/host.rs)
- [Resource table](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/crates/marqdo-host/src/resource.rs)
- [Standard library IO](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/stdlib/io.mq.md)
- [Standard library HTTP](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/stdlib/net/http.mq.md)
- [Standard library Table](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/stdlib/data/table.mq.md)
- [Evaluation semantics](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/spec/evaluation.md)
- [Stdlib deferred split ADR](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/docs/adr/0039-stdlib-extraction-intent.md)
- [Architecture index](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/docs/architecture/README.md)
- [Completion audit](https://github.com/cflmy/marqdo-cli/blob/7fbe069196e61d06a00d64fe8d65049c327373eb/docs/evidence/2026-11-20-rfc-cm-completion-audit.md)
---
**最终设计判断**:核心三域(托管值、可寻址内存区域、受控系统资源)是合理的;现在必须把“资源端点完整性、标准库产品化、CLI/Host 职责收敛”放到第一优先级。不要再为了扩展某个普通函数修改 Rust Builtin;但也不能假装 stdin、目录删除、TLS、系统时钟和熵源在没有 OS 适配器时能够自动涌现。
+17 -7
View File
@@ -133,7 +133,15 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL
`打印/print` to the bundled `io.写值` when the program uses it; stdlib
modules are injected lazily; resource operations are trace `effect` events
and `HostOp::Print` no longer runs on that path. Strict permits only
stdio/stdout; file/network resources remain refused.
stdio/stdout; stdin/stderr/file/network resources remain refused.
- Stdio streams and console productization (ADR-0040): `stdio/stdin` is
read-only over one shared run cursor; `stdio/stdout`/`stdio/stderr` are
write-only with separate host buffers and output-budget accounting; the CLI
copies non-terminal process stdin at composition and embeds program stderr as a
separate JSON string (or writes it to CLI stderr). `stdlib/io/stream.mq.md`
checks short writes and close errors; `stdlib/io/console.mq.md` implements
line input and print through the resource ABI; `stdlib/text/utf8.mq.md`
provides a validating UTF-8 decoder.
- Full-value stdlib formatting (ADR-0029): FunctionRef `module/name` and
PluginHandle `plugin/export` are read-only observation members (host id
private); `stdlib/format.mq.md` reproduces builtin `打印`/`repr` for all 15
@@ -181,13 +189,15 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL
([evidence](evidence/2026-11-20-stdlib-package.md)). The stdlib intentionally
stays in this repository for now (ADR-0039); extraction is a future option
when size/release cadence requires it, not a pending gate.
- Resource streams (RFC-CM CM-A06, ADR-0025): opaque host-issued `Resource`
handles with `resource.open/read/write/close`; bytes move only through
`Memory`; stdout write is capability/output-budget checked and project-file
- Resource streams (RFC-CM CM-A06, ADR-0025/0026/0040): opaque host-issued
`Resource` handles with `resource.open/read/write/close`; bytes move only
through `Memory`; stdin is read-only over one shared run cursor; stdout and
stderr are separate capability/output-budget checked buffers; project-file
read/write is path/grant checked; file writes buffer until `close` and
never flush on drop; a `.mq.md` chunked copy loop reproduces a file without a
Rust `copy_file`; strict permits only the stdio console channel and refuses
file/TCP/provider resources (CM-A11 lane PASS). Semantic fixture 66 and CLI/host tests.
Rust `copy_file`; strict permits only the stdio stdout console channel and
refuses stdin/stderr/file/TCP/provider resources (CM-A11 lane PASS). Semantic
fixture 66 and CLI/host tests.
- Minimal structure observation/construction (RFC-CM CM-A04, ADR-0023):
value-semantics `list.append/concat`, `map.keys/has/get/set/remove`,
Unicode scalar `text.codepoints`/`text.from_codepoint`, and non-negative
@@ -226,7 +236,7 @@ behavior. This table records only what is in this tree now.
| P0 freeze (ADRs, spec deltas, baseline) | **P1 + partial P2 slice done** | ADR-0022 freezes CM-A01/CM-A02/CM-A03; ADR-0023 freezes CM-A04 (minimal List/Map/Text/Table observation and construction); ADR-0024 freezes CM-A05 (recoverable `Result` map convention); `spec/expression.ebnf`, `spec/evaluation.md`, `spec/value-wire.md`, `spec/analysis-v2.md`, `spec/security.md` updated; existing AST/value goldens unchanged |
| P1 Memory + place assignment | **Implemented locally** | `Value::Memory`, run-scoped `MemoryStore` (64 KiB pages, 64 MiB total / 16 MiB region), `memory.new/size/grow`, `m[i]` read, `m[i] = byte`; semantic fixtures 45–47, runtime negatives 13–18, `examples/memory-buffer.mq.md`; evidence [`evidence/2026-11-20-memory-p1.md`](evidence/2026-11-20-memory-p1.md) |
| P2 Marqdo-written allocator/data structures + minimal structure observation | **PASS (local pure lane); stdlib remains in-repo by design** | `.mq.md`-only evidence: bump allocator with u32 codec, linked list, failed-allocation atomicity (48–50); CM-A04 structure primitives and a table filter (51–54, ADR-0023); a first-fit free-list allocator with block reuse, splitting, address-adjacent coalescing and `Result` maps (55, ADR-0024), a `Memory`-backed binary tree with recursive insert/in-order traversal (56), a `.mq.md` CSV row splitter (57), an open-addressing hash table with linear probing, update and `hash.full` (58), a `.mq.md` UTF-8 encoder over `Memory` (59), a validating UTF-8 decoder rejecting truncated, bad-continuation and surrogate/overlong input (60), a `Memory`-backed graph adjacency list (61), a recursive JSON parser for nested arrays/objects, strings with escapes, integers, booleans, null and error Results (62), a `Memory`-backed stack bytecode VM with PUSH/ADD/MUL/JMP/JZ/HALT (63), big-endian 16/32/64-bit plus LEB128 integer codecs (64), a `.mq.md` table sort rebuilding a new table from `table.row/columns/cell` (65), a table join (67), a quoted/escaped CSV row parser (68), table group (69), an Int matrix multiply (70), and a sparse-matrix multiply (73); the first-fit allocator also detects double free (`alloc.double_free`). Evidence: [`memory-p2-closure`](evidence/2026-11-20-memory-p2-closure.md), [`structure-api-p2`](evidence/2026-11-20-structure-api-p2.md), [`result-and-free-list-p2`](evidence/2026-11-20-result-and-free-list-p2.md). Local gate PASS: `crates/cli/tests/p2_closure.rs` runs 21 C01–C06/C11/C12 fixtures plus the stdlib table self-test in `--mode pure` (no foreign/plugin/import grants), scans sources for foreign/plugin/import shortcuts and compares value/stdout to goldens; evidence [`p2-closure-gate`](evidence/2026-11-20-p2-closure-gate.md). The bundled `stdlib/` tree, `std:` imports, digest verification, Analysis identity and frozen-CLI closure test are implemented (ADR-0028). The stdlib intentionally remains in-repo (ADR-0039); a separate repository is a future extraction option, not a gate. Linux Docker CI is green; a strict security runner and Windows/macOS lanes are still pending. |
| P3 Resource/Stream ABI | **PARTIAL** | ADR-0025/0026 CM-A06 slices: opaque `Resource`, `resource.open/read/write/close`, stdout write in pure/project, granted project-file read and buffered project-file write in project, EOF/closed/handle errors, wire/host rejection; strict permits only stdio/stdout and refuses file/network resources. Fixture 66 + CLI/host tests. CM-A07 **Compat-C** is implemented: when a program calls `打印/print`, the loader injects the bundled `stdlib/io.mq.md` and the runtime routes to `io.写值` (all 15 value types, UTF-8 via `Memory`, `resource.write(stdout)`); `HostOp::Print` and the legacy `Value::display()` print path are removed, and a missing bundle fails with `kernel.unavailable`. Strict permits only the existing stdio console channel. Providers are registered through ADR-0033 (in-process) and ADR-0035 (worker helper). Still missing: directories, C ABI provider binding, provider packages, strict provider support. |
| P3 Resource/Stream ABI | **PARTIAL** | ADR-0025/0026/0040 CM-A06 slices: opaque `Resource`, `resource.open/read/write/close`, stdin read, separate stdout/stderr write, granted project-file read and buffered project-file write in project, EOF/closed/handle errors, wire/host rejection; strict permits only stdio/stdout and refuses stdin/stderr/file/network resources. `stdlib/io/stream.mq.md`, `stdlib/io/console.mq.md` and `stdlib/text/utf8.mq.md` are product-level `.mq.md` APIs; fixture 66 + CLI/host tests. CM-A07 **Compat-C** is implemented: when a program calls `打印/print`, the loader injects the bundled `stdlib/io.mq.md` and the runtime routes to `io.写值` (all 15 value types, UTF-8 via `Memory`, `resource.write(stdout)`); `HostOp::Print` and the legacy `Value::display()` print path are removed, and a missing bundle fails with `kernel.unavailable`. Strict permits only the existing stdio console channel. Providers are registered through ADR-0033 (in-process) and ADR-0035 (worker helper). Still missing: directories, C ABI provider binding, provider packages, strict provider support. |
| P4 Table logical/physical migration | **PASS (local gate); T1/T2/T3 future** | ADR-0032/CM-A08: `Value::Table` semantics and GFM AST laziness unchanged; `stdlib/data/table.mq.md` provides select/filter/sort/append/set/join/group/`转GFM` as value-returning `.mq.md` functions with input immutability; zero Rust table algorithms were added. Cross-storage equivalence is pinned by fixture 72 (row Table vs `.mq.md` columnar store, identical headers/rows/cells). The optional T1/T2/T3 physical-storage/column-store migration was not required by the gate and is not implemented. |
| P5 third-party Provider / TCP / HTTP | **PASS (local gate); external network/CI and plugin-Worker resource binding open** | C10: `stdlib/net/http.mq.md` + fixture 71 parse HTTP/1.1 status/headers/chunked bodies and build requests; `http_over_tcp.rs` runs a loopback HTTP exchange wholly in `.mq.md` over `resource.open("tcp",...)`. ADR-0034 IP-literal blocking TCP endpoint (exact IP grant, project-only, 5s timeouts) and ADR-0033 in-process `ResourceProvider` v1 (registration, protocol/kind checks, name grant, mock read/write provider) ADR-0035 host-managed worker provider transport (framed JSON helper, ping/open/read/write/close, trusted-only) and ADR-0036 trusted C ABI provider (shared-library symbols, version/load checks) ADR-0037 local provider-package manifest installation and ADR-0038 C ABI provider worker binding (separate trusted helper process) are tested. External DNS/TLS/UDP, host HTTP, provider packages/signatures and the plugin-Worker resource binding remain out of scope. |
| P6 performance / native backends | **BASELINE ONLY** | reproducible profile harness `tools/perf/bench.py` and [`p6-profile`](evidence/2026-11-20-p6-profile.md) for 7 workloads; no bytecode/JIT/columnar numeric backend or equivalence/optimization gate yet |
+3 -3
View File
@@ -61,9 +61,9 @@ legacy `HostOp::Print` effect does not appear in Compat-B runs.
Strict L1 already permits the console channel. Compat-B needs stdout writes in
strict so the same print implementation works there. Therefore strict mode
permits **only** `stdio/stdout` write with the console grant; project-file,
network, process and device resource opens remain refused, and no new provider
is enabled. This is the existing console capability reached through the
permits **only** `stdio/stdout` write with the console grant; stdin/stderr,
project-file, network, process and device resource opens remain refused, and no new
provider is enabled (ADR-0040 keeps the later stdin/stderr slice separate). This is the existing console capability reached through the
resource ABI, not a new OS capability. File resources stay fail-closed.
### 5. Compatibility and fallback
@@ -0,0 +1,60 @@
# ADR-0040: Stdio stdin/stderr endpoints and stdlib stream APIs
Status: **Accepted**
Date: 2026-11-21
Relates to: ADR-0025, ADR-0026, ADR-0027, ADR-0030, ADR-0031
Specs: `spec/evaluation.md`, `spec/security.md`
Evidence: `docs/evidence/2026-11-21-stdio-streams.md`
## Context
ADR-0025 opened one `stdio/stdout` write endpoint. That left `stdin` and
`stderr` as hidden OS channels, and the bundled `io.写文本` ignored the
`resource.write` `{ok,count}` result. The audit RFC calls for a real resource
boundary, not a Rust `Input`/`Print` builtin.
## Decision
* Add `stdio/stdin` (rights `["read"]`) and `stdio/stderr` (rights
`["write"]`). `stdio/stdout` remains `["write"]`. Rights/kinds are exact;
mixing them is `resource.unsupported`.
* All stdin handles in one run share one cursor. `resource.read` consumes bytes and
reports `{count,eof}`; EOF is a successful zero-byte read, not an error.
Closing a handle does not reset or close the run cursor. There is no blocking
OS read inside the host table.
* stdout and stderr are distinct host buffers. Both charge the existing output
budget, but `output()`/`take_output()` never return stderr and
`error()`/`take_error()` never return stdout. A budget failure appends
nothing to either buffer.
* The CLI reads process stdin once at composition time only when stdin is not a
terminal. This keeps an interactive `marqdo run` from waiting for EOF. JSON run
output adds a separate `stderr` string; text mode writes captured program
stderr to process stderr. Diagnostics and traces remain CLI stderr and never mix
with the JSON stdout document.
* Strict mode remains fail-closed: only `stdio/stdout` with `["write"]` and the
console grant is allowed. stdin, stderr, file, TCP and provider streams are
`permission.denied`.
* The standard library keeps the product APIs in `.mq.md`:
`stdlib/io/stream.mq.md` owns `写全部`/`写文本` with short-write and close
checking; `stdlib/io/console.mq.md` owns `读取行`/`打印` over the resource
ABI; `stdlib/text/utf8.mq.md` owns a validating UTF-8 decoder.
`stdlib/io.mq.md` remains the compatibility facade and print target.
## Non-goals
No process, device, directory, DNS, TLS, UDP, seek, mmap, async or zero-copy
endpoint. The stdin source is the finite process input supplied by the
composition root, not a general interactive console driver. No new language
keyword, `Input` builtin, `HostOp::Print` or second Markdown parser.
## Compatibility
The resource ABI is additive. Existing stdout-only programs, `打印`, file
read/write and provider/tcp streams keep their prior behavior. A program that
used a closed handle still gets the recoverable `resource.closed` result.
## Consequences
CLI programs can now read piped stdin and write program stderr without leaking
either into JSON diagnostics. The remaining OS-shaped gaps are explicit and can be
added only through a separate resource operation decision.
+3 -2
View File
@@ -6,7 +6,7 @@ normative RFC text stays at its reviewable path under `docs/`.
| Document | Status | Implemented part |
|---|---|---|
| [RFC-CM-0001: compute machine, addressable memory and system boundary](../MARQDO_COMPUTE_MACHINE_RFC_0001.md) | Draft / design input | R1 (P1): `Memory` value, `memory.new/size/grow`, byte read/write, Memory-index assignment, budgets, wire/host rejection. Implemented per [ADR-0022](../adr/0022-addressable-memory-and-place-assignment.md); see [STATUS §5](../STATUS.md). |
| [RFC-CM-0002: minimal core, stdlib self-hosting and system capability migration](../MARQDO_COMPUTE_CORE_STDLIB_MIGRATION_RFC_0002.md) | Draft / design input | P0/P1, a **partial P2**, and a **partial P3** first slice (ADR-0025/0026: stdout write, granted project-file read and buffered project-file write via `resource.*`; strict permits only the stdio console channel; ADR-0027/0030/0031 Compat-C print via bundled stdlib, no legacy host formatter). P2: fixtures 48–50 (`.mq.md` bump allocator, u32 codec, linked list/traversal, failed-allocation atomicity), CM-A04 structure primitives for List/Map/Text/Table (ADR-0023, fixtures 51–54), and a `.mq.md` first-fit free-list allocator with block splitting, address-adjacent coalescing and `Result` maps (ADR-0024, fixture 55), plus a `Memory`-backed binary tree with recursive insert/in-order traversal (fixture 56), a `.mq.md` CSV row splitter (fixture 57), an open-addressing hash table with linear probing (fixture 58), a `.mq.md` UTF-8 encoder and validating decoder (`utf8.truncated`/`invalid_continuation`/`invalid_scalar`) (fixtures 59–60), a `Memory`-backed graph adjacency list (fixture 61), a recursive JSON parser/formatter for arrays, objects, strings with escapes, ints/floats/booleans/null and error Results (fixture 62), a `Memory`-backed stack bytecode VM with PUSH/ADD/MUL/JMP/JZ/HALT (fixture 63), big-endian 16/32/64-bit plus LEB128 integer codecs (fixture 64), a `.mq.md` table sort rebuilding the table without mutating the input (fixture 65), a table join built from `table.row/columns/cell` (fixture 67), and a quoted/escaped CSV row parser (fixture 68), table group (fixture 69) and an Int matrix multiply (fixture 70). The stdlib intentionally remains in this repository for now (ADR-0039); no separate external repo, network/generic providers, P4 physical storage migration, P5 providers or P6 performance work. |
| [RFC-CM-0002: minimal core, stdlib self-hosting and system capability migration](../MARQDO_COMPUTE_CORE_STDLIB_MIGRATION_RFC_0002.md) | Draft / design input | P0/P1, a **partial P2**, and a **partial P3** first slice (ADR-0025/0026/0040: stdin read, separate stdout/stderr write, granted project-file read and buffered project-file write via `resource.*`; strict permits only the stdio stdout console channel; ADR-0027/0030/0031 Compat-C print via bundled stdlib, no legacy host formatter). P2: fixtures 48–50 (`.mq.md` bump allocator, u32 codec, linked list/traversal, failed-allocation atomicity), CM-A04 structure primitives for List/Map/Text/Table (ADR-0023, fixtures 51–54), and a `.mq.md` first-fit free-list allocator with block splitting, address-adjacent coalescing and `Result` maps (ADR-0024, fixture 55), plus a `Memory`-backed binary tree with recursive insert/in-order traversal (fixture 56), a `.mq.md` CSV row splitter (fixture 57), an open-addressing hash table with linear probing (fixture 58), a `.mq.md` UTF-8 encoder and validating decoder (`utf8.truncated`/`invalid_continuation`/`invalid_scalar`) (fixtures 59–60), a `Memory`-backed graph adjacency list (fixture 61), a recursive JSON parser/formatter for arrays, objects, strings with escapes, ints/floats/booleans/null and error Results (fixture 62), a `Memory`-backed stack bytecode VM with PUSH/ADD/MUL/JMP/JZ/HALT (fixture 63), big-endian 16/32/64-bit plus LEB128 integer codecs (fixture 64), a `.mq.md` table sort rebuilding the table without mutating the input (fixture 65), a table join built from `table.row/columns/cell` (fixture 67), and a quoted/escaped CSV row parser (fixture 68), table group (fixture 69) and an Int matrix multiply (fixture 70). The stdlib intentionally remains in this repository for now (ADR-0039); no separate external repo, network/generic providers, P4 physical storage migration, P5 providers or P6 performance work. |
The RFCs are architecture constraints, not evidence that a feature exists. The
implementation status lives in `docs/STATUS.md`; frozen behavior is in
@@ -36,7 +36,8 @@ implementation status lives in `docs/STATUS.md`; frozen behavior is in
| CM-A09 provider package manifest v1 | [0037](../adr/0037-provider-package-manifest.md) | Accepted (host-side install) |
| CM-A09 C ABI provider worker binding v1 | [0038](../adr/0038-c-abi-provider-worker-binding.md) | Accepted (trusted helper) |
| Stdlib extraction deferral | [0039](../adr/0039-stdlib-extraction-intent.md) | Accepted (in-repo for now) |
| Table physical storage (T1/T2/T3), C/Worker provider transport, strict file/network resources | — | Not decided; do not implement by implication |
| Stdio stdin/stderr plus stdlib stream/console APIs | [0040](../adr/0040-stdio-stdin-stderr-and-stdlib-streams.md) | Accepted (stdin read, separate stdout/stderr, product stdlib) |
| Table physical storage (T1/T2/T3), strict stdin/stderr/file/network resources | — | Not decided; do not implement by implication |
## What a future phase must not do
+30
View File
@@ -0,0 +1,30 @@
# Evidence: stdio stdin/stderr and stdlib streams
Date: 2026-11-21 (Asia/Shanghai)
Repository: `marqdo-cli`
Decision record: [ADR-0040](../adr/0040-stdio-stdin-stderr-and-stdlib-streams.md)
## Local gate
```text
cargo fmt --all -- --check
cargo test --workspace --no-fail-fast
```
Both ran locally in the writable `.cargo-home`. The workspace suite passed.
## Covered slices
* host: pure-mode stdin chunking and EOF, separate stdout/stderr buffers,
strict refusal of stdin/stderr, closed-handle errors, output budget;
* CLI: piped stdin through `std:io/console.读取行`, separate JSON `stdout` and
`stderr`, EOF without a panic;
* stdlib: `std::io/stream.写全部` loops on `{ok,count}`, rejects zero progress,
`写文本` grows `Memory` from the UTF-8 byte length and checks close;
`std::text/utf8.解码` rejects truncated, invalid-continuation, overlong,
surrogate and out-of-range scalar input.
## Not covered
Directories, process/device resources, DNS/TLS/UDP, seek/mmap, async I/O,
interactive terminal line editing, Windows/macOS native lanes, and remote CI.
+12 -10
View File
@@ -168,18 +168,19 @@ resource.close(handle)
-> {ok:true, value:None} | {ok:false, code, message}
```
* Providers: `stdio/stdout` with `["write"]` (pure/project, and strict's
existing console channel, console grant);
`file` with `["read"]` (project, `fs_read` grant, bounded by the 8 MiB file
limit); `file` with `["write"]` (project, `fs_write` grant, 8 MiB buffered
write flushed on `close`, ADR-0026). Mixed rights and other kinds are
`resource.unsupported`.
* Providers: `stdio/stdin` with `["read"]` and `stdio/stdout`/`stdio/stderr`
with `["write"]` (pure/project; one shared stdin cursor and separate stdout
and stderr buffers; console grant). Strict permits only the existing
`stdio/stdout` channel. `file` with `["read"]` (project, `fs_read` grant,
bounded by the 8 MiB file limit); `file` with `["write"]` (project,
`fs_write` grant, 8 MiB buffered write flushed on `close`, ADR-0026).
Mixed rights and other kinds are `resource.unsupported`.
* `resource.read` checks the destination `Memory` range before calling the
provider, caps the request to the remaining buffer, writes the returned bytes
and reports the actual count plus `eof`. EOF is success, not an error.
* `resource.write` copies bytes out of `Memory` for the call only; a short write
is a count. stdout output is charged to the output budget and delivered to
the program stdout channel. A project-file write stream appends to a bounded
is a count. stdout/stderr output is charged to the output budget and delivered
to separate program channels. A project-file write stream appends to a bounded
host buffer and does not touch the filesystem until `close`; a failed close
leaves the handle open with its buffer, and an unclosed writer never creates
the file.
@@ -247,8 +248,9 @@ refuses to run (see `security.md`).
For a fixed source, mode, capabilities and budget, evaluation is
deterministic. Stdout/stderr from host operations are separated: program
output goes to stdout, diagnostics/traces go to stderr, and `--json` output
never mixes with program output.
stdout and program stderr are captured separately, diagnostics/traces go to the
CLI stderr, and `--json` output never mixes with any program output. JSON
reports `stdout` and `stderr` as separate strings.
## 8. Builtins v1
+8 -7
View File
@@ -27,8 +27,8 @@ Normative implementation:
| Mode | Intended guarantee | Actual status in this build |
|---|---|---|
| `pure` | No external I/O; memory computation and budgets only. | **Enforced.** `Capabilities::pure()` = `console`, `math` only; every FS/foreign/plugin grant flag makes the CLI exit 2 with "pure mode prohibits external I/O grants". |
| `project` | Operator-approved project resources under `CapabilityHost` + bounded resources. | **Enforced for the implemented operations.** Defaults equal `pure`; `--allow-read`, `--allow-write`, `--allow-foreign`, `--allow-plugin` add grants. `resource.open` supports stdout write, granted project-file read/write (ADR-0025/0026), granted in-process provider kinds (ADR-0033) and granted IP-literal TCP streams (ADR-0034); process/device remain unsupported. Sandboxing is cooperative (in-process checks), not OS isolation; in-process providers are trusted, not sandboxed. |
| `strict` | Untrusted documents under real OS isolation. | **Linux: enforced when the runtime probe passes (ADR-0021).** `resource.open` is **refused** in strict except for the existing `stdio/stdout` console channel (console grant); file/TCP/provider/process resources remain refused until a provider passes the isolation lane (CM-A11, ADR-0030/0034). `PlatformCapabilities::detect()` runs a real Bubblewrap smoke test; success reports `Available { mechanism }`, failure reports `Unavailable { reason }`. Strict L1 supports the interpreter, console, native Math and the read-only bundled stdlib (published at `/run/marqdo-stdlib` for the Compat-C print chain); `--allow-read/--allow-write/--allow-foreign/--allow-plugin/--plugin/--trust-native-plugin/--kernel/--trace-file` are rejected with exit 2. macOS/Windows and Linux hosts without the mechanisms remain `Unavailable`; there is **no fallback** to an unrestricted process. |
| `project` | Operator-approved project resources under `CapabilityHost` + bounded resources. | **Enforced for the implemented operations.** Defaults equal `pure`; `--allow-read`, `--allow-write`, `--allow-foreign`, `--allow-plugin` add grants. `resource.open` supports stdin read, separate stdout/stderr write, granted project-file read/write (ADR-0025/0026/0040), granted in-process provider kinds (ADR-0033) and granted IP-literal TCP streams (ADR-0034); process/device remain unsupported. Sandboxing is cooperative (in-process checks), not OS isolation; in-process providers are trusted, not sandboxed. |
| `strict` | Untrusted documents under real OS isolation. | **Linux: enforced when the runtime probe passes (ADR-0021).** `resource.open` is **refused** in strict except for the existing `stdio/stdout` console channel (console grant); stdin/stderr, file/TCP/provider/process resources remain refused until a provider passes the isolation lane (CM-A11, ADR-0030/0034/0040). `PlatformCapabilities::detect()` runs a real Bubblewrap smoke test; success reports `Available { mechanism }`, failure reports `Unavailable { reason }`. Strict L1 supports the interpreter, console, native Math and the read-only bundled stdlib (published at `/run/marqdo-stdlib` for the Compat-C print chain); `--allow-read/--allow-write/--allow-foreign/--allow-plugin/--plugin/--trust-native-plugin/--kernel/--trace-file` are rejected with exit 2. macOS/Windows and Linux hosts without the mechanisms remain `Unavailable`; there is **no fallback** to an unrestricted process. |
`marqdo inspect <file> --platform` prints the detection report, for example:
@@ -55,7 +55,7 @@ set for one run:
| Field | Default (`pure`/`project`) | Used by |
|---|---|---|
| `console: bool` | `true` | `resource.open("stdio","stdout",["write"])` and the stdlib print chain; a missing grant is `permission.denied` and output is also bounded by the output budget |
| `console: bool` | `true` | `resource.open("stdio","stdin",["read"])`, `resource.open("stdio","stdout"/"stderr",["write"])` and the stdlib console chain; a missing grant is `permission.denied` and stdout/stderr are also bounded by the output budget |
| `math: bool` | `true` | `math.*` host ops |
| `fs_read: Vec<String>` | empty | `fs.read` and project `resource.open("file", path, ["read"])` |
| `fs_write: Vec<String>` | empty | `fs.write` and project `resource.open("file", path, ["write"])` |
@@ -143,10 +143,10 @@ no host bypass for tests: tests use the same `StdHost`/`Capabilities` types.
|---|---|---|
| evaluation fuel | `DEFAULT_FUEL = 5_000_000` | 1 unit per statement, 1 per expression node, 1 per branch condition, 1 per loop iteration; exhaustion → `resource.exhausted` |
| call stack depth | `DEFAULT_STACK_LIMIT = 256` | checked before every function call → `resource.exhausted` |
| program output bytes | `DEFAULT_OUTPUT_BYTES = 1 MiB` | `print` output and `fs.read` contents are charged; overflow → `resource.exhausted` |
| program output bytes | `DEFAULT_OUTPUT_BYTES = 1 MiB` | `print`, `stdio/stdout` and `stdio/stderr` output plus `fs.read` contents are charged; overflow → `resource.exhausted` |
| wall clock | none by default | `--timeout <seconds>` sets a deadline; checked between statements/iterations and before loop conditions → `resource.exhausted`. It does **not** preempt a running host call. |
| addressable memory | 64 MiB committed total, 16 MiB per region, 64 KiB pages; `--max-memory` / `--max-region-memory` override | Runtime-owned `MemoryStore` (ADR-0022); create/grow checks both budgets before allocation and failed growth is atomic → `resource.exhausted`. Memory is pure computation and needs no capability grant in any mode. |
| resource streams | stdout writes charged to the existing output budget; file reads and writer buffers bounded by the 8 MiB per-file limit | Host-owned resource table (ADR-0025/0026); output budget is checked before a stdout write is buffered; a failed check produces no bytes; writer buffers flush only on `close`. No separate I/O byte budget yet. |
| resource streams | stdin reads over a shared run cursor; stdout/stderr writes charged to the existing output budget; file reads and writer buffers bounded by the 8 MiB per-file limit | Host-owned resource table (ADR-0025/0026/0040); output budget is checked before a stdout/stderr write is buffered; a failed check produces no bytes; writer buffers flush only on `close`. No separate I/O byte budget yet. |
CLI overrides (`run`/`call`): `--fuel`, `--stack`, `--max-output`,
`--timeout`. Additional fixed limits:
@@ -181,8 +181,9 @@ overflow). In-process math and native plugins are not OS-limited.
* argument values and file contents are **not** recorded, so ordinary secrets
do not leak into traces by design. Paths are also not recorded today.
* In text mode trace entries are printed to stderr (`trace: ...`); in JSON
mode they are a `trace` array next to the program stdout string. Program
output is always separate from diagnostics/trace (stdout vs stderr).
mode they are a `trace` array next to separate program `stdout`/`stderr`
strings. Program output is always separate from diagnostics/trace, and
`stdio/stderr` never shares the `stdout` buffer.
## 5. Foreign kernels: process isolation, not a sandbox
+5 -2
View File
@@ -5,8 +5,11 @@ This directory is the official, offline `marqdo-stdlib` bundle consumed by
module; a mismatch is a hard error.
- `format.mq.md` — value formatting (`显示`, list/map/table display, escaping)
- `text.mq.md` — UTF-8 encoding into `Memory`
- `io.mq.md` — `写文本` / `写值` through `resource.write(stdout)`
- `text.mq.md` — compatibility UTF-8 encoding into `Memory`
- `text/utf8.mq.md` — `.mq.md` UTF-8 encoder and validating decoder
- `io.mq.md` — compatibility facade over `stream` and `console`
- `io/stream.mq.md` — `写全部` / `写文本` with short-write and close checking
- `io/console.mq.md` — `读取行` / `打印` over `stdio/stdin` and `stdio/stdout`
- `demo.mq.md` — side-by-side comparison with builtin `打印`
- `data/table.mq.md` — value-returning table select/filter/sort/append/set/join/group/GFM
- `net/http.mq.md` — HTTP/1.1 response parsing (headers/chunked) and request builder
+17 -14
View File
@@ -1,26 +1,21 @@
---
import:
format: "./format.mq.md"
text: "./text.mq.md"
stream: "./io/stream.mq.md"
console: "./io/console.mq.md"
---
## 写文本
`(文本)`
把文本编码为 UTF-8 并写入已授权的 stdout 资源流。
*stream.写文本(文本)*
**内存 = memory.new(1, 1)**
## 写全部
**长度 = text.utf8编码(文本, 内存, 0)**
`(流, 内存, 起点, 长度)`
**流 = resource.open("stdio", "stdout", ["write"])**
> resource.write(流, 内存, 0, 长度)
> resource.close(流)
*None*
*stream.写全部(流, 内存, 起点, 长度)*
## 写值
@@ -28,8 +23,16 @@ import:
按语言值语义格式化后输出,并追加换行。
**文字 = format.显示(值) + "\n"**
*stream.写文本(format.显示(值) + "\n")*
> 写文本(文字)
## 读取行
*None*
`()`
*console.读取行()*
## 打印
`(值)`
*console.打印(值)*
+119
View File
@@ -0,0 +1,119 @@
---
import:
format: "../format.mq.md"
stream: "./stream.mq.md"
utf8: "../text/utf8.mq.md"
---
## 读取行
`()`
从已授权的 `stdio/stdin` 资源读取一行。按有界块读取并增量解码 UTF-8;
EOF 返回 `{ok:false, code:"io.eof"}`,无效 UTF-8 返回可恢复错误。
**流 = resource.open("stdio", "stdin", ["read"])**
**内存 = memory.new(1, 1)**
**字节 = []**
**结束 = false**
**错误 = ""**
- `not 结束`
**读取 = resource.read(流, 内存, 0, 1)**
1. `not 读取.ok`
**结束 = true**
**错误 = 读取.code**
2. `读取.value.eof and 读取.value.count == 0`
**结束 = true**
3. `内存[0] == 10`
**结束 = true**
4. `true`
**字节 = list.append(字节, 内存[0])**
**关闭 = resource.close(流)**
1. `错误 != ""`
*{ok: false, code: 错误, message: "stdin read failed"}*
2. `not 关闭.ok`
*关闭*
3. `长度(字节) == 0`
*{ok: false, code: "io.eof", message: "end of input"}*
4. `true`
**数据长度 = 长度(字节)**
1. `数据长度 > 0 and 字节[数据长度 - 1] == 13`
**数据长度 = 数据长度 - 1**
2. `true`
**忽略 = 0**
**页 = (数据长度 + 65535) // 65536**
**数据 = memory.new(页, 页)**
**下标 = 0**
- `下标 < 数据长度`
**`数据[下标] = 字节[下标]`**
**下标 = 下标 + 1**
**解码 = utf8.解码(数据, 0, 数据长度)**
1. `not 解码.ok`
*解码*
2. `true`
*{ok: true, value: 解码.value}*
## 打印
`(值)`
**结果 = stream.写文本(format.显示(值) + "\n")**
1. `not 结果.ok`
*结果*
2. `true`
*{ok: true, value: None}*
## 写文本
`(文本)`
*stream.写文本(文本)*
## 写全部
`(流, 内存, 起点, 长度)`
*stream.写全部(流, 内存, 起点, 长度)*
+101
View File
@@ -0,0 +1,101 @@
---
import:
format: "../format.mq.md"
utf8: "../text/utf8.mq.md"
---
## 页数
`(字节)`
**页 = (字节 + 65535) // 65536**
1. `页 == 0`
*1*
2. `true`
*页*
## 写全部
`(流, 内存, 起点, 长度)`
把 `长度` 字节从 `内存` 写到资源流。短写只返回实际数量,调用方继续;
零进度返回 `io.write_zero`,不静默吞掉失败。
**已写 = 0**
**失败 = false**
**本次 = {ok: true, value: {count: 0}}**
**错误结果 = {ok: false, code: "io.write_zero", message: "resource.write made no progress"}**
- `已写 < 长度`
**剩余 = 长度 - 已写**
**本次 = resource.write(流, 内存, 起点 + 已写, 剩余)**
1. `not 本次.ok`
**失败 = true**
**错误结果 = 本次**
**已写 = 长度**
2. `本次.value.count == 0`
**失败 = true**
**已写 = 长度**
3. `true`
**已写 = 已写 + 本次.value.count**
1. `失败`
*错误结果*
2. `已写 < 长度`
*错误结果*
3. `true`
*{ok: true, value: {count: 长度}}*
## 写文本
`(文本)`
把文本编码为 UTF-8 并写入已授权的 stdout 资源流。内存按实际字节长度增长,
写入循环检查 `{ok,count}`;只有全部字节写完并关闭成功才返回成功。
**字节数 = format.字节长度(文本)**
**内存 = memory.new(页数(字节数), 页数(字节数))**
**长度 = utf8.编码(文本, 内存, 0)**
**流 = resource.open("stdio", "stdout", ["write"])**
**写结果 = 写全部(流, 内存, 0, 长度)**
**关闭 = resource.close(流)**
1. `not 写结果.ok`
*写结果*
2. `not 关闭.ok`
*关闭*
3. `true`
*{ok: true, value: {count: 长度}}*
+1 -1
View File
@@ -1,5 +1,5 @@
{
"name": "marqdo-stdlib",
"version": "0.1.0",
"digest": "sha256:832940f35fe5651eabe4ac672e12e2d8f2bf4e649cc430e455425b4f2a983121"
"digest": "sha256:bf11fbdd5fedcaf596308630bdd9590600d9ffac6658c9af483215b4b91aa13f"
}
+224
View File
@@ -0,0 +1,224 @@
## 成功
`(值)`
*{ok: true, value: 值}*
## 失败
`(代码, 消息)`
*{ok: false, code: 代码, message: 消息}*
## 续字节
`(字节)`
*字节 >= 128 and 字节 < 192*
## 编码
`(文本, 内存, 基址)`
把 Unicode 标量序列按 UTF-8 写入 `Memory`,返回字节长度。
这是 `.mq.md` 自己实现的编码逻辑,不是 Rust 内建 UTF-8 转换。
**点 = text.codepoints(文本)**
**下标 = 0**
**偏移 = 0**
- `下标 < 长度(点)`
**码点 = 点[下标]**
1. `码点 < 128`
**`内存[基址 + 偏移] = 码点`**
**偏移 = 偏移 + 1**
2. `码点 < 2048`
**`内存[基址 + 偏移] = 192 + 码点 // 64`**
**`内存[基址 + 偏移 + 1] = 128 + 码点 % 64`**
**偏移 = 偏移 + 2**
3. `码点 < 65536`
**`内存[基址 + 偏移] = 224 + 码点 // 4096`**
**`内存[基址 + 偏移 + 1] = 128 + 码点 // 64 % 64`**
**`内存[基址 + 偏移 + 2] = 128 + 码点 % 64`**
**偏移 = 偏移 + 3**
4. `true`
**`内存[基址 + 偏移] = 240 + 码点 // 262144`**
**`内存[基址 + 偏移 + 1] = 128 + 码点 // 4096 % 64`**
**`内存[基址 + 偏移 + 2] = 128 + 码点 // 64 % 64`**
**`内存[基址 + 偏移 + 3] = 128 + 码点 % 64`**
**偏移 = 偏移 + 4**
**下标 = 下标 + 1**
*偏移*
## 解码2
`(内存, 基址, 长度, 偏移, 文本)`
1. `偏移 + 1 >= 长度`
*[文本, 偏移, "utf8.truncated"]*
**b1 = 内存[基址 + 偏移 + 1]**
1. `not 续字节(b1)`
*[文本, 偏移, "utf8.invalid_continuation"]*
**码点 = (内存[基址 + 偏移] - 192) * 64 + (b1 - 128)**
1. `码点 < 128`
*[文本, 偏移, "utf8.invalid_scalar"]*
*[文本 + text.from_codepoint(码点), 偏移 + 2, ""]*
## 解码3
`(内存, 基址, 长度, 偏移, 文本)`
1. `偏移 + 2 >= 长度`
*[文本, 偏移, "utf8.truncated"]*
**b1 = 内存[基址 + 偏移 + 1]**
**b2 = 内存[基址 + 偏移 + 2]**
1. `not (续字节(b1) and 续字节(b2))`
*[文本, 偏移, "utf8.invalid_continuation"]*
**码点 = (内存[基址 + 偏移] - 224) * 4096 + (b1 - 128) * 64 + (b2 - 128)**
1. `码点 < 2048 or (码点 >= 55296 and 码点 <= 57343)`
*[文本, 偏移, "utf8.invalid_scalar"]*
*[文本 + text.from_codepoint(码点), 偏移 + 3, ""]*
## 解码4
`(内存, 基址, 长度, 偏移, 文本)`
1. `偏移 + 3 >= 长度`
*[文本, 偏移, "utf8.truncated"]*
**b1 = 内存[基址 + 偏移 + 1]**
**b2 = 内存[基址 + 偏移 + 2]**
**b3 = 内存[基址 + 偏移 + 3]**
1. `not (续字节(b1) and 续字节(b2) and 续字节(b3))`
*[文本, 偏移, "utf8.invalid_continuation"]*
**码点 = (内存[基址 + 偏移] - 240) * 262144 + (b1 - 128) * 4096 + (b2 - 128) * 64 + (b3 - 128)**
1. `码点 < 65536 or 码点 > 1114111`
*[文本, 偏移, "utf8.invalid_scalar"]*
*[文本 + text.from_codepoint(码点), 偏移 + 4, ""]*
## 解码
`(内存, 基址, 长度)`
**偏移 = 0**
**文本 = ""**
**错误 = ""**
- `偏移 < 长度 and 错误 == ""`
**首 = 内存[基址 + 偏移]**
1. `首 < 128`
**文本 = 文本 + text.from_codepoint(首)**
**偏移 = 偏移 + 1**
2. `首 < 194`
**错误 = "utf8.invalid_lead"**
3. `首 < 224`
**结果 = 解码2(内存, 基址, 长度, 偏移, 文本)**
**文本 = 结果[0]**
**偏移 = 结果[1]**
**错误 = 结果[2]**
4. `首 < 240`
**结果 = 解码3(内存, 基址, 长度, 偏移, 文本)**
**文本 = 结果[0]**
**偏移 = 结果[1]**
**错误 = 结果[2]**
5. `首 < 245`
**结果 = 解码4(内存, 基址, 长度, 偏移, 文本)**
**文本 = 结果[0]**
**偏移 = 结果[1]**
**错误 = 结果[2]**
6. `true`
**错误 = "utf8.invalid_lead"**
1. `错误 == ""`
*成功(文本)*
2. `true`
*失败(错误, "UTF-8 解码失败")*
## utf8编码
`(文本, 内存, 基址)`
*编码(文本, 内存, 基址)*
## utf8解码
`(内存, 基址, 长度)`
*解码(内存, 基址, 长度)*
+7 -5
View File
@@ -261,12 +261,14 @@ RFC-CM CM-A06 resource streams (ADR-0025) are pinned by:
`stdio/stdout` resource, closed, then re-used to prove
`{ok:false,code:"resource.closed"}`; the expected `stdout` is the decoded
text;
* `crates/marqdo-host/tests/resource_streams.rs` (9): pure-mode stdout write,
output-budget atomicity, strict refusal, project-file chunking/EOF, missing
grant, traversal rejection and unsupported rights/kinds;
* `crates/marqdo-host/tests/resource_streams.rs` (17): pure-mode stdin read,
stdout/stderr separation, output-budget atomicity, strict refusal of
stdin/stderr/file, project-file chunking/EOF, missing grant, traversal
rejection and unsupported rights/kinds;
* `crates/cli/tests/resource_streams.rs`: real `marqdo run` with
`--allow-read data.txt`, three chunked reads into `Memory` plus EOF, the same
document failing with `permission.denied` without the grant, and a
`--allow-read data.txt`, three chunked reads into `Memory` plus EOF, piped
stdin line reads through `std:io/console`, separate stdout/stderr capture,
the same document failing with `permission.denied` without the grant, and a
write-then-read program with `--allow-write/--allow-read out.bin` proving the
file is created only at `resource.close`, and a chunked `.mq.md` file copy
(`resource.read` → `resource.write`) that reproduces `data.txt` in `out.bin`;