diff --git a/.gitignore b/.gitignore index 86e0b12..0180482 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ dist/ # Local-only credentials and environment files .secrets/ .env.local + +# stdlib product test output +stdlib/tests/out/ diff --git a/crates/cli/tests/stdlib_product.rs b/crates/cli/tests/stdlib_product.rs index f52ffe9..8b8e927 100644 --- a/crates/cli/tests/stdlib_product.rs +++ b/crates/cli/tests/stdlib_product.rs @@ -8,6 +8,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::sync::{Mutex, MutexGuard}; fn repo_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) @@ -18,6 +19,16 @@ fn repo_root() -> PathBuf { .to_path_buf() } +static FS_LOCK: Mutex<()> = Mutex::new(()); + +fn lock_fs(name: &str) -> Option> { + if name == "fs.mq.md" { + Some(FS_LOCK.lock().unwrap_or_else(|error| error.into_inner())) + } else { + None + } +} + fn stdlib_dir() -> PathBuf { repo_root().join("stdlib") } @@ -26,6 +37,7 @@ struct TestFile { name: String, stdin: &'static str, stdout: &'static str, + args: &'static [&'static str], } fn product_tests() -> Vec { @@ -34,6 +46,14 @@ fn product_tests() -> Vec { .expect("read stdlib tests") .map(|entry| { let path = entry.unwrap().path(); + if !path.is_file() { + return TestFile { + name: String::new(), + stdin: "", + stdout: "", + args: &[], + }; + } let name = path.file_name().unwrap().to_string_lossy().to_string(); let (stdin, stdout) = match name.as_str() { "console.mq.md" => ("hello\n", ""), @@ -41,21 +61,32 @@ fn product_tests() -> Vec { "stream.mq.md" => ("", "stream\n"), _ => ("", ""), }; + let args = if name == "fs.mq.md" { + &["--allow-read", "out/*", "--allow-write", "out/*"][..] + } else { + &[][..] + }; TestFile { name, stdin, stdout, + args, } }) + .filter(|file| !file.name.is_empty()) .collect(); files.sort_by(|left, right| left.name.cmp(&right.name)); files } fn run_product_test(file: &TestFile) -> serde_json::Value { + if file.name == "fs.mq.md" { + let _ = std::fs::remove_dir_all(stdlib_dir().join("tests/out")); + } let mut child = Command::new(env!("CARGO_BIN_EXE_marqdo")) .args(["run"]) .arg(stdlib_dir().join("tests").join(&file.name)) + .args(file.args) .args(["--json"]) .current_dir(stdlib_dir()) .env("MARQDO_STDLIB_DIR", stdlib_dir()) @@ -90,6 +121,7 @@ fn every_public_stdlib_module_has_a_product_test() { "console.mq.md", "demo.mq.md", "format.mq.md", + "fs.mq.md", "http.mq.md", "io.mq.md", "stream.mq.md", @@ -104,6 +136,7 @@ fn every_public_stdlib_module_has_a_product_test() { #[test] fn the_whole_bundled_library_runs_through_std_imports() { for file in product_tests() { + let _guard = lock_fs(&file.name); let report = run_product_test(&file); assert_eq!(report["success"], true, "{}", file.name); assert_eq!( @@ -114,6 +147,31 @@ fn the_whole_bundled_library_runs_through_std_imports() { } } +#[test] +fn fs_test_reads_writes_and_copies_project_files() { + let _guard = lock_fs("fs.mq.md"); + let report = run_product_test( + product_tests() + .iter() + .find(|file| file.name == "fs.mq.md") + .expect("fs test"), + ); + let items = report["value"]["items"].as_array().expect("list"); + assert_eq!(items[0]["decimal"], "7"); + assert_eq!(items[1]["value"], "你好\n"); + assert_eq!(items[2]["decimal"], "7"); + assert_eq!(items[3]["value"], "你好\n"); + let out = stdlib_dir().join("tests/out"); + assert_eq!( + std::fs::read(out.join("fs-a.txt")).unwrap(), + "你好\n".as_bytes() + ); + assert_eq!( + std::fs::read(out.join("fs-b.txt")).unwrap(), + "你好\n".as_bytes() + ); +} + #[test] fn format_test_pins_value_semantics_and_handle_observation() { let report = run_product_test( diff --git a/docs/STATUS.md b/docs/STATUS.md index bffe02a..2f63b18 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -58,7 +58,7 @@ byte-identical). cargo fmt --all -- --check PASS cargo clippy --workspace --all-targets -- -D warnings PASS cargo test --workspace --locked PASS - 463 passed, 0 failed, 2 ignored; includes 7/7 Linux security tests + 464 passed, 0 failed, 2 ignored; includes 7/7 Linux security tests ``` The documentation and test migration is local work; re-run this block after it @@ -198,9 +198,10 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL - Table stdlib algorithms (CM-A08, ADR-0032): `stdlib/data/table.mq.md` provides select/filter/sort/append/set/join/group/`转GFM` over the logical `Value::Table`; inputs are never mutated; GFM AST laziness and physical T0 - storage are unchanged. + storage are unchanged. `std:fs/file` provides bounded `读文本`/`写文本`/`复制文件` + over the existing `resource.*` ABI; metadata/directory operations remain open. - Bundled standard library (CM-A10, ADR-0028): `stdlib/` ships - `format`/`text`/`io`; explicit `std:` imports resolve only inside the bundle; + `format`/`text`/`io`/`fs`; explicit `std:` imports resolve only inside the bundle; a SHA-256 digest over `*.mq.md` is verified before loading; Analysis records `project.stdlib`; a frozen-binary test asserts the CLI hash is unchanged after running a `std:io` project and that tampering is rejected. `打印/print` is @@ -224,7 +225,7 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL status/index links and SHA-pinned evidence; `crates/cli/tests/architecture_boundaries.rs` pins the workspace graph, `marqdo-runtime -> marqdo-host`, the bundled stdlib digest and the stdio/file/TCP endpoints; `crates/cli/tests/stdlib_product.rs` - runs every public stdlib module (`console/demo/format/http/io/stream/table/text/utf8`) + runs every public stdlib module (`console/demo/format/fs/http/io/stream/table/text/utf8`) through `std:` imports over `stdlib/tests/*.mq.md`. `tools/pack-stdlib.py` now sorts `.mq.md` by the same relative path key as the CLI. Source/test baseline `0d40de7`; later docs-only commits keep code/test bytes identical. Evidence: @@ -246,10 +247,11 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL fixture lane, Unix-gated worker tests). 4. `fs.write`, `net.request`, `process`, `clock`, `random` are not executed broker services; unsupported requests must return `capability.unavailable`. -5. Resource streams cover project-file read/write, stdout, compatible print, - trusted in-process providers and IP-literal TCP; directories, process, - device, DNS/TLS/HTTP, UDP and provider packaging remain open. Strict permits - only stdio and refuses the rest (CM-A11 lane PASS). +5. Resource streams cover project-file read/write (including `std:fs/file` + read/write/copy), stdout, compatible print, trusted in-process providers and + IP-literal TCP; file metadata/directories, process, device, DNS/TLS/HTTP, UDP + and provider packaging remain open. Strict permits only stdio and refuses the + rest (CM-A11 lane PASS). 6. uv/npm/venv provisioning is not implemented. Only a pre-provisioned, digest-pinned directory with a regular-file executable is supported. 7. No Windows/macOS RC archive exists from this host. The release-candidate diff --git a/docs/architecture/CURRENT.md b/docs/architecture/CURRENT.md index fb61513..2b8087d 100644 --- a/docs/architecture/CURRENT.md +++ b/docs/architecture/CURRENT.md @@ -18,7 +18,7 @@ platform". | Dimension | Current state | Source | |---|---|---| | Implemented | Rust workspace `0.1.0`: Markdown/frontend/IR, tree-walk runtime, `Memory`, capability host, Tool/Analysis/Trace, Plugin ABI, Foreign kernels, bundled stdlib, CLI composition root | `Cargo.toml`, `crates/`, `stdlib/` | -| Tested locally | `cargo test --workspace --locked`: **463 passed, 0 failed, 2 ignored** on Linux x86_64 at the baseline | `docs/STATUS.md` §3 | +| Tested locally | `cargo test --workspace --locked`: **464 passed, 0 failed, 2 ignored** on Linux x86_64 at the baseline | `docs/STATUS.md` §3 | | Bundled stdlib released | In-repo bundle `marqdo-stdlib@0.1.0`, SHA-256 digest verified before loading; it is **not** an external registry package | `stdlib/stdlib.json`, `stdlib/README.md` | | Multi-platform verified | Linux local gate is the only executed lane here; Windows/macOS native lanes are **UNVERIFIED**, and the GitHub security lane remains a **VM-only** gate | `docs/STATUS.md` §1, §4 | @@ -86,6 +86,7 @@ review RFC, not a new crate per provider. | stdio stderr | `resource.open("stdio","stderr",["write"])`; separate from stdout and CLI diagnostics | implemented | `resource_streams`, `stdlib_product` | | project file read | `resource.open("file", path, ["read"])`; path/grant checked; bounded | implemented, project mode | `resource_streams`, `stdlib_product` | | project file write | `resource.open("file", path, ["write"])`; buffers until `close`, never flush-on-drop | implemented, project mode | `resource_streams`, `stdlib_product` | +| `std:fs/file` | `fs.读文本`, `fs.写文本`, `fs.复制文件` over `resource.open/read/write/close` and `Memory`; bounded at 8 MiB | implemented (read/write/copy) | `stdlib_product` | | file metadata / directory / rename / remove / mkdir | not part of the current Resource contract | **not implemented**; needs a separate Resource Operation ADR | — | | TCP | `resource.open("tcp", ip:port, ["read","write"])`; exact IP grant, 5 s timeout | implemented; no DNS/TLS/server socket/UDP | `tcp_endpoint`, `http_over_tcp` | | HTTP/1.1 | `stdlib/net/http.mq.md`: status/header/chunked-byte parse, request build | implemented over an already-authorized TCP stream; not a DNS/TLS client | `stdlib_http`, `http_over_tcp`, `stdlib_product` | @@ -111,7 +112,7 @@ that a phase is complete. |---|---|---| | P0 | Stdlib release is a bundle in this repo; extraction is deferred (ADR-0039) | accepted, keep in repo | | P0 | Documentation drift between `STATUS.md`, the architecture index and the evidence | being fixed by this migration and the docs gate | -| P1 | No `std:fs/file` product module; only a bundled first slice and Resource streams | open | +| P1 | `std:fs/file` has read/write/copy; metadata, `exists`, `list`, `mkdir`, `remove` and `rename` need a Resource Operation | open | | P1 | No file stat/list/rename/remove/mkdir/append/seek capability | open; requires a Resource Operation ADR | | P1 | JSON/CSV/UTF-8 decode remain mainly under `tests/conformance` | open as product stdlib API | | P1 | `crates/cli/src/commands.rs` and `crates/marqdo-host/src/host.rs` still mix multiple responsibilities | open refactor, behavior-preserving | diff --git a/docs/evidence/2026-11-21-fs-file-product.md b/docs/evidence/2026-11-21-fs-file-product.md new file mode 100644 index 0000000..ce3ac5d --- /dev/null +++ b/docs/evidence/2026-11-21-fs-file-product.md @@ -0,0 +1,45 @@ +# Evidence: `std:fs/file` product module + +Date: 2026-11-21 (Asia/Shanghai) +Source baseline: `8dd3d5eb76e1acf6a7a015b69515933fb6bbf8e3` +Repository: `marqdo-cli` +Decision input: architecture audit and refactor RFC, P1 file API + +## Implemented + +`stdlib/fs/file.mq.md` exports three `.mq.md` functions over the existing +resource ABI: + +| Function | Contract | +|---|---| +| `读文本(路径)` | `resource.open("file", path, ["read"])` -> one bounded 8 MiB read -> `utf8.解码` | +| `写文本(路径, 文本)` | UTF-8 encode into `Memory` -> `resource.write` -> `resource.close` flushes | +| `复制文件(源, 目标)` | bounded byte read from source -> bounded byte write to destination | + +No Rust builtin, `HostOp` or parser change was added. Path checks, grants, +the 8 MiB read/write ceiling and close-flush behavior remain the host +resource ABI (ADR-0025/0026/0040). Metadata, `exists`, directory listing, +`mkdir`, `remove` and `rename` are **not** implemented; they need a Resource +Operation ADR. + +## Tests + +`stdlib/tests/fs.mq.md` runs through the real `std:fs/file` import with +`--allow-read out/*` and `--allow-write out/*`; it writes, reads back and +copies a UTF-8 file. `crates/cli/tests/stdlib_product.rs` pins the result +`[7, "你好\n", 7, "你好\n"]` and the bytes on disk. The test output directory +`stdlib/tests/out/` is ignored. + +```text +cargo fmt --all -- --check PASS +cargo clippy --workspace --all-targets -- -D warnings PASS +cargo test --workspace 464 passed, 0 failed, 2 ignored +cargo test -p marqdo-cli --test stdlib_product 6 passed +python3 tools/pack-stdlib.py PASS +``` + +## Not covered + +A caller can still open a `file` resource directly with the lower-level +builtins (that is the existing public ABI). Binary files up to the 8 MiB +ceiling are byte-exact for copy; text read requires valid UTF-8 by design. diff --git a/stdlib/README.md b/stdlib/README.md index 19990cc..3dfaa14 100644 --- a/stdlib/README.md +++ b/stdlib/README.md @@ -8,6 +8,7 @@ module; a mismatch is a hard error. - `text.mq.md` — compatibility UTF-8 encoding into `Memory` - `text/utf8.mq.md` — `.mq.md` UTF-8 encoder and validating decoder - `io.mq.md` — compatibility facade over `stream` and `console` +- `fs/file.mq.md` — bounded project-file read/write/copy over `resource.*` - `io/stream.mq.md` — `写全部` / `写文本` with short-write and close checking - `io/console.mq.md` — `读取行` / `打印` over `stdio/stdin` and `stdio/stdout` - `demo.mq.md` — side-by-side comparison with builtin `打印` diff --git a/stdlib/fs/file.mq.md b/stdlib/fs/file.mq.md new file mode 100644 index 0000000..7f7d9e4 --- /dev/null +++ b/stdlib/fs/file.mq.md @@ -0,0 +1,111 @@ +--- +import: + utf8: "../text/utf8.mq.md" +--- + +## 读文本 + +`(路径)` + +以 UTF-8 读取一个有界的项目文件。文件在打开时整文件载入,上限 8 MiB; +结果使用可恢复 `Result` 形状。 + +**流 = resource.open("file", 路径, ["read"])** + +**内存 = memory.new(128, 128)** + +**读取 = resource.read(流, 内存, 0, 8388608)** + +**关闭 = resource.close(流)** + +1. `not 读取.ok` + + *读取* + +2. `not 关闭.ok` + + *关闭* + +3. `true` + + **文本 = utf8.解码(内存, 0, 读取.value.count)** + + 1. `not 文本.ok` + + *文本* + + 2. `true` + + *{ok: true, value: 文本.value}* + +## 写文本 + +`(路径, 文本)` + +把文本按 UTF-8 写入项目文件。路径必须已经通过 `fs_write` 授权; +文件只在 `resource.close` 成功时创建或覆盖,写入缓冲上限 8 MiB。 + +**内存 = memory.new(128, 128)** + +**长度 = utf8.编码(文本, 内存, 0)** + +**流 = resource.open("file", 路径, ["write"])** + +**写入 = resource.write(流, 内存, 0, 长度)** + +**关闭 = resource.close(流)** + +1. `not 写入.ok` + + *写入* + +2. `not 关闭.ok` + + *关闭* + +3. `true` + + *{ok: true, value: {count: 写入.value.count}}* + +## 复制文件 + +`(源, 目标)` + +按字节复制一个有界的项目文件。源读取和目标写入分别需要 +`fs_read`/`fs_write` 授权,结果值与 [`写文本`](#写文本) 相同。 + +**输入 = resource.open("file", 源, ["read"])** + +**内存 = memory.new(128, 128)** + +**读取 = resource.read(输入, 内存, 0, 8388608)** + +**关闭输入 = resource.close(输入)** + +1. `not 读取.ok` + + *读取* + +2. `not 关闭输入.ok` + + *关闭输入* + +3. `true` + + **输出 = resource.open("file", 目标, ["write"])** + + **写入 = resource.write(输出, 内存, 0, 读取.value.count)** + + **关闭输出 = resource.close(输出)** + + 1. `not 写入.ok` + + *写入* + + 2. `not 关闭输出.ok` + + *关闭输出* + + 3. `true` + + *{ok: true, value: {count: 写入.value.count}}* diff --git a/stdlib/stdlib.json b/stdlib/stdlib.json index cd8af1e..19bda8f 100644 --- a/stdlib/stdlib.json +++ b/stdlib/stdlib.json @@ -1,5 +1,5 @@ { "name": "marqdo-stdlib", "version": "0.1.0", - "digest": "sha256:91f1f76daf694efac0c4d8c937d2b2f61d07efd8c105ce407e55690e0a3ad2f5" + "digest": "sha256:5e16dc37ef7fcecf8f6839e4065bf02aa552660c5e872b4c17e1eb71dda9d7e7" } diff --git a/stdlib/tests/fs.mq.md b/stdlib/tests/fs.mq.md new file mode 100644 index 0000000..bed9745 --- /dev/null +++ b/stdlib/tests/fs.mq.md @@ -0,0 +1,16 @@ +--- +import: + fs: "std:fs/file" +--- + +## main + +**写入 = fs.写文本("out/fs-a.txt", "你好\n")** + +**读取 = fs.读文本("out/fs-a.txt")** + +**复制 = fs.复制文件("out/fs-a.txt", "out/fs-b.txt")** + +**复制读取 = fs.读文本("out/fs-b.txt")** + +*[写入.value.count, 读取.value, 复制.value.count, 复制读取.value]*