feat(stdlib): add bounded std:fs/file read, write and copy
This commit is contained in:
1 parent
8dd3d5eb76
commit
cd90f059b4
9 files changed
+248
-11
No files matched your search
@@ -24,3 +24,6 @@ dist/
|
||||
# Local-only credentials and environment files
|
||||
.secrets/
|
||||
.env.local
|
||||
|
||||
# stdlib product test output
|
||||
stdlib/tests/out/
|
||||
@@ -8,6 +8,7 @@
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
|
||||
fn repo_root() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
@@ -18,6 +19,16 @@ fn repo_root() -> PathBuf {
|
||||
.to_path_buf()
|
||||
}
|
||||
|
||||
static FS_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
fn lock_fs(name: &str) -> Option<MutexGuard<'static, ()>> {
|
||||
if name == "fs.mq.md" {
|
||||
Some(FS_LOCK.lock().unwrap_or_else(|error| error.into_inner()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn stdlib_dir() -> PathBuf {
|
||||
repo_root().join("stdlib")
|
||||
}
|
||||
@@ -26,6 +37,7 @@ struct TestFile {
|
||||
name: String,
|
||||
stdin: &'static str,
|
||||
stdout: &'static str,
|
||||
args: &'static [&'static str],
|
||||
}
|
||||
|
||||
fn product_tests() -> Vec<TestFile> {
|
||||
@@ -34,6 +46,14 @@ fn product_tests() -> Vec<TestFile> {
|
||||
.expect("read stdlib tests")
|
||||
.map(|entry| {
|
||||
let path = entry.unwrap().path();
|
||||
if !path.is_file() {
|
||||
return TestFile {
|
||||
name: String::new(),
|
||||
stdin: "",
|
||||
stdout: "",
|
||||
args: &[],
|
||||
};
|
||||
}
|
||||
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
let (stdin, stdout) = match name.as_str() {
|
||||
"console.mq.md" => ("hello\n", ""),
|
||||
@@ -41,21 +61,32 @@ fn product_tests() -> Vec<TestFile> {
|
||||
"stream.mq.md" => ("", "stream\n"),
|
||||
_ => ("", ""),
|
||||
};
|
||||
let args = if name == "fs.mq.md" {
|
||||
&["--allow-read", "out/*", "--allow-write", "out/*"][..]
|
||||
} else {
|
||||
&[][..]
|
||||
};
|
||||
TestFile {
|
||||
name,
|
||||
stdin,
|
||||
stdout,
|
||||
args,
|
||||
}
|
||||
})
|
||||
.filter(|file| !file.name.is_empty())
|
||||
.collect();
|
||||
files.sort_by(|left, right| left.name.cmp(&right.name));
|
||||
files
|
||||
}
|
||||
|
||||
fn run_product_test(file: &TestFile) -> serde_json::Value {
|
||||
if file.name == "fs.mq.md" {
|
||||
let _ = std::fs::remove_dir_all(stdlib_dir().join("tests/out"));
|
||||
}
|
||||
let mut child = Command::new(env!("CARGO_BIN_EXE_marqdo"))
|
||||
.args(["run"])
|
||||
.arg(stdlib_dir().join("tests").join(&file.name))
|
||||
.args(file.args)
|
||||
.args(["--json"])
|
||||
.current_dir(stdlib_dir())
|
||||
.env("MARQDO_STDLIB_DIR", stdlib_dir())
|
||||
@@ -90,6 +121,7 @@ fn every_public_stdlib_module_has_a_product_test() {
|
||||
"console.mq.md",
|
||||
"demo.mq.md",
|
||||
"format.mq.md",
|
||||
"fs.mq.md",
|
||||
"http.mq.md",
|
||||
"io.mq.md",
|
||||
"stream.mq.md",
|
||||
@@ -104,6 +136,7 @@ fn every_public_stdlib_module_has_a_product_test() {
|
||||
#[test]
|
||||
fn the_whole_bundled_library_runs_through_std_imports() {
|
||||
for file in product_tests() {
|
||||
let _guard = lock_fs(&file.name);
|
||||
let report = run_product_test(&file);
|
||||
assert_eq!(report["success"], true, "{}", file.name);
|
||||
assert_eq!(
|
||||
@@ -114,6 +147,31 @@ fn the_whole_bundled_library_runs_through_std_imports() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fs_test_reads_writes_and_copies_project_files() {
|
||||
let _guard = lock_fs("fs.mq.md");
|
||||
let report = run_product_test(
|
||||
product_tests()
|
||||
.iter()
|
||||
.find(|file| file.name == "fs.mq.md")
|
||||
.expect("fs test"),
|
||||
);
|
||||
let items = report["value"]["items"].as_array().expect("list");
|
||||
assert_eq!(items[0]["decimal"], "7");
|
||||
assert_eq!(items[1]["value"], "你好\n");
|
||||
assert_eq!(items[2]["decimal"], "7");
|
||||
assert_eq!(items[3]["value"], "你好\n");
|
||||
let out = stdlib_dir().join("tests/out");
|
||||
assert_eq!(
|
||||
std::fs::read(out.join("fs-a.txt")).unwrap(),
|
||||
"你好\n".as_bytes()
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(out.join("fs-b.txt")).unwrap(),
|
||||
"你好\n".as_bytes()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_test_pins_value_semantics_and_handle_observation() {
|
||||
let report = run_product_test(
|
||||
|
||||
+10
-8
@@ -58,7 +58,7 @@ byte-identical).
|
||||
cargo fmt --all -- --check PASS
|
||||
cargo clippy --workspace --all-targets -- -D warnings PASS
|
||||
cargo test --workspace --locked PASS
|
||||
463 passed, 0 failed, 2 ignored; includes 7/7 Linux security tests
|
||||
464 passed, 0 failed, 2 ignored; includes 7/7 Linux security tests
|
||||
```
|
||||
|
||||
The documentation and test migration is local work; re-run this block after it
|
||||
@@ -198,9 +198,10 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL
|
||||
- Table stdlib algorithms (CM-A08, ADR-0032): `stdlib/data/table.mq.md`
|
||||
provides select/filter/sort/append/set/join/group/`转GFM` over the logical
|
||||
`Value::Table`; inputs are never mutated; GFM AST laziness and physical T0
|
||||
storage are unchanged.
|
||||
storage are unchanged. `std:fs/file` provides bounded `读文本`/`写文本`/`复制文件`
|
||||
over the existing `resource.*` ABI; metadata/directory operations remain open.
|
||||
- Bundled standard library (CM-A10, ADR-0028): `stdlib/` ships
|
||||
`format`/`text`/`io`; explicit `std:` imports resolve only inside the bundle;
|
||||
`format`/`text`/`io`/`fs`; explicit `std:` imports resolve only inside the bundle;
|
||||
a SHA-256 digest over `*.mq.md` is verified before loading; Analysis records
|
||||
`project.stdlib`; a frozen-binary test asserts the CLI hash is unchanged after
|
||||
running a `std:io` project and that tampering is rejected. `打印/print` is
|
||||
@@ -224,7 +225,7 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL
|
||||
status/index links and SHA-pinned evidence; `crates/cli/tests/architecture_boundaries.rs`
|
||||
pins the workspace graph, `marqdo-runtime -> marqdo-host`, the bundled stdlib
|
||||
digest and the stdio/file/TCP endpoints; `crates/cli/tests/stdlib_product.rs`
|
||||
runs every public stdlib module (`console/demo/format/http/io/stream/table/text/utf8`)
|
||||
runs every public stdlib module (`console/demo/format/fs/http/io/stream/table/text/utf8`)
|
||||
through `std:` imports over `stdlib/tests/*.mq.md`. `tools/pack-stdlib.py`
|
||||
now sorts `.mq.md` by the same relative path key as the CLI. Source/test baseline
|
||||
`0d40de7`; later docs-only commits keep code/test bytes identical. Evidence:
|
||||
@@ -246,10 +247,11 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL
|
||||
fixture lane, Unix-gated worker tests).
|
||||
4. `fs.write`, `net.request`, `process`, `clock`, `random` are not executed
|
||||
broker services; unsupported requests must return `capability.unavailable`.
|
||||
5. Resource streams cover project-file read/write, stdout, compatible print,
|
||||
trusted in-process providers and IP-literal TCP; directories, process,
|
||||
device, DNS/TLS/HTTP, UDP and provider packaging remain open. Strict permits
|
||||
only stdio and refuses the rest (CM-A11 lane PASS).
|
||||
5. Resource streams cover project-file read/write (including `std:fs/file`
|
||||
read/write/copy), stdout, compatible print, trusted in-process providers and
|
||||
IP-literal TCP; file metadata/directories, process, device, DNS/TLS/HTTP, UDP
|
||||
and provider packaging remain open. Strict permits only stdio and refuses the
|
||||
rest (CM-A11 lane PASS).
|
||||
6. uv/npm/venv provisioning is not implemented. Only a pre-provisioned,
|
||||
digest-pinned directory with a regular-file executable is supported.
|
||||
7. No Windows/macOS RC archive exists from this host. The release-candidate
|
||||
|
||||
@@ -18,7 +18,7 @@ platform".
|
||||
| Dimension | Current state | Source |
|
||||
|---|---|---|
|
||||
| Implemented | Rust workspace `0.1.0`: Markdown/frontend/IR, tree-walk runtime, `Memory`, capability host, Tool/Analysis/Trace, Plugin ABI, Foreign kernels, bundled stdlib, CLI composition root | `Cargo.toml`, `crates/`, `stdlib/` |
|
||||
| Tested locally | `cargo test --workspace --locked`: **463 passed, 0 failed, 2 ignored** on Linux x86_64 at the baseline | `docs/STATUS.md` §3 |
|
||||
| Tested locally | `cargo test --workspace --locked`: **464 passed, 0 failed, 2 ignored** on Linux x86_64 at the baseline | `docs/STATUS.md` §3 |
|
||||
| Bundled stdlib released | In-repo bundle `marqdo-stdlib@0.1.0`, SHA-256 digest verified before loading; it is **not** an external registry package | `stdlib/stdlib.json`, `stdlib/README.md` |
|
||||
| Multi-platform verified | Linux local gate is the only executed lane here; Windows/macOS native lanes are **UNVERIFIED**, and the GitHub security lane remains a **VM-only** gate | `docs/STATUS.md` §1, §4 |
|
||||
|
||||
@@ -86,6 +86,7 @@ review RFC, not a new crate per provider.
|
||||
| stdio stderr | `resource.open("stdio","stderr",["write"])`; separate from stdout and CLI diagnostics | implemented | `resource_streams`, `stdlib_product` |
|
||||
| project file read | `resource.open("file", path, ["read"])`; path/grant checked; bounded | implemented, project mode | `resource_streams`, `stdlib_product` |
|
||||
| project file write | `resource.open("file", path, ["write"])`; buffers until `close`, never flush-on-drop | implemented, project mode | `resource_streams`, `stdlib_product` |
|
||||
| `std:fs/file` | `fs.读文本`, `fs.写文本`, `fs.复制文件` over `resource.open/read/write/close` and `Memory`; bounded at 8 MiB | implemented (read/write/copy) | `stdlib_product` |
|
||||
| file metadata / directory / rename / remove / mkdir | not part of the current Resource contract | **not implemented**; needs a separate Resource Operation ADR | — |
|
||||
| TCP | `resource.open("tcp", ip:port, ["read","write"])`; exact IP grant, 5 s timeout | implemented; no DNS/TLS/server socket/UDP | `tcp_endpoint`, `http_over_tcp` |
|
||||
| HTTP/1.1 | `stdlib/net/http.mq.md`: status/header/chunked-byte parse, request build | implemented over an already-authorized TCP stream; not a DNS/TLS client | `stdlib_http`, `http_over_tcp`, `stdlib_product` |
|
||||
@@ -111,7 +112,7 @@ that a phase is complete.
|
||||
|---|---|---|
|
||||
| P0 | Stdlib release is a bundle in this repo; extraction is deferred (ADR-0039) | accepted, keep in repo |
|
||||
| P0 | Documentation drift between `STATUS.md`, the architecture index and the evidence | being fixed by this migration and the docs gate |
|
||||
| P1 | No `std:fs/file` product module; only a bundled first slice and Resource streams | open |
|
||||
| P1 | `std:fs/file` has read/write/copy; metadata, `exists`, `list`, `mkdir`, `remove` and `rename` need a Resource Operation | open |
|
||||
| P1 | No file stat/list/rename/remove/mkdir/append/seek capability | open; requires a Resource Operation ADR |
|
||||
| P1 | JSON/CSV/UTF-8 decode remain mainly under `tests/conformance` | open as product stdlib API |
|
||||
| P1 | `crates/cli/src/commands.rs` and `crates/marqdo-host/src/host.rs` still mix multiple responsibilities | open refactor, behavior-preserving |
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Evidence: `std:fs/file` product module
|
||||
|
||||
Date: 2026-11-21 (Asia/Shanghai)
|
||||
Source baseline: `8dd3d5eb76e1acf6a7a015b69515933fb6bbf8e3`
|
||||
Repository: `marqdo-cli`
|
||||
Decision input: architecture audit and refactor RFC, P1 file API
|
||||
|
||||
## Implemented
|
||||
|
||||
`stdlib/fs/file.mq.md` exports three `.mq.md` functions over the existing
|
||||
resource ABI:
|
||||
|
||||
| Function | Contract |
|
||||
|---|---|
|
||||
| `读文本(路径)` | `resource.open("file", path, ["read"])` -> one bounded 8 MiB read -> `utf8.解码` |
|
||||
| `写文本(路径, 文本)` | UTF-8 encode into `Memory` -> `resource.write` -> `resource.close` flushes |
|
||||
| `复制文件(源, 目标)` | bounded byte read from source -> bounded byte write to destination |
|
||||
|
||||
No Rust builtin, `HostOp` or parser change was added. Path checks, grants,
|
||||
the 8 MiB read/write ceiling and close-flush behavior remain the host
|
||||
resource ABI (ADR-0025/0026/0040). Metadata, `exists`, directory listing,
|
||||
`mkdir`, `remove` and `rename` are **not** implemented; they need a Resource
|
||||
Operation ADR.
|
||||
|
||||
## Tests
|
||||
|
||||
`stdlib/tests/fs.mq.md` runs through the real `std:fs/file` import with
|
||||
`--allow-read out/*` and `--allow-write out/*`; it writes, reads back and
|
||||
copies a UTF-8 file. `crates/cli/tests/stdlib_product.rs` pins the result
|
||||
`[7, "你好\n", 7, "你好\n"]` and the bytes on disk. The test output directory
|
||||
`stdlib/tests/out/` is ignored.
|
||||
|
||||
```text
|
||||
cargo fmt --all -- --check PASS
|
||||
cargo clippy --workspace --all-targets -- -D warnings PASS
|
||||
cargo test --workspace 464 passed, 0 failed, 2 ignored
|
||||
cargo test -p marqdo-cli --test stdlib_product 6 passed
|
||||
python3 tools/pack-stdlib.py PASS
|
||||
```
|
||||
|
||||
## Not covered
|
||||
|
||||
A caller can still open a `file` resource directly with the lower-level
|
||||
builtins (that is the existing public ABI). Binary files up to the 8 MiB
|
||||
ceiling are byte-exact for copy; text read requires valid UTF-8 by design.
|
||||
@@ -8,6 +8,7 @@ module; a mismatch is a hard error.
|
||||
- `text.mq.md` — compatibility UTF-8 encoding into `Memory`
|
||||
- `text/utf8.mq.md` — `.mq.md` UTF-8 encoder and validating decoder
|
||||
- `io.mq.md` — compatibility facade over `stream` and `console`
|
||||
- `fs/file.mq.md` — bounded project-file read/write/copy over `resource.*`
|
||||
- `io/stream.mq.md` — `写全部` / `写文本` with short-write and close checking
|
||||
- `io/console.mq.md` — `读取行` / `打印` over `stdio/stdin` and `stdio/stdout`
|
||||
- `demo.mq.md` — side-by-side comparison with builtin `打印`
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
---
|
||||
import:
|
||||
utf8: "../text/utf8.mq.md"
|
||||
---
|
||||
|
||||
## 读文本
|
||||
|
||||
`(路径)`
|
||||
|
||||
以 UTF-8 读取一个有界的项目文件。文件在打开时整文件载入,上限 8 MiB;
|
||||
结果使用可恢复 `Result` 形状。
|
||||
|
||||
**流 = resource.open("file", 路径, ["read"])**
|
||||
|
||||
**内存 = memory.new(128, 128)**
|
||||
|
||||
**读取 = resource.read(流, 内存, 0, 8388608)**
|
||||
|
||||
**关闭 = resource.close(流)**
|
||||
|
||||
1. `not 读取.ok`
|
||||
|
||||
*读取*
|
||||
|
||||
2. `not 关闭.ok`
|
||||
|
||||
*关闭*
|
||||
|
||||
3. `true`
|
||||
|
||||
**文本 = utf8.解码(内存, 0, 读取.value.count)**
|
||||
|
||||
1. `not 文本.ok`
|
||||
|
||||
*文本*
|
||||
|
||||
2. `true`
|
||||
|
||||
*{ok: true, value: 文本.value}*
|
||||
|
||||
## 写文本
|
||||
|
||||
`(路径, 文本)`
|
||||
|
||||
把文本按 UTF-8 写入项目文件。路径必须已经通过 `fs_write` 授权;
|
||||
文件只在 `resource.close` 成功时创建或覆盖,写入缓冲上限 8 MiB。
|
||||
|
||||
**内存 = memory.new(128, 128)**
|
||||
|
||||
**长度 = utf8.编码(文本, 内存, 0)**
|
||||
|
||||
**流 = resource.open("file", 路径, ["write"])**
|
||||
|
||||
**写入 = resource.write(流, 内存, 0, 长度)**
|
||||
|
||||
**关闭 = resource.close(流)**
|
||||
|
||||
1. `not 写入.ok`
|
||||
|
||||
*写入*
|
||||
|
||||
2. `not 关闭.ok`
|
||||
|
||||
*关闭*
|
||||
|
||||
3. `true`
|
||||
|
||||
*{ok: true, value: {count: 写入.value.count}}*
|
||||
|
||||
## 复制文件
|
||||
|
||||
`(源, 目标)`
|
||||
|
||||
按字节复制一个有界的项目文件。源读取和目标写入分别需要
|
||||
`fs_read`/`fs_write` 授权,结果值与 [`写文本`](#写文本) 相同。
|
||||
|
||||
**输入 = resource.open("file", 源, ["read"])**
|
||||
|
||||
**内存 = memory.new(128, 128)**
|
||||
|
||||
**读取 = resource.read(输入, 内存, 0, 8388608)**
|
||||
|
||||
**关闭输入 = resource.close(输入)**
|
||||
|
||||
1. `not 读取.ok`
|
||||
|
||||
*读取*
|
||||
|
||||
2. `not 关闭输入.ok`
|
||||
|
||||
*关闭输入*
|
||||
|
||||
3. `true`
|
||||
|
||||
**输出 = resource.open("file", 目标, ["write"])**
|
||||
|
||||
**写入 = resource.write(输出, 内存, 0, 读取.value.count)**
|
||||
|
||||
**关闭输出 = resource.close(输出)**
|
||||
|
||||
1. `not 写入.ok`
|
||||
|
||||
*写入*
|
||||
|
||||
2. `not 关闭输出.ok`
|
||||
|
||||
*关闭输出*
|
||||
|
||||
3. `true`
|
||||
|
||||
*{ok: true, value: {count: 写入.value.count}}*
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "marqdo-stdlib",
|
||||
"version": "0.1.0",
|
||||
"digest": "sha256:91f1f76daf694efac0c4d8c937d2b2f61d07efd8c105ce407e55690e0a3ad2f5"
|
||||
"digest": "sha256:5e16dc37ef7fcecf8f6839e4065bf02aa552660c5e872b4c17e1eb71dda9d7e7"
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
---
|
||||
import:
|
||||
fs: "std:fs/file"
|
||||
---
|
||||
|
||||
## main
|
||||
|
||||
**写入 = fs.写文本("out/fs-a.txt", "你好\n")**
|
||||
|
||||
**读取 = fs.读文本("out/fs-a.txt")**
|
||||
|
||||
**复制 = fs.复制文件("out/fs-a.txt", "out/fs-b.txt")**
|
||||
|
||||
**复制读取 = fs.读文本("out/fs-b.txt")**
|
||||
|
||||
*[写入.value.count, 读取.value, 复制.value.count, 复制读取.value]*
|
||||
Reference in new issue
Block a user