diff --git a/Cargo.lock b/Cargo.lock index 0d2dc00..074dc54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "1.0.0" @@ -485,6 +494,7 @@ dependencies = [ "marqdo-span", "marqdo-value", "rcgen", + "regex", "rustls", "rustls-native-certs", "rustls-pemfile", @@ -803,6 +813,35 @@ dependencies = [ "yasna", ] +[[package]] +name = "regex" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9807af0e638c12e753aa1b4e601791bf2a8f71274023c8bbeec12fc149950857" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d60ef07df457506a0a1a2073112eb4aae5733dbd2813c3da77d1b0fd441158" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d902046e6a28a1e4c3cab97258a0ce11e994903f01c2d97a9447a0b3df85f4f6" + [[package]] name = "ring" version = "0.17.14" diff --git a/Cargo.toml b/Cargo.toml index e134ba2..bbfc68a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,7 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std" rustls-native-certs = "0.8" rustls-pemfile = "2" rcgen = "0.13" +regex = "1" thiserror = "2" clap = { version = "4", features = ["derive"] } walkdir = "2" diff --git a/crates/cli/tests/stdlib_product.rs b/crates/cli/tests/stdlib_product.rs index 8f8f6a5..468599a 100644 --- a/crates/cli/tests/stdlib_product.rs +++ b/crates/cli/tests/stdlib_product.rs @@ -312,6 +312,12 @@ fn regex_test_matches_anchors_classes_and_search() { assert_eq!(items[9]["value"], false); assert_eq!(items[10]["value"], "regex.invalid"); assert_eq!(items[11]["value"], "regex.unsupported"); + assert_eq!(items[12]["value"], true); + assert_eq!(items[13]["value"], true); + let groups = items[14]["items"].as_array().expect("groups"); + assert_eq!(groups[0]["value"], "aab"); + assert_eq!(groups[1]["value"], "aa"); + assert_eq!(groups[2]["value"], "b"); } #[test] diff --git a/crates/marqdo-host/Cargo.toml b/crates/marqdo-host/Cargo.toml index 24bb716..7c3dda6 100644 --- a/crates/marqdo-host/Cargo.toml +++ b/crates/marqdo-host/Cargo.toml @@ -15,6 +15,7 @@ rustls-native-certs = { workspace = true } marqdo-plugin-abi = { workspace = true } flate2 = { workspace = true } getrandom = { workspace = true } +regex = { workspace = true } glob = { workspace = true } marqdo-span = { workspace = true } marqdo-value = { workspace = true } diff --git a/crates/marqdo-host/src/host.rs b/crates/marqdo-host/src/host.rs index 6e23887..0fceb9c 100644 --- a/crates/marqdo-host/src/host.rs +++ b/crates/marqdo-host/src/host.rs @@ -845,6 +845,82 @@ impl Host for StdHost { self.record_resource_effect("resource.request", ctx, started, "process.run"); return Ok(result); } + if operation == "regex.match" || operation == "regex.captures" { + if operation == "regex.match" { + if args.len() != 2 { + return Err(RuntimeError::new( + ErrorCategory::Arity, + ctx.span, + format!( + "resource operation `regex.match` expects 2 arguments, got {}", + args.len() + ), + )); + } + } else if args.len() != 2 { + return Err(RuntimeError::new( + ErrorCategory::Arity, + ctx.span, + format!( + "resource operation `regex.captures` expects 2 arguments, got {}", + args.len() + ), + )); + } + let pattern_type = args[0].type_name(); + let Value::Text(pattern) = args[0].clone() else { + return Err(RuntimeError::type_error( + ctx.span, + format!("{operation} pattern must be Text, got {pattern_type}"), + )); + }; + let text_type = args[1].type_name(); + let Value::Text(text) = args[1].clone() else { + return Err(RuntimeError::type_error( + ctx.span, + format!("{operation} text must be Text, got {text_type}"), + )); + }; + let regex = match regex::Regex::new(&pattern) { + Ok(regex) => regex, + Err(error) => { + return Ok(result_error( + "regex.invalid", + &format!("invalid regular expression: {error}"), + )) + } + }; + self.record_semantic( + "resource.request", + None, + Some(&ctx.module_id), + ctx.span, + Some(operation), + ); + self.record_resource_effect("resource.request", ctx, started, operation); + let mut map = BTreeMap::new(); + map.insert("ok".to_string(), Value::Bool(true)); + if operation == "regex.match" { + map.insert("value".to_string(), Value::Bool(regex.is_match(&text))); + } else { + let mut payload = BTreeMap::new(); + let captures = regex.captures(&text); + payload.insert("matched".to_string(), Value::Bool(captures.is_some())); + let groups = match captures { + Some(captures) => captures + .iter() + .map(|group| match group { + Some(group) => Value::text(group.as_str()), + None => Value::None, + }) + .collect(), + None => Vec::new(), + }; + payload.insert("groups".to_string(), Value::List(groups)); + map.insert("value".to_string(), Value::Map(payload)); + } + return Ok(Value::Map(map)); + } if operation == "compress.gzip" { if args.len() != 1 { return Err(RuntimeError::new( diff --git a/crates/marqdo-host/tests/resource_streams.rs b/crates/marqdo-host/tests/resource_streams.rs index 866c653..3c8bf27 100644 --- a/crates/marqdo-host/tests/resource_streams.rs +++ b/crates/marqdo-host/tests/resource_streams.rs @@ -1117,3 +1117,44 @@ fn compression_round_trips_and_rejects_invalid_gzip() { assert_eq!(field(&invalid, "ok"), &Value::Bool(false)); assert_eq!(field(&invalid, "code"), &Value::text("compress.invalid")); } + +#[test] +fn regex_operations_support_groups_alternation_and_errors() { + let dir = tempfile::tempdir().unwrap(); + let mut host = StdHost::new(SandboxMode::Project, Capabilities::project(), dir.path()); + let ctx = context(); + let matched = host + .resource_request( + "regex.match", + vec![Value::text("^(a+)(b|c)$"), Value::text("aab")], + &ctx, + ) + .unwrap(); + assert_eq!(field(&matched, "value"), &Value::Bool(true)); + let captures = host + .resource_request( + "regex.captures", + vec![Value::text("^(a+)(b|c)$"), Value::text("aab")], + &ctx, + ) + .unwrap(); + let value = field(&captures, "value"); + assert_eq!(field(value, "matched"), &Value::Bool(true)); + assert_eq!( + field(value, "groups"), + &Value::List(vec![ + Value::text("aab"), + Value::text("aa"), + Value::text("b") + ]) + ); + let invalid = host + .resource_request( + "regex.match", + vec![Value::text("["), Value::text("a")], + &ctx, + ) + .unwrap(); + assert_eq!(field(&invalid, "ok"), &Value::Bool(false)); + assert_eq!(field(&invalid, "code"), &Value::text("regex.invalid")); +} diff --git a/docs/STATUS.md b/docs/STATUS.md index 980d30d..daa74c1 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -58,7 +58,7 @@ byte-identical). cargo fmt --all -- --check PASS cargo clippy --workspace --all-targets -- -D warnings PASS cargo test --workspace --locked PASS - 490 passed, 0 failed, 2 ignored; includes 7/7 Linux security tests + 491 passed, 0 failed, 2 ignored; includes 7/7 Linux security tests ``` The documentation and test migration is local work; re-run this block after it @@ -209,7 +209,9 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL runs an exact `--allow-process PROGRAM` grant with 5 s timeout and 1 MiB caps (ADR-0044, not an OS sandbox); `random.字节` reads explicitly granted `random.bytes` entropy (ADR-0046). File watch, binary offset reads, streaming - incremental JSON/CSV streaming and regex groups/alternation remain open. + incremental JSON/CSV streaming, pure-mode regex groups/alternation and regex + look-around/backreferences remain open; project mode has host-backed groups + and captures (ADR-0050). - Bundled standard library (CM-A10, ADR-0028): `stdlib/` ships `format`/`text`/`io`/`fs`/`data`/`path`/`regex`/`datetime`/`clock`/`env`/`process`/`random`/`compress`; explicit `std:` imports resolve only inside the bundle; a SHA-256 digest over `*.mq.md` is verified before loading; Analysis records diff --git a/docs/adr/0050-host-regex-extension.md b/docs/adr/0050-host-regex-extension.md new file mode 100644 index 0000000..36feb64 --- /dev/null +++ b/docs/adr/0050-host-regex-extension.md @@ -0,0 +1,42 @@ +# ADR-0050: Host-backed extended regular expressions + +Status: **Accepted** +Date: 2026-11-21 +Relates to: ADR-0041 +Specs: `spec/evaluation.md` +Evidence: `docs/evidence/2026-11-21-regex-extension.md` + +## Context + +The pure `.mq.md` regex subset covers literals, `.`, quantifiers, anchors and +character classes, but not groups, alternation or captures. Implementing a +full backtracking engine in `.mq.md` would be a large, hard-to-audit +duplicate. The host already has a safe, well-tested `regex` crate. + +## Decision + +Add project-mode operations: + +```text +resource.request("regex.match", [pattern, text]) -> {ok:true, value:Bool} +resource.request("regex.captures", [pattern, text]) -> {ok:true, value:{matched:Bool, groups:List}} +``` + +* Both require two Text arguments; invalid patterns are recoverable + `regex.invalid`. +* `captures` includes group 0 (the whole match); unmatched optional groups are + `None`. +* No new capability grant: regex compilation/matching is deterministic CPU + work and does not disclose external state. All `resource.request` operations + remain project-mode only, so `pure`/`strict` refuse them. +* `stdlib/regex.mq.md` exposes `扩展匹配`, `扩展搜索` and `捕获`; the existing + pure `匹配`/`搜索` subset stays unchanged for pure mode. +* The `regex` crate is Rust-only, with no OS dependency. No new builtin, + `HostOp` or parser node is added. + +## Consequences + +* Groups, alternation, `\d` classes and captures are available in project mode. +* Pure mode still has only the documented subset; this is an explicit + mode-dependent capability, not a silent downgrade. +* Look-around, backreferences and regex captures in pure `.mq.md` remain open. diff --git a/docs/architecture/CURRENT.md b/docs/architecture/CURRENT.md index d90a042..f38ec71 100644 --- a/docs/architecture/CURRENT.md +++ b/docs/architecture/CURRENT.md @@ -18,7 +18,7 @@ platform". | Dimension | Current state | Source | |---|---|---| | Implemented | Rust workspace `0.1.0`: Markdown/frontend/IR, tree-walk runtime, `Memory`, capability host, Tool/Analysis/Trace, Plugin ABI, Foreign kernels, bundled stdlib, CLI composition root | `Cargo.toml`, `crates/`, `stdlib/` | -| Tested locally | `cargo test --workspace --locked`: **490 passed, 0 failed, 2 ignored** on Linux x86_64 at the baseline | `docs/STATUS.md` §3 | +| Tested locally | `cargo test --workspace --locked`: **491 passed, 0 failed, 2 ignored** on Linux x86_64 at the baseline | `docs/STATUS.md` §3 | | Bundled stdlib released | In-repo bundle `marqdo-stdlib@0.1.0`, SHA-256 digest verified before loading; it is **not** an external registry package | `stdlib/stdlib.json`, `stdlib/README.md` | | Multi-platform verified | Linux local gate is the only executed lane here; Windows/macOS native lanes are **UNVERIFIED**, and the GitHub security lane remains a **VM-only** gate | `docs/STATUS.md` §1, §4 | @@ -95,7 +95,7 @@ review RFC, not a new crate per provider. | `std:data/json` | `json.解析/解析多个/格式化`; nested Value, 128-depth bound, recoverable errors | implemented (recursive, multi-value; no streaming) | `stdlib_product` | | `std:data/csv` | `csv.解析行/解析/格式化行/格式化`; quoted single/multi-row, CRLF, quoted newlines | implemented (no streaming) | `stdlib_product` | | `std:path` | `path.连接/文件名/目录名/扩展名/规范/是否绝对`; lexical only | implemented (no I/O) | `stdlib_product` | -| `std:regex` | `regex.匹配/搜索` with `^ $ . * + ? [...]` escapes | implemented subset; no groups/alternation | `stdlib_product` | +| `std:regex` | pure `匹配/搜索` subset; host `扩展匹配/捕获` with groups/alternation/captures | implemented both paths; pure path has no groups | `resource_streams`, `stdlib_product` | | `std:datetime` | `datetime.格式化(毫秒)` / `datetime.解析(文本)`; UTC ISO-8601, pre-1970 supported | implemented (pure) | `stdlib_product` | | `std:clock` | `clock.现在()` -> `resource.request("clock.now", [])` | implemented (project mode; pure/strict denied) | `resource_streams`, `stdlib_product` | | `std:env` | `env.读取/存在` -> `resource.request("env.get", [name])`; exact `--allow-env` grant | implemented (project mode; no `env.list`) | `resource_streams`, `stdlib_product` | @@ -134,7 +134,7 @@ that a phase is complete. | P1 | CLI split into `commands/{sandbox,run,check,inspect,init,package,tool}.rs`; host split into `host/{resource_ops,providers,effects,dispatch}.rs` plus `process.rs`; `commands.rs` keeps shared project/runtime helpers and `host.rs` keeps the `Host` trait methods and capability checks | split; shared helpers and trait methods remain by design | | P2 | HTTP parsing is Unicode-codepoint oriented; a general byte body is not the only path | open; convert before a general network release | | P2 | DNS, UDP, TCP listener and TLS client endpoints are implemented; no TLS server/mTLS or custom-CA CLI | open | -| P2 | `std:path`, regex subset, `std:datetime`, `std:clock`, `std:env`, `std:process`, `std:random` and gzip compression are implemented; no streaming/zstd/brotli | open | +| P2 | `std:path`, pure+host regex, `std:datetime`, `std:clock`, `std:env`, `std:process`, `std:random` and gzip compression are implemented; no streaming/zstd/brotli | open | | P2 | Memory limits: 16 MiB per region, 64 MiB per run | current budget, not a product limit | | P3 | No concurrency tasks, atomics or cancellable long-running scheduling | open; separate ADR | | P3 | No native optimizer/JIT or columnar numeric backend | open; benchmark-first | @@ -173,5 +173,6 @@ that a phase is complete. - [ADR-0047: DNS/UDP/TCP listener](../adr/0047-net-dns-udp-listener.md) — accepted project-mode network endpoints. - [ADR-0048: TLS client endpoint](../adr/0048-tls-resource-endpoint.md) — accepted validated rustls client; system + embedder DER roots. - [ADR-0049: gzip compression](../adr/0049-compression-resource-operations.md) — accepted bounded gzip byte-list codec. +- [ADR-0050: host regex extension](../adr/0050-host-regex-extension.md) — accepted project-mode groups/alternation/captures. - [Compute-machine RFC-0001](../MARQDO_COMPUTE_MACHINE_RFC_0001.md) and [core/stdlib RFC-0002](../MARQDO_COMPUTE_CORE_STDLIB_MIGRATION_RFC_0002.md) — design input. - [Architecture audit and refactor RFC](Marqdo_CLI_Latest_Architecture_Audit_and_Refactor_RFC.md) — review input for this migration. diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 798545c..c456923 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -54,6 +54,7 @@ The implemented first slices of these RFCs are recorded in `CURRENT.md` and | DNS/UDP/TCP-listener endpoints | [0047](../adr/0047-net-dns-udp-listener.md) | Accepted (project only, exact host grant, no TLS) | | TLS client endpoint | [0048](../adr/0048-tls-resource-endpoint.md) | Accepted (rustls, validated, project only) | | gzip compression operations | [0049](../adr/0049-compression-resource-operations.md) | Accepted (bounded byte-list gzip) | +| Host regex extension | [0050](../adr/0050-host-regex-extension.md) | Accepted (project-mode groups/alternation/captures) | | Table physical storage (T1/T2/T3), strict stdin/stderr/file/network resources | — | **Not decided**; do not implement by implication | ## Evidence and status diff --git a/docs/evidence/2026-11-21-regex-extension.md b/docs/evidence/2026-11-21-regex-extension.md new file mode 100644 index 0000000..087b0d5 --- /dev/null +++ b/docs/evidence/2026-11-21-regex-extension.md @@ -0,0 +1,36 @@ +# Evidence: host-backed regex extension + +Date: 2026-11-21 (Asia/Shanghai) +Source baseline: `94a13880c417739d1d8cc270097c376ef69d87df` +Repository: `marqdo-cli` +Decision record: [ADR-0050](../adr/0050-host-regex-extension.md) + +## Implemented + +- `resource.request("regex.match", [pattern, text])` -> Bool. +- `resource.request("regex.captures", [pattern, text])` -> `{matched, groups}` + with group 0 and `None` for unmatched groups. +- Invalid patterns are recoverable `regex.invalid`; no new grant; project mode + only. +- `stdlib/regex.mq.md` exposes `扩展匹配`, `扩展搜索`, `捕获`; pure + `匹配`/`搜索` subset unchanged. + +## Tests + +`stdlib/tests/regex.mq.md` covers `^(a+)(b|c)$` matching/captures. +`crates/marqdo-host/tests/resource_streams.rs` pins groups/alternation, +captures and invalid-pattern recovery (34 host tests). + +```text +cargo fmt --all -- --check PASS +cargo clippy --workspace --all-targets -- -D warnings PASS +cargo test --workspace 491 passed, 0 failed, 2 ignored +cargo test -p marqdo-host --test resource_streams 34 passed +cargo test -p marqdo-cli --test stdlib_product 16 passed +python3 tools/pack-stdlib.py PASS +``` + +## Not covered + +No pure-mode groups/alternation/captures, look-around, backreferences or +regex replace/split. The host engine is only reachable in project mode. diff --git a/spec/evaluation.md b/spec/evaluation.md index 5a90c3c..20fa6f2 100644 --- a/spec/evaluation.md +++ b/spec/evaluation.md @@ -215,6 +215,9 @@ resource.request("env.get", [name]) -> {ok:true, value:Text} resource.request("random.bytes", [count]) -> {ok:true, value:List} resource.request("compress.gzip", [text]) -> {ok:true, value:List} resource.request("compress.gunzip", [bytes]) -> {ok:true, value:Text} +resource.request("regex.match", [pattern, text]) -> {ok:true, value:Bool} +resource.request("regex.captures", [pattern, text]) -> {ok:true, value:{matched, groups}} + resource.request("process.run", [program, args])-> {ok:true, value:{status, stdout, stderr}} ``` @@ -244,6 +247,10 @@ resource.request("process.run", [program, args])-> {ok:true, value:{status, stdo * `env.get` takes one exact variable name and requires an `--allow-env NAME` grant; un-granted names are `permission.denied`, missing names are recoverable `env.not_found` and non-UTF-8 values are `env.not_unicode` (ADR-0043). +* `regex.match`/`regex.captures` use the host regex engine (groups, + alternation, character classes, captures); invalid patterns are recoverable + `regex.invalid`. They are project-mode only; the pure `.mq.md` subset remains + available (ADR-0050). * `compress.gzip`/`compress.gunzip` codec UTF-8 Text through a byte List; input/output are capped at 8 MiB and bad gzip/UTF-8 is recoverable `compress.invalid`/`compress.not_utf8` (ADR-0049). diff --git a/stdlib/README.md b/stdlib/README.md index 3cfff2e..6a4530c 100644 --- a/stdlib/README.md +++ b/stdlib/README.md @@ -22,7 +22,7 @@ module; a mismatch is a hard error. - `datetime.mq.md` — UTC ISO-8601 format/parse over Unix milliseconds (pure) - `clock.mq.md` — project-mode `clock.now` over `resource.request` - `path.mq.md` — lexical join/basename/dirname/extension/normalize (no I/O) -- `regex.mq.md` — `^ $ . * + ? [...]` matcher and search (no groups) +- `regex.mq.md` — pure subset matcher + host-backed groups/alternation/captures - `net/http.mq.md` — HTTP/1.1 response parsing (headers/chunked) and request builder - `tests/` — product self-tests for every public module (`clock`, `compress`, `console`, `csv`, `datetime`, `demo`, `env`, `format`, `fs`, `http`, diff --git a/stdlib/regex.mq.md b/stdlib/regex.mq.md index 683c39e..89eaa0a 100644 --- a/stdlib/regex.mq.md +++ b/stdlib/regex.mq.md @@ -432,3 +432,27 @@ **下标 = 下标 + 1** *成功(结果)* + +## 扩展匹配 + +`(模式, 文本)` + +使用宿主正则引擎(支持分组、选择、`\\d` 等)判断是否匹配。模式无效时 +返回 `regex.invalid`。仅项目模式;纯 `.mq.md` 子集仍由 `匹配`/`搜索` 提供。 + +*resource.request("regex.match", [模式, 文本])* + +## 扩展搜索 + +`(模式, 文本)` + +*resource.request("regex.match", [模式, 文本])* + +## 捕获 + +`(模式, 文本)` + +返回 `{ok:true, value:{matched:Bool, groups:List}}`。下标 0 是 +完整匹配;未参与匹配的分组是 `None`。 + +*resource.request("regex.captures", [模式, 文本])* diff --git a/stdlib/stdlib.json b/stdlib/stdlib.json index e8549d1..10a5c50 100644 --- a/stdlib/stdlib.json +++ b/stdlib/stdlib.json @@ -1,5 +1,5 @@ { "name": "marqdo-stdlib", "version": "0.1.0", - "digest": "sha256:770a63bbd1d3c8a62471c0f0c3cb9ee5647656c137a528442f16e3bd63505d5f" + "digest": "sha256:0809bdb27b2d4688aae7c8a2fcd270b88e23fcf33fa9edbd127681fa05f3a4a1" } diff --git a/stdlib/tests/regex.mq.md b/stdlib/tests/regex.mq.md index 2b725b0..3167eca 100644 --- a/stdlib/tests/regex.mq.md +++ b/stdlib/tests/regex.mq.md @@ -27,4 +27,8 @@ import: **`不支持 = regex.匹配("(a)", "a")`** -*[星号.value, 加号.value, 锚点.value, 类.value, 否定类.value, 任意.value, 可选.value, 转义.value, 搜索.value, 无效.ok, 无效.code, 不支持.code]* +**扩展 = regex.扩展匹配("^(a+)(b|c)$", "aab")** + +**捕获 = regex.捕获("^(a+)(b|c)$", "aab")** + +*[星号.value, 加号.value, 锚点.value, 类.value, 否定类.value, 任意.value, 可选.value, 转义.value, 搜索.value, 无效.ok, 无效.code, 不支持.code, 扩展.value, 捕获.value.matched, 捕获.value.groups]*