feat(regex): add host-backed groups, alternation and captures

This commit is contained in:
Marqdo Agent committed 2026-11-21 10:58:06 +08:00
1 parent 94a13880c4
commit 28b178fc70
16 files changed
+289 -8

No files matched your search

Generated
+39
View File
@@ -8,6 +8,15 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aho-corasick"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
dependencies = [
"memchr",
]
[[package]]
name = "anstream"
version = "1.0.0"
@@ -485,6 +494,7 @@ dependencies = [
"marqdo-span",
"marqdo-value",
"rcgen",
"regex",
"rustls",
"rustls-native-certs",
"rustls-pemfile",
@@ -803,6 +813,35 @@ dependencies = [
"yasna",
]
[[package]]
name = "regex"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9807af0e638c12e753aa1b4e601791bf2a8f71274023c8bbeec12fc149950857"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d60ef07df457506a0a1a2073112eb4aae5733dbd2813c3da77d1b0fd441158"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d902046e6a28a1e4c3cab97258a0ce11e994903f01c2d97a9447a0b3df85f4f6"
[[package]]
name = "ring"
version = "0.17.14"
+1
View File
@@ -59,6 +59,7 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std"
rustls-native-certs = "0.8"
rustls-pemfile = "2"
rcgen = "0.13"
regex = "1"
thiserror = "2"
clap = { version = "4", features = ["derive"] }
walkdir = "2"
+6
View File
@@ -312,6 +312,12 @@ fn regex_test_matches_anchors_classes_and_search() {
assert_eq!(items[9]["value"], false);
assert_eq!(items[10]["value"], "regex.invalid");
assert_eq!(items[11]["value"], "regex.unsupported");
assert_eq!(items[12]["value"], true);
assert_eq!(items[13]["value"], true);
let groups = items[14]["items"].as_array().expect("groups");
assert_eq!(groups[0]["value"], "aab");
assert_eq!(groups[1]["value"], "aa");
assert_eq!(groups[2]["value"], "b");
}
#[test]
+1
View File
@@ -15,6 +15,7 @@ rustls-native-certs = { workspace = true }
marqdo-plugin-abi = { workspace = true }
flate2 = { workspace = true }
getrandom = { workspace = true }
regex = { workspace = true }
glob = { workspace = true }
marqdo-span = { workspace = true }
marqdo-value = { workspace = true }
+76
View File
@@ -845,6 +845,82 @@ impl Host for StdHost {
self.record_resource_effect("resource.request", ctx, started, "process.run");
return Ok(result);
}
if operation == "regex.match" || operation == "regex.captures" {
if operation == "regex.match" {
if args.len() != 2 {
return Err(RuntimeError::new(
ErrorCategory::Arity,
ctx.span,
format!(
"resource operation `regex.match` expects 2 arguments, got {}",
args.len()
),
));
}
} else if args.len() != 2 {
return Err(RuntimeError::new(
ErrorCategory::Arity,
ctx.span,
format!(
"resource operation `regex.captures` expects 2 arguments, got {}",
args.len()
),
));
}
let pattern_type = args[0].type_name();
let Value::Text(pattern) = args[0].clone() else {
return Err(RuntimeError::type_error(
ctx.span,
format!("{operation} pattern must be Text, got {pattern_type}"),
));
};
let text_type = args[1].type_name();
let Value::Text(text) = args[1].clone() else {
return Err(RuntimeError::type_error(
ctx.span,
format!("{operation} text must be Text, got {text_type}"),
));
};
let regex = match regex::Regex::new(&pattern) {
Ok(regex) => regex,
Err(error) => {
return Ok(result_error(
"regex.invalid",
&format!("invalid regular expression: {error}"),
))
}
};
self.record_semantic(
"resource.request",
None,
Some(&ctx.module_id),
ctx.span,
Some(operation),
);
self.record_resource_effect("resource.request", ctx, started, operation);
let mut map = BTreeMap::new();
map.insert("ok".to_string(), Value::Bool(true));
if operation == "regex.match" {
map.insert("value".to_string(), Value::Bool(regex.is_match(&text)));
} else {
let mut payload = BTreeMap::new();
let captures = regex.captures(&text);
payload.insert("matched".to_string(), Value::Bool(captures.is_some()));
let groups = match captures {
Some(captures) => captures
.iter()
.map(|group| match group {
Some(group) => Value::text(group.as_str()),
None => Value::None,
})
.collect(),
None => Vec::new(),
};
payload.insert("groups".to_string(), Value::List(groups));
map.insert("value".to_string(), Value::Map(payload));
}
return Ok(Value::Map(map));
}
if operation == "compress.gzip" {
if args.len() != 1 {
return Err(RuntimeError::new(
@@ -1117,3 +1117,44 @@ fn compression_round_trips_and_rejects_invalid_gzip() {
assert_eq!(field(&invalid, "ok"), &Value::Bool(false));
assert_eq!(field(&invalid, "code"), &Value::text("compress.invalid"));
}
#[test]
fn regex_operations_support_groups_alternation_and_errors() {
let dir = tempfile::tempdir().unwrap();
let mut host = StdHost::new(SandboxMode::Project, Capabilities::project(), dir.path());
let ctx = context();
let matched = host
.resource_request(
"regex.match",
vec![Value::text("^(a+)(b|c)$"), Value::text("aab")],
&ctx,
)
.unwrap();
assert_eq!(field(&matched, "value"), &Value::Bool(true));
let captures = host
.resource_request(
"regex.captures",
vec![Value::text("^(a+)(b|c)$"), Value::text("aab")],
&ctx,
)
.unwrap();
let value = field(&captures, "value");
assert_eq!(field(value, "matched"), &Value::Bool(true));
assert_eq!(
field(value, "groups"),
&Value::List(vec![
Value::text("aab"),
Value::text("aa"),
Value::text("b")
])
);
let invalid = host
.resource_request(
"regex.match",
vec![Value::text("["), Value::text("a")],
&ctx,
)
.unwrap();
assert_eq!(field(&invalid, "ok"), &Value::Bool(false));
assert_eq!(field(&invalid, "code"), &Value::text("regex.invalid"));
}
+4 -2
View File
@@ -58,7 +58,7 @@ byte-identical).
cargo fmt --all -- --check PASS
cargo clippy --workspace --all-targets -- -D warnings PASS
cargo test --workspace --locked PASS
490 passed, 0 failed, 2 ignored; includes 7/7 Linux security tests
491 passed, 0 failed, 2 ignored; includes 7/7 Linux security tests
```
The documentation and test migration is local work; re-run this block after it
@@ -209,7 +209,9 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL
runs an exact `--allow-process PROGRAM` grant with 5 s timeout and 1 MiB caps
(ADR-0044, not an OS sandbox); `random.字节` reads explicitly granted
`random.bytes` entropy (ADR-0046). File watch, binary offset reads, streaming
incremental JSON/CSV streaming and regex groups/alternation remain open.
incremental JSON/CSV streaming, pure-mode regex groups/alternation and regex
look-around/backreferences remain open; project mode has host-backed groups
and captures (ADR-0050).
- Bundled standard library (CM-A10, ADR-0028): `stdlib/` ships
`format`/`text`/`io`/`fs`/`data`/`path`/`regex`/`datetime`/`clock`/`env`/`process`/`random`/`compress`; explicit `std:` imports resolve only inside the bundle;
a SHA-256 digest over `*.mq.md` is verified before loading; Analysis records
+42
View File
@@ -0,0 +1,42 @@
# ADR-0050: Host-backed extended regular expressions
Status: **Accepted**
Date: 2026-11-21
Relates to: ADR-0041
Specs: `spec/evaluation.md`
Evidence: `docs/evidence/2026-11-21-regex-extension.md`
## Context
The pure `.mq.md` regex subset covers literals, `.`, quantifiers, anchors and
character classes, but not groups, alternation or captures. Implementing a
full backtracking engine in `.mq.md` would be a large, hard-to-audit
duplicate. The host already has a safe, well-tested `regex` crate.
## Decision
Add project-mode operations:
```text
resource.request("regex.match", [pattern, text]) -> {ok:true, value:Bool}
resource.request("regex.captures", [pattern, text]) -> {ok:true, value:{matched:Bool, groups:List<Text|None>}}
```
* Both require two Text arguments; invalid patterns are recoverable
`regex.invalid`.
* `captures` includes group 0 (the whole match); unmatched optional groups are
`None`.
* No new capability grant: regex compilation/matching is deterministic CPU
work and does not disclose external state. All `resource.request` operations
remain project-mode only, so `pure`/`strict` refuse them.
* `stdlib/regex.mq.md` exposes `扩展匹配`, `扩展搜索` and `捕获`; the existing
pure `匹配`/`搜索` subset stays unchanged for pure mode.
* The `regex` crate is Rust-only, with no OS dependency. No new builtin,
`HostOp` or parser node is added.
## Consequences
* Groups, alternation, `\d` classes and captures are available in project mode.
* Pure mode still has only the documented subset; this is an explicit
mode-dependent capability, not a silent downgrade.
* Look-around, backreferences and regex captures in pure `.mq.md` remain open.
+4 -3
View File
@@ -18,7 +18,7 @@ platform".
| Dimension | Current state | Source |
|---|---|---|
| Implemented | Rust workspace `0.1.0`: Markdown/frontend/IR, tree-walk runtime, `Memory`, capability host, Tool/Analysis/Trace, Plugin ABI, Foreign kernels, bundled stdlib, CLI composition root | `Cargo.toml`, `crates/`, `stdlib/` |
| Tested locally | `cargo test --workspace --locked`: **490 passed, 0 failed, 2 ignored** on Linux x86_64 at the baseline | `docs/STATUS.md` §3 |
| Tested locally | `cargo test --workspace --locked`: **491 passed, 0 failed, 2 ignored** on Linux x86_64 at the baseline | `docs/STATUS.md` §3 |
| Bundled stdlib released | In-repo bundle `marqdo-stdlib@0.1.0`, SHA-256 digest verified before loading; it is **not** an external registry package | `stdlib/stdlib.json`, `stdlib/README.md` |
| Multi-platform verified | Linux local gate is the only executed lane here; Windows/macOS native lanes are **UNVERIFIED**, and the GitHub security lane remains a **VM-only** gate | `docs/STATUS.md` §1, §4 |
@@ -95,7 +95,7 @@ review RFC, not a new crate per provider.
| `std:data/json` | `json.解析/解析多个/格式化`; nested Value, 128-depth bound, recoverable errors | implemented (recursive, multi-value; no streaming) | `stdlib_product` |
| `std:data/csv` | `csv.解析行/解析/格式化行/格式化`; quoted single/multi-row, CRLF, quoted newlines | implemented (no streaming) | `stdlib_product` |
| `std:path` | `path.连接/文件名/目录名/扩展名/规范/是否绝对`; lexical only | implemented (no I/O) | `stdlib_product` |
| `std:regex` | `regex.匹配/搜索` with `^ $ . * + ? [...]` escapes | implemented subset; no groups/alternation | `stdlib_product` |
| `std:regex` | pure `匹配/搜索` subset; host `扩展匹配/捕获` with groups/alternation/captures | implemented both paths; pure path has no groups | `resource_streams`, `stdlib_product` |
| `std:datetime` | `datetime.格式化(毫秒)` / `datetime.解析(文本)`; UTC ISO-8601, pre-1970 supported | implemented (pure) | `stdlib_product` |
| `std:clock` | `clock.现在()` -> `resource.request("clock.now", [])` | implemented (project mode; pure/strict denied) | `resource_streams`, `stdlib_product` |
| `std:env` | `env.读取/存在` -> `resource.request("env.get", [name])`; exact `--allow-env` grant | implemented (project mode; no `env.list`) | `resource_streams`, `stdlib_product` |
@@ -134,7 +134,7 @@ that a phase is complete.
| P1 | CLI split into `commands/{sandbox,run,check,inspect,init,package,tool}.rs`; host split into `host/{resource_ops,providers,effects,dispatch}.rs` plus `process.rs`; `commands.rs` keeps shared project/runtime helpers and `host.rs` keeps the `Host` trait methods and capability checks | split; shared helpers and trait methods remain by design |
| P2 | HTTP parsing is Unicode-codepoint oriented; a general byte body is not the only path | open; convert before a general network release |
| P2 | DNS, UDP, TCP listener and TLS client endpoints are implemented; no TLS server/mTLS or custom-CA CLI | open |
| P2 | `std:path`, regex subset, `std:datetime`, `std:clock`, `std:env`, `std:process`, `std:random` and gzip compression are implemented; no streaming/zstd/brotli | open |
| P2 | `std:path`, pure+host regex, `std:datetime`, `std:clock`, `std:env`, `std:process`, `std:random` and gzip compression are implemented; no streaming/zstd/brotli | open |
| P2 | Memory limits: 16 MiB per region, 64 MiB per run | current budget, not a product limit |
| P3 | No concurrency tasks, atomics or cancellable long-running scheduling | open; separate ADR |
| P3 | No native optimizer/JIT or columnar numeric backend | open; benchmark-first |
@@ -173,5 +173,6 @@ that a phase is complete.
- [ADR-0047: DNS/UDP/TCP listener](../adr/0047-net-dns-udp-listener.md) — accepted project-mode network endpoints.
- [ADR-0048: TLS client endpoint](../adr/0048-tls-resource-endpoint.md) — accepted validated rustls client; system + embedder DER roots.
- [ADR-0049: gzip compression](../adr/0049-compression-resource-operations.md) — accepted bounded gzip byte-list codec.
- [ADR-0050: host regex extension](../adr/0050-host-regex-extension.md) — accepted project-mode groups/alternation/captures.
- [Compute-machine RFC-0001](../MARQDO_COMPUTE_MACHINE_RFC_0001.md) and [core/stdlib RFC-0002](../MARQDO_COMPUTE_CORE_STDLIB_MIGRATION_RFC_0002.md) — design input.
- [Architecture audit and refactor RFC](Marqdo_CLI_Latest_Architecture_Audit_and_Refactor_RFC.md) — review input for this migration.
+1
View File
@@ -54,6 +54,7 @@ The implemented first slices of these RFCs are recorded in `CURRENT.md` and
| DNS/UDP/TCP-listener endpoints | [0047](../adr/0047-net-dns-udp-listener.md) | Accepted (project only, exact host grant, no TLS) |
| TLS client endpoint | [0048](../adr/0048-tls-resource-endpoint.md) | Accepted (rustls, validated, project only) |
| gzip compression operations | [0049](../adr/0049-compression-resource-operations.md) | Accepted (bounded byte-list gzip) |
| Host regex extension | [0050](../adr/0050-host-regex-extension.md) | Accepted (project-mode groups/alternation/captures) |
| Table physical storage (T1/T2/T3), strict stdin/stderr/file/network resources | — | **Not decided**; do not implement by implication |
## Evidence and status
@@ -0,0 +1,36 @@
# Evidence: host-backed regex extension
Date: 2026-11-21 (Asia/Shanghai)
Source baseline: `94a13880c417739d1d8cc270097c376ef69d87df`
Repository: `marqdo-cli`
Decision record: [ADR-0050](../adr/0050-host-regex-extension.md)
## Implemented
- `resource.request("regex.match", [pattern, text])` -> Bool.
- `resource.request("regex.captures", [pattern, text])` -> `{matched, groups}`
with group 0 and `None` for unmatched groups.
- Invalid patterns are recoverable `regex.invalid`; no new grant; project mode
only.
- `stdlib/regex.mq.md` exposes `扩展匹配`, `扩展搜索`, `捕获`; pure
`匹配`/`搜索` subset unchanged.
## Tests
`stdlib/tests/regex.mq.md` covers `^(a+)(b|c)$` matching/captures.
`crates/marqdo-host/tests/resource_streams.rs` pins groups/alternation,
captures and invalid-pattern recovery (34 host tests).
```text
cargo fmt --all -- --check PASS
cargo clippy --workspace --all-targets -- -D warnings PASS
cargo test --workspace 491 passed, 0 failed, 2 ignored
cargo test -p marqdo-host --test resource_streams 34 passed
cargo test -p marqdo-cli --test stdlib_product 16 passed
python3 tools/pack-stdlib.py PASS
```
## Not covered
No pure-mode groups/alternation/captures, look-around, backreferences or
regex replace/split. The host engine is only reachable in project mode.
+7
View File
@@ -215,6 +215,9 @@ resource.request("env.get", [name]) -> {ok:true, value:Text}
resource.request("random.bytes", [count]) -> {ok:true, value:List<Int>}
resource.request("compress.gzip", [text]) -> {ok:true, value:List<Int>}
resource.request("compress.gunzip", [bytes]) -> {ok:true, value:Text}
resource.request("regex.match", [pattern, text]) -> {ok:true, value:Bool}
resource.request("regex.captures", [pattern, text]) -> {ok:true, value:{matched, groups}}
resource.request("process.run", [program, args])-> {ok:true, value:{status, stdout, stderr}}
```
@@ -244,6 +247,10 @@ resource.request("process.run", [program, args])-> {ok:true, value:{status, stdo
* `env.get` takes one exact variable name and requires an `--allow-env NAME`
grant; un-granted names are `permission.denied`, missing names are recoverable
`env.not_found` and non-UTF-8 values are `env.not_unicode` (ADR-0043).
* `regex.match`/`regex.captures` use the host regex engine (groups,
alternation, character classes, captures); invalid patterns are recoverable
`regex.invalid`. They are project-mode only; the pure `.mq.md` subset remains
available (ADR-0050).
* `compress.gzip`/`compress.gunzip` codec UTF-8 Text through a byte List;
input/output are capped at 8 MiB and bad gzip/UTF-8 is recoverable
`compress.invalid`/`compress.not_utf8` (ADR-0049).
+1 -1
View File
@@ -22,7 +22,7 @@ module; a mismatch is a hard error.
- `datetime.mq.md` — UTC ISO-8601 format/parse over Unix milliseconds (pure)
- `clock.mq.md` — project-mode `clock.now` over `resource.request`
- `path.mq.md` — lexical join/basename/dirname/extension/normalize (no I/O)
- `regex.mq.md` — `^ $ . * + ? [...]` matcher and search (no groups)
- `regex.mq.md` — pure subset matcher + host-backed groups/alternation/captures
- `net/http.mq.md` — HTTP/1.1 response parsing (headers/chunked) and request builder
- `tests/` — product self-tests for every public module
(`clock`, `compress`, `console`, `csv`, `datetime`, `demo`, `env`, `format`, `fs`, `http`,
+24
View File
@@ -432,3 +432,27 @@
**下标 = 下标 + 1**
*成功(结果)*
## 扩展匹配
`(模式, 文本)`
使用宿主正则引擎(支持分组、选择、`\\d` 等)判断是否匹配。模式无效时
返回 `regex.invalid`。仅项目模式;纯 `.mq.md` 子集仍由 `匹配`/`搜索` 提供。
*resource.request("regex.match", [模式, 文本])*
## 扩展搜索
`(模式, 文本)`
*resource.request("regex.match", [模式, 文本])*
## 捕获
`(模式, 文本)`
返回 `{ok:true, value:{matched:Bool, groups:List<Text|None>}}`。下标 0 是
完整匹配;未参与匹配的分组是 `None`。
*resource.request("regex.captures", [模式, 文本])*
+1 -1
View File
@@ -1,5 +1,5 @@
{
"name": "marqdo-stdlib",
"version": "0.1.0",
"digest": "sha256:770a63bbd1d3c8a62471c0f0c3cb9ee5647656c137a528442f16e3bd63505d5f"
"digest": "sha256:0809bdb27b2d4688aae7c8a2fcd270b88e23fcf33fa9edbd127681fa05f3a4a1"
}
+5 -1
View File
@@ -27,4 +27,8 @@ import:
**`不支持 = regex.匹配("(a)", "a")`**
*[星号.value, 加号.value, 锚点.value, 类.value, 否定类.value, 任意.value, 可选.value, 转义.value, 搜索.value, 无效.ok, 无效.code, 不支持.code]*
**扩展 = regex.扩展匹配("^(a+)(b|c)$", "aab")**
**捕获 = regex.捕获("^(a+)(b|c)$", "aab")**
*[星号.value, 加号.value, 锚点.value, 类.value, 否定类.value, 任意.value, 可选.value, 转义.value, 搜索.value, 无效.ok, 无效.code, 不支持.code, 扩展.value, 捕获.value.matched, 捕获.value.groups]*