Compare commits

..
32 Commits
Author SHA1 Message Date
cflmyandCursor 41587f319e release: v2.1.5 agent reliability workbench
Bump CLI/wasm/ext to 2.1.5; document ResourceView, ToolSpec, CheckpointV2,
EffectLedger, EKC warm-load, and web short/read lane isolation.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 15:39:36 +08:00
cflmyandCursor 298fbd51ea feat(workbench): expose CSV preview text for TablePreview
ResourceView csv render now includes bounded text alongside sanitized
HTML so clients can build interactive tables without re-parsing HTML.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 14:45:05 +08:00
cflmyandCursor 265dfdbcc4 feat(workbench): emit JSON source text and code line locators
ResourceView now carries render.text for JsonPreview/CodePreview and
honors locator.line with capped line locators for navigation.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 14:23:00 +08:00
cflmyandCursor 42a43f5ffd feat(workbench): honor ResourceView locator.page as focus_locator
PDF blob views echo the requested page for client navigation without
claiming text extract or OCR.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 14:18:38 +08:00
cflmyandCursor 037a963547 feat(workbench): emit blob_ref for PDF and image ResourceView
Client visual preview consumes the blob; extract/OCR capabilities stay
false so QA-10 honesty holds without cutting the preview path.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 14:14:17 +08:00
cflmyandCursor a32638e935 fix(workbench): drop trailing blank lines from CSV ResourceView
Avoid an empty last table row when files end with a newline.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 14:03:40 +08:00
cflmyandCursor 43eebe5671 feat(workbench): add bounded JSON and CSV ResourceView renderers
Serve static application/json and text/csv previews with outline/table
limits and flip extract_formats availability for those media types.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 13:55:35 +08:00
cflmyandCursor e833dc05e2 feat(agent): attach effect ledger watermark on CheckpointV2 save
Persist effects_watermark with resume checkpoints and pass resume_id into
step tool calls so mutating tools share a stable logical_run scope.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 13:48:45 +08:00
cflmyandCursor 11bb7ef277 feat(agent): wire EffectLedger through mutating run_tool
Prepare/start/commit write and execute tools with idempotent dedupe, and refuse
blind retry while ledger state is unknown or interrupted mid-flight (MA-T03).

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 13:44:45 +08:00
cflmyandCursor 6bd90bb2d0 feat(agent): ship output channels, ContextPackV2, and effect ledger
Open AgentOutputV1/ContextPackV2 capability bits and add EffectLedgerRecordV1
so CheckpointV2 write auto-resume only unlocks on a safe watermark.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 13:28:03 +08:00
cflmyandCursor 4683f2e067 feat(agent): ToolSpec registry with authorize-before-call
Add ToolSpecV1 register/search/describe/prepare so CALL and native paths share one schema gate; unauthorized and invalid-arg prepares fail with zero executions.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 13:18:55 +08:00
cflmyandCursor 5912164039 feat(agent): CheckpointV2 atomic resume and AgentErrorV1
Replace bare resume writes with temp/fsync/rename, mark legacy loads as manual_review, and expose bounded read-only retry decisions while keeping write auto-resume closed without a ledger.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 11:03:39 +08:00
cflmyandCursor 303c06f773 feat(workbench): freeze agent protocols and ship ResourceView v1
Lock M-A0 Error/Tool/Checkpoint/Output/Context fixtures with capability bits off, and land M-A3 static md/txt/code ResourceView (no-exec) behind resource.view.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 10:59:59 +08:00
cflmyandCursor a1086fd6c3 docs: plan cross-repo agent reliability delivery
Lock M-A*/Q-A* sequencing, F01–F20 mapping, and first-slice gates on top of the 2026-10-27 designs.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 10:49:20 +08:00
cflmy 6ffe5ceb19 docs: expand executable knowledge design notes 2026-10-27 02:10:54 +00:00
cflmy 6b568aff3d docs: design reliable agent capabilities 2026-10-27 02:10:25 +00:00
cflmyandCursor 16f26d7d60 fix(knowledge): warm-load IR for preflight/find/reuse queries
Stop full recompile on every knowledge preflight by writing ekc/graph/ir.json
and loading it unless --refresh or missing projections, cutting chat预检 wall time.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 09:18:27 +08:00
cflmyandCursor b8265a92aa fix(web): stop short/read/control from filling execute slots
Isolating every short invoke on the 2-slot worker pool caused no_slot under
normal traffic; keep short paths on parent (execute already off-process) and
background-reap after observer disconnect so zombies release slots.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 08:56:56 +08:00
cflmyandCursor 9af929971c fix(web): disconnect≠cancel and isolate short/read/control lanes
Observer SSE close no longer cancels execute by default; short/read/control
bypass parent INVOKE_LOCK via isolated admission, with read_lane=1 and gold samples.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 08:44:07 +08:00
cflmyandCursor 2c4fa4bb05 fix(ppa): install only rustc/cargo/std and use disk TMPDIR
Repeated --without flags were ignored by install.sh (last wins), so
rust-docs still filled /tmp and hit quota; prefer --components= and
a home-cache work dir instead of tmpfs.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-26 19:13:48 +08:00
cflmyandCursor 3cabe53da2 release: v2.1.4 isolated HTTP execute (ADR 0009)
Default execute/sse-live/stream to Supervisor worker pool; MR-01 gold;
/_runtime/*; bump CLI/wasm/ext to 2.1.4.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-26 18:33:33 +08:00
cflmy 58e13ee6d3 docs: add isolated web runtime development design 2026-10-26 05:53:02 +00:00
cflmyandCursor 1b11fe063a fix(web): fail-fast when invoke lock is busy; document single-interp limit
Long sse-live chats held INVOKE_LOCK indefinitely and starved short routes
(UI freeze). Timed try-lock returns interpreter busy → HTTP 503; ADR 0008
records that HTTP is multi-goroutine while Marqdo evaluation stays single-flight.
Also lands plan/delta streaming and soft map guards used by qdagent chat.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-26 13:27:53 +08:00
cflmyandCursor b792f44556 feat(web): auto sse-live when chat body has stream:true
Upgrade json invoke routes to the M02 EventBus forwarder for OpenAI-compatible
stream requests, and normalize LLM delta frames to message.delta.payload.delta
so tokens flush before run.completed.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-26 09:47:05 +08:00
cflmyandCursor 9ea5bebfc3 chore(release): bump to 2.1.3 (workbench M00–M06)
Ship ModelSelection/RunContext/DocumentView contracts, LLM selection,
view document CLI, and live/F02 evidence. CLI, wasm, and ext pack aligned.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-25 23:13:27 +08:00
cflmyandCursor c7c329341b feat(workbench): live selection smoke, F02 agent e2e, DocumentView CLI
Hoist llm.ask prompt binding to avoid elevated `p`; prefer reasoning_content
when content is empty; point live/tests at in-tree MARQDO_EXT; add USTC live
gates and mock agent.step selection+resume evidence; view document --out for
read-only browser embed checks. Design stays Partial (not Verified).

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-25 23:05:28 +08:00
cflmyandCursor 8cf5288481 feat(workbench): F14 coalesce, snapshot manifest, and sse drain evidence
Hold analysis locks across compile for concurrent same-snapshot reads,
emit SnapshotManifestV1 for app retention, and add Go drainForward tests.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-25 11:13:53 +08:00
cflmyandCursor 69c4e74872 feat(view): project KnowledgeStep kinds into DocumentView nodes
Strengthen F09 read-model coverage so observe/derive/call/iterate steps
appear alongside artifact cards without executing the document.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-25 11:03:01 +08:00
cflmyandCursor af0bda2e40 test(workbench): add F02 subtask/agent skeleton and cancel bridge evidence
Cover selection env propagation, agent skeleton without llm.llm, step-kinds
DocumentView read purity, and Go cancel-token bridge smoke.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-25 11:02:25 +08:00
cflmyandCursor 090b026547 test(workbench): harden F-matrix evidence and demote Verified claim
Add dual-endpoint LLM isolation, usage revision replace, cancel/F08/F11
asserts, and mark the design status Partial until remaining F gaps close.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-25 11:01:02 +08:00
cflmyandCursor 03396afd02 feat(workbench): land M00–M06 runtime contracts and read-only view
Add capabilities/selection/usage/DocumentView/locator/RunContext with
honest probes, LLM messages+cancel, sse-live cancel bridge, and CI
workbench_contracts gate; record design evidence for verified slices.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-25 10:55:08 +08:00
cflmy 663a8ba2c0 docs: specify runtime and view workbench support 2026-10-24 16:27:08 +00:00
139 changed files with 17116 additions and 395 deletions

No files matched your search

+12 -6
View File
@@ -28,6 +28,7 @@ Canonical release playbook for **cflmy/marqdo**. Read this skill **before** tagg
9. **Extension packs always publish to both** Cloudflare R2 (`https://ext.marqdo.com` via `scripts/upload-ext-r2.py`) **and** GitHub (Release assets / `ext-v*` tag). Credentials only in `~/.marqdo/r2.env` / CI — **never** commit. See [ext-cdn.md](../../../doc/design/ext-cdn.md).
10. **CLI Ubuntu package = PPA** (`ppa:cflmy/marqdo`) via `./scripts/ppa-ship.sh` / `ppa-build-source.sh`. GitHub zips remain for Windows and non-apt Linux.
11. **When a step needs the user** (sudo password, GPG passphrase, Launchpad UI, interactive `dput`): **stop, state what failed, and give the exact commands** to run in their terminal. Do not pretend the step succeeded.
12. **GitHub auto-build must be green before declaring a release done.** For full/cli: wait for the tag-triggered **Release** workflow (`windows` + `linux`) **and** the push-triggered **`executable-knowledge`** workflow on the release commit / `main` tip. Release asset zips alone are **not** sufficient — a green Release job with a red EK gate is an incomplete release. Do not skip or reframe a red `executable-knowledge` as “assets already uploaded.”
## Release modes
@@ -78,6 +79,7 @@ Release progress:
```
Checks: clean tree; `main` ff-only; smoke tests; CHANGELOG Unreleased; no secrets.
Also confirm the latest `main` **`executable-knowledge`** Actions run is green (or that this release commit will be covered after push) — do not proceed to tag if EK is already red on tip without a fix in the same release PR.
## Phase 2 — Docs & version bumps (main)
@@ -129,7 +131,9 @@ git push https://proxy.cflmy.top/github.com/cflmy/marqdo.git vVER
## Phase 5 — GitHub Release notes & assets
1. Wait for Actions (full/cli): both `windows` and `linux` green when ext natives required.
1. Wait for Actions (full/cli):
- Tag **Release** workflow: both `windows` and `linux` green when ext natives required.
- Push **`executable-knowledge`** workflow on the release commit / `main`: **must be success**. Poll with `gh run list --workflow=executable-knowledge.yml --branch main` / `gh run watch`. If EK is red → **stop**; fix plugin/harness on `main`, re-run; do **not** declare the release complete from assets alone.
2. Rewrite release body 中英双语 — [reference.md § Notes](reference.md).
3. Confirm Linux `marqdo-VER-native-x86_64-unknown-linux-gnu.zip` when mode ships ext.
@@ -174,10 +178,11 @@ On Launchpad failure: read the build log, fix `debian/` or bundle script, bump `
## Phase 8 — Post-release
1. `gh release view TAG` — assets + notes.
2. CDN `https://ext.marqdo.com/vVER/` (if ext shipped).
3. PPA build green (if cli shipped).
4. Tell user URLs: GitHub release, CDN, PPA install one-liner.
5. Optional gh-pages only if asked.
2. Confirm **`executable-knowledge`** and **Release** Actions are both green for this tag/commit; if either is red, say so and do not claim “发版完成”.
3. CDN `https://ext.marqdo.com/vVER/` (if ext shipped).
4. PPA build green (if cli shipped).
5. Tell user URLs: GitHub release, CDN, PPA install one-liner.
6. Optional gh-pages only if asked.
## Extension-only release (mode=ext)
@@ -194,7 +199,8 @@ On Launchpad failure: read the build log, fix `debian/` or bundle script, bump `
| `gh` / git TLS or Clash `7890` refused | Unset bad proxy; see [reference § Proxy](reference.md) |
| GitHub blocked | proxy.cflmy.top, then HK jump; **print commands** if agent cannot push |
| Tag exists | Stop; ask user |
| CI red | Fix on main; retag only if user allows |
| CI red (Release assets) | Fix on main; retag only if user allows |
| `executable-knowledge` red | Fix agent plugin / test harness (e.g. `ensure_agent_plugin_built`); push `main`; wait for EK green. **Do not** treat uploaded Release zips as a successful release |
| R2 upload fails | Print `upload-ext-r2.py` command; do not skip silently |
| PPA / sudo / GPG | Print `./scripts/ppa-ship.sh noble` (or `dput …`); do not skip silently |
| Launchpad `Cargo.lock` v4 / old cargo | Ensure bundled `third_party/rust` via current `ppa-build-source.sh`; bump `1ppaN` |
@@ -304,6 +304,15 @@ gh workflow run Release --ref "vVER" # only if workflow_dispatch exists; else f
Current Release workflow is **tag-push only**. Red CI after tag → fix on `main`, then either new patch tag or **user-approved** delete+recreate tag (dangerous; prefer patch).
**Also required before declaring release done:** the push-triggered **`executable-knowledge`** workflow on the release commit / `main` tip must be **success**. Poll with:
```bash
gh run list --workflow=executable-knowledge.yml --branch main --limit 5
gh run watch <run-id>
```
A green Release asset job with a red `executable-knowledge` gate is an **incomplete** release — fix the plugin/harness (e.g. `ensure_agent_plugin_built` for agent-dependent EK tests), push `main`, and wait for EK green. Do not skip.
### HK SSH jump (Paramiko CONNECT)
**When to use:** local → GitHub fails (timeout / TLS / Clash flaky / reverse-proxy `proxy.cflmy.top` connect drops), but this host can reach an overseas box that has stable GitHub (default `hk.cflmy.de`). Proven path for v0.3.9 push/tag/`gh`.
@@ -11,8 +11,20 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- name: Build agent + llm plugins (EK / workbench gates)
run: |
cargo build -p marqdo_plugin_agent --locked
cargo build -p marqdo_plugin_llm --locked
- name: Workbench contracts (M00–M06 / F-subset)
run: cargo test --test workbench_contracts --locked
- name: Offline executable-knowledge gate
run: cargo test --test executable_knowledge --locked
- name: Isolated execute MR-01 (dual 60s SSE + short reads)
run: |
cargo build -p marqdo_plugin_llm --locked
bash scripts/build-web-plugin.sh
cargo test --test isolated_execute_mr01 --locked -- --nocapture
timeout-minutes: 5
- name: no ignored tests counted as success
run: |
if grep -n '#\[ignore\]' tests/executable_knowledge.rs; then
+7 -3
View File
@@ -126,9 +126,13 @@ jobs:
## Marqdo ${{ github.ref_name }}
### 亮点 / Highlights
- **v2.1.2**:可执行知识生产链——证据/闭包 · verify/adjudicate · reuse_execute/bind_execute · document.extract · View KnowledgePresentation + SSE page_token · 上传限流
- **v2.1.2**: Executable-knowledge production chain — evidence/closure, verify/adjudicate, reuse_execute/bind_execute, document.extract, View KnowledgePresentation + SSE page_token, upload caps
- **v2.1.2 ext**:扩展包同步 2.1.2 / Ext pack 2.1.2
- **v2.1.5**:Agent 可靠工作台——ResourceView / ToolSpec / CheckpointV2 / EffectLedger · EKC ir.json 暖机 · Web 短读槽隔离
- **v2.1.5**: Agent reliability workbench — ResourceView / ToolSpec / CheckpointV2 / EffectLedger, EKC ir.json warm-load, web short/read lane isolation
- **v2.1.5 ext**:扩展包同步 2.1.5 / Ext pack 2.1.5
- **v2.1.4**:默认隔离 HTTP execute(ADR 0009)——worker 池 / `--call-lib-path` · `/_runtime/*` · MR-01 双流金样例 · `runtime.isolated_execution=1`
- **v2.1.4**: Default isolated HTTP execute (ADR 0009) — worker pool / `--call-lib-path`, `/_runtime/*`, MR-01 dual-stream gold, `runtime.isolated_execution=1`
- **v2.1.3**:知识工作台运行时——ModelSelection/RunContext/usage · view document 只读嵌入 · LLM selection/messages/cancel · agent selection e2e
- **v2.1.3**: Workbench runtime — ModelSelection/RunContext/usage, view document read-only embed, LLM selection/messages/cancel, agent selection e2e
- **v2.1.1**:可执行知识主链——BoundCall · NarrativeRef · 来源/适用期 · patch/extract · 整代 CURRENT + closure · 发票示例 113
- **v2.1.1**: Executable-knowledge spine — BoundCall, NarrativeRef, sources, patch/extract, generation CURRENT + closure, invoice example 113
- **v1.5.2**:外联 v3 P0/P1——raw stdin + 每 cell 输入 · plan API · snapshot · `view capture` · Persist
+94
View File
@@ -1,5 +1,99 @@
# Changelog
## Unreleased
### Added
### Fixed
### Changed
## v2.1.5 — 2026-10-27
### Highlights
**Agent 可靠工作台 + EKC 热路径**:ResourceView(JSON/CSV/PDF/image `blob_ref`、locator 页/行)· ToolSpec 授权先于调用 · CheckpointV2 / EffectLedger / ContextPackV2 · EKC `ir.json` 暖机与 `--refresh` · Web 短读/控制面与 execute 槽隔离。CLI · wasm · ext **2.1.5**。
```bash
git checkout v2.1.5 && cargo build --release
marqdo ext add web # CDN / Releases → 2.1.5
```
**核心表面 Δ(Core surface Δ)**:**无增删**(仍 19 个构造)——工作台协议与 Agent 运行时合同,不新增 Markdown 标记。
### Added
- **ResourceView v1**:有界 JSON/CSV 渲染;PDF/image `blob_ref`;`locator.page` / 行定位;CSV `text` 供 TablePreview。
- **Agent 合同**:ToolSpec 注册与 authorize-before-call;CheckpointV2 原子 resume;AgentErrorV1;输出通道;ContextPackV2;EffectLedger(含 mutating `run_tool` 水印)。
- **Workbench fixtures/schemas**:`tests/workbench_contracts` 协议金样例。
- **EKC `ir.json` 快照**:`marqdo knowledge` 写入 `.marqdo/ekc/graph/ir.json`;查询命令暖机热加载,免全量重编译。
- **`--refresh`**:`knowledge preflight` / `find` / `reuse` / `impact` / `conflicts` / `stale` / `duplicate` / `verify` 可强制重编译。
### Fixed
- **工程预检热路径**:`knowledge preflight`(及 find/reuse 等)默认不再每次 `ensure_compiled` 全量 compile,修复聊天「工程预检」墙钟过长。
- **Web 槽隔离**:`short`/`read`/`control` 不再填满 execute 槽;disconnect≠cancel。
- **CSV ResourceView**:去掉尾部空行伪影。
### Changed
- 查询类 knowledge CLI:**缺投影才 compile**;CI / 源变更请先 `marqdo knowledge` 或 `--refresh`。
- CLI / wasm / `ext/VERSION` **同步 2.1.5**。
## v2.1.4 — 2026-10-26
### Highlights
**默认隔离 HTTP execute(ADR 0009)**:`class=execute` / `sse-live` / `stream:true` 经 Supervisor worker 池(`marqdo run --call-lib-path`),父进程不再 `CallLibPath` 占 `INVOKE_LOCK`;短读 / 未声明非流式仍 `legacy_serial`(ADR 0008)。`/_runtime/*` 控制面;`runtime.isolated_execution=1`;MR-01 双 60s 假 SSE + 短读金样例。CLI · wasm · ext **2.1.4**。
```bash
git checkout v2.1.4 && cargo build --release
marqdo ext add web # CDN / Releases → 2.1.4
```
**核心表面 Δ(Core surface Δ)**:**无增删**(仍 19 个构造)——隔离运行时与 Web execute 路径,不新增 Markdown 标记。
### Added
- **默认隔离 execute(ADR 0009)**:HTTP `execute` / `sse-live` / `stream:true` 经 Supervisor worker 池(`marqdo run --call-lib-path`),父进程不再 `CallLibPath` 占 `INVOKE_LOCK`;可选 route `class`;`/_runtime/*` 控制面;`runtime.isolated_execution=1`。
- **MR-01 金样例**:`tests/isolated_execute_mr01.rs` + `tests/ext/isolated-execute-site/`(双 60s 假 SSE + 短读并行;槽满拒绝);接入 `executable-knowledge` workflow。
- **Worker CLI**:`marqdo run --call-lib-path` / `--fs-root` / `--args-json` / `--emit-result`;host `runtime` admit/dispatch/cancel/events。
### Fixed
### Changed
- **compile 1000 门禁**:executable-knowledge / workbench 路径恢复与能力探测诚实(isolated 默认开启)。
- CLI / wasm / `ext/VERSION` **同步 2.1.4**。
## v2.1.3 — 2026-10-25
### Highlights
**知识工作台运行时合同(M00–M06)**:`ModelSelectionV1` / `RunContextV1` / `ModelCallUsageV1` / `DocumentViewV1` / locator / snapshot;严格 selection;usage null≠0;`view document` 只读嵌入;LLM messages + cancel;agent.step selection 继承假模型 e2e;USTC 兼容 live smoke(env 注入)。CLI · wasm · ext **2.1.3**。设计证据仍为 Partial(原生适配器 unavailable;完整宿主 F12 留给应用)。
```bash
git checkout v2.1.3 && cargo build --release
marqdo ext add llm # CDN / Releases → 2.1.3
```
**核心表面 Δ(Core surface Δ)**:**无增删**(仍 19 个构造)——工作台合同与 View/LLM/Agent 扩展面,不新增 Markdown 标记。
### Added
- **Workbench M00–M06**:capabilities / selection / RunContext / usage / DocumentView / locator / SnapshotManifest;`tests/workbench_contracts`(含 F02 agent.step+resume、F14 并发);`marqdo view document --out` 只读 HTML 嵌入。
- **LLM**:`messages=` / `selection=` / `strict_selection`;cancel token;reasoning_content 回退;双端点隔离门禁。
- **Live**:`tests/workbench_live`(`OPENAI_*` env only;`MARQDO_EXT` 指向仓内 `ext/`)。
- **Agent skeleton**:子工作簿禁 `llm.llm`,从 env selection `llm.create`。
### Fixed
- **`llm.ask` 升参**:hoist `p` 避免缺参 `missing argument for parameter p`;避免 `~/.marqdo/ext` 陈旧包遮蔽仓内 `ext/`(测试强制 `MARQDO_EXT`)。
### Changed
- CLI / wasm / `ext/VERSION` **同步 2.1.3**。
## v2.0.0 — 2026-10-22
### Highlights
Generated
+17 -16
View File
@@ -1,6 +1,6 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
version = 3
[[package]]
name = "adler2"
@@ -264,9 +264,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
[[package]]
name = "crypto-common"
@@ -539,9 +539,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "js-sys"
version = "0.3.104"
version = "0.3.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a"
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
dependencies = [
"cfg-if",
"futures-util",
@@ -572,7 +572,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "marqdo"
version = "2.1.2"
version = "2.1.5"
dependencies = [
"anyhow",
"assert_cmd",
@@ -589,6 +589,7 @@ dependencies = [
"predicates",
"pulldown-cmark",
"regex",
"serde",
"serde_json",
"sha1",
"sha2",
@@ -610,7 +611,7 @@ dependencies = [
[[package]]
name = "marqdo-wasm"
version = "2.1.2"
version = "2.1.5"
dependencies = [
"marqdo",
"serde_json",
@@ -1290,9 +1291,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasm-bindgen"
version = "0.2.127"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70"
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
dependencies = [
"cfg-if",
"once_cell",
@@ -1303,9 +1304,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.127"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1"
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -1313,22 +1314,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.127"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284"
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn 2.0.119",
"syn 3.0.3",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.127"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
dependencies = [
"unicode-ident",
]
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "marqdo"
version = "2.1.2"
version = "2.1.5"
edition = "2021"
license = "Apache-2.0"
description = "Marqdo — Markdown markers as an interpreted programming language"
@@ -29,6 +29,7 @@ anyhow = "1"
chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
clap = { version = "=4.4.18", features = ["derive"], optional = true }
pulldown-cmark = "0.12"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
thiserror = "1"
tiny_http = { version = "0.12", optional = true }
+9 -5
View File
@@ -281,9 +281,13 @@ marqdo catalog public -o .marqdo
---
## 现状(v2.1.2)
## 现状(v2.1.5)
- **可执行知识生产链(v2.1.2)**:证据/闭包失效 · `verify`/`adjudicate` · `reuse_execute` / `eng_bind_execute`+`publish_gate` · `document.extract` · View `KnowledgePresentation`+SSE `page_token` · 上传限流 · 门禁 45 项(100/1000 单元)· 扩展包同步 **2.1.2**
- **Agent 可靠工作台(v2.1.5)**:ResourceView(JSON/CSV/PDF/image)· ToolSpec 授权先于调用 · CheckpointV2 / EffectLedger / ContextPackV2 · EKC `ir.json` 暖机与 `--refresh` · Web 短读/控制与 execute 槽隔离;扩展包同步 **2.1.5**
- **默认隔离 HTTP execute(v2.1.4 · ADR 0009)**:`execute` / `sse-live` / `stream:true` → Supervisor worker 池(`--call-lib-path`);父进程不再占 `INVOKE_LOCK`;`/_runtime/*`;`runtime.isolated_execution=1`;MR-01 金样例
- **知识工作台运行时(v2.1.3)**:ModelSelection / RunContext / usage null≠0 · `view document` 只读嵌入 · LLM selection/messages/cancel · agent.step selection e2e · live smoke(设计 Partial:原生适配器 unavailable)
- **可执行知识生产链(v2.1.2)**:证据/闭包失效 · `verify`/`adjudicate` · `reuse_execute` / `eng_bind_execute`+`publish_gate` · `document.extract` · View `KnowledgePresentation`+SSE `page_token` · 上传限流 · 门禁 45 项(100/1000 单元)
- **可执行知识主链(v2.1.1)**:BoundCall 双后端 · NarrativeRef · 来源/适用期 · `artifact.patch` / `extract_pure` · 整代 `CURRENT` + closure · 发票示例 `113`
- **链接语法(v2.1.0)**:`[描述](目标)` 描述不执行 · `()` 与 `**代码**` 同权(`加(1)` / `加一 号(31)`)· `access` 取元与 `遍历` 链接循环 · `marqdo labels` 两级检索 / `--baseline` 债务棘轮 · `migrate labels --mark` · 诊断 `label.*` · view 描述正文 + 代码片 + 桩徽章 · **可执行知识 [link-syntax.md](doc/design/link-syntax.md)**(项目索引、LinkInfo、链接图与 lock)
- **外联 v3 P0/P1(v1.5.2)**:raw stdin + 每 cell 独立输入 · plan API · snapshot export/import/replay · `view capture` · Persist 按钮 · 扩展包同步 **1.5.2**
@@ -321,7 +325,7 @@ marqdo catalog public -o .marqdo
- **浏览器 Marqdo(WASM)**:`marqdo wasm build` + 官方 bridge 自启(作者零业务 JS;桥内可含列表/路由/storage/ws/文件/Canvas/音频/Observer/拖放);`lib/browser` + GFM;`web.client_embed`;示例 [browser-hello](examples/browser-hello/) · [browser-app](examples/browser-app/) · [browser-media](examples/browser-media/) · [web-client-site](examples/web-client-site/)([ADR 0002](doc/adr/0002-browser-marqdo-wasm.md) · [D](doc/roadmap/browser-wasm-d.md) · [E](doc/roadmap/browser-wasm-e.md) · [F](doc/roadmap/browser-wasm-f.md))
- 选型:[ADR 0001 — Rust](doc/adr/0001-implementation-language.md) · [ADR 0002 — 浏览器 WASM](doc/adr/0002-browser-marqdo-wasm.md)(C0–C5 完结,见 [roadmap/browser-wasm.md](doc/roadmap/browser-wasm.md))· [ADR 0003 — 异步效应](doc/adr/0003-browser-async-effects.md)
### 如何使用最新 Marqdo(v2.1.2)
### 如何使用最新 Marqdo(v2.1.5)
**安装解释器(任选其一)**
@@ -336,7 +340,7 @@ marqdo catalog public -o .marqdo
```bash
# 1) 源码安装(跟 tag 或 main)
git clone https://github.com/cflmy/marqdo.git && cd marqdo
git checkout v2.1.2 # 或留在 main
git checkout v2.1.5 # 或留在 main
cargo build --release
export PATH="$PWD/target/release:$PATH"
marqdo version
@@ -375,7 +379,7 @@ marqdo wasm build
# → dist/wasm/ … 见 examples/browser-hello/
# 6) Releases:Windows exe/zip/vsix;Linux CLI zip + native `.so` zip
# https://github.com/cflmy/marqdo/releases/tag/v2.1.2
# https://github.com/cflmy/marqdo/releases/tag/v2.1.5
```
开发期也可用 `cargo run -- …` 代替已安装的 `marqdo`:
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "marqdo-wasm"
version = "2.1.2"
version = "2.1.5"
edition = "2021"
license = "Apache-2.0"
description = "Marqdo interpreter as WebAssembly (browser host bridge)"
+23
View File
@@ -1,3 +1,26 @@
marqdo (2.1.5-1ppa1) noble; urgency=medium
* Marqdo 2.1.5: agent reliability workbench — ResourceView / ToolSpec /
CheckpointV2 / EffectLedger; EKC ir.json warm-load; web short/read lanes.
-- Marqdo Maintainers <noreply@marqdo.com> Tue, 27 Oct 2026 07:39:08 +0000
marqdo (2.1.4-1ppa1) noble; urgency=medium
* Marqdo 2.1.4: default isolated HTTP execute (ADR 0009) — Supervisor
worker pool for execute/sse-live/stream, /_runtime/* control plane,
MR-01 dual 60s SSE gold sample, runtime.isolated_execution=1.
-- cflmy <pingan@cflmy.cn> Mon, 26 Oct 2026 10:00:00 +0000
marqdo (2.1.3-1ppa1) noble; urgency=medium
* Marqdo 2.1.3: workbench M00–M06 runtime contracts (ModelSelection,
RunContext, DocumentView, usage null≠0, LLM selection/cancel,
agent.step selection e2e, view document CLI, live smoke).
-- cflmy <pingan@cflmy.cn> Sun, 25 Oct 2026 15:20:00 +0000
marqdo (2.1.2-1ppa1) noble; urgency=medium
* Marqdo 2.1.2: executable-knowledge F0–F3 + View V0–V3 release bar
+7
View File
@@ -19,6 +19,13 @@
| 文档 | 摘要 |
|------|------|
| [roadmap/agent-reliability-crossrepo-2026-10-27.md](roadmap/agent-reliability-crossrepo-2026-10-27.md) | **Proposed:跨仓 Agent 可靠性开发计划**;M-A0–M-A6 ∥ Q-A0–Q-A7 排期、依赖图、F01–F20 映射、MA-T*/QA-* 验收与第一刀切片;协议真源仍见下方设计文档 |
| [design/agent-reliability-tools-resources-2026-10-27.md](design/agent-reliability-tools-resources-2026-10-27.md) | **Proposed:智能体能力补足**;基于 `16f26d7`,增量设计错误恢复、检查点/效应账本、结构化工具、输出通道、上下文与只读资源视图;含跨仓边界、阶段计划和故障验收 |
| [research/agent-capabilities-2026-10-27.md](research/agent-capabilities-2026-10-27.md) | **官方能力调研与源码复核**;OpenAI/Claude/Gemini/LangGraph/Deep Agents/MCP,求道当前问题 F01–F20;区分现有能力、设计推导与待实测项 |
| [design/isolated-web-runtime-development-2026-10-26.md](design/isolated-web-runtime-development-2026-10-26.md) | **隔离执行与响应式 Web 运行时**:进程池、控制/读取分离、取消、事件恢复、发布一致性;实现见 `src/runtime/` |
| [adr/0009-isolated-web-runtime.md](adr/0009-isolated-web-runtime.md) | **ADR 0009**:隔离 Web 运行时决策;金样例通过后替代 ADR 0008 的并行性结论 |
| [跨库缺陷复核(2026-10-26)](../../qdagent/doc/gaps/04-workbench-runtime-and-product-audit-2026-10-26.md) | **最新源码与测试证据**:实际 HTTP 路由、知识发布/阅读断点、旧缺陷处置及 1000 单元编译性能失败;需同级 qdagent checkout |
| [design/qdagent-workbench-runtime-and-view-2026-10-25.md](design/qdagent-workbench-runtime-and-view-2026-10-25.md) | **qdagent 知识工作台框架支持**:M00–M05 合同落地中(`src/workbench/`、MLSP `capabilities` / `view.document` / `view.reference_resolve`、`tests/workbench_contracts.rs`);完整 verified 以切片验收证据为准 |
| [design/view-knowledge-reading-experience-2026-10-24.md](design/view-knowledge-reading-experience-2026-10-24.md) | **View 知识阅读体验重设计(建议)**:正文优先、结构导航、就地知识说明、来源与运行状态、响应式线框、实现契约与验收;含当前呈现和 no-exec 反例 |
| [roadmap/qdagent-knowledge-support-gaps-2026-10-24.md](roadmap/qdagent-knowledge-support-gaps-2026-10-24.md) | **求道知识沉淀与文档编译支撑缺口**:2.1.1 源码复核、来源/依赖失效反例、验证与 Agent 协议、语法提示词、导入底座及跨库分工 |
| [design/document-native-development-guide.md](design/document-native-development-guide.md) | **下一阶段开发指导**:语法审查裁决、正文→命名→复用工作流、AI 上下文与局部编辑、渐进严格、作者成本及 S0–S6 验收 |
+68
View File
@@ -0,0 +1,68 @@
# ADR 0008:Web 热路径单解释器串行化(设计缺陷与缓解)
| | |
|---|---|
| Status | **Accepted**(缺陷已记录;缓解已落地;真并行见 [ADR 0009](0009-isolated-web-runtime.md)) |
| Date | 2026-10-26 |
| Related | [web-invoke-serialization.md](../design/web-invoke-serialization.md) · [ADR 0009](0009-isolated-web-runtime.md) · ADR [0004](0004-web-plugin-go.md) · ADR [0007](0007-web-document-endpoint.md) · [ext-abi.md](../design/ext-abi.md) · [ext-web-go-rewrite.md](../design/ext-web-go-rewrite.md) |
## Context
- Go `plugins/web` 的 HTTP 服务是**多 goroutine** 的:每个请求可并发进入 `handleInvoke` / `handleInvokeLiveSSE`。
- 每个 Endpoint 最终经 C ABI `host_query("call_lib_path", …)` 回到 Rust 宿主,在**同一个** `HostContext`(Marqdo 解释器状态)上执行作者 `.mq.md`。
- `HostContext` **不是** `Sync`:共享可变会话、库装载、写回、事件总线等状态不能安全地被多线程同时求值。
- 因此宿主用进程级 `INVOKE_LOCK`(`src/host/plugin.rs`)串行化所有来自 worker 线程的 `call_lib_path`。
### 常见误解
> 「web 都是单线程的吗?不能多线程吗?」
| 层 | 并发模型 |
|----|----------|
| Go HTTP / SSE flush / 取消 | **多线程(多 goroutine)** |
| Marqdo 解释器 `call_lib_path` | **单飞(一把全局锁)** |
| 浏览器 UI | 与后端锁无关;但若短 API 同步阻塞等待锁,会表现为「整页卡死」 |
**缺陷不在「HTTP 单线程」,而在「多 HTTP 工作线程共享一个非 Sync 解释器」。**
真并行需要多解释器实例(或可共享的隔离运行时),不是去掉 Go 侧并发。
### 事故形态(已观测)
长对话(`return=sse-live` / `stream:true`)在 `CallLibPath` 期间一直持有 `INVOKE_LOCK`。并发的短路由(如 `/v1/models`、设置页探测)在旧实现里对锁 **无限期** `lock()`,HTTP 挂起 → 前端 `fetch` 无超时 → 页面无响应。重启进程可恢复,但属于运维掩盖,不是正确错误处理。
## Decision
1. **承认并文档化**本限制为宿主架构缺陷(单 `HostContext` + 全局 `INVOKE_LOCK`),不得再被表述为「web 插件只能单线程」。
2. **缓解(已实现,非根治)**:
- `call_lib_path` 从 worker 进入时用 **限时 try-lock**(默认 `MARQDO_INVOKE_WAIT_MS=1500`),超时返回错误串 `interpreter busy: …`。
- Go `handleInvoke` 将含 `interpreter busy` 的错误映射为 **HTTP 503**,并设 `Retry-After: 2`。
- **禁止**在 SSE 期间把进程级 `MARQDO_INVOKE_WAIT_MS` 拉长:那会让并发短请求重新卡死。
- 长 SSE 在启动时用同一短预算争锁;一旦拿到则持有至调用结束(对话本来就要跑完)。
3. **客户端契约**:短 API 必须有超时与 503 文案;不得假设「任意时刻都能同步进入解释器」。
4. **根治(未决,另案)**:解释器池 / per-request isolate / 可 `Send` 的会话切片。未落地前,产品行为以本 ADR 的 503 语义为准。
## Consequences
### Positive
- 对话进行中,设置/模型列表等短请求约 1.5s 内失败并可读,UI 不再无限挂起。
- 缺陷有名有姓,后续多解释器工作不会被「修好了」的假象挡住。
### Negative / Risks
- 同一进程仍无法真正并行跑两个 Marqdo Endpoint 体;第二个长 SSE 也会在短预算后失败。
- 极短窗口内,短请求若正好撞上锁交接,可能偶发 503(可重试)。
- `MARQDO_INVOKE_WAIT_MS` 是进程环境变量;误设过大仍会复现卡顿。
### Neutral
- Go 多 goroutine、SSE 桥(`MARQDO_EVENT_FORWARD`)、取消令牌(`MARQDO_CANCEL_TOKEN`)保持不变。
- 不改变 Document / Endpoint 作者面(ADR 0007);只约束宿主与 web 插件错误语义。
## Implementation pointers
| 位置 | 职责 |
|------|------|
| `src/host/plugin.rs` → `acquire_invoke_lock` | 限时争锁;失败 `set_err("interpreter busy…")` |
| `plugins/web/.../routes_invoke.go` → `handleInvoke` | `interpreter busy` → 503 + `Retry-After` |
| 设计长文 | [web-invoke-serialization.md](../design/web-invoke-serialization.md) |
+33
View File
@@ -0,0 +1,33 @@
# ADR 0009: Isolated Web Runtime (worker process pool)
| 项目 | 约定 |
|---|---|
| 状态 | **Accepted**;HTTP **execute** 路径正式替代 [ADR 0008](0008-web-invoke-serialization.md) 的并行性结论。`legacy_serial`(未声明且非流式)仍引用 0008 单飞 / 父进程 `CallLibPath`。 |
| 日期 | 2026-10-26(修订 2026-10-27:disconnect≠cancel;control/read/short 无父锁) |
| 设计 | [isolated-web-runtime-development-2026-10-26.md](../design/isolated-web-runtime-development-2026-10-26.md) |
| 金样例 | `tests/isolated_execute_mr01.rs`(MR-01 双 60s SSE + short reads;同文件 MR-02 槽满拒绝;disconnect-keeps-running;control-under-dual-execute) |
## 背景
ADR 0008 确认:Go HTTP 多线程,但所有 Endpoint 共享一把解释器锁与进程级 SSE 桥环境变量。1500ms busy → 503 只是缓解,不是并行。
## 决策
1. **首选架构**:预热的一次性 Marqdo worker 进程池;每个租约独占一个进程与 HostContext。
2. **HTTP 默认**:`class=execute` 或 `return=sse-live` / body `stream:true` → `admit` → `dispatch`(`--call-lib-path` worker);**禁止**父进程 `CallLibPath` / `Setenv` / `legacySSEBridgeMu`。声明的 `short`/`read`/`control` 走父进程短路径(execute 已离进程,父锁通常空闲);未声明非流式 → `legacy_serial`。
3. **控制/读取分离**:`/_runtime/*`(cancel、events、task get)不进入解释器锁。短读勿挤占 execute worker 槽(池默认 2);产品 cancel 优先 `/_runtime`。
4. **订阅 ≠ 取消**:`sse-live` 观察者 `ctx.Done()` 默认**只结束订阅**,不 `runtime.cancel`;后台仍 reap worker 以释放槽。仅显式 header `X-Marqdo-Cancel-On-Disconnect: 1` 才 disconnect=cancel。同步 `waitIsolatedJSON` 可保留 disconnect 取消。重连:`GET /_runtime/tasks/{id}/events` + `Last-Event-ID` / `after_seq`。
5. **协议**:`InvokeRequestV2` / worker env + NDJSON `MARQDO_EVENT_FORWARD` + `--emit-result`(首期);`/_runtime/*` 控制面。
6. **能力探测**:`runtime.isolated_execution=1`、`runtime.task_control_v2`、`runtime.event_replay`、`view.analysis_cache_v2`、`runtime.read_lane=1`(短读不与 execute 抢父锁)。
7. **发布**:`expected_generation` CAS + publish journal;逻辑文档 ID 与物理 generation 路径解耦。
## 后果
- Execute / live SSE 不再抢进程级 SSE 桥;双流可并行(MR-01)。
- 短读走父进程:在 dual execute 下通常仍 200(父锁空闲);勿把 short/read/control 塞进 2 槽 execute 池。
- Legacy 未声明非流式路径仍可走父进程锁(ADR 0008)。
- 产品(qdagent)`turn_events` / Last-Event-ID / cancel CAS 接线由 qdagent 验收;框架侧 disconnect≠cancel 与 `/_runtime` 无锁控制面已落地。
## 验收(摘要)
MR-01 绿:双假上游 60s 流不混字、短读全 200、两 worker `--call-lib-path`、第三路槽满 429/503、`runtime.isolated_execution=1`、`runtime.read_lane=1`。观察者断连后任务仍跑完并可经 `/_runtime` 重放。其余见设计文档 §12。
@@ -0,0 +1,348 @@
# Marqdo 智能体能力补足:恢复、工具、上下文与资源视图
| 项目 | 约定 |
|---|---|
| 状态 | **Proposed**;本次交付设计,不代表以下新接口已实现 |
| 日期 / 基线 | 2026-10-27(Asia/Shanghai);Marqdo `16f26d7` / 2.1.4;qdagent `68b757e` |
| 调研与现状 | [官方资料与源码证据 F01–F20](../research/agent-capabilities-2026-10-27.md) |
| 产品配套 | [qdagent 智能体能力与预览设计](../../../qdagent/doc/design/agent-capabilities-and-preview-2026-10-27.md)(同级 checkout) |
| 继承 | [ext/agent](ext-agent.md)、[隔离 Web 运行时](isolated-web-runtime-development-2026-10-26.md)、[工作台合同](qdagent-workbench-runtime-and-view-2026-10-25.md) |
## 1. 目标与架构边界
让同一套文档原生 Agent 在模型错误、工具失败、网络中断后有界恢复;让工具可发现、可验证、可审计;让结果同时服务模型上下文与用户阅读。保留 `.mq.md` 工作簿、EKC 与验证证据的权威地位,不把框架改成第二套产品聊天服务。
现有 `src/runtime/` 已实现隔离进程、任务/事件与控制通道,本设计在其上补 Agent 步骤语义。现有 `ext/ai/agent.mq.md` 已有 step/plan/run、CALL/READ、观察压缩,继续扩展。框架不持有求道“笔记”业务表、中文按钮、默认回答字数或具体供应商账户。
| 议题 | Marqdo 所有 | qdagent 所有 |
|---|---|---|
| 恢复 | 错误分类、预算、检查点、工具效应、运行/事件语义 | 恢复按钮、会话关联、凭证修复入口、产品政策 |
| 工具 | ToolSpec/Call/Result、schema、权限执行、并发、MCP 传输 | 业务工具、默认集合、连接配置、用户授权范围 |
| 回答 | 结果分通道、输出预算、角色消息、finish reason | 简洁/标准/详细策略、摘要与产物卡、ASK/BUILD 行为 |
| 资源 | ResourceRef、只读内容/派生预览、定位、转换插件接口 | 笔记/文件/产物 ID 映射、owner 权限、预览组件和布局 |
| 知识 | preflight/reuse/verify/compile、记忆原语 | 何时提议沉淀、用户画像与可见管理 |
| 调试 | span、结构化事件、用量与错误码 | 可读过程、运行详情、产品评测语料 |
```mermaid
flowchart LR
APP[qdagent 产品策略与授权] --> RUN[ext/ai/agent step / plan / run]
RUN --> CTX[ContextPack 与预算]
RUN --> LLM[LLM Adapter]
RUN --> DISPATCH[Tool Dispatcher]
DISPATCH --> LOCAL[受控 Marqdo 函数]
DISPATCH --> MCP[MCP Client]
DISPATCH --> RES[资源读取 / 转换]
RUN --> REC[Checkpoint / Effect Ledger]
REC --> RT[既有 runtime task store / event log]
RUN --> OUTPUT[answer / artifact / progress / diagnostic]
RES --> VIEW[DocumentView / ResourceView]
OUTPUT --> APP
VIEW --> APP
```
## 2. 现状与增量
| 已有能力 | 本轮不能误判为缺失 | 要补的具体语义 |
|---|---|---|
| runtime.task_control_v2/event_replay/read_lane | 不再新建一套队列与 SSE 框架 | Agent 步骤恢复与传输重放的边界 |
| resume_save/load | 不再新建平行 checkpoint 命名空间 | 原子提交、版本摘要、作用域、效应关联 |
| CALL、ARGS、READ、plan patch | 不废弃文档式编排 | 类型化 schema、校验错误、有限修复、稳定 call_id |
| MCP stdio client | 不从零重写连接器 | timeout/cancel、resource、远程 transport 的能力探测 |
| DocumentViewV1 / Locator | 不另写 `.mq.md` 解析器 | 普通资源类型、明确 renderer、历史与相对资源处理 |
| LLM messages/usage | 不把 provider 行为交给产品猜测 | 输出预算、原生工具适配探测、角色边界与分通道 |
新模块路径及协议名以下均是**建议落点**。首批开发先固化 schema 与 fixture,再开放 feature bit;不得只更新 capabilities 把未实现能力标真。
## 3. 错误和恢复合同
### 3.1 ErrorV1
现有字符串错误保留兼容适配,新工具与 Agent 结果返回:
```json
{
"schema": "AgentErrorV1",
"code": "tool.timeout",
"category": "transient",
"stage": "tool",
"message": "读取资源超时",
"retryable": true,
"retry_after_ms": 1200,
"effect_state": "none",
"recovery_action": "retry_step",
"call_id": "call_01",
"details_ref": "resource:diagnostic_01"
}
```
`retryable` 仅是候选信息,最终动作由可信策略结合效应、截止时间、预算、取消状态决定。远端工具声称 readOnly/idempotent 不构成授权依据。凭证和原始堆栈不进入公开 message。
| 错误类别 / 例子 | 框架动作 | 不允许的动作 |
|---|---|---|
| transient:429、部分 5xx、连接失败 | 只在安全步骤重试,遵守 Retry-After | 无限重试、复用同一流片段直接追加 |
| invalid_arguments:schema 不匹配 | 把字段级错误作为 observation,最多 1 次参数修复 | 启动工具后才校验;静默丢弃字段 |
| validation_failed:编译/文档校验 | 保留候选与诊断,局部修复后重新验证 | 将未验证草稿标为成功 |
| context_overflow | 压缩可重建上下文,保留任务/限制/未决效应 | 丢失最新用户指令、授权或版本 |
| auth / forbidden | 直接 needs_input/blocked,交应用处理 | 换身份、扩大范围或匿名绕过 |
| conflict / stale_revision | 重新读取版本并重新规划 | 覆盖用户修改、对写操作盲重试 |
| capability_unavailable | 返回可行替代操作清单 | 假称已搜索、已渲染或已执行 |
| policy_denied | 显式拒绝该动作 | 把限制当工具错误让模型规避 |
| budget_exhausted | 保存状态与已完成结果,返回 partial | 自动增预算或伪造 completed |
| unknown_effect / worker_crash | 对账、查询外部状态,不能判断则 needs_reconciliation | 重跑整个工作簿 |
| cancelled | 停止新调用、保留可读产物、传播取消 | 在后台悄悄修复或恢复 |
初始建议:传输最多 2 次重试(总 3 次尝试),指数退避基准 0.5/1 秒并加 jitter;有 Retry-After 时使用服务端建议并受总 deadline 限制。参数修复最多 1 次,内容修复最多 2 次。预算由应用传入,框架只提供默认上限与执行机制。所有重试、修复和摘要模型调用计入同一成本账本。
错误指纹 `hash(tool_id, normalized_arguments, error_code, relevant_revision)` 连续出现 2 次且没有新证据时中止该策略,允许一次有依据的替代路径;不能通过改写错误文本绕过循环检测。
### 3.2 四类“恢复”必须分开
1. **续订事件**:客户端从 cursor 重放和订阅同一个 task,不发起新推理。
2. **重试步骤**:相同逻辑 operation、同幂等键;新 invocation attempt,复用成功的前序结果。
3. **恢复工作流**:从兼容 checkpoint 建立新 runtime task / attempt,关联 `resumed_from`,不把旧终态改回 running。
4. **重新生成答案**:新的输出版本;已有成功工具结果可复用,用户明确重新执行才重新产生效应。
当前 TaskState 把 interrupted/needs_reconciliation 视为终态,沿用此约定。新增 `AgentRun` 逻辑关联只管理目标和 attempt 链;调度权仍在原 runtime。恢复请求带 `expected_revision`、checkpoint digest 与幂等键,相同请求返回同一新 attempt,冲突返回 409 对应的框架错误。
执行失败不必丢掉有用内容:`outcome.status = succeeded | partial | failed | needs_input | cancelled`,伴随 `answer_ref/artifacts/error/recovery_options`。HTTP 调用成功与 Agent 业务 succeeded 不再混为同一字段。
## 4. 检查点与工具效应
### 4.1 CheckpointV2
在 `plugins/agent/src/resume.rs` 扩展已有实现,统一根目录由宿主提供,客户端不能传任意绝对路径。保留 legacy load,但旧 checkpoint 无效应信息时默认 `manual_review`,不能自动重做写操作。
| 字段组 | 内容 |
|---|---|
| 身份 | schema、checkpoint_id、logical_run_id、task_id、attempt_id、step_id、parent_checkpoint_id |
| 作用域 | principal_ref、workspace_id、session_id、授权 scope digest |
| 输入 | task revision、workbook revision/hash、输入资源 revisions、knowledge generation |
| 运行配置 | ModelSnapshot public ref、tool registry digest、policy revision、runtime compatibility |
| 进度 | completed_steps、pending_operation、observation refs、context summary ref |
| 约束 | 剩余 token/tool/time/cost 预算、deadline、cancel epoch |
| 一致性 | effect ledger watermark、event seq、manifest digest、committed_at |
写入协议:写唯一临时文件→flush/fsync→原子 rename→提交 checkpoint 指针;持久化配置下目录 fsync 按平台支持实现。所有引用 blob 在指针提交前落盘。损坏记录报 `checkpoint.corrupt` 并尝试上一个完整版本,不能读取半个 JSON 当无历史。
恢复前校验身份/授权、源版本、工具 schema 与模型兼容性。权限撤销立即生效;checkpoint 不是永久授权。源文件改变返回 `checkpoint.input_changed`,重新计划受影响步骤,不能偷偷使用最新文件继续旧验证结果。
### 4.2 Effect ledger
```text
prepared → started → committed
↘ failed_no_effect
↘ unknown → reconciled_committed | reconciled_no_effect | needs_input
```
每个 operation 持久记录 `operation_id/idempotency_key/args_digest/resource_scope/expected_revision/effect_class/result_ref`。幂等键作用域是 principal + workspace + logical_run + step + operation,不用随机 attempt ID 替换稳定 operation key。对同 key 不同 payload 拒绝。
读工具可重试;确定幂等写由服务端去重;可补偿写仅在补偿明确安全时执行;未知写先查询 receipt/版本。对外部系统不能承诺通用 exactly-once。关键 crash 窗口是“远端已成功、本地尚未记账”,必须返回 unknown 并对账。
本地文件使用 expected revision + staging + atomic replace;变更提交后检查点失败也靠 operation receipt 恢复。`tool.completed` 必须在结果与 receipt 可读取后提交,再对外发事件。事件发送失败不回滚已发生的业务效应。
不要复制 runtime 事件存储:Agent ledger 与 artifact manifest 是步骤事实,现有 runtime event log 是传输/调度事实;以 ID 关联,采用可重建投影与 outbox。跨 DB/文件不能声称一个事务完成,使用 manifest 提交点和恢复扫描补齐索引。
## 5. 结构化工具与调度
### 5.1 ToolSpecV1
```json
{
"schema": "ToolSpecV1",
"id": "workspace.read",
"version": "1",
"description": "读取授权资源的指定行范围",
"input_schema": {
"type": "object",
"properties": {
"resource_id": {"type": "string"},
"revision": {"type": "string"},
"start_line": {"type": "integer", "minimum": 1},
"max_lines": {"type": "integer", "minimum": 1, "maximum": 500}
},
"required": ["resource_id"],
"additionalProperties": false
},
"output_schema_ref": "ResourceReadV1",
"effect": "read",
"scopes": ["workspace:read"],
"timeout_ms": 10000,
"max_result_bytes": 65536,
"idempotency": "safe",
"concurrency_group": "resource-read",
"availability": "ready"
}
```
定义由宿主/可信模块注册;用户认证、scope 与资源根由宿主注入,不接受模型提供 principal 或任意 root。`read`、`write`、`execute`、`external_write` 是策略分类,不是靠工具命名推断。
ToolCallV1:`call_id/tool_id/tool_version/arguments/operation_id/parent_span_id`。ToolResultV1:`ok/status/summary/structured_content/resources/citations/error/usage/truncated/next_cursor/effect_receipt_ref`。超限结果先写 artifact,再返回摘要和范围读取句柄;不把几十 MB 文本送进 prompt。
### 5.2 一条执行通道
```text
发现 → 加载 schema → schema 校验 → 授权与范围解析 → 预算/取消检查
→ 幂等查询 → 执行 → 结果 schema/大小校验 → receipt/结果提交 → 事件 → observation
```
provider 原生 function call 与文档 `CALL:name + ARGS:{...}` 都归一到该通道。保留旧白名单适配,已有 `.mq.md ##` 可根据形参推导初始 schema,再允许显式描述收紧。转换不允许调用未暴露的框架内部函数。禁止对模型参数使用 eval/任意代码反序列化。
`tools.search` 返回已授权且当前可用的工具摘要;`tools.describe` 按 ID 返回完整 schema。核心 5–8 个工具可常驻,其余按需加载;此数量是产品起始配置,不写死框架。每次真正调用仍重新检查授权;搜索索引可列 unavailable 原因供诊断,但不能让未授权工具名字泄露私有连接信息。
初始支持 JSON Schema 的明确子集并公开能力;不支持的 schema 拒绝注册,不静默放宽。原生工具能力、并行工具调用、图片输入、结构化输出分别探测;纯文本兼容 provider 继续走 CALL,但并发和可靠性承诺按测试能力开放。
### 5.3 并发、MCP 与程序化执行
独立只读调用可并发,默认最多 4 个;修改同一资源的调用串行且用 revision CAS。依赖链、审批与取消检查保持顺序。一个并发调用失败不丢弃其他已成功结果。
MCP 扩展现有 client:先完善 timeout/cancel、错误映射、工具分页与 schema、resource read/link,再按 capability 增加远程 Streamable HTTP 与连接认证。stdio 不应允许模型选择任意宿主可执行文件。连接元数据归应用,凭证通过 secret handle 注入 worker,工具输出和 trace 不记录密钥。
工具 annotations 只是提示,可信策略确定真实效应。外部结果全部为数据,不能提升为 standing/system。MCP resource URI 转成宿主授权 ResourceRef,不允许前端直接请求任意 file:// 或内部网地址。
程序化工具编排和子 Agent 为后续切片:在沙箱中仅暴露注册 dispatcher,逐次执行同样的授权、预算和效应日志;禁止通过一段脚本绕过逐工具控制。子任务继承更小权限和共享总预算,不默认开放写同一资源。
## 6. 输出通道与上下文
### 6.1 AgentOutputV1
```json
{
"schema": "AgentOutputV1",
"status": "succeeded",
"answer": {"format": "markdown", "text": "已整理为三点,完整笔记见右侧。"},
"artifacts": [{"resource_id": "res_note_01", "revision": "sha256:example"}],
"citations": [],
"validation": {"status": "passed", "report_ref": "resource:verify_01"},
"finish_reason": "completed",
"recovery_options": []
}
```
这是结果传输协议,不要求模型每次逐字生成这张 JSON。文档式编排仍可产出 `.mq.md`,框架把稳定产物、摘要与事件归一化。普通文本回答可没有 artifact 和 validation;未运行的内容标 `not_applicable/not_run`,不能填 passed。
channel 至少区分 `answer/artifact/progress/diagnostic`。只有 answer delta 可追加到聊天正文;工作簿 token 和工具日志进入其他通道。进度是可观察动作摘要,不要求或泄露模型内部思维链。
应用传 `ResponsePolicy`:detail level、目标长度、最大输出 tokens、是否允许 artifact、用户显式需求。框架负责 budget 和 finish_reason,中文字符数与最终文案由应用决定。达到 token 上限返回 `length_limited` 和完整/不完整标记,不能裁剪 JSON/代码后假装成功。
优先从已验证产物结构投影摘要。必要时增加一次受预算控制的摘要模型调用;摘要不得新增未经证据支持的结论。修复成功后 accepted candidate、结果正文、artifact revision 与 verification digest 必须全部一致。
### 6.2 ContextPackV2
按优先级组装:可信运行规则→最新用户目标/约束→用户选中资源范围→未决操作/成功步骤摘要→检索证据→近期对话→可回读历史。保持角色结构,不把用户/工具文本简单拼进系统指令。
每个 item 携带 `origin/trust/resource_ref/revision/token_estimate/priority`。预算初值建议保留模型窗口 20% 给输出与协议开销;上下文使用达到可用预算 70% 时触发整理,具体比例配置化。优先剔除可重读的完整工具输出,保留摘要、引用和失败恢复锚点。
Compaction 必须保留:目标、用户限制、当前计划、已完成步骤、待决写入、授权范围、输入版本、已接受产物与引用、余量。summary 绑定其 source event range 和 digest,不能当成新的已验证知识。工具输入过时须重读,不能依赖压缩摘要执行写入。
已有 EKC/agent memory 继续提供知识发现和复用。长期用户记忆的创建/修改/删除政策归 qdagent;网页中的指令不能自动变成长期偏好。
## 7. 统一资源与只读视图
### 7.1 ResourceRefV1 / ResourceViewV1
ResourceRef 统一携带 `resource_id/workspace_id/kind/media_type/revision/digest/title/size/origin/locator`;opaque ID 由宿主映射,不将浏览器 path 直接当文件权限。revision 是内容版本,generation 是知识发布代,两者不混用。
```json
{
"schema": "ResourceViewV1",
"resource": {
"resource_id": "res_01", "workspace_id": "ws_01", "kind": "file",
"revision": "sha256:example", "media_type": "text/markdown", "title": "设计说明"
},
"renderer": {"id": "markdown", "version": "1", "trust": "sanitized"},
"render": {"format": "sanitized_html", "fragment": "<p>正文</p>"},
"outline": [],
"locators": [],
"completeness": "complete",
"no_exec": true,
"diagnostics": [],
"next_cursor": null
}
```
继续把 `.mq.md` 交 `view.document`,ResourceView 只包装/扩展,不能另解释一套 Marqdo 语义。普通 `.md/.txt/代码/JSON/CSV` 走媒体类型处理,避免全仓语义编译。浏览器组件由 qdagent 实现,框架只提供可读内容、受控片段或短期授权 blob 引用。
`fragment`、`text`、`blob_ref` 是互斥载荷,按严格 format enum 解码。未知格式返回 unsupported,不能用 `format.includes('html')` 猜测 HTML。为空与加载失败必须有独立状态。
### 7.2 格式矩阵
| 格式 | 框架提供 | 前端消费 | 未具备能力时 |
|---|---|---|---|
| `.mq.md` | 静态 DocumentView、outline、节点/引用 | 正文、语义链接、源码切换 | 只读源码 + 明确诊断,不执行 |
| Markdown / 笔记正文 | 共享静态 Markdown→受控结构/HTML、相对资源引用 | 标题、列表、表格、代码、链接 | 纯文本 |
| 文本/代码 | 分页行片段、编码信息、行定位 | 行号与语法高亮 | 编码提示/下载 |
| JSON / CSV | 有界解析、路径/行列定位、分页 | 树/表/源码 | 原始文本,标格式错误 |
| 图片 | 授权 blob、尺寸/元信息 | 缩放/旋转/替代文本 | 下载,不暗示已做 OCR |
| PDF | blob、页元信息,可选提取插件 | PDF.js 页渲染与定位 | 文件卡;提取不可用不阻止支持的视觉预览 |
| DOCX/XLSX/PPTX | 受限转换插件,可选 PDF/HTML 派生物 | 按派生格式预览 | 未安装转换器,保留下载 |
| HTML/SVG | 默认源码;允许时经严格清洗或受限派生物 | 隔离预览 | 不以内联可执行页面显示 |
| 未知二进制/压缩包 | 元数据 | 下载卡 | 不自动解压/执行 |
提取、预览、执行各有独立 capability。办公转换、OCR 使用受限 worker,禁宏与网络、限制时间/内存/页数/解压膨胀比;结果绑定原件 digest、转换器版本和参数。首期不承诺保真编辑。
### 7.3 安全、定位、缓存
所有读取先校验宿主授予的 workspace/resource scope,canonicalize 并处理 symlink 逃逸;读取句柄与校验对象一致,防路径检查后替换。读取原件的身份权限不可被缓存绕过。
预览绝不调用 `run/verify` 或有执行副作用的 view 路径;正文中的 Marqdo 调用链接只展示语义与导航,执行需单独工具调用与授权。远程图片、相对链接、资源 URL 经受控解析;下载入口不允许任意内网 fetch。
HTML 清洗在框架产出侧和浏览器插入侧都有明确边界;用成熟 allowlist 实现而非正则。禁止 script/event handler、危险 URI、任意 style/form/嵌入对象。前端配 CSP;对需要交互的产物另建无权限 sandbox,不复用文档渲染器。
缓存 key:workspace + resource ID + revision + renderer version + render options + policy version;命中前重新授权。历史 revision 不存在就返回 `version_unavailable`,不能默默展示当前版。locator 支持 node/span/line/page/cell,失效时标 stale 并提供用户主动查看最新版的操作。
建议首期限制:文本片段 256 KiB、代码单次 500 行、CSV 单次 200 行;更大内容分页,完整文件保留。数值需压测校准,不作为现有系统性能声明。
## 8. 事件、可观测性与能力协商
复用 RuntimeEventV2 的 task/run/attempt/seq/event_id。拟增加事件类型:`agent.step.started/completed`、`tool.started/completed/failed`、`recovery.scheduled/exhausted`、`checkpoint.committed`、`artifact.updated`、`answer.delta/committed`、`context.compacted`。现有 `message.delta` 由兼容适配映射 answer channel,不能同时重复消费两套事件。
终态提交包含 accepted output manifest、last_seq、result digest;断流客户端能通过快照补齐最终答案。stream delta 是预览,最终 committed revision 是权威。统计每次尝试的 usage,摘要和修复调用分别标 purpose;未知计量保持 unknown。
拟新增 capability:`agent.error_v1`、`agent.checkpoint_v2`、`agent.tool_schema_v1`、`agent.output_channels_v1`、`agent.context_pack_v2`、`resource.view_v1`,以及各 media renderer 的独立版本/可用状态。保持现有 probe,新增字段向后兼容;未知 feature 当不可用。
trace 记录因果 ID、耗时、错误类别、参数摘要、资源版本和 effect receipt。正文与密钥按配置脱敏,不默认把用户全文件复制进日志。输出中不包含模型私有推理;可审计的是实际动作与结果。
## 9. 分阶段实现与文件落点
| 包 | 修改/新增建议 | 交付内容 | 依赖 / 退出条件 |
|---|---|---|---|
| M-A0 | `src/workbench/capabilities.rs`、`tests/workbench_contracts.rs`、新增合同 fixtures | 上述协议、旧适配、状态映射 | 先冻结跨仓字段,未知字段/版本可降级 |
| M-A1 | `ext/ai/agent.mq.md`、`plugins/agent/src/resume.rs`、新 recovery/effects 支撑模块 | 分类错误、有界修复、accepted candidate、原子检查点 | M-A0;故障注入不重复写、旧记录不冒险自动恢复 |
| M-A2 | `plugins/agent/src/lib.rs`、`mcp_client.rs`、新 tool registry/dispatcher | schema 工具、调用统一、分页与结果资源 | M-A0;未授权调用为 0,失败结果结构正确 |
| M-A3 | `src/workbench/document_view.rs`、新 resource_view、媒体适配模块 | ResourceRef/View、普通格式静态阅读、no-exec | M-A0;资源定位与安全语料通过 |
| M-A4 | `ext/ai/agent.mq.md`、`plugins/llm/src/lib.rs`、`src/workbench/run_context.rs` | 输出分通道、预算、ContextPackV2 | M-A1/2;原生和纯文本 provider fixture 都可用 |
| M-A5 | `src/runtime/` 的既有存储/事件适配、MCP 与转换插件 | 恢复对账、PDF/Office 插件、远程 MCP | M-A1/2/3;持久模式 crash/restart 通过 |
| M-A6 | plan/subtask/sandbox 扩展 | 程序化工具、有界委派与可选浏览器 | 后置;须证明任务收益和预算/效应可控 |
先交 M-A0 与 qdagent P0 的静态阅读/错误结果修复;不要让完整 checkpoint v2 阻塞笔记预览。M-A1 可先上只读重试,写恢复待 ledger 通过再开放。新特性默认关闭直到对应合同实测通过,旧路径继续可读。
## 10. 验收与评测
所有目标是 proposed gate,尚未实测。基础机制用确定性 fake provider/tool;真实 provider 做少量单独 smoke 并记录费用,不能把随机模型输出作为唯一门禁。
| 编号 | 故障/任务 | 必须观察到的结果 |
|---|---|---|
| MA-T01 | 第一次 429、第二次成功 | 尊重退避、计量 2 次、最终只有一个 accepted answer |
| MA-T02 | 连续 schema 参数错误 | 一次修复后停止;工具未在无效参数下执行 |
| MA-T03 | 工具写成功但 receipt 提交前杀 worker | unknown_effect→查询对账;写计数不增加 |
| MA-T04 | checkpoint 写一半杀进程 | 读取上个完整 checkpoint 或明确 corrupt,不产生伪成功 |
| MA-T05 | 终态事件丢失、cursor 重放 | 最终快照恢复完整消息,工具执行次数不增加 |
| MA-T06 | 恢复请求重复/并发 | 同一幂等键只产生一个恢复 attempt |
| MA-T07 | 用户取消与重试定时器竞争 | 取消生效后无新工具调用,旧任务不复活 |
| MA-T08 | 执行中资源/权限改变 | stale/conflict/forbidden,不覆盖新版本 |
| MA-T09 | 恶意文本伪造 CALL/system | 不跨越工具白名单和授权范围 |
| MA-T10 | `.mq.md` 预览含写文件/联网语句 | 哨兵文件/网络计数不变;只读渲染成功或安全降级 |
| MA-T11 | 资源格式、超长输出、路径逃逸 | 正确 renderer/分页;越界拒绝;无 HTML 注入 |
| MA-T12 | 修复原文 A 得到通过验证的 B | answer/artifact/verification 都绑定 B digest |
| MA-T13 | 上下文压缩后继续 | 最新约束、待决效应、资源版本与剩余预算不丢失 |
| MA-T14 | 旧 CALL、旧 DocumentView 消费端 | 基础行为可用,能力缺失有明确返回 |
先运行受影响的 Rust/Go/Marqdo 合同与既有隔离 runtime 金样例,再做跨仓 HTTP/browser gate。性能目标:已缓存 100 KiB 文本视图服务端 p95 < 300 ms、核心工具发现 p95 < 100 ms,基准机器/并发/冷暖缓存必须记录;第三方网络延迟单独列出。
## 11. 迁移、回滚与未决项
兼容迁移按“新增协议→双读→新写→验证→逐步移除旧适配”推进。`.mq.md` 源码与旧运行产物不批量重写;旧 resume JSON 可读取但不能自动升级成带副作用保证的检查点。v2 checkpoint/ledger 保留 schema 版本,新老版本不并发写同一运行。
回滚关闭新工具自动发现、输出通道或转换器时,已提交资源和最终答案仍可读;执行恢复中的任务先 drain/停止接纳,不能切回会盲重做写操作的旧路径。迁移前备份可变索引与 manifest,回滚不删除用户产物。
实现前验证项:现有 runtime 持久存储可提供的原子提交边界、各平台 fsync/rename 行为、LLM adapter 原生 tool call 流式兼容、转换插件部署体积。协议优先按本文保守语义实现,验证不能满足则 feature 保持不可用,不阻塞已经完成的只读预览切片。
+6 -5
View File
@@ -53,22 +53,23 @@ Marqdo 把仓库投影为 **可查询、可验证、可复用** 的工程知识
落盘(G01 分离后):
- **权威语义图**:`.marqdo/graph/edges.json` · `nodes.json`(`SemanticProject`,含 `authority` 字段)· `steps.json`
- **EKC 检索投影**(非执行依据):`.marqdo/ekc/graph/graph.json` · `edges.json` · `index.json`
- **EKC 检索投影**(非执行依据):`.marqdo/ekc/graph/graph.json` · `edges.json` · `index.json` · **`ir.json`**(完整 IR 快照,供查询热路径)
## 5. CLI
```bash
marqdo knowledge [PATH] -o .marqdo # compile(默认)
marqdo knowledge preflight "task"
marqdo knowledge [PATH] -o .marqdo # compile(默认;写投影 + ir.json)
marqdo knowledge preflight "task" # 暖机读 ir.json;缺图才 compile
marqdo knowledge preflight "task" --refresh # 强制全量重编译后再预检
marqdo knowledge record REUSE
marqdo knowledge learn --task ... --failure ...
marqdo find "authentication"
marqdo find "authentication" # 同上:缺图才 compile;--refresh 强制
marqdo reuse "implement oauth login"
marqdo impact path/to/file.mq.md
marqdo conflicts | stale | duplicate [--fail] | verify
```
统一 `--json`。`duplicate` 默认阈值 `0.75`;`--fail` 作 CI Gate。
统一 `--json`。查询类命令(`preflight` / `find` / `reuse` / `impact` / `conflicts` / `stale` / `duplicate` / `verify`)默认 **读已编译投影**;CI 或源变更后应先 `marqdo knowledge` 或加 `--refresh`。`duplicate` 默认阈值 `0.75`;`--fail` 作 CI Gate。
## 6. Reuse 协议
@@ -0,0 +1,326 @@
# 隔离执行与响应式 Web 运行时:详细开发设计
| 项目 | 约定 |
|---|---|
| 状态 | **Verified**(execute 路径);HTTP `execute` / `sse-live` / `stream:true` 默认走 Supervisor worker 池(不再经父进程 `CallLibPath` / `INVOKE_LOCK`);`runtime.isolated_execution=1`;MR-01 金样例 `tests/isolated_execute_mr01.rs`;未声明非流式仍 `legacy_serial`(ADR 0008) |
| 基线 | Marqdo `1b11fe0` / 2.1.3;qdagent `64b97e5`;2026-10-26 |
| 问题依据 | [单解释器缺陷](web-invoke-serialization.md)、[ADR 0008](../adr/0008-web-invoke-serialization.md)、[本轮跨库复核 R01–R14](../../../qdagent/doc/gaps/04-workbench-runtime-and-product-audit-2026-10-26.md) |
| 产品配套 | [qdagent 可靠工作台开发设计](../../../qdagent/doc/design/reliable-workbench-development-2026-10-26.md) |
| 继承合同 | [工作台 M00–M06](qdagent-workbench-runtime-and-view-2026-10-25.md) 中 ModelSelection、ModelSnapshot、Usage、DocumentView、Locator;保留文档原生作者面 |
## 1. 目标、边界与不变量
目标是让用户在长对话、导入和构建过程中持续打开模型列表、读取来源、修改下一轮设置并可靠停止任务。高并发容量按资源预算显式限制;不能让所有请求进入一把解释器锁再超时。
Marqdo 负责通用执行隔离、资源调度、事件/取消、语义分析和制品校验。qdagent 负责用户/工作区授权、业务状态与发布政策。产品仍通过 `.mq.md` 调用 Marqdo Agent,不增加第二套模型循环或知识编译器。
必须保持的系统不变量:
1. 同一个可变 HostContext 同时最多一个求值者;不删除锁、不增加虚假的 `unsafe impl Sync` 来获得并行。
2. 一个执行租约只属于一个 run/attempt;凭证、事件、取消、cwd、权限和模型选择不可流入另一请求。
3. 控制与读取具有独立容量,不被 LLM 等待占满;停止不需要重新进入正在执行的解释器。
4. 事件可以重投,外部副作用不默认重做;不宣称跨网络 exactly-once。
5. 已确认终态不能被晚到完成/取消覆盖;取消不回滚已经完成的外部写入。
6. 阅读不执行 `.mq.md`,缓存不绕过授权或版本校验;缓存命中必须实际复用结果。
7. 正文、依赖、证据和编译代属于同一不可变快照;不会把两个代的结果拼成一次阅读或执行。
8. 任何能力开关都以实际探测和测试为依据;新 API/schema 在落地前都属于本文提案。
## 2. 架构选择
| 方案 | 优点 | 当前代价 | 本轮选择 |
|---|---|---|---|
| 继续全局锁,只增加 Go handler | 改动小 | 长任务依然阻塞所有 Endpoint,串扰变量也未消失 | 只保留作兼容模式 |
| 同进程多 HostContext | 潜在内存复用较好 | GLOBAL_HOST、插件回调、环境、cwd、事件 sink 生命周期需系统重做 | 后续优化,不是首个交付门槛 |
| 独立 Marqdo worker 进程 + 控制/读取通道 | 隔离现有全局状态,可强制回收卡死任务,复用现有语言与 Agent | IPC、进程启动、内存、发布写协调 | **首选实现** |
| 引入外部通用队列和新业务 Runtime | 容易扩到集群 | 运维复杂度及第二套编排风险 | 单机阶段不引入 |
首期采用**预热的一次性执行进程池**:进程可预装运行时和非用户特定模块,分配任务后独占,结束即退出并异步补充。这样不用假设清空部分字段就能安全复用 HostContext。长寿命多租约 worker 只有通过状态泄漏、插件资源与 RSS 回归后才开启。只读分析服务可以长寿命运行,但不得加载有副作用的求值插件。
```mermaid
flowchart TD
UI[浏览器 / CLI / MCP] --> WEB[Go HTTP / 认证入口 / 限流]
WEB --> CTRL[任务控制与事件服务]
WEB --> SHORT[短业务请求容量]
WEB --> READ[只读分析服务]
SHORT --> POLICY[qdagent .mq.md 授权与业务规则]
POLICY --> CTRL
CTRL --> DB[(任务索引 / outbox / 事件日志)]
CTRL --> SUP[Rust Supervisor / 有界调度]
SUP --> A[Marqdo worker A / 一个租约]
SUP --> B[Marqdo worker B / 一个租约]
A --> AGENT[既有 Marqdo Agent / LLM / sandbox]
B --> AGENT
A --> STAGE[每任务暂存目录]
B --> STAGE
STAGE --> COMMIT[按资源串行提交与 revision CAS]
COMMIT --> SNAP[(不可变知识代 / 发布 manifest)]
READ --> SNAP
A --> CTRL
B --> CTRL
CTRL --> UI
```
“短业务请求容量”依旧可执行产品 `.mq.md`,但有独立 worker 预算和短 deadline。控制服务只做通用接纳、状态 CAS、取消与日志投递,不把模型选型、知识晋升等业务逻辑搬进 Go。读取服务接收经过授权的资源句柄;不得直接相信浏览器提交的绝对路径。
## 3. 路由分类与接纳
### 3.1 执行类别
在 Endpoint 装配中新增可选 metadata(字段名待 schema 固定):
| 类别 | 示例 | 资源预算与规则 |
|---|---|---|
| `control` | 任务查询、事件订阅、取消、health | 不执行用户代码;认证后访问任务存储/租约注册表 |
| `read` | view.document、引用解析、已发布模型列表投影 | 只读快照;不写用户源码;可缓存 |
| `short` | 保存设置、创建导入、接纳任务 | 无 LLM 长等待;默认 deadline 2s,较慢动作转为任务 |
| `execute` | ASK/BUILD/生成/验证 | 有界队列、专属进程、显式任务 deadline |
| `commit` | 切换知识代、合并受控文件变更 | 按 workspace/kb 的短临界区;执行前检查 revision |
旧 Endpoint 未声明时仍走 legacy_serial;不能自动猜测任意函数是纯读。纯度来自静态能力和注册配置,实际写权限仍由宿主限制。文件上传字节传输不占解释器,应先落隔离存储,再进入短接纳路径。
### 3.2 接纳步骤
1. HTTP 层解析受限请求,验证身份、格式、速率与连接数量。
2. 产品短入口解析 workspace/model/policy;授权版本与模型配置 revision 一并冻结。返回规范化任务请求,不执行 Agent。
3. 控制服务在同一数据库事务中登记作用域化 idempotency key、payload hash、task 和 dispatch outbox;重复请求读回已有 task,同键异内容返回 409。
4. 提交事务后才返回 202/Location。调度器消费 outbox 分配租约,租约带 attempt/epoch。
5. 对同步兼容 API,先完成接纳,再挂接结果;等待客户端超时不等于安全重做。
队列入库前容量不足返回 429 + Retry-After;服务不可用返回 503;接纳失败不发送 started。控制通道须保留槽位,不能因为数据流连接占满而无法取消。
## 4. Supervisor、生命周期与调度
### 4.1 Worker 生命周期
`starting → ready → leased → running → draining → exited`;启动失败/心跳丢失走 `unhealthy → terminating → exited`。任务状态由控制服务维护,进程状态不能直接当作业务成功。
- worker 启动握手报告 CLI/ext/native、protocol_version、能力、构建摘要;不匹配不进入 ready。
- 预热阶段不读取用户凭证、不挂载工作区写权限。分配租约后挂载当前任务的 capability roots、只读快照和独立 staging。
- 每任务具有随机 lease_id、run_id、attempt_id、epoch、绝对 deadline 和资源限制。
- Supervisor 持有进程句柄、子进程组、事件通道与取消句柄。worker 不能自行修改其他 worker 状态。
- 返回业务结果后仍须 drain 事件/用量并提交最终 checkpoint,控制服务持久化终态后确认 ACK,之后释放租约。
- 超时或断连清理不能遗留孤儿子任务;正常优雅退出后再删除临时文件。异常由独立 janitor 按租约过期回收。
### 4.2 有界、公平与资源预算
初始验证配置建议 4 vCPU/8 GiB、执行槽 2、短请求槽 1、读取服务 1。具体容量由每类任务的 RSS p95 和 CPU profile 决定,不能把这个数当所有机器的默认。执行槽上限:
`min(CPU约束, floor((可用内存 - 系统/控制/缓存预留) / worker_RSS_p95))`。
运行队列按 workspace 轮转,再按 session FIFO;一个 session 默认只允许一个活动 turn,跨 session 可并行。相同知识库的生成可并行,发布只占短提交锁。interactive 优先于后台索引,但后台使用老化提升或最低服务配额,防永久饥饿。
全局队列初始上限 64、单工作区 16、单会话待处理 4,均可配置;队列超限及时返回,不能悄悄丢任务。显示队列位置或“前方有任务”,没有可靠完成时间估计时不显示虚构倒计时。
### 4.3 副作用与重试
任务声明 `read_only / staged_write / external_effect`,由工具权限校验而非模型自报。read_only 可按政策重试;staged_write 丢弃未提交 staging 后重试;external_effect 出现超时/失联时标记 `effect_unknown`,等待核对,不自动重复支付、发信或写入。
稳定的操作键使用 `task_id + logical_step_id`,attempt_id 只标记传输尝试,不应导致同一外部动作生成新的去重键。工具不支持幂等查询时,只能明确报告不确定。工作簿记录已完成与未知副作用;“已停止”不意味着“全部撤销”。
## 5. IPC 与请求上下文
首期使用本机 Unix socket 或匿名管道,采用长度前缀 JSON 控制帧与独立事件通道。协议禁止无上限 JSON/单行日志;控制帧初始上限 1 MiB,大文档通过授权 blob handle 传递。普通 stdout/stderr 仅诊断,不与协议混流。当前 NDJSON 文件桥保留为 legacy fallback。
新增 `InvokeRequestV2` 示例(提案):
```json
{
"schema": "InvokeRequestV2",
"schema_version": 2,
"task_id": "task_01",
"run_id": "run_01",
"attempt_id": "attempt_01",
"lease": {"id": "lease_01", "epoch": 3},
"entry": {"module": "agent/qdagent.mq.md", "function": "run"},
"args_ref": "blob:sha256:...",
"context": {
"workspace_id": "ws_01",
"snapshot_id": "generation_01",
"selection_ref": "model-snapshot_01",
"capabilities_ref": "grant_01",
"deadline_at": "2026-10-26T08:10:00Z"
}
}
```
`grant_01` 是服务端验证的短寿命授权句柄,内容包括 principal、scope、根目录、允许效应、授权 revision、失效时间。worker 不使用浏览器任意传入的 fn/path。权限撤回后,新的工具调用必须重新验证 lease/grant,已开始的外部调用记录撤回时间与实际结果。
`RunContextV1` 保留原有字段与序列化兼容,内部增补显式 Context 对象;扩展 V2 由能力协商启用。event_sink、usage_sink、cancel_token 使用句柄,不使用外部可控路径。子任务通过 IPC/Command.env 显式继承,禁止父进程 `setenv`/`set_current_dir` 承担请求状态。
凭证在租约启动时通过受保护 IPC 或继承 FD 注入;不落任务 JSON、argv、日志或通用事件。模型快照可含 credential_ref,不含 api_key。即使选择相同 provider,不同请求的授权也不得合并。
## 6. 取消协议与状态 CAS
任务基础状态:`accepted → queued → running → finalizing → succeeded/failed`;queued/running 可进入 `cancel_requested → cancelled`;worker 失联进入 `interrupted`,副作用不明时进入 `needs_reconciliation`。这些是提案,产品可以将 succeeded 映射为 completed。
取消顺序:
1. 验证调用者对 task 的权限,CAS 写 cancel_requested、revision 与 requested_at;终态返回原状态,未知 ID 返回 404。
2. 向匹配 lease/epoch 的控制通道发送 Cancel;不经任务解释器锁。queued 任务原子移出待派发队列。
3. worker 在每次 LLM/工具调用、重试、子任务启动和提交之前检查 token;网络请求使用可中断句柄,不能仅在一行输出结束后才检查。
4. 收到 ACK 仍不等于任务停止。等待 worker 报 stopped、落未完成副作用和用量,再 CAS 到 cancelled。
5. 默认 2s 协作宽限、额外 3s 进程退出宽限,之后回收进程组。强制退出记 interrupted/needs_reconciliation,不能伪造干净 cancelled。
完成与取消竞争的线性化点是控制存储的 CAS:终态先提交则 cancel 返回 already_terminal;取消先提交则晚到 result 只可作为证据追加,不能重新变 succeeded。若外部动作已发生但响应晚到,记录 effect outcome,不撤销取消事实。
兼容文件 token 统一:不存在/空/`0` 为未取消,`1`/`cancelled` 为取消;读取失败返回诊断并按明确策略处理。删除 token 只能发生在持有者退出之后。移除全局 `MARQDO_CANCEL` 对多个任务的隐式作用域。
## 7. 事件、SSE 与背压
### 7.1 事件权威
生产者发出 `(run_id, attempt_id, local_seq)`,控制服务去重、持久化并分配 task 内单调 seq/event_id;客户端游标来自持久化日志,不是 handler 重连后从 1 开始计数。终态事件唯一,usage 可以在终态后以更高 usage revision 补录,但不能再次结束任务。
```json
{
"schema": "RuntimeEventV2",
"schema_version": 2,
"event_id": "task_01:42",
"seq": 42,
"task_id": "task_01",
"run_id": "run_01",
"attempt_id": "attempt_01",
"type": "message.delta",
"payload": {"message_id": "msg_01", "offset": 24, "text": "结果"}
}
```
不要直接破坏 RuntimeEventV1;新增字段/新版本由能力协商确定。普通用户可见 progress 只描述工具、阶段、文件等结构化事实;模型内部 reasoning 不作为完成证据,也不默认写入公开事件。
### 7.2 App SSE
新增任务 API 的事件订阅用 `GET /tasks/{id}/events`(最终产品前缀见配套设计),发送 SSE `id`/`event`/`data`;每个 data 是一个完整 JSON。客户端支持 Last-Event-ID、CRLF、多行 data、任意 UTF-8 分块和 EOF 未完成帧。重复事件按 ID 去重;message offset 检查缺口,缺口时恢复快照。
心跳 15s,客户端空闲探测建议 45s;二者可配置。事件保留建议 24h,最终结果/审计按项目政策更长保留。游标早于保留边界返回 410 `event.cursor_expired` 和快照入口,不能默默从最新开始丢字。
订阅断开默认只关闭订阅。App 显式停止通过 cancel API,切页/移动网络重连可恢复同一任务。传统同步 OpenAI 调用可保持 disconnect=cancel 的兼容策略,但必须显式区分,不能统一把所有浏览器断流当取消。
### 7.3 OpenAI 兼容流
`/v1/chat/completions stream=true` 输出 OpenAI chunk 与 `[DONE]`;App 运行事件由独立入口或协商的协议提供。禁止仅凭任意 JSON body 中 stream=true 将不相关路由自动改为 RuntimeEvent 协议。已发 header 后的错误使用该协议允许的错误帧;发 header 前可用 429/503/401 等正确状态码。
token delta 与最终完整答复分开:最终事件持有结果引用/摘要和 finish reason,不再次追加整篇正文。无 token 能力的适配器报告 `evented_batch`,界面用阶段状态,不把整包人为拆字伪装成实时。
### 7.4 背压和旧桥修复
worker→事件服务与服务→订阅者均有界。建议每订阅者 256 KiB 待发送内存,超出后断开慢消费者并允许按游标重连;事件正文已持久化所以不能静默丢 token。允许合并短时间 delta,必须保留 message offset;usage/终态/错误不丢弃。磁盘压力时停止接纳新任务,已有任务使用保留终态空间并按策略中断,不能无限写事件。
legacy NDJSON tailer 只消费换行已完整到达的字节;保留残片,offset 按真实消费字节移动。超长行、I/O error、文件旋转显式报协议错误。测试至少分三次写一个 JSON、中文多字节边界、恰好 EOF、超过上限、取消与尾帧竞争。禁止吞掉 Scanner.Err() 继续宣告成功。
## 8. 只读快照与真正有效的缓存
### 8.1 快照一致性
分析服务构建 SnapshotKey:规范化 project identity、允许扫描范围、源文件闭包摘要、依赖/来源/配置摘要、compiler/lib/ext 版本、policy/schema、解析选项。输入由编译器枚举,不另造“前 200 个根目录文件”规则。
读取一次快照内的正文、语义和来源,完成后复核输入 manifest;文件在分析中发生变化则重试一次或返回 `snapshot.changed`。对不可变发布代直接读其 manifest。mtime/size 只用于快速发现候选变更,不能代替内容摘要;watcher 丢事件后以完整扫描修复。
### 8.2 缓存结构
- `AnalysisCache<SnapshotKey, Arc<Compilation>>`:实际编译结果。
- `DocumentCache<(SnapshotKey, document_digest, renderer_version, view_options), DocumentView>`:安全 fragment、outline、nodes。
- 同 key single-flight 的 follower 等待同一个 Result,不再次 compile。
- RAII guard 在 error/panic/cancel 时清除 inflight 并唤醒 waiters;等待者有自己的 deadline。
- compile 失败只返回给当前等待者,短暂错误缓存需显式 TTL;不得调用“finish_success”标记失败结果可复用。
- LRU 按字节而非条数上限管理;初始预算按部署内存的 10% 且设硬上限,长文档超限不缓存或分页;记录淘汰原因。
- 每次读取先授权再查缓存;权限敏感的响应按 scope/policy 分区。全局缓存只共享不含用户秘密且访问已验证的不可变语义数据。
性能证据必须统计 `compile_invocations`、输入字节、缓存结果命中、leader/follower、wait_ms、evictions;现有 hit counter 不能证明减少编译。
### 8.3 HTML 与来源定位
沿用 DocumentViewV1,明确 `render.format=plain_text|sanitized_html`。HTML 通过解析器白名单清洗标签/属性/URL scheme,剥离脚本、事件属性、危险 CSS/嵌入与可执行 SVG;相对资源经受限资源端点解析。纯文本始终转义,不以 `<` 字符判断类型。
node_id、revision、generation 必须从定位到读取到浏览器滚动完整传递。不存在历史快照返回 version_unavailable;返回当前版是用户显式操作且标明版本变化。no_exec 测试分别验证“无语言副作用”和“浏览器无脚本执行”,二者不能互相替代。
### 8.4 首次编译的性能改造
本轮 `v3_corpus_1000_compile_smoke` 在 debug 下 243.012s,超过既有 180s 门禁;缓存不能缩短一个全新项目的第一次编译。先为 `knowledge::compile` 的语义编译、语义落盘、检索 extract、检索落盘、manifest 分别加 span,再细分 `ProjectIndex::build`、源枚举/解析、导入挂接、绑定、效应传播、闭包摘要。记录文件数/总字节、节点/边数、parse 次数与序列化字节量。
源码中 `sema::compile_project` 在每个文件查找 module path、每个 module 过滤全量 symbols,属于值得测量的重复扫描;未取得 profile 前不将其认定为 243s 的唯一根因。优化候选按证据选择:建立 path→module 与 module→symbols 索引;同一输入快照共享 parsed module;绑定/依赖图用稳定索引;效应传播用 worklist;闭包按 SCC DAG 计算并保留循环依赖语义;减少每制品重复序列化和小文件写入。
任何优化须保持排序确定性、SymbolId/ArtifactId 与证据摘要规则,禁止通过跳过来源校验、删掉复杂 fixture 或改大超时过关。用 100/1000/10000 单元、无依赖/链/菱形/循环四种图分别测时间与 RSS;给出随 V/E 增长的曲线。release 和 debug 分开;已有 debug 门禁恢复通过只是第一步,§12 的 release 预算仍须独立验收。
## 9. 写入、发布与数据库边界
执行 worker 只写任务 staging;需要写工程时通过沙箱产生文件 diff。发布以 kb 资源锁 + expected_generation CAS 提交。锁只覆盖校验当前版本和切换 manifest,不覆盖 LLM、全文编译或网络调用。
完整发布代先写临时目录、校验摘要、完成必要 fsync,再原子 rename 到不可变 generation 目录;最后在同一文件系统原子替换 CURRENT。源码、编译投影和证据在该 manifest 下对齐。SQLite 中的产品“published”是投影,不参与虚构的文件/数据库跨系统原子事务。通过 publish journal + outbox 恢复:CURRENT 已切换但 DB 未更新时重放;未切换时旧代继续有效。
project_id 与逻辑文档 ID 不随 staging/generation 的物理目录名变化。manifest 保存稳定 project identity、逻辑相对路径和实际 blob/generation 映射;导入引用按项目规则解析,不能因复制到发布目录而重新生成全部符号身份。框架若尚不能区分逻辑身份与物理根,先补该能力,再启用 generation 目录发布;迁移 fixture 必须断言未修改的知识 ID 和跨文档绑定保持稳定。
任务表/幂等/CAS/outbox 需要通用事务能力:先审计 `ext/data` 是否支持事务、条件更新、受影响行数、复合唯一键;缺少则在 Marqdo 数据层补齐,不能用 `.mq.md` 的先读后写模拟原子性。SQLite 使用短事务和有界 busy timeout;WAL/连接池是否开启需按现有插件适配测试确认。单主本地盘为首期部署范围,跨主机共享 SQLite/NFS 不在首期承诺内。
## 10. 临时缓解与迁移
### 10.1 进程池完成前
- 保留 1500ms busy 预算和 Retry-After;展示繁忙,而非无限 spinner。
- 在接纳任何 SSE 之前取得覆盖整个 bridge 生命周期的 legacy 单飞许可,第二条流在修改环境变量/发送 200 前失败;普通请求不得绕过许可观察被污染的状态。
- 断连只标记该任务取消,清理由监督者等待执行结束后负责;无法确认停止时报告 stopping/interrupted。
- 上述步骤只降低 R02/R03 风险,仍不算并行 Endpoint。
### 10.2 功能开关(新增提案)
`runtime.isolated_execution`、`runtime.task_control_v2`、`runtime.event_replay`、`view.analysis_cache_v2`、`runtime.read_lane`。探测返回 feature version 和限制。二进制版本相同不等于所有 native 插件都有该能力。
先离线 fixture → 本地假模型 → 新 workspace → 只读 ASK → staging BUILD → 真正发布。禁止对有副作用的请求做双执行 shadow;允许 shadow 只读分析并比较摘要。
回滚先停接新任务、drain 已接纳任务、保留任务/事件和新 schema;不能将活动任务交给不认识 lease 的旧进程。只读当前发布代可以保留,写入能力按版本协商关闭。ADR 0008 仅在并行金样例与回滚演练通过后由新 ADR 正式替代。
## 11. 开发包与依赖
下列新模块路径是建议,不代表文件已存在。
| 包 | 建议落点与改动 | 前置 | 验收 |
|---|---|---|---|
| M-R0 基线/缓解 | `host/plugin.rs`、`httpx/routes_invoke.go`;固定并发屏障、旧桥许可、取消清理 | 无 | 双流 busy 不改写 A 的桥;断连无提前清理 |
| M-R1 请求上下文 | `workbench/run_context.rs`、`host/subtask.rs`、LLM 插件;统一 token 与显式 sink | M-R0 | 父子运行模型/事件/取消不串;空 token 不取消 |
| M-R2 Worker/IPC | 新 `runtime/{supervisor,worker,protocol,lease}.rs`;独立进程与握手 | M-R1 | 两任务并行、超时回收、子进程无孤儿 |
| M-R3 接纳/控制 | 新 `runtime/{task_store,scheduler,control}.rs` 与 ext/data 事务 primitives | M-R2 | 原子幂等、队列公平、取消与结果 CAS |
| M-R4 事件协议 | `workbench/event_sink.rs`、Web stream/IPC、协议适配器 | M-R3 | 断点恢复/去重/半帧/慢消费/磁盘错误 |
| M-R5 读服务/缓存 | `analysis_cache.rs`、`document_view.rs`、`locator.rs`、MLSP transport | M-R1,可独立推进 | 100 同 key 真正只编译一次;嵌套变更正确失效 |
| M-R5b 首次编译性能 | `knowledge/compile.rs`、`sema/ir.rs`、ProjectIndex;分阶段 profile 与按热点优化 | 基线语义回归 | 现有 1000 单元门禁恢复;release 与规模曲线报告,身份和语义不变 |
| M-R6 提交底座 | `knowledge` generation/manifest、publish journal、条件更新 | M-R3 | 崩溃点恢复,旧代不被半发布污染 |
| M-R7 整合与发布 | 版本探测、CI、release、文档/ADR | M-R4–R6 + 产品接入 | §12 全部必要门禁与回滚演练 |
先以本地文件和单机进程解决明确缺陷;不要把集群、远程 worker、复杂 Actor 系统设为第一阶段依赖。
## 12. 验收、性能预算与可观测性
### 12.1 正确性矩阵
| 编号 | 场景 | 必须观察到的结果 |
|---|---|---|
| MR-01 | A/B 两个 60s 假模型流 + 20 次/s 模型/设置读取 | 两流不混字、不混模型;短请求 200;无 interpreter busy |
| MR-02 | B 在 A 运行时接纳失败/断连 | A 事件和取消句柄完全不变 |
| MR-03 | 同时取消 A、完成 B;取消重复/晚到 | A 正确停;B 成功;终态不回退;审计只结算一次 |
| MR-04 | 帧拆字节、多行、EOF 半帧、重连 | 无丢字/重字;协议错误可见;游标恢复精确 |
| MR-05 | 慢消费者、磁盘满、worker 崩溃 | 内存有界;事件不静默丢失;状态可恢复 |
| MR-06 | 100 个相同读取并发 | 实际编译一次;后续结果复用;失败 leader 不挂死 follower |
| MR-07 | 改子目录依赖、第 201 个文件、配置、来源 | SnapshotKey 改变;缓存失效;旧业务句柄拒绝 |
| MR-08 | 上传有写文件/网络代码、HTML 事件属性 | 阅读不求值;浏览器脚本不执行;正文可读 |
| MR-09 | 发布每个持久化边界 kill | 仅旧完整代或新完整代可见;DB 投影最终追上 |
| MR-10 | 并发幂等请求、取消队列、权限撤回 | 单一 task;同键异内容 409;无越权派发 |
| MR-11 | kill 后副作用未知 | 不自动重放,显示 needs_reconciliation |
| MR-12 | 降级/升级、混合 CLI/ext/native | 缺能力明确禁用,不报已支持 |
### 12.2 性能目标(待测验收预算)
基准机:Linux 4 vCPU/8 GiB SSD;release 构建;1000 个知识单元、文档总量 10 MiB、500 条历史 run;假上游固定 60s、每秒 20 个增量;30s 预热、5min 采样、3 轮,报告 p50/p95/p99、错误率、CPU、RSS、磁盘。另测 10k 单元和 100 MiB 以确定容量上限,不能直接沿用小数据阈值。
| 指标 | 初始预算 | 说明 |
|---|---|---|
| 短读/任务接纳 | p95 ≤200ms,p99 ≤500ms | 两个长任务运行时;过载拒绝另统计 |
| 取消受理 | p95 ≤200ms | 独立于任务停止耗时 |
| 协作取消收敛 | p95 ≤2s | 假上游/可中断工具;不可中断外部动作按不确定处理 |
| 已就绪 worker 调度 | p95 ≤100ms | 不含排队、模型首 token 时间 |
| 事件到订阅者新增延迟 | p95 ≤100ms | 从已收到 provider delta 计,不承诺上游 TTFT |
| 1000 单元首次分析 | p95 ≤2s | 候选目标,超标先剖析,不降低正确性 |
| 热 DocumentView | p95 ≤100ms | 真实复用结果;长文档另测分页 |
| 慢消费者缓冲 | 每订阅 ≤256 KiB | 总连接数也必须受限 |
| 压测结束内存 | 稳态可回收、无持续增长 | 记录每次重复轮次 RSS;不能只看一次峰值 |
新指标至少有 queue_depth/wait、worker_active/recycle、admission_rejected、cancel_ack/stop、orphan_count、event_lag/replay/duplicate、compile_invocations/cache_result_hit、publish_conflict/recovery。指标标签禁止 task_id、原始路径、prompt 等高基数/敏感值;trace 中可记录脱敏 task/attempt/lease 与构建摘要。
现有测试继续保留:`cargo test --offline --locked --test workbench_contracts --test executable_knowledge`、LLM 插件单测、Go httpx 测试。新增集成门禁必须启动真实 HTTP→IPC→worker 路径并使用本地假模型;不能以手写 fixture 或写 token 文件代替端到端行为。本轮执行结果统一记录在[跨库复核 §5](../../../qdagent/doc/gaps/04-workbench-runtime-and-product-audit-2026-10-26.md)。
@@ -0,0 +1,435 @@
# qdagent 知识工作台所需的多模型运行与 View 嵌入支持
日期:2026-10-25。状态:Partial / in progress(M00–M06 合同门禁 + F02 agent.step/resume 假模型 e2e + USTC live selection smoke + DocumentView 浏览器只读核对已落地;F 矩阵未全绿,不得整文宣称 Verified;原生非兼容适配器仍 unavailable)。
源码基线:Marqdo `dc853fb`(2.1.2),qdagent `643dd31`,已同步远端。应用侧配套:[qdagent 多提供商与知识工作台](../../../qdagent/doc/design/multi-provider-and-knowledge-workbench-2026-10-25.md)。跨仓相对链接以兄弟仓库工作区布局为准。
沿用 [文档原生开发指导](document-native-development-guide.md)、[可执行知识规范](../next/Marqdo_Executable_Knowledge_Development.md) 与 [View 阅读体验设计](view-knowledge-reading-experience-2026-10-24.md)。本规格补充嵌入、调用身份和运行观测,不另造语言或第二套知识真相。
## 1. 框架目标与责任边界
让任何宿主应用可以按请求构造模型、追踪所有真实模型调用、以只读 View 展示知识文档,并从引用定位到版本一致的知识与来源。qdagent 的设置页、模型目录持久化、账单展示、会话导航、工作区授权和画像管理由应用实现。
| 能力 | Marqdo 拥有 | qdagent 拥有 |
|---|---|---|
| 多提供商 | 协议适配、模型 handle、能力协商、调用参数、归一化错误 | 提供商账户/端点/模型 CRUD、默认选择、凭证管理 |
| 模型路由 | Runtime 执行显式选择和允许的辅助模型策略 | 用户选择、工作区模型允许列表、路由授权 |
| 用量 | 每次请求/重试的身份、usage 来源与完整性、子任务传播 | 持久化账本、价格版本、费用估算、筛选/导出/预算策略 |
| 实时运行 | 逐事件输出、取消传播、背压、终态 | HTTP 会话鉴权、轮次映射、持久化事件重放、断线 UX |
| View | 文档语义、正文/目录/步骤、纯读取、稳定定位、可嵌入输出 | 容器/主题/深链/阅读历史/产品操作 |
| 知识引用 | 解析/歧义/版本/来源定位、验证多维事实 | 消息引用标识、授权、历史资料保存与入口 |
| 工作区 | 按显式 project 编译/查询,缓存及输出隔离 | 工作区 registry、root 解析、用户权限、文件管理 |
| 画像与笔记 | 通用文档渲染、来源关联、验证/编译原语 | 内容策略、提案/确认、生命周期、展示和上下文选择 |
不在语言核心添加 provider/workspace/profile 关键字;沿用 `.mq.md` 函数、结构化参数、扩展插件和 MLSP。不得让“可读”取决于用户手填 hash、node id、source span 等机器字段。
## 2. 2.1.2 已完成的基础与剩余缺口
### 2.1 当前可复用基础
| 源码 | 已有内容 | 本规格的增量 |
|---|---|---|
| [ext/ai/llm](../../ext/ai/llm.mq.md) | `create(model,base_url,api_key,backend,name)`、命名 handle、ask/stream | 无需重做基础 handle;增加隔离策略、协议能力、完整消息与调用身份 |
| [LLM plugin](../../plugins/llm/src/lib.rs) | 兼容 chat completions HTTP、SSE 文本读取、非流 usage 三项 | SSE usage/异常终态/每次调用观测、可取消持续输出 |
| [Agent](../../ext/ai/agent.mq.md) | Agent 唯一 loop、router_model、EFI、绑定/发布门闩 | 主/路由/修复/生成工作簿的模型及计量上下文传播 |
| [语义 IR](../../src/sema/ir.rs) | Artifact、KnowledgeStep、KnowledgeRef、SourceLocation、BoundCall | View 面向阅读的完整投影及 ID/版本映射 |
| [MLSP](../../src/mlsp.rs) | `artifact.inspect`、`link.resolve`、verify/bind/execute、document.extract | 只读 document/引用解析及 capabilities 版本握手 |
| [KnowledgePresentation](../../src/view/knowledge_present.rs) | schema v1,artifact nodes、依赖、generation/page token | 保留兼容,新增正文/目录/refs/source spans/结构化诊断 |
| [View output](../../src/view/output.rs) | `--no-exec` 在构建前跳过 capture | 将纯读取能力暴露给嵌入客户端,不复用会执行的 live 默认路径 |
| [document extract](../../src/document_extract.rs) | TXT/MD,partial,PDF/DOCX/OCR 显式不支持 | 能力查询及稳定来源定位;不要求本阶段重造 OCR |
| [数据资源](../../ext/data/db.mq.md) | SQLite migrate/exec/事务、唯一/索引声明 | 应用可直接承载 registry/账本/索引;不需另加产品专用数据库框架 |
2.1.2 已把知识辅助块移到正文后并折叠,跳过 module/type 卡片,显示“结构步骤非实际执行时序”。旧 View 截图中的最初顺序问题已有改进;但这不意味着已提供能直接放进 qdagent 右栏的独立阅读 SDK。
### 2.2 直接影响集成的事实
1. `llm.complete_result` 当前仍走 `openai.chat_completions`,Ollama 也是兼容路径;`backend` 名称不是任意原生协议适配器。
2. `plugins/llm::chat_completions` 从 `prompt` 构造一条 user 消息,没有完整消息列表契约;不能靠模型选择 UI 弥补历史丢失。
3. `usage_from` 缺字段填零;Agent `model_turn(stream=True)` 直接 `tokens=0`,最终 `token_cost=tokens`。现有输出无法支撑准确货币或流式用量显示。
4. 原生 LLM SSE parser 返回事件数组;`ext/ai` 有流事件机制,但应用所用 [web invoke](../../plugins/web/internal/httpx/routes_invoke.go) 在 `CallLibPath` 返回后才写 `{sse}`。需要打通实际增量路径,不能只换 Content-Type。
5. Agent 生成子工作簿的模板仍能构造默认 `llm.llm`;只传主 agent handle 不足以证明嵌套调用沿用了用户选定模型。
6. `build_file_view` 默认传 `execute=true`;live GET 路径使用它。嵌入阅读绝不能直接暴露这个入口。
7. Presentation 目前按 artifacts 投影,缺全量 prose/heading/step/ref 的统一阅读树;`render_knowledge_html` 另从 steps/refs 渲染,容易与 JSON 事实发生覆盖差异。
8. `source_digest` 当前由语义 artifact hashes 汇总,`semantic_generation` 是语义散列截短,`page_token` 来自原文件。它们不能直接分别冒充原文 revision、`.marqdo/CURRENT` generation 和跨请求完整版本。
9. `sse_accepts_packet` 对缺 token 的 legacy 包放行;正式嵌入客户端必须采用严格版本匹配,而不是继承这项兼容宽容。
10. `artifact.inspect/link.resolve` 每次走当前 semantic_project;现有入口不保证可以读任意历史 generation。历史引用需要明确的保留与解析合同。
## 3. 共用合同、版本与能力发现
合同名字固定为 `ModelSelectionV1`、`ModelCallUsageV1`、`RuntimeEventV1`、`DocumentViewV1`、`KnowledgeLocatorV1`;它们是这次扩展的 v1,与当前 KnowledgePresentation `schema_version:1` 不混用。新增 read model 不强行给已有 JSON 改含义。
新增 MLSP `capabilities`(拟议):
```json
{
"schema_version": 1,
"runtime_version": "implementation-version",
"features": {
"llm.explicit_selection": 1,
"llm.call_usage": 1,
"runtime.event_stream": 1,
"runtime.cancel": 1,
"view.document": 1,
"view.reference_resolve": 1
},
"adapters": [{"id": "openai-compatible", "available": true}],
"extract_formats": [{"format": "text", "available": true}],
"loaded_plugins": [{"name": "llm", "capabilities": ["call_usage_v1"]}]
}
```
只返回实际探测且加载可用的功能。不能仅从 CLI 版本推断 native plugin 符号存在;字段未知时应用降级或解释不支持。协议允许新增可选字段;删除/改变状态语义必须提升 major contract version。能力查询不调用供应商、不执行项目代码。
MLSP 继续使用现有 `{id,ok,result|error}` envelope;qdagent `/api/app` 转为自己的 `{ok,schema_version,data|error}`,不要要求它们共享同一层 envelope。所有合同提供成功、partial、unknown、unsupported、stale、cancelled、错误 fixtures 与 JSON Schema;应用和框架从相同样本跑消费测试。
## 4. M01:请求级模型身份与适配器
### 4.1 显式选择合同
应用将配置解析成可信 `ModelSelectionV1` 后传入 runtime,核心字段:
```json
{
"schema_version": 1,
"selection_id": "sel-1",
"provider_id": "provider-1",
"model_id": "model-2",
"provider_revision": 3,
"model_revision": 5,
"adapter": "openai-compatible",
"endpoint_ref": "endpoint-local-1",
"credential_ref": "credential-local-1",
"credential_version": 2,
"upstream_model_id": "example-model",
"parameters": {"max_output_tokens": 2048},
"fallback_policy": "disabled"
}
```
endpoint/credential 引用由宿主 resolver 解析为调用局部配置;短期直接向现有 `llm.create` 传参数亦可,但这不是可序列化的 public snapshot。API key 不能出现在 events、debug JSON、生成工作簿、prompt、错误 URI 或持久化模型对象中。
当前显式 args 优先的能力保留;新增 strict selection 模式:字段缺失报错,不回退 process env/frontmatter。无凭证端点通过 `auth_mode:none` 显式表达,避免 falsy key 触发环境 fallback。legacy 命令行未传 selection 时仍可沿用原有 env 行为;本地便利兼容不扩散到应用请求。
每个运行绑定 selection snapshot,后续设置改变不修改它。主模型与 router_model 分开但同属受控 selection 集;manual 模式所有需要 LLM 的步骤固定选择,REUSE 无模型调用仍允许。fallback 必须由宿主明确授予、列出允许目标,不能因额度/网络错误静默转到另一个供应商。
### 4.2 子任务和工作簿传播
统一 `RunContextV1 = run_id,parent_run_id?,turn_id?,workspace_scope,selection_context,usage_sink,event_sink,cancellation_token`。进入 ask/step/plan、repair、summary、learn、子工作簿时显式继承;新 run/call id 关联父节点。
生成工作簿只包含不可重放滥用的运行上下文引用,不写密钥;执行时宿主注入/解析。独立导出工作簿在缺凭证/选择时清晰要求配置,不偷偷取默认模型。在任何 child/spawn/resume 路径遗漏 selection 都应由 strict 模式拒绝并给出定位诊断。
同时审查环境变量不只是凭证:project root、temp request、输出目录、event sink 也必须请求局部,不能在并发运行中依靠进程可变状态串联。
### 4.3 适配器与消息
适配器最小职责:验证 selection、报告能力、序列化请求、解析增量结果、usage 归一化、取消传输、错误归一化。可选 discovery/probe;无模型列表 API 仍允许手工配置。
V1 消息包含 role 和 content parts,parts 最低 text,附件通过受控引用;tool call/result 要保持配对及 role。不支持的 part/parameter 返回 `llm.unsupported_input/parameter`,不能悄悄转成空字符串。适配器决定协议格式,Agent 决定行为;qdagent 不拥有供应商协议代码。
阶段一验证两个不同兼容端点,分别至少两个模型,以及无密钥本地兼容服务。阶段二增加原生非兼容协议:每个适配器需具名能力、流式/非流式/错误/取消/usage fixtures 和独立接入说明后才能在 capabilities 中置 available。不能把所有厂商默认走兼容接口后统称全支持。
统一参数采用 `max_output_tokens,temperature?,stop?,reasoning?` 等语义键,能力不支持时明确拒绝;不随意透传客户端未知参数。实际 upstream model 若与请求 id 不同,在 result 中保留 requested/resolved 两者。供应商版本与价格随时变化,本文仅锁定内部合同;实施原生适配器时另附当时官方协议核验记录。
## 5. M02:用量、事件流与取消
### 5.1 ModelCallUsageV1
```json
{
"schema_version": 1,
"call_id": "call-1",
"attempt_id": "attempt-1",
"parent_call_id": null,
"run_id": "run-1",
"turn_id": "turn-1",
"workspace_scope": "ws-1",
"purpose": "answer",
"selection_id": "sel-1",
"provider_id": "provider-1",
"model_id": "model-2",
"requested_model": "example-model",
"resolved_model": "example-model",
"status": "succeeded",
"usage_state": "final",
"measurement_source": "provider",
"input_tokens": 1200,
"output_tokens": 300,
"total_tokens": 1500,
"cached_input_tokens": 800,
"reasoning_output_tokens": null,
"revision": 2,
"provider_request_id": "opaque-provider-id",
"started_at": "2026-10-25T01:00:00Z",
"finished_at": "2026-10-25T01:00:03Z"
}
```
字段要求:tokens 非负整数或 null;`usage_state = pending|partial|final|unknown`;`measurement_source = provider|estimated|none`。估算可作为单独 estimate 字段返回,不能覆盖实测字段。供应商没有报告时用 null/unknown,传输失败不能自动解释成零消耗。
`status = started|succeeded|failed|cancelled|interrupted`,status 和 usage_state 独立。每个真实发出的 HTTP attempt 一个唯一 id;预检拒绝只产生 call-rejected 事件,不伪造已发送请求。框架重试必须可观测:所有 attempts 包括失败/有部分输出的 attempt;重复 delivery 的 usage revision 不算新请求。
累计 SSE usage 在同 attempt 内按 revision 发布完整快照,不让上层猜是 delta 还是 total。provider 总量与 input/output 不一致时保留原始读数及 `usage.inconsistent` 诊断,不自行强制改账;细分字段要标 inclusion 关系以避免缓存、reasoning 与总量重复相加。只保留脱敏的必要 provider usage 扩展字段,不把完整原始响应当 public usage。
纯知识复用用 run summary 中 `llm_calls=0`、`measurement_source=none` 表示,而不是伪造一条耗费为零的模型请求。框架输出 `call_count,known_tokens,unknown_call_count`,宿主据此显示覆盖程度。
### 5.2 事件流合同
```text
run.started
→ model.resolved
→ model.call.started
→ message.delta / tool.progress / usage.updated(零到多次)
→ model.call.finished(每个 attempt 各一条)
→ document.produced / citation.produced(仅有真实对象时)
→ run.completed | run.failed | run.cancelled | run.interrupted
```
`RuntimeEventV1` 公共字段:`schema_version,event_id,seq,run_id,parent_run_id?,turn_id?,workspace_scope,timestamp,type,payload`。seq 每个 run 单调;不同子 run 可以并发,宿主按所属 turn 另分配产品流序号。runtime terminal 事件仅一次;后续补全计量用有 revision 的 `usage.updated`,不能改变内容终态。
事件 sink 支持有界队列和背压;慢客户端不能无限堆积 token。丢弃可合并 progress 时记录 gap;内容与终态事件不可无声丢失。连接终止不会自动重跑;持久化/重放由应用,但框架需在实时回调后仍提供可核对的 final result/call summaries。
同一真实调用所有数据使用同一 run/call identity,不能流与最终结果各生成一套 id。工具参数/结果按 policy 脱敏,不把 secret、完整私有文件、模型内部推理链写入默认进度事件。
### 5.3 打通传输而不重造 Agent
需要同时打通 LLM provider reader → Agent event sink → host callback → web streaming handler → 浏览器。现有 `sys.stream_publish` 可以作为底座,但要验证它是否绑定正确请求且在 `CallLibPath` 返回前可写出;不能用进程全局事件广播混流。
可实施路径:新增 web 流式 invoke/job primitive,由框架注册 run handle 和 event sink;调用返回 handle 后宿主可订阅/取消。采用插件 ABI 增量接口及能力检测,旧 ABI 未支持时拒绝新 streaming 模式,保留旧完整返回。qdagent 只挂产品鉴权、event id 持久化和 HTTP 端点,不添加另一个推理循环。
SSE parser 必须处理任意 TCP 分块、UTF-8 跨块、CRLF、多行 data、注释 heartbeat、尾帧、usage-only 帧、[DONE]、连接在结束前关闭。未知可选事件可忽略且记诊断;已知事件无效、异常截断不能静默当成功完成。公共 framing 不包含供应商私有认证信息。
### 5.4 取消与重试
取消 token 传到 HTTP reader、循环、工具边界、子任务。取消后不得启动新模型调用/新工具;正在进行的不可抢占操作返回 `cancelling` 和是否仍运行,结束后才报告 cancelled。明确“停止生成”不能撤销已经完成的外部写入。
调用超时、连接错误、鉴权失败、限流、协议错误分开;错误字段包含 `code,retryable,retry_after_ms?,call_id,attempt_id,usage_state`。不要将原始包含密钥/完整 prompt 的错误直接传到 UI。限流/短暂网络错误按有界策略重试;已产生输出的重试不能接在同一条文字后假装连续,也不能把未知结果的外部工具操作自动重做。
费用价格由应用管理。框架保留 legacy `tokens/token_cost` 并标注 deprecated/单位为 token,新功能只消费 ModelCallUsageV1,避免破坏旧脚本的同时停止误用。
## 6. M03:无执行的嵌入式 View
### 6.1 新入口与纯度约束
新增内部公共服务 `analyze_document → project_read_model → render_fragment`;MLSP 暴露 `view.document`,可选 CLI `view document --json --no-exec` 与它共享实现。不要让新入口先进入 `build_file_view` 再隐藏按钮/清空输出。
读取允许:授权范围内原文/已编译快照/已有验证记录。读取禁止:运行模块入口、加载文档声明插件、执行外国代码、LLM、自动抓取 URL、写回、重新验证执行测试、隐式发布/重编译写盘。静态分析/内存投影允许;缺完整依赖时明确 partial。任何需要执行的行为必须另走显式 execute/verify 产品流程。
当前 `compile_project` 声明不加载插件/不求值/不抓取来源,可复用;缓存写入若需要,作为单独的宿主管理路径,不能把“读文档”隐含写项目目录。
### 6.2 DocumentViewV1 结构
请求由宿主传入受控 project 和文档定位,宿主必须已授权;框架仍做路径边界校验。
```json
{
"method": "view.document",
"params": {
"project": "/authorized/project",
"document": "knowledge/example.mq.md",
"revision": "sha256:requested-source",
"generation": "generation-id-or-null",
"mode": "read",
"node_id": "optional-node",
"include": ["fragment", "outline", "nodes", "refs", "diagnostics"]
}
}
```
```json
{
"schema": "DocumentViewV1",
"schema_version": 1,
"document": {
"id": "project-relative-document-id",
"title": "知识标题",
"source_revision": "sha256:raw-bytes-digest",
"semantic_digest": "sha256:semantic-facts",
"generation_id": null,
"presentation_token": "opaque-token"
},
"execution": {"state": "not_run", "run_id": null},
"render": {"format": "sanitized_html", "fragment": "<article class=\"mq-document\">…</article>", "renderer_version": "1"},
"outline": [],
"nodes": [],
"refs": [],
"diagnostics": [],
"completeness": "complete",
"capabilities": {"read_only": true, "has_semantics": true}
}
```
`generation_id` 来自实际编译投影;即时分析但未编译发布时为 null,同时给 semantic_digest。`source_revision` 是原始文件字节摘要,不能复用当前基于 artifacts 的 `source_digest`;`presentation_token` 覆盖 source revision、generation/semantic digest、依赖版本、renderer/schema、读取策略与请求范围。应用缓存还必须含用户授权范围。
`completeness = complete|partial`;语法损坏时允许返回原文与结构化诊断,但不能展示猜测出来的正常知识图。失败用 error envelope,不能空字符串伪装“文档无内容”。大文档 response 支持 section cursor 和 truncated 标识,限制要可配置并在响应中标明。
### 6.3 统一正文、目录、节点与引用
节点包含 `id,parent_id,kind,label,order,source_span,artifact_id?,ref_ids[],children[]`。kind 包括 heading/prose/table/code/observe/derive/call/iterate/narrate/navigation;正文树来自 parser/classifier/sema 的共同投影,不能只枚举 functions 后把每个步骤标 call。
outline 描述读者文档层级,不简单把函数和对象全铺成同一级。HTML 中节点带 `data-mq-node`,引用带 `data-mq-ref`;实现 payload 按需展开,说明作为正文。动态调用/无法解析引用保持原文并解释原因,不虚构执行边。
首批 renderer 共享 `src/view/document.rs` 的文档表面,standalone/full page 和嵌入模式调用同一 read model;只改变容器和导航外观,不产生第二种语义解释。替换当前 `KnowledgePresentation` 与 `render_knowledge_html` 双路径重复组装。
稳定 ID 的保证范围要写明:
- 同一 revision 再次读取,node/ref id 必须相同。
- 仅前插空行/叙述时,已有命名符号保持 symbol id;步骤 id 由结构身份映射,不能只靠行号。
- 同一行多个节点需要 column/offset/span 区分,不能都映射成 `stmt-N`。
- 匿名步骤重排/重复同名内容可能无法唯一映射;返回 unmapped/ambiguous,不能声称任意编辑下永恒稳定。
- source span 明确 1-based line/column 和 UTF-8 byte offset;JavaScript UTF-16 偏移不直接当 Rust 字节偏移。
旧 `#stmt-{line}` 可保留为当前版本兼容锚点,但不得作为跨版本唯一身份。
### 6.4 嵌入渲染、主题与访问性
fragment 不包含 html/body、应用导航、运行按钮、内联可执行 script、全局 CSS;CSS 以 `.mq-document` namespace 限定。主题暴露 `--mq-bg/surface/text/muted/border/accent/font-body/font-code`,宿主映射自己的设计 tokens;不把独立 View 的全局 `*`/body 样式注入 qdagent。
首批推荐 fragment + 结构化引用 + 宿主事件代理;无需为 iframe 和 React 各重做 parser。可选 iframe embedded shell 必须只读、无重复导航,宿主/子页通信校验 origin/source/nonce/schema,仅允许 open-reference/selection/ready 等已定义消息,不能从内容发任意运行指令。
所有文本/属性转义,允许的 Markdown HTML 白名单净化;链接 scheme 限制;外部资源不默认跨网加载;代码/公式不得执行。静态导出、独立页面、嵌入三者都测试恶意文本/闭合标签/危险 URL。不要把 KnowledgePresentation 内 JSON 的转义当作所有正文 XSS 防护。
标题/列表/表格用语义 HTML;说明链接有可访问名称,焦点可见,视觉排序和键盘顺序一致。折叠只隐藏技术细节,不隐藏理解结论所需的正文。320–720px 容器宽、暗色、200% 缩放和 print 都要能读,不出现遮住正文的关系连线。无 JS 时仍能读正文和普通链接。
## 7. M04:知识、来源与历史引用定位
### 7.1 KnowledgeLocatorV1
统一 locator:`project_id,document_id,source_revision,generation_id?,artifact_id?,node_id?,source_ref_id?,span?,relation`。relation 区分 cites/supports/derived_from/executes。普通链接导航、NarrativeRef 和 BoundCall 都能定位,但语义不同;点击说明永不成为 BoundCall 的执行。
新增 `view.reference_resolve`:接受 locator 或当前文档 ref_id,返回 `resolution = resolved|stale|unresolved|ambiguous|version_unavailable|unsupported`、目标 locator、候选(若有)、定位精度、诊断、读者标签。框架内部路径不作为浏览器 href,应用根据授权映射为 document/source route。
扩展 `artifact.inspect` 的版本参数,旧无版本调用仍查当前;新调用带 expected source/generation mismatch 时不能自动降到当前。来源读取接口可由宿主取实际文件,再调用框架 locator 映射;框架不自动 fetch 外部网页。`mq://` 必须按框架规则解析,不能交给浏览器或把 URI 直接当文件路径。
### 7.2 历史版本与保留责任
生成 artifact/generation 元数据不等于保存了原文历史。框架须输出可保存的 snapshot manifest:原文 revision、依赖 revision、source refs、artifact hashes、renderer/schema version;应用决定哪些源版本随引用保留及保留周期。已有 Git 或内容寻址存储可提供 snapshot resolver;不强制应用部署 Git。
source revision 定位到存储快照后,框架基于该快照解析;缺失任何必要版本时返回 version_unavailable/partial,不读取最新文件填充旧快照。如果只留 artifact 元数据、原始资料已删除,仍可展示历史元数据但明确“原文不可读取”。
知识改名、移动或 source anchor 改变可返回候选映射,只有唯一可靠匹配才自动导航;否则用户选择最新版/候选。保持旧引用与新引用两个对象,防止历史回答证据在后台悄悄改变。
### 7.3 验证和运行事实
节点详情至少携带 purpose/description、signature、applicability、provenance、effects、dependencies,以及 parse/resolve/contract/tests/evidence/execution 各自状态与记录来源。sourced 只能说“已关联来源”;static verify 成功不表示执行成功或适用业务正确。
真实执行证据独立结构 `run_id,artifact_digest,closure_digest,source_revision,generation_id,inputs_digest,executed_at,result_state,evidence_ref`,展示时匹配版本。旧运行结果不涂成当前文档的绿色通过。知识复用必须存在实际 bind/execute trace,不能凭 find 命中显示“已执行”。
View 不运行验证测试以填满状态。若用户点“验证”,应用走已有 verify/policy 门禁并记录新 evidence,之后读取才刷新详情。
### 7.4 错误与授权协作
| 内部结果 | 应用建议呈现 | 不能做 |
|---|---|---|
| unresolved | 无法定位该知识,可读原引用 | 跳转到第一个同名 artifact |
| ambiguous | 显示已授权候选和位置 | 自动选择或暴露不可见候选 |
| stale | 该知识/来源已变化,提供版本选择 | 静默替换旧引用 |
| version_unavailable | 回答所用版本不可用,可查看最新版 | 用最新版伪装历史证据 |
| partial | 部分依赖或来源缺失,说明范围 | 默认绿色通过 |
| permission denied(应用) | 无权访问 | 在错误中泄漏标题、路径、片段 |
用户权限归应用;框架服务在调用边界接受 scoped project/source resolver,不把传来一个 root 视为可遍历整个服务器。所有路径经过 canonicalize/symlink 检查;缓存的语义事实可共享,带原文的响应必须按权限复核,不能只靠 artifact id 缓存授权。
## 8. M05:工作区、笔记与导入所需的通用底座
不为 qdagent 画像加语言特殊节点。通用能力应支持普通文档元信息读取、来源范围、内容修订、diff/patch 的 expected digest、提取格式 capability。
document.extract 继续使用 DocumentSource→ExtractedDocument,报告 format/status/segments/source locator/diagnostics。TXT/MD 可支持;PDF/DOCX/OCR 没有适配器时明确 unsupported/failed,不能将空文本当成功。partial 保留已提取片段及原文范围、遗漏原因;发布政策由应用决定。
同一资料可产生多个知识单元,provenance 必须到具体片段,不能所有函数只指向文件根。原始资料 digest 改动的失效由现有 provenance/closure 保证;应用用户画像也可用普通来源引用,但不自动进入公共知识发布。
ProjectRoot/CompileOptions 显式传递,工作区 A/B 的 source id、输出 generation、缓存、临时目录必须隔离。避免每读一个节点就全项目编译:引入按 project snapshot 复用的分析服务,针对来源/依赖变化失效;并发同 snapshot 请求合并,不能分别写同一个 CURRENT。
上传 multipart/大小限制已有 web 底座,框架只提供安全提取和可消费结果;浏览器上传、作业生命周期、发布确认、权限和历史保存仍在 qdagent。多文件任务桥接采用结构化参数,禁止拼接用户文件名进 shell/JSON 或共用固定请求临时文件。
## 9. 实现落点与兼容策略
| 模块 | 改动方向 | 兼容要求 |
|---|---|---|
| `plugins/llm/src/lib.rs` | adapter boundary、消息数组、usage、event sink、取消 | 保留已有符号;新能力有 probe 和合同版本 |
| `ext/ai/llm.mq.md` 与 `ext/ai/llm/*` | strict selection、结构化 result、public snapshot | 无 selection 的旧调用仍可运行;不得回显密钥 |
| `ext/ai/agent.mq.md` | run context、子工作簿传播、逐调用观测 | 不复制 Agent loop,EFI/verify/publish_gate 不被绕过 |
| `plugins/agent/src/*` | 子任务/恢复上下文、cancel、event identity | resume 不重发已结算请求;不可恢复说明 interrupted |
| `src/host/*`、插件 SDK、web invoke | 请求局部 event/cancel 桥接 | ABI 旧宿主不伪支持;并发请求不混流 |
| `src/view/knowledge_present.rs` | DocumentView read model/兼容 adapter | 旧 presentation v1 不被新字段语义污染 |
| `src/view/document.rs,render.rs,html.rs,mod.rs,output.rs` | 共享纯读取、fragment、scope CSS、定位 | live/debug 与 read 明确分离;旧 no-exec 门禁保留 |
| `src/mlsp.rs` | capabilities、view/document/reference、版本 inspect | 使用现有 envelope;未知 feature 可探测 |
| `src/sema/ir.rs,provenance.rs` | span、snapshot/locator、依赖与来源失效 | 不改变核心语法,不把读请求变为执行 |
| `src/document_extract.rs` | capability 和片段定位完整性 | 保留不支持格式的显式失败 |
测试分层对应 plugin HTTP fixtures、Agent 假模型、MLSP read contracts、web 持续流集成、独立/嵌入 DOM 和浏览器交互。应用共享样本文件必须注明 schema 版本;两仓 PR 在各自文档索引相互链接对应实现说明。
迁移顺序为新增合同/接口 → 旧输出 adapter → qdagent 按 capability 接入 → 切默认 → 后续版本再考虑弃用。不要同时移除 legacy 行为且要求应用一次完成全链升级。发布包对 CLI/ext/native 插件做组合测试,缺插件给功能不可用诊断而非 unknown function 直到用户点击才暴露。
## 10. 交付切片与验收门槛
| 切片 | 前置 | 验收输出 | 支撑应用任务 |
|---|---|---|---|
| M00 合同冻结/探测 | 无 | schemas、fixtures、capabilities、版本和插件清单 | Q00、所有升级探测 |
| M01 多模型隔离 | M00 | 兼容适配器、strict handles、history、子任务选择一致 | Q01/Q02 |
| M02 调用观测与流 | M01 | 每 attempt usage、真流式、取消、错误/重放一致 | Q03/Q06 |
| M03 只读嵌入 View | M00,可与模型线独立 | DocumentViewV1、fragment、目录/节点、无执行证明 | Q04 |
| M04 引用与快照 | M03 | 历史 locator、来源详情、版本失效/歧义 | Q05 |
| M05 文档工作区底座 | M03/M04 + 已有 extract | 范围隔离、提取定位、并发与缓存 | Q07/Q08 |
| M06 集成与性能 | 全部所选切片 | 旧门禁 + 新 contracts + 浏览器与负载报告 | Q09 |
M01 支持多个兼容提供商不等待原生协议全做完;但原生适配器需另列完成状态。M03 不等待计费系统。M04 未完成历史快照前,应用可以显示当前文档,但必须把历史引用标“版本不可用”,不能宣布引用链闭合。
每个切片的 Definition of Done:实际接口可调用、失败合同覆盖、无副作用/隔离测试、文档示例由测试消费、版本能力可探测、qdagent integration fixture 通过。只有可运行的验收证据才可将 proposed 改为 verified。
## 11. 框架测试矩阵
| 编号 | 用例 | 断言 |
|---|---|---|
| F01 | 两端点各两模型、并行 A/B、无密钥本地端点 | 请求 payload/端点/鉴权正确,环境不串扰 |
| F02 | step/plan/router/repair/child/resume | strict selection 和 parent identity 完整,不退回默认模型 |
| F03 | multi-turn、system/tool roles、超限/不支持模态 | 顺序和边界完整,不吞最新 user,不静默降级 |
| F04 | 非流/流式/usage-only/缺 usage/错误后部分 usage | null 与 0 区分,终态和用量独立 |
| F05 | 相同累计 usage 多次、重放、新重试 | 同 attempt revision 覆盖,不重复相加;真重试新增 attempt |
| F06 | 慢供应商、慢消费者、断连、取消、超时 | 完成前有 delta;队列有界;取消不启动后续工具/模型 |
| F07 | 外部工具已写后取消、恢复未知状态 | 不声称撤销,不自动重做可能有副作用的动作 |
| F08 | 带写文件/网络/foreign/插件的文档被阅读 | read API 不加载/运行这些效果,无项目写回 |
| F09 | prose/heading/6 类步骤、同一行多引用、中文 span | HTML/outline/节点/ref 同源,定位唯一准确 |
| F10 | 仅说明变化、依赖变化、旧 SSE 包、无 token | token 正确失效,嵌入 strict 拒绝过期/无身份包 |
| F11 | 来源改删、历史 generation 未留原文、同名歧义 | 明确 stale/unavailable/ambiguous,无猜测替代 |
| F12 | no-js/390px/暗色/长表格/键盘 | 正文可读,可返回,有清晰标题和焦点 |
| F13 | 原始 HTML/危险 URL/闭合 script/越界路径 | 无脚本执行/越权读取/泄漏;不靠应用猜净化 |
| F14 | 两项目同相对路径/并发 100 读取 | scope 缓存隔离,同 snapshot 合并分析 |
| F15 | 编译成功但验证/证据未知、旧执行记录 | 各状态独立;不把 static/sourced 标 verified |
| F16 | CLI/ext/native 版本错配 | capability 早发现不可用,而非运行时静默失败 |
性能目标需实测而非发版文字:固定机器与 release/debug 分开记录,1000 单元分析冷/热、单文档 fragment 体积/时间、并发读取 P95/内存、UI 首屏。应用目标见 Q 文档;如果全量 compile 超时,优先分析重复全库扫描、依赖计算与缓存,不通过删除 fixture 或抬高阈值宣布可用。
## 12. 本次验证结果与已知阻碍
### 12.1 M00–M06 落地证据(2026-10-25 续)
命令(本机):
```bash
cargo test --offline --locked --test workbench_contracts
cargo test --offline --locked -p marqdo_plugin_llm
cargo test --offline --locked --test executable_knowledge f1_agent_eng_bind_gate_mq_offline -- --exact
```
已交付并通过自动化子集(**非** F01–F16 全绿):
| 切片 | 证据 |
|---|---|
| M00 | fixtures + JSON Schema 消费;`capabilities`;`force_unloaded` / `force_*_off`(F16) |
| M01 | strict selection;`RunContextV1` 并行不串扰;llm plugin messages/roles;workbook 禁 `llm.llm`;**F02** `m01_f02_agent_step_selection_and_resume`(mock HTTP + `llm.create` 抗 env poison + resume save/load) |
| M02 | usage null≠0;revision 替换(F05);cancel token(F06/F07);event_sink;web `sse-live` 断连 cancel |
| M03 | `view.document`;CJK outline/nodes;path escape;静态 a11y 子集;F08 无副作用标记;**浏览器** `workbench-docview.html`(`article.mq-document`、0 script、无 `javascript:`) |
| M04 | stale / version_unavailable / **ambiguous**;`SnapshotManifestV1` 输出(retention=application);`artifact.version_mismatch`;embed strict SSE |
| M05 | extract `:B`;analysis_cache begin/finish;**F14 并发 100** 同 snapshot + 跨 project 隔离 |
| M06 | CI 预构建 agent+llm + `workbench_contracts`;Go `drainForward` / cancel-token 桥 |
| Live | `tests/workbench_live.rs`:USTC OpenAI-compatible(`OPENAI_*` env only)`qwen3.8-chat` / `deepseek-flash` selection ask → `pong` + usage;`MARQDO_EXT` 指向仓内 `ext/` 避免 `~/.marqdo/ext` 陈旧遮蔽 |
已知限制(发版前须书面接受或继续补):
- F02 repair/plan **全路径**假模型仍薄于 step+resume;RunContext 三级继承 + skeleton 禁 `llm.llm` + 子任务 env 已有。
- 历史正文由应用按 SnapshotManifest 保留;框架缺快照 → `version_unavailable`(不静默降级)。
- 完整宿主嵌入 F12 / 产品鉴权留给 qdagent;原生适配器 / PDF/OCR 仍 unavailable。
### 12.2 历史基线记录(设计起草时)
起草时 `executable_knowledge` 曾出现 `f1` 缺插件与 `v3_corpus_1000` debug 超时;其后 `ensure_agent_plugin_built` + CI 预构建已修复 f1 路径。1000 单元性能仍按剖面单独记录。
+163
View File
@@ -0,0 +1,163 @@
# Web 热路径:单解释器串行化(设计缺陷说明)
| | |
|---|---|
| 状态 | **Accepted**(缺陷记录 + 缓解已落地) |
| 日期 | 2026-10-26 |
| 规范决策 | [ADR 0008](../adr/0008-web-invoke-serialization.md) |
| 相关 | [ext-web-go-rewrite.md](ext-web-go-rewrite.md) · [ext-abi.md](ext-abi.md) · [ADR 0004](../adr/0004-web-plugin-go.md) · [ADR 0007](../adr/0007-web-document-endpoint.md) |
本文把「页面卡死 / 像单线程」拆成可验证的架构事实、错误语义与演进边界。
**结论先行:HTTP 是多线程的;卡住是因为所有 Endpoint 共享一把解释器锁。**
增量实施提案见 [隔离执行与响应式 Web 运行时(2026-10-26)](isolated-web-runtime-development-2026-10-26.md):独立进程池、控制/读取容量、请求级事件与取消、缓存和恢复。提案通过验收前,本文件及 ADR 0008 的现行单飞约束不变。
---
## 1. 问题陈述
### 1.1 用户可见症状
1. 发起一次长对话(OpenAI 兼容 `POST …/v1/chat/completions` + `stream:true` / `sse-live`)。
2. 对话尚未结束时,打开设置、拉 `/v1/models`、或其它短 JSON API。
3. **旧行为**:浏览器请求一直 pending,整页像死机;只能杀 `marqdo run serve…` 恢复。
4. **目标行为**:短请求在约 1.5s 内得到 **HTTP 503**,错误体标明 `interpreter busy`;对话 SSE 继续;结束后短请求立刻恢复 200。
### 1.2 错误归因(不要再说错)
| 错误说法 | 正确说法 |
|----------|----------|
| 「web 插件是单线程的」 | Go `net/http` 与 SSE 刷写是多 goroutine |
| 「忘记启动进程所以卡死」 | 进程在跑;是 **锁争用** 导致短请求永不返回 |
| 「把 SSE 的等待时间调大就能修好」 | 进程级拉长 `MARQDO_INVOKE_WAIT_MS` 会让短请求一起卡死 |
| 「去掉 INVOKE_LOCK 即可并行」 | `HostContext` 非 Sync,无锁并行会数据竞争 / 未定义行为 |
---
## 2. 调用栈与锁边界
```
Browser
│ GET /v1/models POST /v1/chat/completions (SSE)
▼
Go plugins/web (多 goroutine)
handleInvoke / handleInvokeLiveSSE
│
▼ C ABI host_query("call_lib_path", {path, args})
Rust src/host/plugin.rs
┌─────────────────────────────┐
│ INVOKE_LOCK (进程级 Mutex) │ ← 同一时刻只允许一个 worker 进入
└─────────────────────────────┘
│
▼
HostContext:装载/求值作者 .mq.md、写回、事件总线、LLM 插件回调…
```
要点:
1. **锁在宿主,不在 Go 业务路由表。** 增加更多 Go handler 不会自动并行执行 Marqdo。
2. **SSE 持锁时间 ≈ 整次 Endpoint 求值时间**(含多次 LLM 往返),可达数十秒到数分钟。
3. **事件转发**(`MARQDO_EVENT_FORWARD` NDJSON)与 **取消**(`MARQDO_CANCEL_TOKEN`)在持锁期间仍可由其它 goroutine 写文件/读桥;它们不进入解释器,故不受 `INVOKE_LOCK` 保护,也不应误设长等待。
---
## 3. 为何不能简单「多线程跑 Marqdo」
`HostContext` 持有(非穷尽):
- 当前模块图、函数表、写回缓冲与 FS 根;
- 插件注册与 `CURRENT_HOST` / 全局 host 指针;
- Agent / LLM 运行时与 stream 发布钩子;
- 与文档写回、会话态相关的可变结构。
这些状态在实现上按**单线程求值**建模。把它标成 `Sync` 或去掉 `INVOKE_LOCK` 而没有隔离运行时,属于未定义行为,不是性能优化。
**真并行的合法方向(未实现):**
1. **解释器池**:每请求或每会话一个 `HostContext`(或可重置的 isolate),路由层选闲置实例;
2. **只读快照 + 写队列**:热路径只读克隆,写回串行合并(适合纯查询 Endpoint,不适合带写回的 agent);
3. **把长任务移出解释器**:Marqdo 只做编排,重活在独立进程/队列(与「代码即文档」主路径权衡后再定)。
在上述任一方案落地并经金样例验收前,**产品必须以「单飞解释器」为约束设计 UX**。
---
## 4. 缓解契约(已实现)
### 4.1 宿主:限时争锁
文件:`src/host/plugin.rs`
- 仅当从 worker 线程进入(`CURRENT_HOST` 为空、走全局 host)时争 `INVOKE_LOCK`。
- `acquire_invoke_lock`:`try_lock` 循环,预算默认 **1500ms**(`MARQDO_INVOKE_WAIT_MS`,最小 1)。
- 超时:`set_err("interpreter busy: another request holds the Marqdo invoke lock (waited …ms)")`,返回非 0。
- **不要**在 SSE 入口把该环境变量设成数分钟:它是进程级的,会污染并发短请求。
### 4.2 Web 插件:503
文件:`plugins/web/internal/httpx/routes_invoke.go`
- 普通 JSON invoke:错误信息含 `interpreter busy` → **503** + `Retry-After: 2` + `{ok:false, error:…}`。
- Live SSE:若启动时即 busy,以失败事件/错误结束该流,**不得**无限挂起握手。
### 4.3 客户端:超时与文案
短 API(模型列表、设置、registry 探测等)必须:
- 设置合理 `fetch` 超时(秒级,例如 8s);
- 将 503 / `interpreter busy` 展示为「服务正忙(对话进行中)」类可读提示,而不是空白转圈。
(qdagent 前端 `frontend/src/lib/api.ts` 已按此契约处理。)
### 4.4 验收探针
在本地 serve 上:
```bash
# 终端 A:启动长 SSE 对话(持锁)
curl -N -m 120 -H 'Authorization: Bearer …' -H 'Content-Type: application/json' \
-d '{"stream":true,…}' http://127.0.0.1:PORT/v1/chat/completions
# 终端 B:对话进行中
curl -sS -w 'http=%{http_code} time=%{time_total}\n' --max-time 5 \
-H 'Authorization: Bearer …' http://127.0.0.1:PORT/v1/models
# 期望:http=503,time≈1.5,body 含 interpreter busy
# 对话结束后再打同一 URL → http=200,time≪1s
```
---
## 5. 运维与调参
| 变量 | 默认 | 含义 |
|------|------|------|
| `MARQDO_INVOKE_WAIT_MS` | `1500` | worker 进入解释器前最长等待;超时 → busy |
建议:
- 开发机 / 产品默认保持 1–3s。
- **禁止**为「让第二个聊天排队」而把默认调到分钟级——那等于恢复卡死。
- 需要排队时,应在 **应用层**(会话队列、UI 禁用「再发一条」),而不是拉长宿主锁等待。
---
## 6. 与其它机制的边界
| 机制 | 与 INVOKE_LOCK 的关系 |
|------|----------------------|
| `MARQDO_EVENT_FORWARD` | SSE 期间写 NDJSON;不替代争锁;不延长等待 |
| `MARQDO_CANCEL_TOKEN` | 断开连接时可取消 LLM;释放锁仍取决于解释器何时回到 FFI |
| `web.serve` / 静态资源 | 通常不经 `call_lib_path`,不受本锁影响 |
| 插件内部 Tokio/Go 并发 | 可以;只要不再次 `call_lib_path` 进同一 HostContext |
---
## 7. 演进检查清单(根治前勿宣称「已支持并行 Endpoint」)
- [ ] 多 `HostContext` 或 isolate 的生命周期与 ABI(装载、shutdown、写回冲突)有设计稿与金样例。
- [ ] 池化后 `call_lib_path` 不再依赖进程级单锁,或锁粒度降到实例级。
- [ ] 两个并发长 SSE + 一个短 `/v1/models` 的压力测试:短请求稳定 200,长请求互不污染会话写回。
- [ ] 更新本文件与 ADR 0008 Status(例如改为 Superseded by ADR NNNN)。
在清单完成前,对外与文档中应明确:**Marqdo Web Endpoint 求值在单进程内串行;HTTP 层并发,解释器层单飞。**
+124 -1
View File
@@ -450,4 +450,127 @@ AI 优先使用既有知识,而非反复解释和重新生成
如何让一个 AI 在面对新任务时,能够像编译器复用已有函数一样,直接复用知识库中已经编译的知识,而不必重新理解和生成它?
如果 Marqdo 能真正解决这个问题,你原来的三句话就不再只是设计理念,而是一个可以通过实验验证的语言能力。
如果 Marqdo 能真正解决这个问题,你原来的三句话就不再只是设计理念,而是一个可以通过实验验证的语言能力。
# Marqdo 可执行知识体系:设计规范与完整范例
我已经基于 Marqdo 当前仓库的语法规范、`[]()` 设计、知识分层、对象模型和 EKC,整理了两份详细文档。
这次设计保留了你的原始理念:
> 代码即文档,文档即知识库,知识即可执行。
核心目标不是让 AI 更容易生成 Marqdo 代码,而是让 AI 尽可能不必重新生成已经存在的知识,转而通过编译器确定性地发现、引用、验证和执行既有知识。
第一份:完整架构设计规范
涵盖语义制品、知识引用、编译器架构、解析优先级、效应安全、版本管理、EKC 集成、Agent 复用协议和实施验收。
阅读设计文档
第二份:多文件可执行知识范例
以虚构书店定价为例,从原始政策文档开始,逐步构造知识定义、对象、计算能力、业务入口、AI 问答、知识复用和错误验证。
阅读完整示例
两份文档打包
下载 ZIP
## 一、这份设计最重要的三个决定
# Marqdo 可执行知识体系:核心设计纲领
**核心理念:代码即文档,文档即知识库,知识即可执行。**
## 1. 知识是可编译的语义制品
文档经过 Marqdo 编译器处理后,不只产生可执行逻辑,还应产生具有稳定身份、类型、签名、契约、依赖关系、来源和验证状态的语义制品(Semantic Artifact)。
知识可以表现为事实、函数、能力、约束、数据集或类型。它们无需全部被强制编码为类。
## 2. `[]()` 是知识使用的统一入口
`[描述](目标)` 保留 Markdown 的可读性。
- `[]` 面向人类,描述意图、意义或用途。
- `()` 面向编译器,包含既有 Marqdo 表达式或者可解析的知识目标。
- 描述不改变执行结果,也不能作为可信性依据。
- 编译器负责将目标解析为确定的符号身份,并验证使用条件。
既支持隐藏不易阅读的代码,也支持引用已经编译的知识。
## 3. AI 复用知识,而不是重新解释知识
AI 面对新任务时,应该遵循:
**Discover → Resolve → Verify → Reuse → Execute → Trace**
当知识缺失或不适用时,才进入 ADAPT 或 CREATE 流程。
检索可以模糊,但最终绑定必须确定。验证必须区分程序可执行性、契约符合性和现实事实正确性。
## 4. 编译器是知识系统的基础
AST、Semantic IR、符号索引、链接关系和知识验证应当由同一套语义体系产生。
解释器、字节码 VM、View、EKC 和 Agent 消费统一的编译结果,而不是各自重新解析文档。
## 5. 最终验收标准
一个 AI 能够发现现有知识、确定其身份、检查适用条件、直接执行既有能力、引用结果和来源,并在知识发生变化时识别受影响的调用者。
只有这条链条成立,Marqdo 才真正实现知识的可执行复用。
## 二、示例将展示什么
我选择了一个具有明确规则和复用价值的业务案例,而不只是使用“李白出生年份”这样的静态事实。
用户购买两件单价 60 元的商品,金卡折扣为 90%,满 100 元免运费。
AI 要处理的业务请求
> 请计算这笔金卡订单的最终应付款,并解释依据。
传统路径
检索政策 → AI 阅读规则 → 重新推导计算 → 生成新代码或答案
Marqdo 目标路径
发现定价能力 → 绑定已编译函数 → 检查输入与政策 → 执行 → 引用结果
目标语法示意(v2.1+,不是 v2.0.0 实测)
```
**应付 = [订单最终应付款](定价.计算应付金额(60, 2, "金卡"))**
```
根据演示政策计算
# ¥108
这套例子进一步展示了参数和返回值、类与继承、表格、条件分支、循环、跨文档调用、自然语言遮蔽,以及知识来源改变时的影响分析。
## 三、与当前版本的兼容边界
我特别保留了一个重要区分:设计目标不等于当前已经实现的功能。
根据这次读取的仓库内容,v2.0.0 已支持本地表达式遮蔽、对象、方法、控制流、表格集合、导入以及现有 EKC 基础;但无实例类函数调用、跨文档知识自动解析、完整 LinkInfo 和语义制品绑定仍需要进一步开发。
因此示例文档明确标注了目标语法和现有兼容写法,没有将尚未验证的代码描述为已经跑通。
## 四、我的主要架构建议
我认为接下来的开发不应该从 `[]()` 的外观继续扩展,而应该先实现:
统一符号索引 → 稳定知识身份 → 引用绑定 → 契约验证 → EKC 复用。
还有一点值得特别强调:我不赞成为了建立知识系统,再单独实现一个与 Marqdo 解释器并列的“知识编译器”。那样会重新制造你希望消除的表示碎片。
更合适的架构是让现有 Marqdo 编译管线产出统一的语义中间表示,然后由运行时、View、EKC 和 Agent 各自消费。
这意味着 Marqdo 的知识体系不是一个外挂的 RAG 框架,而是语言本身的语义能力。
我认为这才是整个设计中最值得投入资源,也最有可能形成独立研究贡献的部分。
@@ -0,0 +1,102 @@
# 智能体能力调研与求道差距复核(2026-10-27)
| 项目 | 说明 |
|---|---|
| 状态 | Research / 设计输入;不是功能交付声明 |
| 调研日期 | 2026-10-27,Asia/Shanghai |
| 源码基线 | Marqdo `16f26d7d60ce9fbc62feb0e8e83f93951f7b53fe`;qdagent `68b757e02ebfb64833b5262b0c8a683512581e8b` |
| 框架设计 | [Agent 恢复、工具与资源协议](../design/agent-reliability-tools-resources-2026-10-27.md) |
| 产品设计 | [求道智能体能力与预览体验](../../../qdagent/doc/design/agent-capabilities-and-preview-2026-10-27.md)(同级仓库) |
## 1. 方法与结论边界
先以 `git pull --ff-only` 同步两仓,再追踪真实调用链与前端入口;对比公开官方文档,不以营销介绍、工具数量或旧设计中的 done 标签代替实现证据。本次是静态源码复核与官方资料研究,没有启动部署实例,没有调用付费模型,没有进行竞品账号实测。因此,下表“源码确认”仅代表代码路径可见,“设计判断”与“待实测”不能写成已复现或已交付。
主流能力的共同方向是:可恢复的执行、结构化工具与资源、按需上下文、可追溯结果、独立产物展示。不同产品和模型支持范围不同;这里不是市场占有率调查,也不声称所有智能体都具备全部能力。求道应保留文档原生、EKC、验证后沉淀的定位,吸收机制,不引入第二套 Agent Runtime。
## 2. 官方资料与可借鉴机制
以下页面均在调研日实际打开;MCP 特意引用固定 `2025-06-18` 版本,作为兼容基线,不宣称它是最新协议。厂商专有能力必须探测,不能因为端点兼容 `/v1` 就认为全部可用。
| 编号 | 官方来源 | 已核实内容 | 对求道的设计启示 |
|---|---|---|---|
| S01 | [OpenAI Agents:集成与可观测性](https://developers.openai.com/api/docs/guides/agents/integrations-observability) | MCP 接入与运行、模型、工具等 tracing | 统一 call/attempt/span 关联;产品只显示阶段摘要,详细事件可展开 |
| S02 | [OpenAI:Memory 与 Compaction](https://developers.openai.com/cookbook/examples/agents_sdk/building_reliable_agents_memory_compaction) | 区分当前任务上下文压缩、跨运行记忆、可审阅产物 | 历史不要无限拼接;完整产物不等于聊天全文 |
| S03 | [LangGraph Persistence](https://docs.langchain.com/oss/python/langgraph/persistence) | checkpointer 保存 thread 状态,store 支撑跨 thread 数据 | 恢复执行状态和用户长期知识是两种生命周期 |
| S04 | [LangGraph Interrupts](https://docs.langchain.com/oss/python/langgraph/interrupts) | 恢复时节点代码可能重新执行 | 检查点不能单独保证写操作只执行一次,必须记录效应与幂等键 |
| S05 | [Claude Tool search](https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool) | 工具定义按需进入模型上下文 | 工具多不代表每轮注入全部定义;检索后再校验权限 |
| S06 | [Claude Programmatic tool calling](https://platform.claude.com/docs/en/agents-and-tools/tool-use/programmatic-tool-calling) | 在执行环境中编排工具、筛选结果 | 大批量只读处理可减少模型往返;后置于基础工具可靠性 |
| S07 | [Deep Agents overview](https://docs.langchain.com/oss/python/deepagents/overview) | 文件系统、上下文管理、子任务与可选规划能力 | 规划应按任务启用;简单问答不强制复杂工作流 |
| S08 | [Gemini Tools](https://ai.google.dev/gemini-api/docs/tools) | 检索、URL、文件搜索、代码执行等工具面 | 覆盖“找资料—读内容—计算—形成产物”,不能只有工程检索 |
| S09 | [Gemini File search](https://ai.google.dev/gemini-api/docs/file-search) | 文件检索与引用信息 | 命中要能定位原文,检索摘要不能代替源文件版本 |
| S10 | [MCP Tools](https://modelcontextprotocol.io/specification/2025-06-18/server/tools) | schema、结构化结果、资源链接;annotations 有信任边界 | 工具结果包含短摘要和资源引用;注解不是权限凭证 |
| S11 | [MCP Resources](https://modelcontextprotocol.io/specification/2025-06-18/server/resources) | URI 资源、读取和应用决定的上下文注入 | “预览”“加入上下文”“执行”必须是独立动作 |
| S12 | [Claude Artifacts](https://support.claude.com/en/articles/17153992-what-are-artifacts-and-how-do-i-use-them) | 产物在会话旁独立打开,可继续查看与编辑 | 右栏应承载正文;不应只提供离开当前会话的入口 |
| S13 | [Mozilla PDF.js](https://mozilla.github.io/pdf.js/) | Web PDF 解析和渲染 | PDF 视觉预览和 OCR/知识提取分开交付 |
| S14 | [DOMPurify 官方仓库](https://github.com/cure53/DOMPurify) | DOM 清洗、标签/属性配置;插入前清洗 | 替换前端正则“清洗器”,配合同源资源代理和 CSP |
上述设计启示是本次推导,相关性能收益、成本下降比例不照搬厂商基准。工程路线不绑定某个最新模型或套餐。
## 3. 源码事实清单
下列 qdagent 路径相对该仓库;Marqdo 路径相对本仓库。基线 commit 已固定,后续实现需按符号再定位。
| 编号 | 当前实现证据 | 结论与限制 |
|---|---|---|
| F01 | qdagent `agent/qdagent.mq.md::契约提示`、`ask` | ASK 和 BUILD 都要求完整六段式 `.mq.md`,包含 main、验证和元数据;普通回答变长有明确结构性诱因。不能断言这是唯一原因 |
| F02 | 同文件 `ask` 的门禁修复分支 | `r2` 送入第二次门禁后,成功分支仍 `result = raw`;可能展示修复前内容,设计必须绑定同一 accepted candidate |
| F03 | 同文件 `ask` 返回表 | `gated_error` 路径仍可能保留 `status=ok` 且返回 `ok=True`;必须明确业务失败与输送成功的不同含义 |
| F04 | 同文件 `eng_tools` | 默认白名单是 `tool_find/reuse/bind/verify/summarize` 五个工程工具;不是整个框架只有五个工具,也不代表已覆盖网页、笔记、文件编辑 |
| F05 | `ext/ai/agent.mq.md` | 有 step/plan/run、READ、CALL、观察压缩、工作簿和恢复接口;应扩展这些原语,不另造循环 |
| F06 | `plugins/agent/src/resume.rs::resume_save` | JSON 直接 `fs::write`;现有恢复接口不等于具备原子检查点、工具效应对账及任意步骤自动续跑 |
| F07 | `src/runtime/`、`src/workbench/capabilities.rs` | 已有隔离执行、取消、事件回放与读通道;这一轮是 Agent 级恢复补足,不重新规划整个进程池 |
| F08 | qdagent `session-store.ts::resumeTurn` | 终态 completed/failed/interrupted 直接返回;已有事件续订,但终态消息补齐与中断后的执行恢复需分别处理 |
| F09 | qdagent `DesktopFrame.tsx::RightNotesPanel` | 实际只有“查看全部笔记”导航至 `/notes`,没有侧栏笔记正文 |
| F10 | 同文件 `RightFilesPanel`、`WorkspacePage.tsx::openFile` | 文件点击已调 `openTarget`;右栏取首个 workspace,根目录最多 40 个非目录项。不能笼统写“所有按钮都跳转” |
| F11 | `DesktopFrame.tsx` 的 reader/rightbar 条件 | `DocumentReader` 与右侧 tab 容器互斥,且使用两套开关/宽度状态;不是统一的资源浏览面板 |
| F12 | qdagent `NotesPage.tsx` detail | 剥离 frontmatter 后以 `whiteSpace: pre-wrap` 显示文本;没有 Markdown/Marqdo 正文结构渲染 |
| F13 | qdagent `DocumentReader.tsx` | 已有 DocumentView、引用版本传递、请求代数防迟到覆盖;`profile_item` target 未进入读取分支,格式不是通用资源分派 |
| F14 | 同文件 `sanitizeHtmlFragment` | 主要靠正则移除 script/事件/javascript URL;不能当作完整 HTML 白名单安全边界。未在本次做漏洞利用实验 |
| F15 | `src/workbench/document_view.rs` | 有只读文档视图,注释与实现明确避开执行路径;已解析语义应继续复用 |
| F16 | `src/workbench/capabilities.rs` | text/markdown 提取可用,pdf/docx/ocr 标 unavailable;提取能力与浏览器 PDF 预览能力须分别探测 |
| F17 | qdagent `lib/web_search.mq.md` | 既有网页搜索桥使用 legacy 脚本和固定 `last_web_search.json`;存在并发覆盖设计风险,不能直接作为新工具通道 |
| F18 | `plugins/agent/src/mcp_client.rs` | 已有 stdio connect/list/call/close;不能把本轮描述为“从零添加 MCP”,远程连接和资源能力需另验 |
| F19 | qdagent `compose_messages` / `ask` | 对话按角色标记拼成字符串注入 standing;会话语义、上下文预算和外部内容信任层级仍需强化 |
| F20 | qdagent `lib/workbench.mq.md::documents_read` | 临时文件名以秒和环境 RANDOM 构造,RANDOM 缺失又退到秒;设计仍需真正唯一临时文件。workspace 缺省 `.` 不应作为新授权资源接口规则 |
这些是本轮优先级依据。10-26 release notes 所列修复作为已有基础保留,但“Implemented(本仓)”不能替代浏览器、真实 HTTP、进程中断与模型输出的联合验收。
## 4. 能力覆盖与差距优先级
| 能力 | 当前基础 | 本轮目标 | 优先级 / 归属 |
|---|---|---|---|
| 自动纠错 | 一次文档门禁修复、plan patch | 分类错误、有限修复、正确结果替换、可恢复失败 | P0 / 两仓 |
| 执行恢复 | 任务隔离、取消、事件回放、JSON checkpoint | 传输续订与执行续跑分离;效应对账 | P0 / Marqdo 原语,qdagent UX |
| 回答长度 | 六段式文档、完整历史 | 直接回答、摘要+产物、显式详述三种表达 | P0 / qdagent 策略,Marqdo 输出通道 |
| 资源阅读 | DocumentView / Locator | 笔记、工作文件、引用、产物统一入口 | P0 / 两仓 |
| 侧栏预览 | 两套状态与导航入口 | 单一右栏,点击即预览,聊天不中断 | P0 / qdagent |
| 工具使用 | 五工具默认白名单、MCP/库原语 | schema 注册、按需发现、资源读取/搜索/写入/验证 | P1 / 两仓 |
| 联网研究 | legacy 搜索脚本 | search+fetch、来源时间、限量正文、可定位引用 | P1 / 两仓 |
| 上下文与记忆 | EFI pack、profile、history | 分层上下文、预算、可编辑记忆、资源 pin | P1 / 两仓 |
| 多格式 | 文档视图、文本提取 | text/md/mq/code/json/csv/image;PDF 后续 | P0→P1 / 两仓 |
| 长任务与计划 | plan 工作簿 | 里程碑、预算、暂停/继续、产物进度 | P1 / 两仓 |
| 多智能体与浏览器 | 不作为当前完成前提 | 有界委派、隔离执行、受控浏览器工具 | P2 / 框架主导 |
| 可观测评测 | usage、事件与既有测试 | 工具/恢复指标、实际任务语料、回归门禁 | P0 起持续 / 两仓 |
## 5. 采用与暂缓的决策
采用:有界失败恢复;工具结果摘要+资源引用;结构化参数;步骤效应账本;产物与聊天投影分离;资源预览与“加入上下文”分离;用户可检查的引用;按需工具发现。原有源码、知识、工作簿与验证证据仍是权威,衍生视图可重建。
暂缓:全自动跨站电脑操作、任意外部 MCP 自动信任、默认多智能体协作、所有 Office 格式保真编辑、每次问答都规划/运行代码。它们不能先于当前恢复、工具授权和预览闭环。
不采用:把长回答直接 substring 截断、把失败原文当最终成功、前端断流即重发任务、把所有文件当 HTML、用网页内容改写 system 指令、把预览等同于执行可执行文档。
## 6. 必须补做的动态验证
1. 在真实聊天中验证六段式输出与门禁失败/修复的最终消息和持久产物是否一致。
2. 丢弃最后一帧、断网、刷新、worker 崩溃分别注入,确认事件重放不重做工具副作用。
3. 使用真实 HTTP 入口验证右栏笔记、嵌套文件、历史引用、跨 workspace 资源;不只调用兄弟辅助函数。
4. 用恶意/畸形 HTML、路径与符号链接、过大文件验证预览隔离;no-exec 通过哨兵副作用测试。
5. 对无凭证模型、text-only provider、不可用转换器做明确降级;记录版本与环境,禁止以 mock 成功声称上线成功。
后续设计文档中的时间、轮数、字符数、成功率阈值均为拟定验收目标,需用该语料校准。
@@ -0,0 +1,194 @@
# 跨仓 Agent 可靠性开发计划(2026-10-27)
| 项目 | 约定 |
|---|---|
| 状态 | **Proposed / planning**;本文是排期与验收编排,**不是**已实现声明 |
| 日期 | 2026-10-27(Asia/Shanghai) |
| 基线 | Marqdo `6ffe5ce` / 2.1.4;qdagent `c4c976c`(设计文档基线曾为 `16f26d7` / `68b757e`;排期以当前 HEAD 为准) |
| 协议真源 | [智能体能力补足设计](../design/agent-reliability-tools-resources-2026-10-27.md) |
| 产品真源 | [qdagent 智能体能力与预览](../../../qdagent/doc/design/agent-capabilities-and-preview-2026-10-27.md)(同级 checkout) |
| 证据 | [官方调研与 F01–F20](../research/agent-capabilities-2026-10-27.md) |
| 产品侧镜像 | [qdagent 开发计划镜像](../../../qdagent/doc/roadmap/agent-capabilities-development-2026-10-27.md) |
本文只编排依赖、落点、门禁与发布次序。**不发明第二套 Agent Framework**;不修改 `doc/next/`;不把 capability bit 在合同实测前标真。协议字段以设计文档为准,路线图只引用不另造 schema。
## 1. 目标与架构边界
让同一套文档原生 Agent 在模型错误、工具失败、网络中断后有界恢复;工具可发现、可验证、可审计;结果同时服务模型上下文与用户阅读。保留 `.mq.md` 工作簿、EKC 与验证证据的权威地位。
| 议题 | Marqdo 所有 | qdagent 所有 |
|---|---|---|
| 恢复 | ErrorV1、预算、CheckpointV2、效应账本、运行/事件语义 | 恢复按钮、会话关联、凭证修复入口、产品政策 |
| 工具 | ToolSpec/Call/Result、schema、权限执行、并发、MCP 传输 | 业务工具、默认集合、连接配置、用户授权范围 |
| 回答 | AgentOutput 分通道、输出预算、ContextPackV2 | 简洁/标准/详细策略、摘要与产物卡、ASK/BUILD |
| 资源 | ResourceRef/View、只读预览、转换插件接口 | 笔记/文件/产物 ID 映射、owner 权限、预览布局 |
| 知识 | preflight/reuse/verify/compile、记忆原语 | 何时提议沉淀、用户画像与可见管理 |
| 调试 | span、结构化事件、用量与错误码 | 可读过程、运行详情、产品评测语料 |
## 2. 现状(不再重做)
| 已落地基础 | 说明 |
|---|---|
| Agent A0–A4、O1–O5 | 文档原生 step/plan/run、沙盒与优化波次;扩展 `ext/ai/agent`,不另造循环 |
| 隔离 Web 运行时(ADR 0009) | `src/runtime/` Verified;本轮补 Agent 步骤语义,不重建进程池 |
| Workbench 合同切片 | DocumentView / Locator / capabilities;Partial,随本轮补 F 矩阵 |
| 可执行知识 S0–S6 | 本地确定性子集已门禁;**不重开 S0–S6** |
本轮增量(设计已锁、代码待做):`AgentErrorV1`、`CheckpointV2` + effect ledger、`ToolSpecV1` 统一调度、`ResourceViewV1`、`AgentOutputV1` 通道、`ContextPackV2`。
**产品措辞约束**:本地门禁绿 ≠ 远程/多租户安全;≠「知识验证 + Agent 复用闭环」全产品完成。
## 3. 跨仓依赖与日历顺序
```mermaid
flowchart TD
MA0[M-A0 contracts] --> QA0[Q-A0 fixtures]
MA0 --> MA3[M-A3 ResourceView]
MA0 --> MA1[M-A1 Error and Checkpoint]
MA0 --> MA2[M-A2 ToolSpec]
QA0 --> QA1[Q-A1 answer outcome]
QA0 --> QA2[Q-A2 right panel]
MA3 --> QA2
QA1 --> QA3[Q-A3 recovery UX]
MA1 --> QA3
MA2 --> QA4[Q-A4 product tools]
MA1 --> MA4[M-A4 channels ContextPack]
MA2 --> MA4
MA4 --> QA4
MA3 --> QA5[Q-A5 formats]
MA5[M-A5 ledger PDF MCP] --> QA5
MA4 --> QA6[Q-A6 plan memory]
MA5 --> QA7[Q-A7 connectors]
MA6[M-A6 subagents] --> QA7
```
### 3.1 强制顺序原则
1. 先 **M-A0 ∥ Q-A0** 冻结跨仓字段与缺陷 fixtures。
2. 立刻 **Q-A1 + Q-A2 + M-A3(文本/md/mq)**——**不得**被完整 CheckpointV2 阻塞。
3. **M-A1** 可先上只读重试 + 原子 checkpoint;**Q-A3** 终态快照可先行,写恢复等 effect ledger。
4. 再 **M-A2 ∥ Q-A4** → **M-A4 / Q-A6** → **M-A5 / Q-A5** → **M-A6 / Q-A7**。
5. 四类恢复必须分开:事件续订 / 步骤重试 / 工作流 resume(新 attempt + `resumed_from`)/ 答案再生成。
## 4. 阶段包明细
### Wave 0 — 合同
| 包 | 修改/新增建议 | 交付内容 | 退出门槛 |
|---|---|---|---|
| **M-A0** | `src/workbench/capabilities.rs`、`tests/workbench_contracts.rs`、新增合同 fixtures | Error/Tool/Resource/Output/Context 协议、旧适配、状态映射;capability **默认 false** | 跨仓字段冻结;未知字段/版本可降级 |
| **Q-A0** | 缺陷样例、HTTP 路由清单、answer/resource fixtures;设计入口 | 固化 F01–F20 相关 fixtures;`mock`/`live` 证据区分 | 与 M-A0 字段一致;不把 mock 当 live |
### Wave 1 — 可见痛点
| 包 | 修改/新增建议 | 交付内容 | 退出门槛 |
|---|---|---|---|
| **Q-A1** | `agent/qdagent.mq.md`、`gateway/openai`、api 映射 | accepted candidate;`outcome` ≠ HTTP `ok`;摘要投影;ASK 不强制六节 | 修复后只展示 B;未通过不标成功(F01/F02/F03) |
| **M-A3(首批)** | `src/workbench/document_view.rs`、新 `resource_view`、媒体适配 | ResourceRef/View;md/txt/code/mq 静态阅读;no-exec | 资源定位与安全语料通过(F15) |
| **Q-A2** | `DesktopFrame`、panel store、`DocumentReader`、ResourcePanel/Preview | 统一右栏;笔记/文件点击不导航;同 renderer | 所有入口同预览(F09–F14) |
### Wave 2 — 恢复
| 包 | 修改/新增建议 | 交付内容 | 退出门槛 |
|---|---|---|---|
| **M-A1** | `ext/ai/agent.mq.md`、`plugins/agent/src/resume.rs`、recovery/effects 模块 | ErrorV1、有界修复、accepted candidate 绑定、CheckpointV2 原子写 | 故障注入不重复写;旧记录不冒险自动恢复(F06) |
| **Q-A3** | `execution/turns`、session store、`api.ts` | 终态快照补齐、恢复动作、错误卡 | 事件恢复 0 工具调用;写恢复须 ledger(F08) |
### Wave 3 — 工具与通道
| 包 | 修改/新增建议 | 交付内容 | 退出门槛 |
|---|---|---|---|
| **M-A2** | `plugins/agent/src/lib.rs`、`mcp_client.rs`、tool registry/dispatcher | ToolSpec/Call/Result;CALL+native 单通道;分页与结果资源 | 未授权调用为 0;失败结果结构正确(F04/F05/F18) |
| **M-A4** | `ext/ai/agent.mq.md`、`plugins/llm`、`src/workbench/run_context.rs` | AgentOutput 通道、预算、ContextPackV2 | 原生与纯文本 provider fixture 可用(F01/F19) |
| **Q-A4** | `agent/tools/`、工具目录、ContextAttachment | 业务工具;保留旧五工程工具别名 | 找→读→改→验证→预览真实闭环 |
| **Q-A6** | 计划折叠、预算 UI、profile 忘记、候选沉淀 | ResponsePolicy UX、记忆边界 | 压缩与恢复不丢用户约束 |
### Wave 4 — 持久与格式
| 包 | 修改/新增建议 | 交付内容 | 退出门槛 |
|---|---|---|---|
| **M-A5** | `src/runtime/` 存储/事件适配、MCP 与转换插件 | ledger↔runtime 对账、PDF/Office 插件、远程 MCP | crash/restart 通过;诚实探测(F16–F18) |
| **Q-A5** | JSON/CSV/图片/PDF UI、网页来源与页/行引用 | 格式扩展与转换进度 | 缺插件可读降级;PDF 不假称 OCR |
| **M-A6 / Q-A7** | plan/subtask/sandbox;MCP 配置与委派 | 程序化工具、有界委派、可选浏览器 | **后置**;须证明收益与预算/效应可控 |
## 5. F01–F20 → 包映射
| ID | 摘要 | Marqdo | qdagent | 主要验收 |
|---|---|---|---|---|
| F01 | ASK/BUILD 强制六节 | M-A4 输出通道 | Q-A1 ResponsePolicy | QA-01、MA-T12 |
| F02 | 修复后仍返回 raw | M-A1 accepted 绑定 | Q-A1 | QA-03、MA-T12 |
| F03 | `gated_error` 仍 `status=ok` | M-A0/M-A1 ErrorV1 | Q-A1 envelope | QA-04 |
| F04 | 默认仅五工程工具 | M-A2 registry | Q-A4 目录 | QA-17 |
| F05 | 已有 step/plan/run | **扩展** agent.mq.md | — | 不另造 loop |
| F06 | `resume_save` 裸 `fs::write` | M-A1 CheckpointV2 | Q-A3 UX | MA-T03/T04 |
| F07 | runtime 已有 | 复用,不重建 | 接线恢复 API | — |
| F08 | `resumeTurn` 终态直返 | 事件 vs 再执行分离 | Q-A3 快照补齐 | QA-13、MA-T05 |
| F09–F12 | 右栏/笔记预览缺陷 | M-A3 ResourceView | Q-A2 | QA-06–QA-09 |
| F13 | DocumentReader 非通用分派 | DocumentView 保留 mq;ResourceView 其它 | 预览分派 | QA-09 |
| F14 | 正则 HTML 清洗 | 生产侧 allowlist | DOMPurify + CSP | QA-11、MA-T11 |
| F15 | DocumentView no-exec | 保持;ResourceView 禁 run/verify | — | MA-T10 |
| F16 | pdf/docx/ocr unavailable | M-A5 插件 + 诚实探测 | Q-A5 | QA-10 |
| F17 | legacy web_search 临时文件 | 优先框架 net 工具 | 迁移工具 | QA-17、F20 |
| F18 | MCP stdio 已有 | M-A2/5 timeout/cancel/remote | 连接器配置 | MA-T07 |
| F19 | 历史字符串 standing | M-A4 ContextPackV2 | compose_messages | QA-19、MA-T13 |
| F20 | 弱临时名;workspace `.` | 安全临时文件 + ResourceRef | 资源解析 | QA-08、MA-T11 |
## 6. 验收与发布
### 6.1 Marqdo 门禁(MA-T01–T14)
以设计文档 §10 为准;基础机制用确定性 fake provider/tool;真实 provider 少量单独 smoke 并记费用。摘要:
| 编号 | 必须观察到 |
|---|---|
| MA-T01 | 429 后成功:退避、计量、单一 accepted answer |
| MA-T02 | schema 连续错:一次修复后停;无效参数不执行工具 |
| MA-T03 | 写成功但 receipt 前提杀:unknown_effect→对账;写计数不增 |
| MA-T04 | checkpoint 写一半杀进程:完整上一检查点或明确 corrupt |
| MA-T05 | 终态事件丢失:快照恢复完整消息;工具次数不增 |
| MA-T06 | 重复/并发恢复:同一幂等键一个 attempt |
| MA-T07 | 取消 vs 重试:取消后无新工具调用 |
| MA-T08 | 权限/版本变更:stale/conflict/forbidden |
| MA-T09 | 伪造 CALL/system:不越白名单 |
| MA-T10 | `.mq.md` 预览含写/联网:哨兵不变;只读或安全降级 |
| MA-T11 | 格式/超长/路径逃逸:正确 renderer;越界拒绝;无注入 |
| MA-T12 | 修复 A→B:answer/artifact/verification 均绑定 B digest |
| MA-T13 | 上下文压缩后:约束、待决效应、版本、预算不丢 |
| MA-T14 | 旧 CALL/DocumentView:基础可用;缺失能力明确返回 |
先跑受影响的 Rust/Go/Marqdo 合同与隔离 runtime 金样例,再做跨仓 HTTP/browser gate。性能目标(须记录机器/并发/冷暖缓存):已缓存 100 KiB 文本视图服务端 p95 &lt; 300 ms;核心工具发现 p95 &lt; 100 ms。
### 6.2 qdagent 门禁(QA-01–QA-20)
以产品设计 §13 为准;路径必须 `serve → api → 权威业务函数 →` 真实前端。零容忍:重复写效应、预览执行副作用、跨 workspace 越权。
实现时:`python3 scripts/check-contracts.py`、相关 workbench 测试、前端 `npm test` / `npm run build`,再跑 Playwright;无服务时 E2E 须明确 skip/失败原因。
### 6.3 Feature flags 与灰度
建议开关:`agent_output_v2`、`resource_panel_v2`、`agent_tools_v1`、`agent_recovery_v2`。启用取决于 runtime capability + 服务端政策,不由前端单独决定。
灰度:内部单 workspace → 多 workspace → 真 provider/断网 → 默认开启。关闭恢复时必须停止接纳新恢复请求并保留 attempt 状态,**禁止**切回会盲重做写操作的旧路径。
兼容迁移:新增协议 → 双读 → 新写 → 验证 → 逐步移除旧适配。旧 resume JSON 可读但不能自动升级成带副作用保证的检查点。
## 7. 伴生轨(登记,不阻塞主线)
| 轨 | 动作 |
|---|---|
| 可执行知识 S0–S6 | **不重开**;消费 BoundCall/EKC;话术保持「本地子集」 |
| 知识支撑 F0(M02/M03/M05) | **并行伴生**:provenance/closure invalidate、真 verify API、Skill 无冲突 |
| Workbench Partial | 随 M-A0/M-A3 补 F 矩阵与 adapters;不重写 M00–M05 |
| 语言 next-phase P3 / Wave B1 评测 | **延后** |
| Web Document/Endpoint | 遵守 ADR 0007;禁止 compose_/json bag 回退 |
## 8. 下一步开工(第一刀实现切片)
确认本路线图入库后,**另开实现回合**(本文件仅文档):
1. **M-A0 + Q-A0**:合同与 fixtures
2. **Q-A1**:raw→accepted、outcome 一致性
3. **Q-A3 子集**:终态快照补齐
4. **Q-A2 + M-A3 子集**:右栏真实预览(笔记 / md / mq / txt / code)
发布判据是 MA-T* / QA-* 与真实 HTTP/browser 证据,不是接口返回 200 或代码中出现某符号名。
+1 -1
View File
@@ -1 +1 @@
2.1.2
2.1.5
+790 -149
View File
File diff suppressed because it is too large. Load diff
+90 -23
View File
@@ -23,16 +23,27 @@ Load `.env` from cwd (optional `path=`). Does not override existing variables.
## collect
+ `events`
Collapse a stream event list to final answer text (`done.result` or `error.message`). Prefer this over raw SSE fields. Alias: `stream_result`.
Collapse a stream event list to final answer text.
Accepts semantic events (`delta` / `text` / `done.result` / `error.message`) and
native-plugin raw SSE frames (`type=data` with OpenAI chunk JSON in `data`).
Alias: `stream_result`.
**answer = ""**
- [ev](遍历 events 为 ev)
**t = > json.get value=`ev` key="type"**
1. `t` == "done"
**answer = > json.get value=`ev` key="result"**
**done_r = > json.get value=`ev` key="result"**
1. `done_r`
**answer = `done_r`**
2. *
**_ = 1**
2. `t` == "error"
**answer = > json.get value=`ev` key="message"**
**em = > json.get value=`ev` key="message"**
1. `em`
**answer = `em`**
2. *
**_ = 1**
3. `t` == "delta"
4. `t` == "text"
**chunk = > json.get value=`ev` key="text"**
@@ -40,9 +51,45 @@ Collapse a stream event list to final answer text (`done.result` or `error.messa
**answer = answer + chunk**
2. *
**_ = 1**
5. *
5. `t` == "data"
**raw = > json.get value=`ev` key="data"**
1. `raw`
1. `raw` == "[DONE]"
**_ = 1**
2. *
**parsed = > json.parse text=`raw`**
1. `parsed`
**choices = > json.get value=`parsed` key="choices"**
1. `choices`
**c0 = [1](access(choices, "1"))**
**delta = > json.get value=`c0` key="delta"**
1. `delta`
**piece = > json.get value=`delta` key="content"**
1. `piece`
**answer = answer + piece**
2. *
**_ = 1**
2. *
**msg = > json.get value=`c0` key="message"**
1. `msg`
**piece = > json.get value=`msg` key="content"**
1. `piece`
**answer = answer + piece**
2. *
**_ = 1**
2. *
**_ = 1**
2. *
**_ = 1**
2. *
**_ = 1**
2. *
**_ = 1**
6. *
**_ = 1**
1. not `answer`
**answer = ""**
*answer*
## stream_result
@@ -156,6 +203,7 @@ Prompt artifact helper: `path=` loads a file; `text=` strips `type: prompt` fron
Module convenience: create a default handle and return **answer text**. Pass `path=` to load a prompt document, `prompt=` text, or omit both to use the entry `.mq.md` as a prompt artifact (`type: prompt` body via `sys.module_source`).
**p = `prompt`**
1. `path`
**p = > prompt_load path=`path`**
2. `prompt`
@@ -268,20 +316,26 @@ Prefer declaring `model` / `api_key` / `base_url` in frontmatter Binding, then u
## ask
+ `prompt`=None
+ `path`=None
+ `messages`=None
+ `selection`=None
+ `strict_selection`=False
Semantic ask → **LLMResult** (`text`, `model`, `finish`, `backend`, `usage`, `name`, `tool_calls`). Prefer `[text](result)` for the answer string. Omit `prompt=` / `path=` to use the entry document (`sys.module_source` + `prompt_body`).
Semantic ask → **LLMResult** (`text`, `model`, `finish`, `backend`, `usage`, `name`, `tool_calls`). Prefer `[text](result)` for the answer string. Omit `prompt=` / `path=` to use the entry document (`sys.module_source` + `prompt_body`). Pass `messages=` for multi-turn history (M01). `selection=` + `strict_selection=True` freeze request identity without env fallback.
1. `path`
**p = `prompt`**
1. `messages`
**p = None**
2. `path`
**p = > prompt_load path=`path`**
2. `prompt`
3. `prompt`
**p = `prompt`**
3. *
4. *
**raw = > sys.module_source**
1. not `raw`
**p = ""**
2. *
**p = > prompt_body src=`raw`**
**pack = > self.complete_result prompt=`p` stream=False echo=False**
**pack = > self.complete_result prompt=`p` messages=`messages` selection=`selection` strict_selection=`strict_selection` stream=False echo=False**
**text = > json.get value=`pack` key="text"**
**usage = > json.get value=`pack` key="usage"**
**finish = > json.get value=`pack` key="finish"**
@@ -312,11 +366,16 @@ inside the table is unsafe.
**nm = > json.get value=`self` key="name"**
1. `usage`
**tt = > json.get value=`usage` key="total_tokens"**
**ustate = > json.get value=`usage` key="usage_state"**
2. *
**tt = 0**
1. not `tt`
**tt = 0**
2. *
**tt = None**
**ustate = "unknown"**
1. `ustate` == "unknown"
Keep null — never coerce missing usage to zero tokens (M02 / F04).
**tt = None**
2. not `tt`
**tt = None**
3. *
**_ = 1**
1. `err`
**calls = 0**
@@ -325,9 +384,9 @@ inside the table is unsafe.
`result` =
| text | model | finish | backend | usage | name | llm_calls | tokens | tool_calls | error |
|------|-------|--------|---------|-------|------|-----------|--------|------------|-------|
| `text` | `mdl` | `finish` | `b` | `usage` | `nm` | `calls` | `tt` | `tc` | `err` |
| text | model | finish | backend | usage | name | llm_calls | tokens | tool_calls | error | selection |
|------|-------|--------|---------|-------|------|-----------|--------|------------|-------|-----------|
| `text` | `mdl` | `finish` | `b` | `usage` | `nm` | `calls` | `tt` | `tc` | `err` | `selection` |
*result*
@@ -335,8 +394,10 @@ inside the table is unsafe.
+ `prompt`=None
+ `path`=None
+ `echo`=False
+ `stream_kind`="delta"
Semantic stream: return the event list (use `llm.collect` to reduce to text). Omit `prompt=` / `path=` to use the entry document.
`stream_kind=plan` tags EventBus frames for the parent plan protocol (process trail).
1. `path`
**p = > prompt_load path=`path`**
@@ -349,30 +410,35 @@ Semantic stream: return the event list (use `llm.collect` to reduce to text). Om
2. *
**p = > prompt_body src=`raw`**
*> self.complete prompt=`p` stream=True echo=`echo`*
*> self.complete prompt=`p` stream=True echo=`echo` stream_kind=`stream_kind`*
## complete_result
+ `prompt`
+ `prompt`=None
+ `messages`=None
+ `selection`=None
+ `strict_selection`=False
+ `stream`=False
+ `echo`=False
+ `stream_kind`="delta"
Transport pack via `ext/ai/llm/openai` (Ollama uses the same OpenAI-compatible path). Non-stream → `{text,usage,finish}`; stream → `{events}`.
Transport pack via `ext/ai/llm/openai` (Ollama uses the same OpenAI-compatible path). Non-stream → `{text,usage,finish}`; stream → `{events}`. Prefer `messages=` for multi-turn; optional `selection=` (ModelSelectionV1) with `strict_selection=True` for request-scoped identity (M01).
*> openai.chat_completions base_url=[base_url](access(self, "base_url")) api_key=[api_key](access(self, "api_key")) model=[model](access(self, "model")) prompt=`prompt` stream=`stream` echo=`echo` suffix=[suffix](access(self, "suffix")) bearer=[bearer](access(self, "bearer"))*
*> openai.chat_completions base_url=[base_url](access(self, "base_url")) api_key=[api_key](access(self, "api_key")) model=[model](access(self, "model")) prompt=`prompt` messages=`messages` selection=`selection` strict_selection=`strict_selection` stream=`stream` echo=`echo` stream_kind=`stream_kind` suffix=[suffix](access(self, "suffix")) bearer=[bearer](access(self, "bearer"))*
## complete
+ `prompt`
+ `stream`=False
+ `echo`=False
+ `stream_kind`="delta"
Internal / compatibility primitive. Prefer `ask` / `stream` in new code.
1. `stream`
**pack = > self.complete_result prompt=`prompt` stream=True echo=`echo`**
**pack = > self.complete_result prompt=`prompt` stream=True echo=`echo` stream_kind=`stream_kind`**
**_ret = > json.get value=`pack` key="events"**
*_ret*
2. *
**pack = > self.complete_result prompt=`prompt` stream=False echo=`echo`**
**pack = > self.complete_result prompt=`prompt` stream=False echo=`echo` stream_kind=`stream_kind`**
**_ret = > json.get value=`pack` key="text"**
*_ret*
@@ -380,7 +446,8 @@ Internal / compatibility primitive. Prefer `ask` / `stream` in new code.
+ `prompt`
+ `stream`=False
+ `echo`=False
+ `stream_kind`="delta"
Alias for `complete` (compat).
*> self.complete prompt=`prompt` stream=`stream` echo=`echo`*
*> self.complete prompt=`prompt` stream=`stream` echo=`echo` stream_kind=`stream_kind`*
+17 -6
View File
@@ -73,28 +73,39 @@ Extract OpenAI-shaped `choices[0].message.tool_calls` as a list (or empty list).
+ `base_url`
+ `api_key`
+ `model`
+ `prompt`
+ `prompt`=None
+ `messages`=None
+ `selection`=None
+ `strict_selection`=False
+ `auth_mode`=None
+ `stream`=False
+ `echo`=False
+ `stream_kind`="delta"
+ `suffix`=/chat/completions
+ `bearer`="Bearer "
POST `/chat/completions` (or stream SSE). Prefers `plugins/llm` ABI (`llm_chat_completions`) when the native plugin is resolvable; otherwise `lib/net` HTTP/SSE. Non-stream → `{text,usage,finish,tool_calls}`; stream → `{events,finish}`. Soft transport / HTTP failures return `finish=error` packs (never abort the VM).
Pass either `messages=` (role/content list) or legacy `prompt=` (single user message). Optional `selection=` is a ModelSelectionV1 snapshot; `strict_selection=True` refuses env fallbacks inside the plugin.
`stream_kind` tags live EventBus frames: `delta` (answer tokens) or `plan` (parent protocol — UI process trail, not the bubble).
**p = > plugin.native_path name="llm"**
1. `p`
> plugin.load path=`p`
*> llm_chat_completions base_url=`base_url` api_key=`api_key` model=`model` prompt=`prompt` stream=`stream` echo=`echo` suffix=`suffix` bearer=`bearer`*
*> llm_chat_completions base_url=`base_url` api_key=`api_key` model=`model` prompt=`prompt` messages=`messages` selection=`selection` strict_selection=`strict_selection` auth_mode=`auth_mode` stream=`stream` echo=`echo` stream_kind=`stream_kind` suffix=`suffix` bearer=`bearer`*
2. *
**url = base_url + suffix**
**auth = bearer + api_key**
**headers = > table.put in=None at="Authorization" value=`auth`**
`messages` =
1. not `messages`
`messages` =
| @ | role | content |
|---|------|---------|
| 1 | user | `prompt` |
| @ | role | content |
|---|------|---------|
| 1 | user | `prompt` |
2. *
**_ = 1**
`req` =
+50 -3
View File
@@ -20,14 +20,25 @@ description: >-
## 收集
+ `事件`
Collapse stream events to text. Supports semantic `delta`/`done.result` and
native-plugin raw frames (`type=data` + OpenAI chunk JSON).
**answer = ""**
- [ev](遍历 事件 为 ev)
**t = [type](access(ev, "type"))**
1. `t` == "done"
**answer = [result](access(ev, "result"))**
**done_r = [result](access(ev, "result"))**
1. `done_r`
**answer = `done_r`**
2. *
**_ = 1**
2. `t` == "error"
**answer = [message](access(ev, "message"))**
**em = [message](access(ev, "message"))**
1. `em`
**answer = `em`**
2. *
**_ = 1**
3. `t` == "delta"
4. `t` == "text"
**chunk = [text](access(ev, "text"))**
@@ -35,9 +46,45 @@ description: >-
**answer = answer + chunk**
2. *
**_ = 1**
5. *
5. `t` == "data"
**raw = [data](access(ev, "data"))**
1. `raw`
1. `raw` == "[DONE]"
**_ = 1**
2. *
**parsed = > json.parse text=`raw`**
1. `parsed`
**choices = [choices](access(parsed, "choices"))**
1. `choices`
**c0 = [1](access(choices, "1"))**
**delta = [delta](access(c0, "delta"))**
1. `delta`
**piece = [content](access(delta, "content"))**
1. `piece`
**answer = answer + piece**
2. *
**_ = 1**
2. *
**msg = [message](access(c0, "message"))**
1. `msg`
**piece = [content](access(msg, "content"))**
1. `piece`
**answer = answer + piece**
2. *
**_ = 1**
2. *
**_ = 1**
2. *
**_ = 1**
2. *
**_ = 1**
2. *
**_ = 1**
6. *
**_ = 1**
1. not `answer`
**answer = ""**
*answer*
## 流式结果
+13
View File
@@ -86,6 +86,19 @@ Like `select` but returns `{ rows, total }` — the total counts rows matching `
**url = [url](access(self, "url"))**
*> web_db_update url=`url` table=`table` id=`id` row=`row` txn=`txn`*
## update_if
+ `table`
+ `id`
+ `row`
+ `expect_col`
+ `expect_val`
+ `txn`=None
Conditional update (CAS) for task/revision control: `UPDATE … WHERE id=? AND expect_col=expect_val`. Returns `{ ok, updated, matched }`; `updated=0` means conflict — do not simulate atomicity with read-then-write in `.mq.md`.
**url = [url](access(self, "url"))**
*> web_db_update_if url=`url` table=`table` id=`id` row=`row` expect_col=`expect_col` expect_val=`expect_val` txn=`txn`*
## delete
+ `table`
+ `id`
+7 -4
View File
@@ -16,15 +16,17 @@ import:
+ `request`="json"
+ `response`="json"
+ `auth`=""
+ `class`=""
Declared route handle. Prefer `web.route` / Artifact `type: endpoint` discovery via `web.serve`.
Optional `class`: `execute` | `short` | `read` | `control` (empty → heuristic / `legacy_serial`).
> cap.load
`out` =
| _type | app | path | method | page | fn | request | response | auth |
|-------|-----|------|--------|------|----|---------|----------|------|
| route | `app` | `path` | `method` | `page` | `fn` | `request` | `response` | `auth` |
| _type | app | path | method | page | fn | request | response | auth | class |
|-------|-----|------|--------|------|----|---------|----------|------|-------|
| route | `app` | `path` | `method` | `page` | `fn` | `request` | `response` | `auth` | `class` |
*out*
@@ -57,6 +59,7 @@ Register this declared route on an app handle.
**method = [method](access(self, "method"))**
**request = [request](access(self, "request"))**
**response = [response](access(self, "response"))**
*> web_app_invoke app=`a` path=`path` method=`method` fn=`fn` body=`request` return=`response`*
**class = [class](access(self, "class"))**
*> web_app_invoke app=`a` path=`path` method=`method` fn=`fn` body=`request` return=`response` class=`class`*
3. *
*> table.put in=None at="ok" value=False*
+3 -1
View File
@@ -38,10 +38,12 @@ Construct a Document handle. Prefer `type: web` Artifact files discovered by `se
+ `request`="json"
+ `response`="json"
+ `auth`=""
+ `class`=""
Register a declared Document page or Endpoint (`fn` = `lib.member`).
Optional `class`: `execute` | `short` | `read` | `control`.
**r = > route_mod.route app=`app` path=`path` method=`method` page=`page` fn=`fn` request=`request` response=`response` auth=`auth`**
**r = > route_mod.route app=`app` path=`path` method=`method` page=`page` fn=`fn` request=`request` response=`response` auth=`auth` class=`class`**
*> `r`.register app=`app`*
## render
+266
View File
@@ -0,0 +1,266 @@
//! ContextPackV2 assemble + compact (M-A4).
use serde_json::{json, Value};
fn opt_text<'a>(args: &'a Value, key: &str) -> Option<&'a str> {
args.get(key)
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
}
fn item_tokens(item: &Value) -> u64 {
item.get("token_estimate")
.and_then(|v| v.as_u64())
.unwrap_or_else(|| {
item.get("text")
.and_then(|v| v.as_str())
.map(|s| (s.chars().count() as u64).div_ceil(2))
.unwrap_or(32)
})
}
fn normalize_item(raw: &Value, default_priority: i64) -> Result<Value, String> {
let origin = raw
.get("origin")
.and_then(|v| v.as_str())
.unwrap_or("unknown");
let trust = raw
.get("trust")
.and_then(|v| v.as_str())
.unwrap_or(match origin {
"system" | "policy" => "trusted",
"user" => "user",
"tool" => "tool",
"web" => "untrusted",
_ => "unknown",
});
// External / web content must never be trust=trusted.
if matches!(origin, "web" | "tool") && trust == "trusted" {
return Err("tool/web origin cannot have trust=trusted".into());
}
Ok(json!({
"origin": origin,
"trust": trust,
"resource_ref": raw.get("resource_ref").cloned().unwrap_or(Value::Null),
"revision": raw.get("revision").cloned().unwrap_or(Value::Null),
"token_estimate": item_tokens(raw),
"priority": raw.get("priority").and_then(|v| v.as_i64()).unwrap_or(default_priority),
"role": raw.get("role").cloned().unwrap_or(json!(origin)),
"text": raw.get("text").cloned().unwrap_or(Value::Null),
"kind": raw.get("kind").cloned().unwrap_or(json!("content")),
"retain": raw.get("retain").and_then(|v| v.as_bool()).unwrap_or(false),
}))
}
/// Build ContextPackV2 from items + optional budget.
pub fn context_pack_build(args: &Value) -> Result<Value, String> {
let raw_items = args
.get("items")
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
let mut items = Vec::new();
for (i, raw) in raw_items.iter().enumerate() {
items.push(normalize_item(raw, i as i64)?);
}
// Stable priority order: lower priority number first (more important).
items.sort_by(|a, b| {
let pa = a.get("priority").and_then(|v| v.as_i64()).unwrap_or(0);
let pb = b.get("priority").and_then(|v| v.as_i64()).unwrap_or(0);
pa.cmp(&pb)
});
let budget = args
.get("budget_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(8_000);
let reserve_pct = args
.get("output_reserve_pct")
.and_then(|v| v.as_u64())
.unwrap_or(20)
.min(50);
let available = budget.saturating_mul(100 - reserve_pct) / 100;
let used: u64 = items.iter().map(item_tokens).sum();
Ok(json!({
"schema": "ContextPackV2",
"schema_version": 2,
"items": items,
"budget_tokens": budget,
"available_tokens": available,
"used_tokens": used,
"output_reserve_pct": reserve_pct,
"compact_threshold_pct": args.get("compact_threshold_pct").and_then(|v| v.as_u64()).unwrap_or(70),
"goal": args.get("goal").cloned().unwrap_or(Value::Null),
"constraints": args.get("constraints").cloned().unwrap_or_else(|| json!([])),
}))
}
/// Compact pack when over threshold; must retain anchors listed in design §6.2.
pub fn context_pack_compact(args: &Value) -> Result<Value, String> {
let mut pack = if args.get("schema").and_then(|v| v.as_str()) == Some("ContextPackV2") {
args.clone()
} else {
context_pack_build(args)?
};
let available = pack
.get("available_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(6400);
let threshold_pct = pack
.get("compact_threshold_pct")
.and_then(|v| v.as_u64())
.unwrap_or(70);
let threshold = available.saturating_mul(threshold_pct) / 100;
let used = pack
.get("used_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(0);
if used <= threshold {
if let Some(obj) = pack.as_object_mut() {
obj.insert("compacted".into(), json!(false));
obj.insert("reason".into(), json!("under_threshold"));
}
return Ok(pack);
}
let items = pack
.get("items")
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
// Retain: goal/constraints (pack fields), retain=true, high priority (<=2),
// pending writes, failures — drop bulky tool dumps first (kind=tool_output, low priority).
let mut kept = Vec::new();
let mut dropped = Vec::new();
for item in items {
let retain = item.get("retain").and_then(|v| v.as_bool()).unwrap_or(false);
let priority = item.get("priority").and_then(|v| v.as_i64()).unwrap_or(99);
let kind = item.get("kind").and_then(|v| v.as_str()).unwrap_or("");
let trust = item.get("trust").and_then(|v| v.as_str()).unwrap_or("");
let must = retain
|| priority <= 2
|| matches!(kind, "goal" | "constraint" | "pending_write" | "plan" | "failure")
|| trust == "trusted";
if must {
kept.push(item);
} else if kind == "tool_output" || priority >= 8 {
dropped.push(item);
} else {
kept.push(item);
}
}
// If still over budget, drop lowest-priority non-retain from kept.
let mut kept_tokens: u64 = kept.iter().map(item_tokens).sum();
if kept_tokens > threshold {
kept.sort_by(|a, b| {
let pa = a.get("priority").and_then(|v| v.as_i64()).unwrap_or(0);
let pb = b.get("priority").and_then(|v| v.as_i64()).unwrap_or(0);
pb.cmp(&pa) // high priority number first for dropping
});
let mut next = Vec::new();
for item in kept.into_iter().rev() {
// rebuild ascending later
let retain = item.get("retain").and_then(|v| v.as_bool()).unwrap_or(false);
let priority = item.get("priority").and_then(|v| v.as_i64()).unwrap_or(99);
let t = item_tokens(&item);
if (retain || priority <= 1) || kept_tokens <= threshold {
next.push(item);
} else if kept_tokens > threshold {
kept_tokens = kept_tokens.saturating_sub(t);
dropped.push(item);
} else {
next.push(item);
}
}
next.sort_by(|a, b| {
let pa = a.get("priority").and_then(|v| v.as_i64()).unwrap_or(0);
let pb = b.get("priority").and_then(|v| v.as_i64()).unwrap_or(0);
pa.cmp(&pb)
});
kept = next;
kept_tokens = kept.iter().map(item_tokens).sum();
}
let digest_src = format!(
"kept={} dropped={} used={}",
kept.len(),
dropped.len(),
kept_tokens
);
let digest = format!("sha256-lite:{}", simple_hash(&digest_src));
if let Some(obj) = pack.as_object_mut() {
obj.insert("items".into(), json!(kept));
obj.insert("used_tokens".into(), json!(kept_tokens));
obj.insert("compacted".into(), json!(true));
obj.insert(
"compaction".into(),
json!({
"dropped_count": dropped.len(),
"source_event_range": args.get("source_event_range").cloned().unwrap_or(Value::Null),
"digest": digest,
"retained": ["goal", "constraints", "pending_write", "plan", "trusted_policy"],
}),
);
// Goal / constraints fields stay untouched.
if obj.get("goal").map(|v| v.is_null()).unwrap_or(true) {
if let Some(g) = opt_text(args, "goal") {
obj.insert("goal".into(), json!(g));
}
}
}
Ok(pack)
}
fn simple_hash(s: &str) -> u64 {
let mut h: u64 = 0xcbf29ce484222325;
for b in s.as_bytes() {
h ^= u64::from(*b);
h = h.wrapping_mul(0x100000001b3);
}
h
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn build_sorts_and_rejects_trusted_web() {
let pack = context_pack_build(&json!({
"budget_tokens": 1000,
"items": [
{"origin": "user", "text": "goal", "priority": 0, "kind": "goal", "retain": true},
{"origin": "tool", "text": "big dump", "priority": 9, "kind": "tool_output", "token_estimate": 500}
]
}))
.unwrap();
assert_eq!(pack["schema"], "ContextPackV2");
assert_eq!(pack["items"][0]["priority"], 0);
let bad = context_pack_build(&json!({
"items": [{"origin": "web", "trust": "trusted", "text": "x"}]
}));
assert!(bad.is_err());
}
#[test]
fn compact_keeps_goal_drops_tool_output() {
let pack = context_pack_build(&json!({
"budget_tokens": 200,
"output_reserve_pct": 20,
"compact_threshold_pct": 50,
"goal": "finish task",
"items": [
{"origin": "user", "text": "do it", "priority": 0, "kind": "goal", "retain": true, "token_estimate": 20},
{"origin": "tool", "text": "x".repeat(400), "priority": 9, "kind": "tool_output", "token_estimate": 200}
]
}))
.unwrap();
let compacted = context_pack_compact(&pack).unwrap();
assert_eq!(compacted["compacted"], true);
let items = compacted["items"].as_array().unwrap();
assert!(items.iter().any(|i| i["kind"] == "goal"));
assert!(!items.iter().any(|i| i["kind"] == "tool_output"));
assert_eq!(compacted["goal"], "finish task");
}
}
+623
View File
@@ -0,0 +1,623 @@
//! EffectLedgerRecordV1 — operation receipts for write-safe resume (M-A5 subset).
//!
//! State machine (design §4.2):
//! prepared → started → committed
//! ↘ failed_no_effect
//! ↘ unknown → reconciled_committed | reconciled_no_effect | needs_input
use std::fs::{self, File, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use serde_json::{json, Value};
const DEFAULT_LEDGER_DIR: &str = ".marqdo/agent-effects";
const STATES: &[&str] = &[
"prepared",
"started",
"committed",
"failed_no_effect",
"unknown",
"reconciled_committed",
"reconciled_no_effect",
"needs_input",
];
fn opt_text<'a>(args: &'a Value, key: &str) -> Option<&'a str> {
args.get(key)
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
}
fn arg_text<'a>(args: &'a Value, key: &str) -> Result<&'a str, String> {
opt_text(args, key).ok_or_else(|| format!("missing text `{key}`"))
}
fn now_rfc3339() -> String {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
fn resolve_path(rel: &str) -> PathBuf {
let p = PathBuf::from(rel);
if p.is_absolute() {
return p;
}
if p.exists() {
return p;
}
let base = (|| {
let q = crate::host_query_json("cwd")?;
q.as_str()
.map(PathBuf::from)
.ok_or_else(|| "cwd not text".to_string())
})()
.or_else(|_| std::env::current_dir().map_err(|e| e.to_string()))
.unwrap_or_else(|_| PathBuf::from("."));
base.join(p)
}
fn ledger_root(args: &Value) -> Result<PathBuf, String> {
let rel = opt_text(args, "ledger_dir").unwrap_or(DEFAULT_LEDGER_DIR);
let p = PathBuf::from(rel);
if p.is_absolute() {
return Err(
"ledger_dir must be host-relative; absolute client paths are rejected".into(),
);
}
if rel.contains("..") {
return Err("ledger_dir must not contain '..'".into());
}
Ok(resolve_path(rel))
}
fn safe_id(id: &str) -> String {
id.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' || c == '_' {
c
} else {
'_'
}
})
.collect()
}
fn record_path(root: &Path, operation_id: &str) -> PathBuf {
root.join(format!("{}.json", safe_id(operation_id)))
}
fn ensure_dir(root: &Path) -> Result<(), String> {
fs::create_dir_all(root).map_err(|e| format!("effects mkdir: {e}"))
}
fn atomic_write_json(path: &Path, doc: &Value) -> Result<(), String> {
let parent = path
.parent()
.ok_or_else(|| "ledger path has no parent".to_string())?;
ensure_dir(parent)?;
let tmp_name = format!(
".{}.tmp-{}-{}",
path.file_name()
.and_then(|s| s.to_str())
.unwrap_or("op.json"),
std::process::id(),
now_rfc3339().replace(':', "")
);
let tmp = parent.join(tmp_name);
let text = serde_json::to_string_pretty(doc).map_err(|e| e.to_string())?;
{
let mut f = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&tmp)
.map_err(|e| format!("effects temp open: {e}"))?;
f.write_all(text.as_bytes())
.map_err(|e| format!("effects temp write: {e}"))?;
f.flush().map_err(|e| format!("effects temp flush: {e}"))?;
f.sync_all()
.map_err(|e| format!("effects temp fsync: {e}"))?;
}
if let Ok(dir) = File::open(parent) {
let _ = dir.sync_all();
}
fs::rename(&tmp, path).map_err(|e| {
let _ = fs::remove_file(&tmp);
format!("effects atomic rename: {e}")
})?;
Ok(())
}
fn load_record(path: &Path) -> Result<Option<Value>, String> {
if !path.is_file() {
return Ok(None);
}
let text = fs::read_to_string(path).map_err(|e| format!("effects read: {e}"))?;
let doc: Value = serde_json::from_str(&text).map_err(|e| format!("effects corrupt: {e}"))?;
Ok(Some(doc))
}
fn valid_transition(from: &str, to: &str) -> bool {
matches!(
(from, to),
("prepared", "started")
| ("started", "committed")
| ("started", "failed_no_effect")
| ("started", "unknown")
| ("unknown", "reconciled_committed")
| ("unknown", "reconciled_no_effect")
| ("unknown", "needs_input")
// Idempotent no-ops / crash retries of the same terminal.
| ("committed", "committed")
| ("failed_no_effect", "failed_no_effect")
| ("reconciled_committed", "reconciled_committed")
| ("reconciled_no_effect", "reconciled_no_effect")
| ("needs_input", "needs_input")
// Re-prepare of same op is handled separately via idempotency.
| ("prepared", "prepared")
)
}
fn effect_class_ok(c: &str) -> bool {
matches!(c, "read" | "write" | "execute" | "external_write" | "none")
}
/// Prepare (or return existing) an EffectLedgerRecordV1.
pub fn effects_prepare(args: &Value) -> Result<Value, String> {
let operation_id = arg_text(args, "operation_id")?;
let idempotency_key = arg_text(args, "idempotency_key")?;
let args_digest = arg_text(args, "args_digest")?;
let effect_class = opt_text(args, "effect_class").unwrap_or("write");
if !effect_class_ok(effect_class) {
return Err(format!("invalid effect_class={effect_class}"));
}
let root = ledger_root(args)?;
ensure_dir(&root)?;
let path = record_path(&root, operation_id);
if let Some(existing) = load_record(&path)? {
let same_key = existing
.get("idempotency_key")
.and_then(|v| v.as_str())
== Some(idempotency_key);
let same_digest = existing.get("args_digest").and_then(|v| v.as_str()) == Some(args_digest);
if same_key && same_digest {
return Ok(json!({
"ok": true,
"deduped": true,
"record": existing,
}));
}
if same_key && !same_digest {
return Err(
"idempotency_key collision: same key with different args_digest".into(),
);
}
// Different operation_id path can't collide; same path different key is corrupt reuse.
return Err("operation_id already exists with different idempotency_key".into());
}
// Also scan for same idempotency_key under other operation_ids in this ledger dir.
if let Ok(rd) = fs::read_dir(&root) {
for ent in rd.flatten() {
let p = ent.path();
if p.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
if let Ok(Some(doc)) = load_record(&p) {
if doc.get("idempotency_key").and_then(|v| v.as_str()) == Some(idempotency_key) {
let dig = doc.get("args_digest").and_then(|v| v.as_str());
if dig == Some(args_digest) {
return Ok(json!({
"ok": true,
"deduped": true,
"record": doc,
}));
}
return Err(
"idempotency_key collision: same key with different args_digest".into(),
);
}
}
}
}
let doc = json!({
"schema": "EffectLedgerRecordV1",
"schema_version": 1,
"operation_id": operation_id,
"idempotency_key": idempotency_key,
"args_digest": args_digest,
"resource_scope": args.get("resource_scope").cloned().unwrap_or(Value::Null),
"expected_revision": args.get("expected_revision").cloned().unwrap_or(Value::Null),
"effect_class": effect_class,
"result_ref": args.get("result_ref").cloned().unwrap_or(Value::Null),
"state": "prepared",
"logical_run_id": args.get("logical_run_id").cloned().unwrap_or(Value::Null),
"step_id": args.get("step_id").cloned().unwrap_or(Value::Null),
"attempt_id": args.get("attempt_id").cloned().unwrap_or(Value::Null),
"prepared_at": now_rfc3339(),
"updated_at": now_rfc3339(),
});
atomic_write_json(&path, &doc)?;
Ok(json!({
"ok": true,
"deduped": false,
"record": doc,
"path": path.to_string_lossy(),
}))
}
/// Transition an existing record to a new state.
pub fn effects_transition(args: &Value) -> Result<Value, String> {
let operation_id = arg_text(args, "operation_id")?;
let to = arg_text(args, "state")?;
if !STATES.contains(&to) {
return Err(format!("invalid state={to}"));
}
let root = ledger_root(args)?;
let path = record_path(&root, operation_id);
let mut doc = load_record(&path)?
.ok_or_else(|| format!("operation_id not found: {operation_id}"))?;
let from = doc
.get("state")
.and_then(|v| v.as_str())
.unwrap_or("prepared")
.to_string();
if !valid_transition(&from, to) {
return Err(format!("illegal transition {from} → {to}"));
}
if let Some(obj) = doc.as_object_mut() {
obj.insert("state".into(), json!(to));
obj.insert("updated_at".into(), json!(now_rfc3339()));
if let Some(r) = args.get("result_ref") {
if !r.is_null() {
obj.insert("result_ref".into(), r.clone());
}
}
if let Some(n) = args.get("note") {
if !n.is_null() {
obj.insert("note".into(), n.clone());
}
}
if to == "unknown" || to.starts_with("reconciled") {
obj.insert(
"reconcile_hint".into(),
args.get("reconcile_hint")
.cloned()
.unwrap_or_else(|| json!("check receipt / resource revision")),
);
}
}
atomic_write_json(&path, &doc)?;
Ok(json!({
"ok": true,
"from": from,
"to": to,
"record": doc,
}))
}
pub fn effects_get(args: &Value) -> Result<Value, String> {
let operation_id = arg_text(args, "operation_id")?;
let root = ledger_root(args)?;
let path = record_path(&root, operation_id);
match load_record(&path)? {
Some(doc) => Ok(json!({
"ok": true,
"found": true,
"record": doc,
"path": path.to_string_lossy(),
})),
None => Ok(json!({
"ok": true,
"found": false,
"operation_id": operation_id,
"path": path.to_string_lossy(),
})),
}
}
pub fn effects_list(args: &Value) -> Result<Value, String> {
let root = ledger_root(args)?;
let mut records = Vec::new();
if root.is_dir() {
let mut paths: Vec<_> = fs::read_dir(&root)
.map_err(|e| e.to_string())?
.flatten()
.map(|e| e.path())
.filter(|p| p.extension().and_then(|e| e.to_str()) == Some("json"))
.collect();
paths.sort();
for p in paths {
if let Ok(Some(doc)) = load_record(&p) {
if let Some(run) = opt_text(args, "logical_run_id") {
if doc.get("logical_run_id").and_then(|v| v.as_str()) != Some(run) {
continue;
}
}
if let Some(st) = opt_text(args, "state") {
if doc.get("state").and_then(|v| v.as_str()) != Some(st) {
continue;
}
}
records.push(doc);
}
}
}
Ok(json!({
"ok": true,
"count": records.len(),
"records": records,
"ledger_dir": root.to_string_lossy(),
}))
}
/// Compute a watermark digest over committed / reconciled records for a run.
pub fn effects_watermark(args: &Value) -> Result<Value, String> {
let listed = effects_list(args)?;
let records = listed
.get("records")
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
let mut parts = Vec::new();
let mut open_unknown = 0u64;
let mut pending_writes = 0u64;
for r in &records {
let state = r.get("state").and_then(|v| v.as_str()).unwrap_or("");
let oid = r.get("operation_id").and_then(|v| v.as_str()).unwrap_or("");
let cls = r.get("effect_class").and_then(|v| v.as_str()).unwrap_or("");
parts.push(format!("{oid}:{state}"));
if state == "unknown" || state == "needs_input" {
open_unknown += 1;
}
if matches!(cls, "write" | "external_write" | "execute")
&& matches!(state, "prepared" | "started" | "unknown" | "needs_input")
{
pending_writes += 1;
}
}
parts.sort();
let joined = parts.join("|");
let digest = format!("sha256-lite:{:x}", fnv64(&joined));
let write_auto_resume_safe = open_unknown == 0 && pending_writes == 0;
Ok(json!({
"ok": true,
"schema": "EffectLedgerWatermarkV1",
"schema_version": 1,
"watermark": digest,
"record_count": records.len(),
"open_unknown": open_unknown,
"pending_writes": pending_writes,
"write_auto_resume_safe": write_auto_resume_safe,
"logical_run_id": args.get("logical_run_id").cloned().unwrap_or(Value::Null),
}))
}
/// Reconcile an `unknown` record after inspecting the external receipt / revision.
pub fn effects_reconcile(args: &Value) -> Result<Value, String> {
let outcome = arg_text(args, "outcome")?; // committed | no_effect | needs_input
let state = match outcome {
"committed" => "reconciled_committed",
"no_effect" => "reconciled_no_effect",
"needs_input" => "needs_input",
other => return Err(format!("invalid reconcile outcome={other}")),
};
let mut targs = args.clone();
if let Some(obj) = targs.as_object_mut() {
obj.insert("state".into(), json!(state));
}
effects_transition(&targs)
}
fn fnv64(s: &str) -> u64 {
let mut h: u64 = 0xcbf29ce484222325;
for b in s.as_bytes() {
h ^= u64::from(*b);
h = h.wrapping_mul(0x100000001b3);
}
h
}
#[cfg(test)]
mod tests {
use super::*;
fn scratch(tag: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!(
"mq-effects-{}-{}-{}",
tag,
std::process::id(),
now_rfc3339().replace(':', "")
));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn prepare_dedupe_and_collision() {
let dir = scratch("prep");
let prev = std::env::current_dir().unwrap();
std::env::set_current_dir(&dir).unwrap();
let a = effects_prepare(&json!({
"operation_id": "op1",
"idempotency_key": "idem1",
"args_digest": "sha256:a",
"effect_class": "write",
"ledger_dir": "led",
"logical_run_id": "run1",
}))
.unwrap();
assert_eq!(a["deduped"], false);
assert_eq!(a["record"]["state"], "prepared");
let again = effects_prepare(&json!({
"operation_id": "op1",
"idempotency_key": "idem1",
"args_digest": "sha256:a",
"ledger_dir": "led",
}))
.unwrap();
assert_eq!(again["deduped"], true);
let clash = effects_prepare(&json!({
"operation_id": "op2",
"idempotency_key": "idem1",
"args_digest": "sha256:b",
"ledger_dir": "led",
}));
assert!(clash.unwrap_err().contains("collision"));
std::env::set_current_dir(prev).unwrap();
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn transition_and_watermark_gates_write_resume() {
let dir = scratch("wm");
let prev = std::env::current_dir().unwrap();
std::env::set_current_dir(&dir).unwrap();
effects_prepare(&json!({
"operation_id": "op_w",
"idempotency_key": "idem_w",
"args_digest": "sha256:w",
"effect_class": "write",
"ledger_dir": "led",
"logical_run_id": "run1",
}))
.unwrap();
effects_transition(&json!({
"operation_id": "op_w",
"state": "started",
"ledger_dir": "led",
}))
.unwrap();
// Crash window: unknown — write auto-resume must stay closed.
effects_transition(&json!({
"operation_id": "op_w",
"state": "unknown",
"ledger_dir": "led",
}))
.unwrap();
let wm1 = effects_watermark(&json!({
"ledger_dir": "led",
"logical_run_id": "run1",
}))
.unwrap();
assert_eq!(wm1["write_auto_resume_safe"], false);
assert_eq!(wm1["open_unknown"], 1);
effects_reconcile(&json!({
"operation_id": "op_w",
"outcome": "committed",
"ledger_dir": "led",
"result_ref": "receipt:1",
}))
.unwrap();
let wm2 = effects_watermark(&json!({
"ledger_dir": "led",
"logical_run_id": "run1",
}))
.unwrap();
assert_eq!(wm2["write_auto_resume_safe"], true);
assert_eq!(wm2["open_unknown"], 0);
assert!(wm2["watermark"].as_str().unwrap().starts_with("sha256-lite:"));
std::env::set_current_dir(prev).unwrap();
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn ma_t03_unknown_then_reconcile_no_second_commit_path() {
// Write "succeeds" locally only after reconcile; retry while unknown must not
// look write_auto_resume_safe (MA-T03 watermark gate).
let dir = scratch("mat03");
let prev = std::env::current_dir().unwrap();
std::env::set_current_dir(&dir).unwrap();
effects_prepare(&json!({
"operation_id": "op_kill",
"idempotency_key": "run|s|write_probe|sha256-lite:1",
"args_digest": "sha256-lite:1",
"effect_class": "write",
"ledger_dir": "led",
"logical_run_id": "run_kill",
}))
.unwrap();
effects_transition(&json!({
"operation_id": "op_kill",
"state": "started",
"ledger_dir": "led",
}))
.unwrap();
// Crash before commit:
effects_transition(&json!({
"operation_id": "op_kill",
"state": "unknown",
"ledger_dir": "led",
}))
.unwrap();
let wm = effects_watermark(&json!({
"ledger_dir": "led",
"logical_run_id": "run_kill",
}))
.unwrap();
assert_eq!(wm["write_auto_resume_safe"], false);
// Idempotent re-prepare returns same unknown record (no new op).
let again = effects_prepare(&json!({
"operation_id": "op_kill",
"idempotency_key": "run|s|write_probe|sha256-lite:1",
"args_digest": "sha256-lite:1",
"effect_class": "write",
"ledger_dir": "led",
"logical_run_id": "run_kill",
}))
.unwrap();
assert_eq!(again["deduped"], true);
assert_eq!(again["record"]["state"], "unknown");
effects_reconcile(&json!({
"operation_id": "op_kill",
"outcome": "committed",
"ledger_dir": "led",
"result_ref": "effect:op_kill",
}))
.unwrap();
let wm2 = effects_watermark(&json!({
"ledger_dir": "led",
"logical_run_id": "run_kill",
}))
.unwrap();
assert_eq!(wm2["write_auto_resume_safe"], true);
assert_eq!(wm2["pending_writes"], 0);
std::env::set_current_dir(prev).unwrap();
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn illegal_skip_started() {
let dir = scratch("bad");
let prev = std::env::current_dir().unwrap();
std::env::set_current_dir(&dir).unwrap();
effects_prepare(&json!({
"operation_id": "op_x",
"idempotency_key": "idem_x",
"args_digest": "sha256:x",
"ledger_dir": "led",
}))
.unwrap();
let err = effects_transition(&json!({
"operation_id": "op_x",
"state": "committed",
"ledger_dir": "led",
}))
.unwrap_err();
assert!(err.contains("illegal transition"));
std::env::set_current_dir(prev).unwrap();
let _ = fs::remove_dir_all(&dir);
}
}
+274
View File
@@ -0,0 +1,274 @@
//! AgentErrorV1 — classified errors and bounded recovery decisions (M-A1).
use serde_json::{json, Value};
/// Build an AgentErrorV1 document from caller fields (compat: missing → defaults).
pub fn make_error(args: &Value) -> Result<Value, String> {
let code = args
.get("code")
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.ok_or_else(|| "missing text `code`".to_string())?;
let category = args
.get("category")
.and_then(|v| v.as_str())
.unwrap_or_else(|| infer_category(code));
let stage = args
.get("stage")
.and_then(|v| v.as_str())
.unwrap_or("agent");
let message = args
.get("message")
.and_then(|v| v.as_str())
.unwrap_or(code);
let retryable = args
.get("retryable")
.and_then(|v| v.as_bool())
.unwrap_or_else(|| default_retryable(category));
let effect_state = args
.get("effect_state")
.and_then(|v| v.as_str())
.unwrap_or("none");
let recovery_action = args
.get("recovery_action")
.and_then(|v| v.as_str())
.unwrap_or_else(|| default_recovery(category, effect_state));
let mut out = json!({
"schema": "AgentErrorV1",
"schema_version": 1,
"code": code,
"category": category,
"stage": stage,
"message": message,
"retryable": retryable,
"effect_state": effect_state,
"recovery_action": recovery_action,
});
if let Some(obj) = out.as_object_mut() {
if let Some(v) = args.get("retry_after_ms") {
obj.insert("retry_after_ms".into(), v.clone());
} else {
obj.insert("retry_after_ms".into(), Value::Null);
}
for key in ["call_id", "details_ref", "fingerprint"] {
if let Some(v) = args.get(key) {
if !v.is_null() {
obj.insert(key.into(), v.clone());
}
}
}
}
Ok(out)
}
/// Decide whether a step may auto-retry (read-only / no-effect only in M-A1).
///
/// Returns `{ok, action, reason, attempts_used, max_attempts}`.
pub fn decide_retry(args: &Value) -> Result<Value, String> {
let error = args
.get("error")
.cloned()
.ok_or_else(|| "missing map `error`".to_string())?;
let attempts = args
.get("attempts")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32;
let max_transport = args
.get("max_transport_attempts")
.and_then(|v| v.as_u64())
.unwrap_or(3) as u32; // 1 initial + 2 retries
let max_arg_repair = args
.get("max_arg_repairs")
.and_then(|v| v.as_u64())
.unwrap_or(1) as u32;
let effect_class = args
.get("effect_class")
.and_then(|v| v.as_str())
.unwrap_or("read");
let cancelled = args
.get("cancelled")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let fingerprint_repeats = args
.get("fingerprint_repeats")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32;
if cancelled {
return Ok(decision("stop", "cancelled", attempts, max_transport));
}
let category = error
.get("category")
.and_then(|v| v.as_str())
.unwrap_or("unknown");
let retryable = error
.get("retryable")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let effect_state = error
.get("effect_state")
.and_then(|v| v.as_str())
.unwrap_or("none");
let recovery = error
.get("recovery_action")
.and_then(|v| v.as_str())
.unwrap_or("none");
// Write / unknown effect: never auto-retry in M-A1 (ledger required).
if effect_class != "read" || matches!(effect_state, "unknown" | "started" | "committed") {
return Ok(decision(
"manual_review",
"write_or_unknown_effect_requires_ledger",
attempts,
max_transport,
));
}
if fingerprint_repeats >= 2 {
return Ok(decision(
"stop",
"error_fingerprint_loop",
attempts,
max_transport,
));
}
match category {
"transient" if retryable && attempts + 1 < max_transport => Ok(decision(
"retry_step",
"transient_read_safe",
attempts,
max_transport,
)),
"invalid_arguments" if recovery == "repair_args" && attempts < max_arg_repair => Ok(
decision("repair_args", "one_arg_repair", attempts, max_arg_repair),
),
"auth" | "forbidden" | "policy_denied" => Ok(decision(
"needs_input",
category,
attempts,
max_transport,
)),
"budget_exhausted" => Ok(decision("partial", category, attempts, max_transport)),
"cancelled" => Ok(decision("stop", "cancelled", attempts, max_transport)),
"unknown_effect" | "worker_crash" => Ok(decision(
"manual_review",
category,
attempts,
max_transport,
)),
_ => Ok(decision("stop", "not_auto_retryable", attempts, max_transport)),
}
}
fn decision(action: &str, reason: &str, attempts: u32, max_attempts: u32) -> Value {
json!({
"ok": true,
"schema": "AgentRetryDecisionV1",
"schema_version": 1,
"action": action,
"reason": reason,
"attempts_used": attempts,
"max_attempts": max_attempts,
"auto": matches!(action, "retry_step" | "repair_args"),
})
}
fn infer_category(code: &str) -> &'static str {
let c = code.to_ascii_lowercase();
if c.contains("timeout") || c.contains("429") || c.contains("5xx") || c.contains("connect") {
"transient"
} else if c.contains("schema") || c.contains("argument") {
"invalid_arguments"
} else if c.contains("auth") {
"auth"
} else if c.contains("forbid") {
"forbidden"
} else if c.contains("policy") {
"policy_denied"
} else if c.contains("budget") {
"budget_exhausted"
} else if c.contains("cancel") {
"cancelled"
} else if c.contains("conflict") || c.contains("stale") {
"conflict"
} else if c.contains("effect") || c.contains("crash") {
"unknown_effect"
} else {
"unknown"
}
}
fn default_retryable(category: &str) -> bool {
matches!(category, "transient")
}
fn default_recovery(category: &str, effect_state: &str) -> &'static str {
if effect_state == "unknown" {
return "reconcile";
}
match category {
"transient" => "retry_step",
"invalid_arguments" => "repair_args",
"auth" | "forbidden" | "policy_denied" | "needs_input" => "needs_input",
"budget_exhausted" => "return_partial",
"cancelled" => "stop",
"unknown_effect" | "worker_crash" => "manual_review",
_ => "stop",
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn make_error_defaults() {
let err = make_error(&json!({"code": "tool.timeout", "message": "t"})).unwrap();
assert_eq!(err["schema"], "AgentErrorV1");
assert_eq!(err["category"], "transient");
assert_eq!(err["retryable"], true);
assert_eq!(err["recovery_action"], "retry_step");
}
#[test]
fn read_transient_retries_once() {
let err = make_error(&json!({"code": "tool.timeout"})).unwrap();
let d = decide_retry(&json!({
"error": err,
"attempts": 0,
"effect_class": "read"
}))
.unwrap();
assert_eq!(d["action"], "retry_step");
assert_eq!(d["auto"], true);
}
#[test]
fn write_never_auto_retries() {
let err = make_error(&json!({"code": "tool.timeout"})).unwrap();
let d = decide_retry(&json!({
"error": err,
"attempts": 0,
"effect_class": "write"
}))
.unwrap();
assert_eq!(d["action"], "manual_review");
assert_eq!(d["auto"], false);
}
#[test]
fn fingerprint_loop_stops() {
let err = make_error(&json!({"code": "tool.timeout"})).unwrap();
let d = decide_retry(&json!({
"error": err,
"attempts": 0,
"effect_class": "read",
"fingerprint_repeats": 2
}))
.unwrap();
assert_eq!(d["action"], "stop");
}
}
+150 -1
View File
@@ -1,14 +1,19 @@
//! Agent plugin (C ABI v2): layout helpers + session bag + context via host_query + agent-kb + v2 memory + adaptive route.
mod context_pack;
mod corpus;
mod effects;
mod eng;
mod error;
mod kb;
mod mcp_client;
mod mcp_server;
mod memory;
mod output;
mod resume;
mod route;
mod sandbox;
mod tools;
use std::collections::HashMap;
use std::ffi::CString;
@@ -667,7 +672,7 @@ unsafe extern "C" fn agent_marqdo_skill(
}
}
fn tool_names(tools: &serde_json::Value) -> Vec<String> {
pub(crate) fn tool_names(tools: &serde_json::Value) -> Vec<String> {
let mut out = Vec::new();
let rows = match tools {
serde_json::Value::Array(a) => a.as_slice(),
@@ -829,6 +834,86 @@ kb_ffi!(agent_resume_save, resume::resume_save, "agent_resume_save");
kb_ffi!(agent_resume_load, resume::resume_load, "agent_resume_load");
kb_ffi!(agent_resume_clear, resume::resume_clear, "agent_resume_clear");
kb_ffi!(agent_resume_list, resume::resume_list, "agent_resume_list");
kb_ffi!(agent_error_make, error::make_error, "agent_error_make");
kb_ffi!(agent_error_decide_retry, error::decide_retry, "agent_error_decide_retry");
kb_ffi!(agent_tools_register, tools::tools_register, "agent_tools_register");
kb_ffi!(agent_tools_clear, tools::tools_clear, "agent_tools_clear");
kb_ffi!(agent_tools_search, tools::tools_search, "agent_tools_search");
kb_ffi!(agent_tools_describe, tools::tools_describe, "agent_tools_describe");
kb_ffi!(agent_tools_prepare_call, tools::tools_prepare_call, "agent_tools_prepare_call");
kb_ffi!(agent_tools_wrap_result, tools::tools_wrap_result, "agent_tools_wrap_result");
kb_ffi!(
agent_tools_args_digest,
tools::tools_args_digest,
"agent_tools_args_digest"
);
kb_ffi!(
agent_tools_needs_ledger,
tools::tools_needs_ledger,
"agent_tools_needs_ledger"
);
kb_ffi!(
agent_tools_register_eng_defaults,
tools::tools_register_eng_defaults,
"agent_tools_register_eng_defaults"
);
kb_ffi!(
agent_tools_seed_allowlist,
tools::seed_from_allowlist_args,
"agent_tools_seed_allowlist"
);
kb_ffi!(agent_output_build, output::output_build, "agent_output_build");
kb_ffi!(
agent_output_apply_policy,
output::output_apply_policy,
"agent_output_apply_policy"
);
kb_ffi!(
agent_output_channel_event,
output::output_channel_event,
"agent_output_channel_event"
);
kb_ffi!(
agent_output_project_conclusion,
output::output_project_conclusion,
"agent_output_project_conclusion"
);
kb_ffi!(
agent_context_pack_build,
context_pack::context_pack_build,
"agent_context_pack_build"
);
kb_ffi!(
agent_context_pack_compact,
context_pack::context_pack_compact,
"agent_context_pack_compact"
);
kb_ffi!(
agent_effects_prepare,
effects::effects_prepare,
"agent_effects_prepare"
);
kb_ffi!(
agent_effects_transition,
effects::effects_transition,
"agent_effects_transition"
);
kb_ffi!(agent_effects_get, effects::effects_get, "agent_effects_get");
kb_ffi!(
agent_effects_list,
effects::effects_list,
"agent_effects_list"
);
kb_ffi!(
agent_effects_watermark,
effects::effects_watermark,
"agent_effects_watermark"
);
kb_ffi!(
agent_effects_reconcile,
effects::effects_reconcile,
"agent_effects_reconcile"
);
kb_ffi!(agent_eng_preflight, eng::eng_preflight, "agent_eng_preflight");
kb_ffi!(agent_eng_reuse, eng::eng_reuse, "agent_eng_reuse");
kb_ffi!(agent_eng_bind_execute, eng::eng_bind_execute, "agent_eng_bind_execute");
@@ -878,6 +963,70 @@ pub unsafe extern "C" fn marqdo_plugin_init(host: *const MarqdoHostApi) -> c_int
("agent_resume_load", "id,resume_dir", agent_resume_load),
("agent_resume_clear", "id,resume_dir", agent_resume_clear),
("agent_resume_list", "resume_dir", agent_resume_list),
("agent_error_make", "code,message,category", agent_error_make),
("agent_error_decide_retry", "error,attempts,effect_class", agent_error_decide_retry),
("agent_tools_register", "spec,specs", agent_tools_register),
("agent_tools_clear", "", agent_tools_clear),
("agent_tools_search", "query,tools", agent_tools_search),
("agent_tools_describe", "id,tools", agent_tools_describe),
("agent_tools_prepare_call", "tool_id,arguments,tools", agent_tools_prepare_call),
("agent_tools_wrap_result", "ok,status,summary", agent_tools_wrap_result),
("agent_tools_args_digest", "arguments", agent_tools_args_digest),
("agent_tools_needs_ledger", "effect", agent_tools_needs_ledger),
(
"agent_tools_register_eng_defaults",
"",
agent_tools_register_eng_defaults,
),
("agent_tools_seed_allowlist", "tools", agent_tools_seed_allowlist),
("agent_output_build", "status,text,validation", agent_output_build),
(
"agent_output_apply_policy",
"status,text,policy",
agent_output_apply_policy,
),
(
"agent_output_channel_event",
"channel,delta",
agent_output_channel_event,
),
(
"agent_output_project_conclusion",
"document",
agent_output_project_conclusion,
),
(
"agent_context_pack_build",
"items,budget_tokens",
agent_context_pack_build,
),
(
"agent_context_pack_compact",
"items,budget_tokens",
agent_context_pack_compact,
),
(
"agent_effects_prepare",
"operation_id,idempotency_key,args_digest",
agent_effects_prepare,
),
(
"agent_effects_transition",
"operation_id,state",
agent_effects_transition,
),
("agent_effects_get", "operation_id", agent_effects_get),
("agent_effects_list", "ledger_dir", agent_effects_list),
(
"agent_effects_watermark",
"ledger_dir",
agent_effects_watermark,
),
(
"agent_effects_reconcile",
"operation_id,outcome",
agent_effects_reconcile,
),
("agent_kb_add_alias", "kb_dir,slug,alias", agent_kb_add_alias),
("agent_kb_promote", "kb_dir,goal,workbook", agent_kb_promote),
("agent_kb_quarantine", "kb_dir,slug", agent_kb_quarantine),
+289
View File
@@ -0,0 +1,289 @@
//! AgentOutputV1 + ResponsePolicy (M-A4).
use serde_json::{json, Value};
fn opt_text<'a>(args: &'a Value, key: &str) -> Option<&'a str> {
args.get(key)
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
}
/// Normalize / build an AgentOutputV1 document.
pub fn output_build(args: &Value) -> Result<Value, String> {
let status = opt_text(args, "status").unwrap_or("succeeded");
if !matches!(
status,
"succeeded" | "partial" | "failed" | "needs_input" | "cancelled"
) {
return Err(format!("invalid status={status}"));
}
let answer = args.get("answer").cloned().unwrap_or_else(|| {
json!({
"format": opt_text(args, "format").unwrap_or("markdown"),
"text": opt_text(args, "text").unwrap_or(""),
})
});
let validation = args.get("validation").cloned().unwrap_or_else(|| {
json!({
"status": "not_run",
"report_ref": Value::Null
})
});
let vstatus = validation
.get("status")
.and_then(|v| v.as_str())
.unwrap_or("not_run");
if !matches!(
vstatus,
"passed" | "failed" | "not_applicable" | "not_run"
) {
return Err(format!("invalid validation.status={vstatus}"));
}
// Never claim passed without an explicit report when status says passed.
if vstatus == "passed" && validation.get("report_ref").map(|v| v.is_null()).unwrap_or(true) {
if args.get("allow_passed_without_report").and_then(|v| v.as_bool()) != Some(true) {
return Err("validation.status=passed requires report_ref".into());
}
}
let finish = opt_text(args, "finish_reason").unwrap_or(match status {
"succeeded" => "completed",
"partial" => "length_limited",
"cancelled" => "cancelled",
"needs_input" => "needs_input",
_ => "error",
});
Ok(json!({
"schema": "AgentOutputV1",
"schema_version": 1,
"status": status,
"answer": answer,
"artifacts": args.get("artifacts").cloned().unwrap_or_else(|| json!([])),
"citations": args.get("citations").cloned().unwrap_or_else(|| json!([])),
"validation": validation,
"finish_reason": finish,
"recovery_options": args.get("recovery_options").cloned().unwrap_or_else(|| json!([])),
"channels": args.get("channels").cloned().unwrap_or_else(|| json!({
"answer": true,
"artifact": true,
"progress": true,
"diagnostic": true
})),
}))
}
/// Apply ResponsePolicy budgets to an answer text → possibly partial output.
pub fn output_apply_policy(args: &Value) -> Result<Value, String> {
let mut out = if args.get("schema").and_then(|v| v.as_str()) == Some("AgentOutputV1") {
args.clone()
} else {
output_build(args)?
};
let policy = args.get("policy").cloned().unwrap_or(json!({}));
let detail = policy
.get("detail")
.and_then(|v| v.as_str())
.unwrap_or("standard");
let max_chars = match policy.get("max_chars").and_then(|v| v.as_u64()) {
Some(n) if n > 0 => n as usize,
_ => match detail {
"concise" => 250,
"detailed" => 8_000,
_ => 700,
},
};
let max_tokens = policy
.get("max_output_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(0) as usize;
let artifact_allowed = policy
.get("artifact_allowed")
.and_then(|v| v.as_bool())
.unwrap_or(true);
let text = out
.pointer("/answer/text")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
// Soft char budget (CJK-aware enough: count Unicode scalars).
let chars: Vec<char> = text.chars().collect();
let mut finish = out
.get("finish_reason")
.and_then(|v| v.as_str())
.unwrap_or("completed")
.to_string();
let mut status = out
.get("status")
.and_then(|v| v.as_str())
.unwrap_or("succeeded")
.to_string();
let mut answer_text = text.clone();
if chars.len() > max_chars {
answer_text = chars.into_iter().take(max_chars).collect();
// Prefer not to cut mid-code fence if possible — soft trim at last newline.
if let Some(pos) = answer_text.rfind('\n') {
if pos > max_chars / 2 {
answer_text.truncate(pos);
}
}
finish = "length_limited".into();
if status == "succeeded" {
status = "partial".into();
}
}
// Token budget is advisory here (chars/4 approx) when max_output_tokens set.
if max_tokens > 0 {
let approx = answer_text.chars().count().div_ceil(2); // rough CJK token estimate
if approx > max_tokens {
let keep = max_tokens.saturating_mul(2);
answer_text = answer_text.chars().take(keep).collect();
finish = "length_limited".into();
status = "partial".into();
}
}
if let Some(obj) = out.as_object_mut() {
obj.insert("status".into(), json!(status));
obj.insert("finish_reason".into(), json!(finish));
if let Some(ans) = obj.get_mut("answer").and_then(|v| v.as_object_mut()) {
ans.insert("text".into(), json!(answer_text));
}
if !artifact_allowed {
obj.insert("artifacts".into(), json!([]));
}
obj.insert(
"response_policy".into(),
json!({
"detail": detail,
"max_chars": max_chars,
"max_output_tokens": max_tokens,
"artifact_allowed": artifact_allowed,
}),
);
}
Ok(out)
}
/// Split a channel event. Only `answer` is meant for chat body append.
pub fn output_channel_event(args: &Value) -> Result<Value, String> {
let channel = opt_text(args, "channel").ok_or_else(|| "missing channel".to_string())?;
if !matches!(channel, "answer" | "artifact" | "progress" | "diagnostic") {
return Err(format!("invalid channel={channel}"));
}
let delta = opt_text(args, "delta").unwrap_or("");
Ok(json!({
"schema": "AgentChannelEventV1",
"schema_version": 1,
"channel": channel,
"delta": delta,
"append_to_chat": channel == "answer",
"resource_id": args.get("resource_id").cloned().unwrap_or(Value::Null),
"meta": args.get("meta").cloned().unwrap_or(json!({})),
}))
}
/// Project `# 结论` (or first non-empty paragraph) from a marqdo document for chat.
pub fn output_project_conclusion(args: &Value) -> Result<Value, String> {
let doc = opt_text(args, "document")
.or_else(|| opt_text(args, "text"))
.unwrap_or("");
let mut in_conclusion = false;
let mut buf = String::new();
for line in doc.lines() {
let t = line.trim();
if t == "# 结论" || t.eq_ignore_ascii_case("# conclusion") {
in_conclusion = true;
continue;
}
if in_conclusion {
if t.starts_with("# ") {
break;
}
if t.is_empty() {
if !buf.is_empty() {
buf.push('\n');
}
continue;
}
if !buf.is_empty() {
buf.push('\n');
}
buf.push_str(t);
}
}
if buf.trim().is_empty() {
// Fallback: first non-frontmatter paragraph.
let mut past_fm = !doc.starts_with("---");
let mut fm_done = past_fm;
for line in doc.lines() {
if !fm_done {
if line.trim() == "---" {
if past_fm {
fm_done = true;
} else {
past_fm = true;
}
}
continue;
}
let t = line.trim();
if t.is_empty() || t.starts_with('#') || t.starts_with('|') {
continue;
}
buf = t.to_string();
break;
}
}
Ok(json!({
"ok": true,
"text": buf.trim(),
"channel": "answer",
"source": "conclusion_projection",
}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn build_and_policy_limits() {
let out = output_build(&json!({
"status": "succeeded",
"text": "短答",
"validation": {"status": "not_applicable", "report_ref": null}
}))
.unwrap();
assert_eq!(out["schema"], "AgentOutputV1");
assert_eq!(out["answer"]["text"], "短答");
let long: String = "字".repeat(400);
let limited = output_apply_policy(&json!({
"status": "succeeded",
"text": long,
"validation": {"status": "not_applicable"},
"policy": {"detail": "concise", "max_chars": 100}
}))
.unwrap();
assert_eq!(limited["status"], "partial");
assert_eq!(limited["finish_reason"], "length_limited");
assert!(limited["answer"]["text"].as_str().unwrap().chars().count() <= 100);
}
#[test]
fn channel_answer_only_appends_chat() {
let e = output_channel_event(&json!({"channel": "progress", "delta": "检索中"})).unwrap();
assert_eq!(e["append_to_chat"], false);
let a = output_channel_event(&json!({"channel": "answer", "delta": "hi"})).unwrap();
assert_eq!(a["append_to_chat"], true);
}
#[test]
fn project_conclusion() {
let doc = "---\ntitle: t\n---\n\n# 任务\n问\n\n# 结论\n答案是 42。\n\n# main\nprint\n";
let p = output_project_conclusion(&json!({"document": doc})).unwrap();
assert!(p["text"].as_str().unwrap().contains("42"));
}
}
+348 -16
View File
@@ -1,6 +1,8 @@
//! Agent plan/run resume checkpoints under `.marqdo/agent-resume/`.
//! M-A1: CheckpointV2 atomic write (temp→fsync→rename); legacy load → manual_review.
use std::fs;
use std::fs::{self, File, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use serde_json::{json, Value};
@@ -27,7 +29,9 @@ fn resolve_path(rel: &str) -> PathBuf {
}
fn opt_text<'a>(args: &'a Value, key: &str) -> Option<&'a str> {
args.get(key).and_then(|v| v.as_str()).filter(|s| !s.is_empty())
args.get(key)
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
}
fn arg_text<'a>(args: &'a Value, key: &str) -> Result<&'a str, String> {
@@ -43,9 +47,19 @@ fn opt_i64(args: &Value, key: &str, default: i64) -> i64 {
}
}
fn resume_root(args: &Value) -> PathBuf {
/// Host-relative resume roots only — reject absolute client paths (M-A1).
fn resume_root(args: &Value) -> Result<PathBuf, String> {
let rel = opt_text(args, "resume_dir").unwrap_or(DEFAULT_RESUME_DIR);
resolve_path(rel)
let p = PathBuf::from(rel);
if p.is_absolute() {
return Err(
"resume_dir must be host-relative; absolute client paths are rejected".into(),
);
}
if rel.contains("..") {
return Err("resume_dir must not contain '..'".into());
}
Ok(resolve_path(rel))
}
fn checkpoint_path(root: &Path, id: &str) -> PathBuf {
@@ -70,11 +84,106 @@ fn ensure_dir(root: &Path) -> Result<(), String> {
fs::create_dir_all(root).map_err(|e| format!("resume mkdir: {e}"))
}
pub fn resume_save(args: &Value) -> Result<Value, String> {
let id = arg_text(args, "id")?;
let root = resume_root(args);
ensure_dir(&root)?;
let path = checkpoint_path(&root, id);
/// Atomic durable write: unique temp → write → flush/fsync → rename.
fn atomic_write_json(path: &Path, doc: &Value) -> Result<(), String> {
let parent = path
.parent()
.ok_or_else(|| "checkpoint path has no parent".to_string())?;
ensure_dir(parent)?;
let tmp_name = format!(
".{}.tmp-{}-{}",
path.file_name()
.and_then(|s| s.to_str())
.unwrap_or("ckpt.json"),
std::process::id(),
now_rfc3339().replace(':', "")
);
let tmp = parent.join(tmp_name);
let text = serde_json::to_string_pretty(doc).map_err(|e| e.to_string())?;
{
let mut f = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&tmp)
.map_err(|e| format!("resume temp open: {e}"))?;
f.write_all(text.as_bytes())
.map_err(|e| format!("resume temp write: {e}"))?;
f.flush().map_err(|e| format!("resume temp flush: {e}"))?;
f.sync_all()
.map_err(|e| format!("resume temp fsync: {e}"))?;
}
// Best-effort directory fsync (platform may ignore).
if let Ok(dir) = File::open(parent) {
let _ = dir.sync_all();
}
fs::rename(&tmp, path).map_err(|e| {
let _ = fs::remove_file(&tmp);
format!("resume atomic rename: {e}")
})?;
Ok(())
}
fn wants_checkpoint_v2(args: &Value) -> bool {
matches!(
args.get("schema").and_then(|v| v.as_str()),
Some("CheckpointV2")
) || args.get("schema_version").and_then(|v| v.as_u64()) == Some(2)
|| args.get("checkpoint_v2").and_then(|v| v.as_bool()) == Some(true)
}
fn build_checkpoint_v2(args: &Value, id: &str) -> Value {
let committed = now_rfc3339();
let mut doc = json!({
"schema": "CheckpointV2",
"schema_version": 2,
"checkpoint_id": args.get("checkpoint_id").cloned().unwrap_or_else(|| json!(id)),
"logical_run_id": args.get("logical_run_id").cloned().unwrap_or_else(|| json!(id)),
"task_id": args.get("task_id").cloned().unwrap_or_else(|| json!(id)),
"attempt_id": args.get("attempt_id").cloned().unwrap_or_else(|| json!("attempt_1")),
"step_id": args.get("step_id").cloned().unwrap_or(Value::Null),
"parent_checkpoint_id": args.get("parent_checkpoint_id").cloned().unwrap_or(Value::Null),
"principal_ref": args.get("principal_ref").cloned().unwrap_or(Value::Null),
"workspace_id": args.get("workspace_id").cloned().unwrap_or(Value::Null),
"session_id": args.get("session_id").cloned().unwrap_or(Value::Null),
"completed_steps": args.get("completed_steps").cloned().unwrap_or_else(|| json!([])),
"pending_operation": args.get("pending_operation").cloned().unwrap_or(Value::Null),
"deadline": args.get("deadline").cloned().unwrap_or(Value::Null),
"committed_at": args.get("committed_at").cloned().unwrap_or_else(|| json!(committed)),
// Effect watermark — absent/null means no ledger yet (write auto-resume closed).
"effect_ledger_watermark": args.get("effect_ledger_watermark").cloned().unwrap_or(Value::Null),
"event_seq": args.get("event_seq").cloned().unwrap_or(Value::Null),
"manifest_digest": args.get("manifest_digest").cloned().unwrap_or(Value::Null),
});
if let Some(obj) = doc.as_object_mut() {
// Preserve legacy author fields for dual-read.
for key in [
"goal", "task", "status", "workbook", "events", "route", "result", "round", "meta",
"id",
] {
if let Some(v) = args.get(key) {
if !v.is_null() {
obj.insert(key.to_string(), v.clone());
}
}
}
if !obj.contains_key("id") {
obj.insert("id".into(), json!(id));
}
if !obj.contains_key("updated_at") {
obj.insert("updated_at".into(), json!(committed.clone()));
}
if !obj.contains_key("round") {
obj.insert("round".into(), json!(opt_i64(args, "round", 0)));
}
if !obj.contains_key("status") {
obj.insert("status".into(), json!("running"));
}
}
doc
}
fn build_legacy_doc(args: &Value, id: &str) -> Value {
let mut doc = json!({
"id": id,
"updated_at": now_rfc3339(),
@@ -96,19 +205,84 @@ pub fn resume_save(args: &Value) -> Result<Value, String> {
obj.insert("status".into(), json!("running"));
}
}
let text = serde_json::to_string_pretty(&doc).map_err(|e| e.to_string())?;
fs::write(&path, text).map_err(|e| format!("resume save: {e}"))?;
doc
}
pub fn resume_save(args: &Value) -> Result<Value, String> {
let id = arg_text(args, "id")?;
let root = resume_root(args)?;
ensure_dir(&root)?;
let path = checkpoint_path(&root, id);
let v2 = wants_checkpoint_v2(args);
let doc = if v2 {
build_checkpoint_v2(args, id)
} else {
build_legacy_doc(args, id)
};
atomic_write_json(&path, &doc)?;
Ok(json!({
"ok": true,
"id": id,
"path": path.to_string_lossy(),
"updated_at": doc.get("updated_at").cloned().unwrap_or(Value::Null),
"schema": doc.get("schema").cloned().unwrap_or(Value::Null),
"schema_version": doc.get("schema_version").cloned().unwrap_or(Value::Null),
"updated_at": doc.get("updated_at").or_else(|| doc.get("committed_at")).cloned().unwrap_or(Value::Null),
"atomic": true,
}))
}
fn watermark_allows_write_auto(wm: &Value) -> bool {
match wm {
Value::Null => false,
Value::String(s) => !s.is_empty(),
Value::Object(obj) => {
// Prefer explicit safety bit from effects_watermark; bare digest objects
// without the bit are treated as unsafe (open unknown / pending writes).
match obj.get("write_auto_resume_safe").and_then(|v| v.as_bool()) {
Some(b) => b,
None => obj
.get("watermark")
.and_then(|v| v.as_str())
.map(|s| !s.is_empty())
.unwrap_or(false)
&& obj.get("open_unknown").and_then(|v| v.as_u64()).unwrap_or(1) == 0
&& obj.get("pending_writes").and_then(|v| v.as_u64()).unwrap_or(1) == 0,
}
}
_ => false,
}
}
fn annotate_recovery(mut doc: Value) -> Value {
let is_v2 = doc.get("schema").and_then(|v| v.as_str()) == Some("CheckpointV2")
&& doc.get("schema_version").and_then(|v| v.as_u64()) == Some(2);
let wm = doc
.get("effect_ledger_watermark")
.cloned()
.unwrap_or(Value::Null);
let ledger_safe = watermark_allows_write_auto(&wm);
let (recovery_mode, write_auto_resume) = if is_v2 && ledger_safe {
("checkpoint_v2", true)
} else if is_v2 {
// V2 without a safe ledger watermark: read resume OK; write auto-resume closed.
("checkpoint_v2_read_only", false)
} else {
// Legacy: no effect info → manual_review; never auto redo writes.
("manual_review", false)
};
if let Some(obj) = doc.as_object_mut() {
obj.insert("recovery_mode".into(), json!(recovery_mode));
obj.insert("write_auto_resume".into(), json!(write_auto_resume));
if !is_v2 {
obj.insert("legacy_adaptation".into(), json!("manual_review"));
}
}
doc
}
pub fn resume_load(args: &Value) -> Result<Value, String> {
let id = arg_text(args, "id")?;
let root = resume_root(args);
let root = resume_root(args)?;
let path = checkpoint_path(&root, id);
if !path.is_file() {
return Ok(json!({
@@ -119,7 +293,31 @@ pub fn resume_load(args: &Value) -> Result<Value, String> {
}));
}
let text = fs::read_to_string(&path).map_err(|e| format!("resume load: {e}"))?;
let mut doc: Value = serde_json::from_str(&text).map_err(|e| format!("resume json: {e}"))?;
let parsed: Value = match serde_json::from_str(&text) {
Ok(v) => v,
Err(e) => {
return Ok(json!({
"ok": false,
"found": true,
"id": id,
"path": path.to_string_lossy(),
"error": {
"schema": "AgentErrorV1",
"schema_version": 1,
"code": "checkpoint.corrupt",
"category": "unknown_effect",
"stage": "checkpoint",
"message": format!("corrupt checkpoint json: {e}"),
"retryable": false,
"effect_state": "unknown",
"recovery_action": "manual_review",
},
"recovery_mode": "manual_review",
"write_auto_resume": false,
}));
}
};
let mut doc = annotate_recovery(parsed);
if let Some(obj) = doc.as_object_mut() {
obj.insert("ok".into(), json!(true));
obj.insert("found".into(), json!(true));
@@ -130,7 +328,7 @@ pub fn resume_load(args: &Value) -> Result<Value, String> {
pub fn resume_clear(args: &Value) -> Result<Value, String> {
let id = arg_text(args, "id")?;
let root = resume_root(args);
let root = resume_root(args)?;
let path = checkpoint_path(&root, id);
let existed = path.is_file();
if existed {
@@ -145,7 +343,7 @@ pub fn resume_clear(args: &Value) -> Result<Value, String> {
}
pub fn resume_list(args: &Value) -> Result<Value, String> {
let root = resume_root(args);
let root = resume_root(args)?;
if !root.is_dir() {
return Ok(json!({ "ok": true, "items": [], "resume_dir": root.to_string_lossy() }));
}
@@ -156,6 +354,14 @@ pub fn resume_list(args: &Value) -> Result<Value, String> {
if p.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
// Skip leftover temp files from crashed writers.
if p.file_name()
.and_then(|s| s.to_str())
.map(|s| s.starts_with('.'))
.unwrap_or(false)
{
continue;
}
let id = p
.file_stem()
.and_then(|s| s.to_str())
@@ -179,3 +385,129 @@ pub fn resume_list(args: &Value) -> Result<Value, String> {
"resume_dir": root.to_string_lossy(),
}))
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{SystemTime, UNIX_EPOCH};
fn scratch(label: &str) -> PathBuf {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let dir = std::env::temp_dir().join(format!("marqdo-resume-{label}-{nanos}"));
fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn absolute_resume_dir_rejected() {
let err = resume_root(&json!({"resume_dir": "/tmp/evil"})).unwrap_err();
assert!(err.contains("host-relative"));
}
#[test]
fn atomic_v2_roundtrip_and_legacy_manual_review() {
let dir = scratch("v2");
// Use relative path via env cwd — chdir into scratch for this test.
let prev = std::env::current_dir().unwrap();
std::env::set_current_dir(&dir).unwrap();
let rel = "ckpts";
let saved = resume_save(&json!({
"id": "run1",
"schema": "CheckpointV2",
"schema_version": 2,
"goal": "ping",
"status": "running",
"round": 1,
"resume_dir": rel,
"attempt_id": "a1",
}))
.unwrap();
assert_eq!(saved["ok"], true);
assert_eq!(saved["atomic"], true);
assert_eq!(saved["schema"], "CheckpointV2");
let loaded = resume_load(&json!({"id": "run1", "resume_dir": rel})).unwrap();
assert_eq!(loaded["found"], true);
assert_eq!(loaded["schema"], "CheckpointV2");
assert_eq!(loaded["recovery_mode"], "checkpoint_v2_read_only");
assert_eq!(loaded["write_auto_resume"], false);
// Legacy document without schema.
let _ = resume_save(&json!({
"id": "legacy1",
"goal": "pong",
"status": "interrupted",
"round": 2,
"resume_dir": rel,
}))
.unwrap();
let legacy = resume_load(&json!({"id": "legacy1", "resume_dir": rel})).unwrap();
assert_eq!(legacy["recovery_mode"], "manual_review");
assert_eq!(legacy["write_auto_resume"], false);
assert_eq!(legacy["legacy_adaptation"], "manual_review");
// V2 + safe watermark → write_auto_resume opens.
let _ = resume_save(&json!({
"id": "run_safe",
"schema": "CheckpointV2",
"schema_version": 2,
"status": "running",
"round": 3,
"resume_dir": rel,
"attempt_id": "a2",
"effect_ledger_watermark": {
"schema": "EffectLedgerWatermarkV1",
"watermark": "sha256-lite:abc",
"write_auto_resume_safe": true,
"open_unknown": 0,
"pending_writes": 0
}
}))
.unwrap();
let safe = resume_load(&json!({"id": "run_safe", "resume_dir": rel})).unwrap();
assert_eq!(safe["recovery_mode"], "checkpoint_v2");
assert_eq!(safe["write_auto_resume"], true);
// Unsafe watermark (open unknown) keeps write auto-resume closed.
let _ = resume_save(&json!({
"id": "run_unsafe",
"schema": "CheckpointV2",
"schema_version": 2,
"status": "running",
"round": 4,
"resume_dir": rel,
"attempt_id": "a3",
"effect_ledger_watermark": {
"watermark": "sha256-lite:def",
"write_auto_resume_safe": false,
"open_unknown": 1
}
}))
.unwrap();
let unsafe_ck = resume_load(&json!({"id": "run_unsafe", "resume_dir": rel})).unwrap();
assert_eq!(unsafe_ck["recovery_mode"], "checkpoint_v2_read_only");
assert_eq!(unsafe_ck["write_auto_resume"], false);
std::env::set_current_dir(prev).unwrap();
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn corrupt_checkpoint_reports_error() {
let dir = scratch("corrupt");
let prev = std::env::current_dir().unwrap();
std::env::set_current_dir(&dir).unwrap();
fs::create_dir_all("ckpts").unwrap();
fs::write("ckpts/bad.json", "{not-json").unwrap();
let loaded = resume_load(&json!({"id": "bad", "resume_dir": "ckpts"})).unwrap();
assert_eq!(loaded["found"], true);
assert_eq!(loaded["ok"], false);
assert_eq!(loaded["error"]["code"], "checkpoint.corrupt");
assert_eq!(loaded["recovery_mode"], "manual_review");
std::env::set_current_dir(prev).unwrap();
let _ = fs::remove_dir_all(&dir);
}
}
+794
View File
@@ -0,0 +1,794 @@
//! ToolSpecV1 registry + call validation (M-A2).
//! Discovery → schema validate → authorize. Execution of Marqdo `##` stays in subtask;
//! this module never evals model-supplied code.
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
use serde_json::{json, Map, Value};
fn registry() -> &'static Mutex<HashMap<String, Value>> {
static REG: OnceLock<Mutex<HashMap<String, Value>>> = OnceLock::new();
REG.get_or_init(|| Mutex::new(HashMap::new()))
}
fn opt_text<'a>(args: &'a Value, key: &str) -> Option<&'a str> {
args.get(key)
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
}
fn arg_text<'a>(args: &'a Value, key: &str) -> Result<&'a str, String> {
opt_text(args, key).ok_or_else(|| format!("missing text `{key}`"))
}
/// Supported JSON Schema subset for tool input (reject unsupported constructs).
fn validate_schema_subset(schema: &Value) -> Result<(), String> {
let obj = schema
.as_object()
.ok_or_else(|| "input_schema must be object".to_string())?;
let ty = obj
.get("type")
.and_then(|v| v.as_str())
.unwrap_or("object");
if ty != "object" {
return Err(format!("unsupported root type={ty}; only object"));
}
if let Some(props) = obj.get("properties") {
let props = props
.as_object()
.ok_or_else(|| "properties must be object".to_string())?;
for (name, prop) in props {
let p = prop
.as_object()
.ok_or_else(|| format!("property {name} must be object"))?;
let pty = p.get("type").and_then(|v| v.as_str()).unwrap_or("string");
match pty {
"string" | "integer" | "number" | "boolean" | "array" | "object" => {}
other => {
return Err(format!(
"unsupported property type {other} for {name}; refuse register"
))
}
}
// Reject constructs we do not implement.
for banned in ["$ref", "oneOf", "anyOf", "allOf", "not", "if", "then", "else"] {
if p.contains_key(banned) {
return Err(format!(
"unsupported schema keyword {banned} on {name}; refuse register"
));
}
}
}
}
for banned in ["$ref", "oneOf", "anyOf", "allOf", "not", "if", "then", "else"] {
if obj.contains_key(banned) {
return Err(format!(
"unsupported schema keyword {banned}; refuse register"
));
}
}
Ok(())
}
fn normalize_spec(mut spec: Value) -> Result<Value, String> {
let obj = spec
.as_object_mut()
.ok_or_else(|| "spec must be object".to_string())?;
obj.insert("schema".into(), json!("ToolSpecV1"));
obj.entry("schema_version".to_string())
.or_insert(json!(1));
let id = obj
.get("id")
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.ok_or_else(|| "ToolSpecV1 requires id".to_string())?
.to_string();
obj.entry("version".to_string()).or_insert(json!("1"));
obj.entry("description".to_string())
.or_insert(json!(""));
let input = obj
.get("input_schema")
.cloned()
.unwrap_or_else(|| json!({"type": "object", "properties": {}, "additionalProperties": false}));
validate_schema_subset(&input)?;
obj.insert("input_schema".into(), input);
obj.entry("effect".to_string()).or_insert(json!("read"));
let effect = obj.get("effect").and_then(|v| v.as_str()).unwrap_or("read");
if !matches!(effect, "read" | "write" | "execute" | "external_write") {
return Err(format!("invalid effect={effect}"));
}
obj.entry("scopes".to_string()).or_insert(json!([]));
obj.entry("timeout_ms".to_string()).or_insert(json!(10_000));
obj.entry("max_result_bytes".to_string())
.or_insert(json!(65_536));
obj.entry("idempotency".to_string())
.or_insert(json!("safe"));
obj.entry("availability".to_string())
.or_insert(json!("ready"));
let _ = id;
Ok(spec)
}
fn validate_args_against_schema(schema: &Value, arguments: &Value) -> Result<(), String> {
let args = arguments
.as_object()
.ok_or_else(|| "arguments must be object".to_string())?;
let sch = schema.as_object().ok_or_else(|| "input_schema invalid".to_string())?;
let props = sch
.get("properties")
.and_then(|v| v.as_object())
.cloned()
.unwrap_or_default();
let additional = sch
.get("additionalProperties")
.and_then(|v| v.as_bool())
.unwrap_or(true);
if !additional {
for key in args.keys() {
if !props.contains_key(key) {
return Err(format!("unknown argument `{key}`"));
}
}
}
if let Some(req) = sch.get("required").and_then(|v| v.as_array()) {
for r in req {
let name = r.as_str().unwrap_or("");
if name.is_empty() {
continue;
}
if !args.contains_key(name) || args.get(name).map(|v| v.is_null()).unwrap_or(true) {
return Err(format!("missing required argument `{name}`"));
}
}
}
for (key, val) in args {
if let Some(prop) = props.get(key) {
if let Some(pty) = prop.get("type").and_then(|v| v.as_str()) {
let ok = match pty {
"string" => val.is_string(),
"integer" => val.as_i64().is_some() || val.as_u64().is_some(),
"number" => val.as_f64().is_some(),
"boolean" => val.is_boolean(),
"array" => val.is_array(),
"object" => val.is_object(),
_ => true,
};
if !ok {
return Err(format!("argument `{key}` type mismatch (want {pty})"));
}
}
if let Some(min) = prop.get("minimum").and_then(|v| v.as_f64()) {
if let Some(n) = val.as_f64() {
if n < min {
return Err(format!("argument `{key}` below minimum {min}"));
}
}
}
if let Some(max) = prop.get("maximum").and_then(|v| v.as_f64()) {
if let Some(n) = val.as_f64() {
if n > max {
return Err(format!("argument `{key}` above maximum {max}"));
}
}
}
}
}
Ok(())
}
fn authorized(allowlist: Option<&Value>, tool_id: &str) -> bool {
match allowlist {
None => true, // registry-only call sites may omit; still need registered spec
Some(tools) => crate::tool_names(tools).iter().any(|n| n == tool_id),
}
}
/// Register one ToolSpecV1 (or list). Host/trusted only — model cannot invent principal/root.
pub fn tools_register(args: &Value) -> Result<Value, String> {
let specs = if let Some(arr) = args.get("specs").and_then(|v| v.as_array()) {
arr.clone()
} else if args.get("spec").is_some() || args.get("id").is_some() {
let raw = args.get("spec").cloned().unwrap_or_else(|| args.clone());
vec![raw]
} else {
return Err("tools.register requires spec or specs[]".into());
};
let mut registered = Vec::new();
let mut guard = registry()
.lock()
.map_err(|_| "tools registry lock poisoned".to_string())?;
for raw in specs {
let spec = normalize_spec(raw)?;
let id = spec
.get("id")
.and_then(|v| v.as_str())
.unwrap()
.to_string();
guard.insert(id.clone(), spec);
registered.push(id);
}
Ok(json!({
"ok": true,
"registered": registered,
"count": registered.len(),
}))
}
pub fn tools_clear(args: &Value) -> Result<Value, String> {
let _ = args;
let mut guard = registry()
.lock()
.map_err(|_| "tools registry lock poisoned".to_string())?;
let n = guard.len();
guard.clear();
Ok(json!({"ok": true, "cleared": n}))
}
pub fn tools_search(args: &Value) -> Result<Value, String> {
let query = opt_text(args, "query").unwrap_or("").to_ascii_lowercase();
let allow = args.get("tools");
let guard = registry()
.lock()
.map_err(|_| "tools registry lock poisoned".to_string())?;
let mut items = Vec::new();
for (id, spec) in guard.iter() {
let avail = spec
.get("availability")
.and_then(|v| v.as_str())
.unwrap_or("ready");
let auth = authorized(allow, id);
if !auth {
// Do not leak unauthorized private connector names in search hits.
continue;
}
if !query.is_empty() {
let desc = spec
.get("description")
.and_then(|v| v.as_str())
.unwrap_or("");
if !id.to_ascii_lowercase().contains(&query)
&& !desc.to_ascii_lowercase().contains(&query)
{
continue;
}
}
items.push(json!({
"id": id,
"version": spec.get("version").cloned().unwrap_or(json!("1")),
"description": spec.get("description").cloned().unwrap_or(json!("")),
"effect": spec.get("effect").cloned().unwrap_or(json!("read")),
"availability": avail,
"authorized": true,
}));
}
items.sort_by(|a, b| {
let ai = a.get("id").and_then(|v| v.as_str()).unwrap_or("");
let bi = b.get("id").and_then(|v| v.as_str()).unwrap_or("");
ai.cmp(bi)
});
Ok(json!({
"ok": true,
"schema": "ToolSearchResultV1",
"schema_version": 1,
"items": items,
"count": items.len(),
}))
}
pub fn tools_describe(args: &Value) -> Result<Value, String> {
let id = arg_text(args, "id")?;
let allow = args.get("tools");
if !authorized(allow, id) {
return Ok(json!({
"ok": false,
"found": false,
"authorized": false,
"error": {
"schema": "AgentErrorV1",
"schema_version": 1,
"code": "tool.forbidden",
"category": "forbidden",
"stage": "tool",
"message": format!("tool `{id}` not in allowlist"),
"retryable": false,
"effect_state": "none",
"recovery_action": "needs_input",
}
}));
}
let guard = registry()
.lock()
.map_err(|_| "tools registry lock poisoned".to_string())?;
match guard.get(id) {
Some(spec) => Ok(json!({
"ok": true,
"found": true,
"authorized": true,
"spec": spec,
})),
None => Ok(json!({
"ok": false,
"found": false,
"authorized": true,
"error": {
"schema": "AgentErrorV1",
"schema_version": 1,
"code": "tool.not_found",
"category": "capability_unavailable",
"stage": "tool",
"message": format!("tool `{id}` not registered"),
"retryable": false,
"effect_state": "none",
"recovery_action": "stop",
}
})),
}
}
/// Validate + authorize a ToolCallV1. Does not execute side effects.
pub fn tools_prepare_call(args: &Value) -> Result<Value, String> {
let tool_id = arg_text(args, "tool_id")
.or_else(|_| arg_text(args, "id"))
.or_else(|_| arg_text(args, "name"))?;
let allow = args.get("tools");
if !authorized(allow, tool_id) {
return Ok(tool_result_err(
"tool.forbidden",
"forbidden",
format!("unauthorized call to `{tool_id}`"),
false,
));
}
let guard = registry()
.lock()
.map_err(|_| "tools registry lock poisoned".to_string())?;
let spec = match guard.get(tool_id) {
Some(s) => s.clone(),
None => {
return Ok(tool_result_err(
"tool.not_found",
"capability_unavailable",
format!("tool `{tool_id}` not registered"),
false,
));
}
};
drop(guard);
let avail = spec
.get("availability")
.and_then(|v| v.as_str())
.unwrap_or("ready");
if avail != "ready" {
return Ok(tool_result_err(
"tool.unavailable",
"capability_unavailable",
format!("tool `{tool_id}` availability={avail}"),
false,
));
}
let arguments = match args.get("arguments") {
None | Some(Value::Null) => json!({}),
Some(v) => v.clone(),
};
let schema = spec
.get("input_schema")
.cloned()
.unwrap_or_else(|| json!({"type":"object"}));
if let Err(e) = validate_args_against_schema(&schema, &arguments) {
return Ok(tool_result_err(
"tool.invalid_arguments",
"invalid_arguments",
e,
true,
));
}
let call_id = opt_text(args, "call_id")
.map(|s| s.to_string())
.unwrap_or_else(|| format!("call_{}", std::process::id()));
let operation_id = opt_text(args, "operation_id")
.map(|s| s.to_string())
.unwrap_or_else(|| format!("op_{tool_id}_{call_id}"));
let version = spec
.get("version")
.and_then(|v| v.as_str())
.unwrap_or("1");
let effect = spec
.get("effect")
.and_then(|v| v.as_str())
.unwrap_or("read");
let digest = args_digest(&arguments);
let logical_run = opt_text(args, "logical_run_id").unwrap_or("-");
let step = opt_text(args, "step_id").unwrap_or("-");
let idempotency_key = opt_text(args, "idempotency_key")
.map(|s| s.to_string())
.unwrap_or_else(|| format!("{logical_run}|{step}|{tool_id}|{digest}"));
let needs_ledger = matches!(effect, "write" | "execute" | "external_write");
let call = json!({
"schema": "ToolCallV1",
"schema_version": 1,
"call_id": call_id,
"tool_id": tool_id,
"tool_version": version,
"arguments": arguments,
"operation_id": operation_id,
"parent_span_id": args.get("parent_span_id").cloned().unwrap_or(Value::Null),
});
Ok(json!({
"ok": true,
"status": "prepared",
"call": call,
"spec": {
"id": tool_id,
"effect": effect,
"timeout_ms": spec.get("timeout_ms").cloned().unwrap_or(json!(10000)),
"max_result_bytes": spec.get("max_result_bytes").cloned().unwrap_or(json!(65536)),
},
"args_digest": digest,
"idempotency_key": idempotency_key,
"needs_ledger": needs_ledger,
"summary": format!("prepared authorized call to `{tool_id}`"),
}))
}
/// Stable digest of tool arguments for effect ledger idempotency (M-A5a).
pub fn tools_args_digest(args: &Value) -> Result<Value, String> {
let arguments = args
.get("arguments")
.cloned()
.or_else(|| args.get("value").cloned())
.unwrap_or_else(|| json!({}));
Ok(json!({
"ok": true,
"args_digest": args_digest(&arguments),
}))
}
fn args_digest(arguments: &Value) -> String {
let text = serde_json::to_string(arguments).unwrap_or_else(|_| "{}".into());
format!("sha256-lite:{:x}", fnv1a64(&text))
}
fn fnv1a64(s: &str) -> u64 {
let mut h: u64 = 0xcbf29ce484222325;
for b in s.as_bytes() {
h ^= u64::from(*b);
h = h.wrapping_mul(0x100000001b3);
}
h
}
/// True when effect_class requires EffectLedger before/after execution.
pub fn tools_needs_ledger(args: &Value) -> Result<Value, String> {
let effect = opt_text(args, "effect")
.or_else(|| opt_text(args, "effect_class"))
.unwrap_or("read");
Ok(json!({
"ok": true,
"needs_ledger": matches!(effect, "write" | "execute" | "external_write"),
"effect": effect,
}))
}
/// Wrap a successful / failed tool execution into ToolResultV1.
pub fn tools_wrap_result(args: &Value) -> Result<Value, String> {
let ok = args.get("ok").and_then(|v| v.as_bool()).unwrap_or(false);
let status = opt_text(args, "status").unwrap_or(if ok { "succeeded" } else { "failed" });
let summary = opt_text(args, "summary").unwrap_or("");
let mut out = json!({
"schema": "ToolResultV1",
"schema_version": 1,
"ok": ok,
"status": status,
"summary": summary,
"structured_content": args.get("structured_content").cloned().unwrap_or(Value::Null),
"resources": args.get("resources").cloned().unwrap_or_else(|| json!([])),
"citations": args.get("citations").cloned().unwrap_or_else(|| json!([])),
"error": args.get("error").cloned().unwrap_or(Value::Null),
"usage": args.get("usage").cloned().unwrap_or(Value::Null),
"truncated": args.get("truncated").and_then(|v| v.as_bool()).unwrap_or(false),
"next_cursor": args.get("next_cursor").cloned().unwrap_or(Value::Null),
"effect_receipt_ref": args.get("effect_receipt_ref").cloned().unwrap_or(Value::Null),
});
// Truncate oversized summary into artifact hint (do not dump MB into prompt).
let max_bytes = args
.get("max_result_bytes")
.and_then(|v| v.as_u64())
.unwrap_or(65_536) as usize;
if summary.len() > max_bytes {
if let Some(obj) = out.as_object_mut() {
obj.insert("truncated".into(), json!(true));
obj.insert(
"summary".into(),
json!(format!(
"{}… [truncated; use resource handle]",
&summary[..max_bytes.min(summary.len())]
)),
);
}
}
Ok(out)
}
/// Seed the five engineering tools used by qdagent (aliases remain CALL:tool_*).
pub fn tools_register_eng_defaults(args: &Value) -> Result<Value, String> {
let _ = args;
let specs = eng_default_specs();
tools_register(&json!({"specs": specs}))
}
fn eng_default_specs() -> Vec<Value> {
vec![
json!({
"id": "tool_find",
"version": "1",
"description": "Engineering find via EKC (read-only)",
"input_schema": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
"additionalProperties": false
},
"effect": "read",
"scopes": ["knowledge:read"],
"availability": "ready"
}),
json!({
"id": "tool_reuse",
"version": "1",
"description": "Engineering reuse / preflight discovery (read-only)",
"input_schema": {
"type": "object",
"properties": {"task": {"type": "string"}},
"required": ["task"],
"additionalProperties": false
},
"effect": "read",
"scopes": ["knowledge:read"],
"availability": "ready"
}),
json!({
"id": "tool_bind",
"version": "1",
"description": "Bind and execute a verified knowledge artifact",
"input_schema": {
"type": "object",
"properties": {
"query": {"type": "string"},
"policy": {"type": "string"},
"project": {"type": "string"}
},
"required": ["query"],
"additionalProperties": false
},
"effect": "execute",
"scopes": ["knowledge:execute"],
"availability": "ready"
}),
json!({
"id": "tool_verify",
"version": "1",
"description": "Run L3 knowledge verification checks",
"input_schema": {
"type": "object",
"properties": {},
"additionalProperties": false
},
"effect": "read",
"scopes": ["knowledge:verify"],
"availability": "ready"
}),
json!({
"id": "tool_summarize",
"version": "1",
"description": "Summarize a Context Pack / preflight map",
"input_schema": {
"type": "object",
"properties": {"preflight": {"type": "object"}},
"required": ["preflight"],
"additionalProperties": false
},
"effect": "read",
"scopes": ["knowledge:read"],
"availability": "ready"
}),
]
}
fn tool_result_err(code: &str, category: &str, message: String, retryable: bool) -> Value {
let recovery = if category == "invalid_arguments" {
"repair_args"
} else if category == "forbidden" {
"needs_input"
} else {
"stop"
};
json!({
"ok": false,
"status": "failed",
"summary": message,
"schema": "ToolResultV1",
"schema_version": 1,
"structured_content": null,
"resources": [],
"citations": [],
"error": {
"schema": "AgentErrorV1",
"schema_version": 1,
"code": code,
"category": category,
"stage": "tool",
"message": message,
"retryable": retryable,
"retry_after_ms": null,
"effect_state": "none",
"recovery_action": recovery,
},
"usage": null,
"truncated": false,
"next_cursor": null,
"effect_receipt_ref": null,
})
}
/// Seed stub specs from a CALL allowlist (args.tools).
pub fn seed_from_allowlist_args(args: &Value) -> Result<Value, String> {
let tools = args
.get("tools")
.ok_or_else(|| "tools.seed_allowlist requires tools".to_string())?;
seed_from_allowlist(tools)
}
fn seed_from_allowlist(tools: &Value) -> Result<Value, String> {
let names = crate::tool_names(tools);
let mut specs = Vec::new();
for name in names {
// Minimal stub if not already registered — CALL whitelist compatibility.
let already = {
let guard = registry()
.lock()
.map_err(|_| "tools registry lock poisoned".to_string())?;
guard.contains_key(&name)
};
if already {
continue;
}
let mut m = Map::new();
m.insert("id".into(), json!(name));
m.insert("version".into(), json!("1"));
m.insert(
"description".into(),
json!(format!("Allowlist-derived tool `{name}`")),
);
m.insert(
"input_schema".into(),
json!({"type":"object","properties":{},"additionalProperties":true}),
);
m.insert("effect".into(), json!("execute"));
m.insert("scopes".into(), json!(["call:allowlist"]));
m.insert("availability".into(), json!("ready"));
specs.push(Value::Object(m));
}
if specs.is_empty() {
return Ok(json!({"ok": true, "registered": [], "count": 0}));
}
tools_register(&json!({"specs": specs}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn register_search_describe_and_forbid() {
let _ = tools_clear(&json!({}));
let reg = tools_register_eng_defaults(&json!({})).unwrap();
assert_eq!(reg["count"], 5);
let allow = json!(["tool_find", "tool_reuse"]);
let search = tools_search(&json!({"tools": allow, "query": "find"})).unwrap();
assert_eq!(search["count"], 1);
assert_eq!(search["items"][0]["id"], "tool_find");
// Unauthorized tool must not appear.
let search2 = tools_search(&json!({"tools": allow, "query": "bind"})).unwrap();
assert_eq!(search2["count"], 0);
let denied = tools_describe(&json!({"id": "tool_bind", "tools": allow})).unwrap();
assert_eq!(denied["authorized"], false);
assert_eq!(denied["ok"], false);
let ok = tools_describe(&json!({"id": "tool_find", "tools": allow})).unwrap();
assert_eq!(ok["found"], true);
assert_eq!(ok["spec"]["schema"], "ToolSpecV1");
}
#[test]
fn prepare_rejects_bad_args_and_unauthorized() {
let _ = tools_clear(&json!({}));
tools_register_eng_defaults(&json!({})).unwrap();
let allow = json!(["tool_find"]);
let bad = tools_prepare_call(&json!({
"tool_id": "tool_find",
"tools": allow,
"arguments": {}
}))
.unwrap();
assert_eq!(bad["ok"], false);
assert_eq!(bad["error"]["category"], "invalid_arguments");
let unauth = tools_prepare_call(&json!({
"tool_id": "tool_bind",
"tools": allow,
"arguments": {"query": "x"}
}))
.unwrap();
assert_eq!(unauth["ok"], false);
assert_eq!(unauth["error"]["category"], "forbidden");
let good = tools_prepare_call(&json!({
"tool_id": "tool_find",
"tools": allow,
"arguments": {"query": "foo"},
"call_id": "c1"
}))
.unwrap();
assert_eq!(good["ok"], true);
assert_eq!(good["call"]["schema"], "ToolCallV1");
assert_eq!(good["call"]["tool_id"], "tool_find");
assert_eq!(good["needs_ledger"], false);
assert!(good["args_digest"].as_str().unwrap().starts_with("sha256-lite:"));
let allow2 = json!(["tool_bind"]);
let bind = tools_prepare_call(&json!({
"tool_id": "tool_bind",
"tools": allow2,
"arguments": {"query": "x"},
"call_id": "c2",
"logical_run_id": "run1",
"step_id": "s1"
}))
.unwrap();
assert_eq!(bind["ok"], true);
assert_eq!(bind["needs_ledger"], true);
assert_eq!(bind["spec"]["effect"], "execute");
assert!(bind["idempotency_key"]
.as_str()
.unwrap()
.starts_with("run1|s1|tool_bind|"));
}
#[test]
fn needs_ledger_only_for_mutating_effects() {
assert_eq!(
tools_needs_ledger(&json!({"effect": "read"})).unwrap()["needs_ledger"],
false
);
assert_eq!(
tools_needs_ledger(&json!({"effect": "write"})).unwrap()["needs_ledger"],
true
);
assert_eq!(
tools_needs_ledger(&json!({"effect": "execute"})).unwrap()["needs_ledger"],
true
);
}
#[test]
fn refuse_unsupported_schema_keywords() {
let err = tools_register(&json!({
"id": "bad",
"input_schema": {
"type": "object",
"oneOf": [{"type": "object"}]
}
}));
assert!(err.is_err());
}
}
+476 -24
View File
@@ -44,21 +44,40 @@ fn read_limited(mut r: impl Read) -> Result<String, String> {
String::from_utf8(buf).map_err(|e| e.to_string())
}
/// Prefer null over zero when the provider omitted a field (ModelCallUsageV1 / F04).
fn token_or_null(v: Option<i64>) -> Value {
match v {
Some(n) if n >= 0 => json!(n),
_ => Value::Null,
}
}
fn usage_from(data: &Value) -> Value {
let u = data.get("usage").cloned().unwrap_or(Value::Null);
let pt = u.get("prompt_tokens").and_then(|v| v.as_i64()).unwrap_or(0);
let ct = u
.get("completion_tokens")
.and_then(|v| v.as_i64())
.unwrap_or(0);
if u.is_null() || !u.is_object() {
return json!({
"prompt_tokens": Value::Null,
"completion_tokens": Value::Null,
"total_tokens": Value::Null,
"usage_state": "unknown",
"measurement_source": "none",
});
}
let pt = u.get("prompt_tokens").and_then(|v| v.as_i64());
let ct = u.get("completion_tokens").and_then(|v| v.as_i64());
let tt = u
.get("total_tokens")
.and_then(|v| v.as_i64())
.unwrap_or(pt + ct);
.or_else(|| match (pt, ct) {
(Some(a), Some(b)) => Some(a + b),
_ => None,
});
json!({
"prompt_tokens": pt,
"completion_tokens": ct,
"total_tokens": tt,
"prompt_tokens": token_or_null(pt),
"completion_tokens": token_or_null(ct),
"total_tokens": token_or_null(tt),
"usage_state": "final",
"measurement_source": "provider",
})
}
@@ -72,10 +91,101 @@ fn tool_calls_from(data: &Value) -> Value {
.unwrap_or_else(|| json!([]))
}
fn parse_sse_events(reader: impl BufRead, echo: bool) -> Result<Vec<Value>, String> {
fn cancel_requested() -> bool {
// Prefer request-scoped token handle. Empty / "0" ≠ cancel (M-R1).
// When a token path is set, ignore process-global MARQDO_CANCEL so parallel
// tasks cannot cancel each other via shared env.
if let Ok(tok) = std::env::var("MARQDO_CANCEL_TOKEN") {
if !tok.is_empty() {
let path = std::path::Path::new(&tok);
if path.is_file() {
if let Ok(body) = std::fs::read_to_string(path) {
let t = body.trim();
return t == "1" || t.eq_ignore_ascii_case("cancelled");
}
}
return false;
}
}
// Legacy single-task CLI only when no token handle is bound.
std::env::var("MARQDO_CANCEL")
.map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
.unwrap_or(false)
}
/// Mirror host `net::push_openai_delta_events`: append semantic frames to
/// `MARQDO_EVENT_FORWARD` so subtask/web sse-live can flush tokens before the
/// lib call returns (OpenAI-style incremental SSE).
fn forward_event_bus_frame(ev: &Value) {
let Ok(path) = std::env::var("MARQDO_EVENT_FORWARD") else {
return;
};
if path.is_empty() {
return;
}
use std::io::Write;
if let Ok(mut f) = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&path)
{
let _ = writeln!(f, "{ev}");
let _ = f.flush();
}
}
/// Publish `{type:delta|plan|reasoning|error, text|message}` from one OpenAI chunk.
/// `content_type` is usually `delta` (answer tokens) or `plan` (parent protocol LLM).
fn forward_openai_chunk(data: &str, content_type: &str) {
let data = data.trim();
if data.is_empty() || data == "[DONE]" {
return;
}
let Ok(v) = serde_json::from_str::<Value>(data) else {
return;
};
if let Some(msg) = v.get("error").and_then(|e| {
e.get("message")
.and_then(|m| m.as_str())
.or_else(|| e.as_str())
}) {
forward_event_bus_frame(&json!({"type": "error", "message": msg}));
return;
}
let reasoning = v
.pointer("/choices/0/delta/reasoning_content")
.and_then(|c| c.as_str())
.unwrap_or("");
if !reasoning.is_empty() {
forward_event_bus_frame(&json!({"type": "reasoning", "text": reasoning}));
}
let content = v
.pointer("/choices/0/delta/content")
.and_then(|c| c.as_str())
.unwrap_or("");
if !content.is_empty() {
let ty = if content_type.is_empty() {
"delta"
} else {
content_type
};
forward_event_bus_frame(&json!({"type": ty, "text": content}));
}
}
fn parse_sse_events(
reader: impl BufRead,
echo: bool,
content_type: &str,
) -> Result<Vec<Value>, String> {
let mut events = Vec::new();
let mut data_lines: Vec<String> = Vec::new();
let mut accumulated = String::new();
for line in reader.lines() {
if cancel_requested() {
events.push(json!({"type": "cancelled", "message": "MARQDO_CANCEL"}));
return Ok(events);
}
let line = line.map_err(|e| e.to_string())?;
if echo {
eprint!("{line}\n");
@@ -85,9 +195,23 @@ fn parse_sse_events(reader: impl BufRead, echo: bool) -> Result<Vec<Value>, Stri
let data = data_lines.join("\n");
data_lines.clear();
if data.trim() == "[DONE]" {
// Parent owns run lifecycle `done`; only forward for answer streams.
if content_type == "delta" || content_type.is_empty() {
let done = json!({"type": "done", "result": accumulated});
forward_event_bus_frame(&done);
}
events.push(json!({"type": "done", "data": "[DONE]"}));
break;
}
forward_openai_chunk(&data, content_type);
if let Ok(v) = serde_json::from_str::<Value>(&data) {
if let Some(c) = v
.pointer("/choices/0/delta/content")
.and_then(|x| x.as_str())
{
accumulated.push_str(c);
}
}
events.push(json!({"type": "data", "data": data}));
}
continue;
@@ -99,21 +223,123 @@ fn parse_sse_events(reader: impl BufRead, echo: bool) -> Result<Vec<Value>, Stri
if !data_lines.is_empty() {
let data = data_lines.join("\n");
if data.trim() == "[DONE]" {
if content_type == "delta" || content_type.is_empty() {
let done = json!({"type": "done", "result": accumulated});
forward_event_bus_frame(&done);
}
events.push(json!({"type": "done", "data": "[DONE]"}));
} else {
forward_openai_chunk(&data, content_type);
events.push(json!({"type": "data", "data": data}));
}
}
Ok(events)
}
fn chat_completions(args: &Value) -> Result<Value, String> {
let base_url = arg_text(args, "base_url")?;
let api_key = arg_text(args, "api_key").unwrap_or("");
let model = arg_text(args, "model")?;
fn build_messages(args: &Value) -> Result<Value, String> {
if let Some(arr) = args.get("messages").and_then(|v| v.as_array()) {
if arr.is_empty() {
return Err("messages array must not be empty".into());
}
// Validate minimal role/content shape; reject unsupported part types loudly.
for (i, m) in arr.iter().enumerate() {
let role = m.get("role").and_then(|r| r.as_str()).unwrap_or("");
if role.is_empty() {
return Err(format!("llm.unsupported_input: messages[{i}].role missing"));
}
if let Some(parts) = m.get("content").and_then(|c| c.as_array()) {
for (j, p) in parts.iter().enumerate() {
let ty = p.get("type").and_then(|t| t.as_str()).unwrap_or("text");
if ty != "text" {
return Err(format!(
"llm.unsupported_input: messages[{i}].content[{j}] type={ty}"
));
}
}
}
}
return Ok(Value::Array(arr.clone()));
}
let prompt = arg_text(args, "prompt")?;
Ok(json!([{"role": "user", "content": prompt}]))
}
fn chat_completions(args: &Value) -> Result<Value, String> {
if cancel_requested() {
return Ok(json!({
"text": "",
"usage": {
"prompt_tokens": Value::Null,
"completion_tokens": Value::Null,
"total_tokens": Value::Null,
"usage_state": "unknown",
"measurement_source": "none",
},
"finish": "cancelled",
"error": "cancelled",
"tool_calls": [],
"transport": "plugin",
}));
}
// Strict selection mode: refuse env fallback when selection.strict is set.
let strict = opt_bool(args, "strict_selection", false)
|| args
.get("selection")
.and_then(|s| s.get("strict"))
.and_then(|v| v.as_bool())
.unwrap_or(false);
let selection = args.get("selection").cloned();
if strict {
if selection.is_none() {
return Err("llm.selection_missing:selection".into());
}
let sel = selection.as_ref().unwrap();
for key in ["selection_id", "provider_id", "model_id", "upstream_model_id"] {
if sel
.get(key)
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.is_none()
{
return Err(format!("llm.selection_missing:{key}"));
}
}
}
let base_url = match arg_text(args, "base_url") {
Ok(u) => u,
Err(_) if strict => return Err("llm.selection_missing:base_url".into()),
Err(e) => return Err(e),
};
let auth_mode = opt_text(args, "auth_mode")
.or_else(|| {
selection
.as_ref()
.and_then(|s| s.get("auth_mode"))
.and_then(|v| v.as_str())
})
.unwrap_or("api_key");
let api_key = if auth_mode == "none" {
""
} else {
arg_text(args, "api_key").unwrap_or("")
};
if strict && auth_mode != "none" && api_key.is_empty() {
return Err("llm.selection_missing:api_key".into());
}
let model = selection
.as_ref()
.and_then(|s| s.get("upstream_model_id"))
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
.or_else(|| arg_text(args, "model").ok().map(|s| s.to_string()))
.ok_or_else(|| "missing text `model`".to_string())?;
let messages = build_messages(args)?;
let stream = opt_bool(args, "stream", false);
let echo = opt_bool(args, "echo", false);
// Answer tokens → "delta"; parent plan protocol → "plan" (UI process trail).
let stream_kind = opt_text(args, "stream_kind").unwrap_or("delta");
let suffix = opt_text(args, "suffix").unwrap_or("/chat/completions");
let bearer = opt_text(args, "bearer").unwrap_or("Bearer ");
@@ -123,11 +349,20 @@ fn chat_completions(args: &Value) -> Result<Value, String> {
}
let mut req_body = Map::new();
req_body.insert("model".into(), Value::String(model.to_string()));
req_body.insert(
"messages".into(),
json!([{"role": "user", "content": prompt}]),
);
req_body.insert("model".into(), Value::String(model.clone()));
req_body.insert("messages".into(), messages);
if let Some(params) = selection
.as_ref()
.and_then(|s| s.get("parameters"))
.and_then(|p| p.as_object())
{
if let Some(max) = params.get("max_output_tokens").and_then(|v| v.as_u64()) {
req_body.insert("max_tokens".into(), json!(max));
}
if let Some(temp) = params.get("temperature") {
req_body.insert("temperature".into(), temp.clone());
}
}
if stream {
req_body.insert("stream".into(), Value::Bool(true));
}
@@ -162,7 +397,8 @@ fn chat_completions(args: &Value) -> Result<Value, String> {
"transport": "plugin",
}));
}
let events = parse_sse_events(BufReader::new(r.into_reader()), echo)?;
let events =
parse_sse_events(BufReader::new(r.into_reader()), echo, stream_kind)?;
Ok(json!({
"events": events,
"finish": "stop",
@@ -193,11 +429,19 @@ fn chat_completions(args: &Value) -> Result<Value, String> {
if status != 200 {
return Ok(json!({
"text": "",
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
"usage": {
"prompt_tokens": Value::Null,
"completion_tokens": Value::Null,
"total_tokens": Value::Null,
"usage_state": "unknown",
"measurement_source": "none",
},
"finish": "error",
"error": format!("HTTP {status}: {text}"),
"tool_calls": [],
"transport": "plugin",
"requested_model": model,
"resolved_model": model,
}));
}
let data: Value = serde_json::from_str(&text).map_err(|e| e.to_string())?;
@@ -208,9 +452,12 @@ fn chat_completions(args: &Value) -> Result<Value, String> {
.and_then(|c0| c0.get("message"))
.cloned()
.unwrap_or(json!({}));
// Some gateways (reasoning models) return null content + reasoning_content.
let content = msg
.get("content")
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.or_else(|| msg.get("reasoning_content").and_then(|v| v.as_str()))
.unwrap_or("")
.to_string();
let fr = data
@@ -221,32 +468,62 @@ fn chat_completions(args: &Value) -> Result<Value, String> {
.and_then(|v| v.as_str())
.unwrap_or("stop")
.to_string();
let resolved = data
.get("model")
.and_then(|v| v.as_str())
.unwrap_or(model.as_str())
.to_string();
Ok(json!({
"text": content,
"usage": usage_from(&data),
"finish": fr,
"tool_calls": tool_calls_from(&data),
"transport": "plugin",
"requested_model": model,
"resolved_model": resolved,
"selection": selection.as_ref().map(|s| {
let mut snap = s.clone();
if let Some(obj) = snap.as_object_mut() {
obj.remove("api_key");
}
snap
}),
}))
}
Err(ureq::Error::Status(code, r)) => {
let text = read_limited(r.into_reader()).unwrap_or_default();
Ok(json!({
"text": "",
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
"usage": {
"prompt_tokens": Value::Null,
"completion_tokens": Value::Null,
"total_tokens": Value::Null,
"usage_state": "unknown",
"measurement_source": "none",
},
"finish": "error",
"error": format!("HTTP {code}: {text}"),
"tool_calls": [],
"transport": "plugin",
"requested_model": model,
"resolved_model": model,
}))
}
Err(e) => Ok(json!({
"text": "",
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
"usage": {
"prompt_tokens": Value::Null,
"completion_tokens": Value::Null,
"total_tokens": Value::Null,
"usage_state": "unknown",
"measurement_source": "none",
},
"finish": "error",
"error": format!("http POST {url}: {e}"),
"tool_calls": [],
"transport": "plugin",
"requested_model": model,
"resolved_model": model,
})),
}
}
@@ -274,5 +551,180 @@ unsafe extern "C" fn llm_ping(
marqdo_plugin!(marqdo_plugin_sdk::ABI_VERSION, [
("llm_ping", "", llm_ping),
("llm_chat_completions", "base_url,api_key,model,prompt,stream,echo,suffix,bearer", llm_chat_completions),
("llm_chat_completions", "base_url,api_key,model,prompt,messages,selection,strict_selection,auth_mode,stream,echo,suffix,bearer", llm_chat_completions),
]);
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn f03_messages_preserve_roles_and_reject_image() {
let msgs = build_messages(&json!({
"messages": [
{"role": "system", "content": "rules"},
{"role": "user", "content": "你好"},
{"role": "assistant", "content": "先前"},
{"role": "user", "content": "最新用户"}
]
}))
.unwrap();
let arr = msgs.as_array().unwrap();
assert_eq!(arr.len(), 4);
assert_eq!(arr[3]["role"], "user");
assert_eq!(arr[3]["content"], "最新用户");
let err = build_messages(&json!({
"messages": [{"role": "user", "content": [{"type": "image", "url": "x"}]}]
}))
.unwrap_err();
assert!(err.contains("unsupported_input"), "{err}");
}
#[test]
fn f01_strict_selection_does_not_invent_env_model() {
std::env::remove_var("MARQDO_CANCEL");
std::env::remove_var("MARQDO_CANCEL_TOKEN");
let err = chat_completions(&json!({
"strict_selection": true,
"base_url": "http://127.0.0.1:9",
"api_key": "k",
"prompt": "hi"
// missing selection → must fail before any HTTP
}))
.unwrap_err();
assert!(err.contains("selection_missing"), "{err}");
}
#[test]
fn f06_cancel_short_circuits_before_http() {
// Prefer token file over process-global MARQDO_CANCEL to avoid parallel test races.
let tok = std::env::temp_dir().join(format!("llm-cancel-{}", std::process::id()));
std::fs::write(&tok, "cancelled").unwrap();
std::env::set_var("MARQDO_CANCEL_TOKEN", &tok);
let out = chat_completions(&json!({
"base_url": "http://127.0.0.1:9",
"api_key": "k",
"model": "m",
"prompt": "hi"
}));
std::env::remove_var("MARQDO_CANCEL_TOKEN");
let _ = std::fs::remove_file(&tok);
let out = out.unwrap();
assert_eq!(out["finish"], "cancelled");
}
#[test]
fn f06_empty_cancel_token_is_not_cancelled() {
// sse-live creates an empty token file; that must NOT abort the turn.
let tok = std::env::temp_dir().join(format!("llm-cancel-empty-{}", std::process::id()));
std::fs::write(&tok, "").unwrap();
std::env::set_var("MARQDO_CANCEL_TOKEN", &tok);
std::env::remove_var("MARQDO_CANCEL");
assert!(!cancel_requested());
std::env::remove_var("MARQDO_CANCEL_TOKEN");
let _ = std::fs::remove_file(&tok);
}
/// F01: two local endpoints receive distinct models/auth — no cross-talk.
#[test]
fn f01_dual_endpoint_isolation() {
use std::io::{Read, Write};
use std::net::TcpListener;
use std::sync::{Arc, Mutex};
use std::thread;
fn spawn_mock(tag: &'static str, seen: Arc<Mutex<Vec<String>>>) -> String {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut raw = Vec::new();
let mut buf = [0u8; 4096];
loop {
let n = stream.read(&mut buf).unwrap_or(0);
if n == 0 {
break;
}
raw.extend_from_slice(&buf[..n]);
if let Some(pos) = raw.windows(4).position(|w| w == b"\r\n\r\n") {
let header = String::from_utf8_lossy(&raw[..pos]);
let cl = header
.lines()
.find_map(|l| l.strip_prefix("Content-Length:"))
.and_then(|v| v.trim().parse::<usize>().ok())
.unwrap_or(0);
let body_start = pos + 4;
while raw.len() < body_start + cl {
let n = stream.read(&mut buf).unwrap_or(0);
if n == 0 {
break;
}
raw.extend_from_slice(&buf[..n]);
}
break;
}
}
let req = String::from_utf8_lossy(&raw).to_string();
seen.lock().unwrap().push(format!("{tag}:{req}"));
let body = r#"{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}"#;
let resp = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
let _ = stream.write_all(resp.as_bytes());
}
});
format!("http://{addr}")
}
let seen_a = Arc::new(Mutex::new(Vec::new()));
let seen_b = Arc::new(Mutex::new(Vec::new()));
let url_a = spawn_mock("A", seen_a.clone());
let url_b = spawn_mock("B", seen_b.clone());
let out_a = chat_completions(&json!({
"base_url": url_a,
"api_key": "key-a",
"model": "model-a",
"auth_mode": "api_key",
"messages": [{"role":"user","content":"from-a"}],
"selection": {
"selection_id": "sel-a",
"provider_id": "p-a",
"model_id": "m-a",
"upstream_model_id": "model-a",
"strict": true
},
"strict_selection": true
}))
.unwrap();
let out_b = chat_completions(&json!({
"base_url": url_b,
"api_key": "key-b",
"model": "model-b",
"auth_mode": "api_key",
"messages": [{"role":"user","content":"from-b"}],
"selection": {
"selection_id": "sel-b",
"provider_id": "p-b",
"model_id": "m-b",
"upstream_model_id": "model-b",
"strict": true
},
"strict_selection": true
}))
.unwrap();
assert_eq!(out_a["finish"], "stop", "{out_a}");
assert_eq!(out_b["finish"], "stop", "{out_b}");
let a = seen_a.lock().unwrap().join("\n");
let b = seen_b.lock().unwrap().join("\n");
assert!(a.contains("model-a"), "A saw wrong model: {a}");
assert!(a.contains("key-a") || a.contains("Authorization"), "{a}");
assert!(!a.contains("model-b"), "A must not see B's model: {a}");
assert!(b.contains("model-b"), "B saw wrong model: {b}");
assert!(!b.contains("model-a"), "B must not see A's model: {b}");
}
}
+23 -2
View File
@@ -63,6 +63,11 @@ func ParseFrontmatter(src string) (meta map[string]string, body string) {
body = body[i+3:]
body = strings.TrimPrefix(body, "\r")
body = strings.TrimPrefix(body, "\n")
// Drop blank line(s) between closing --- and first content.
for strings.HasPrefix(body, "\n") || strings.HasPrefix(body, "\r\n") {
body = strings.TrimPrefix(body, "\r\n")
body = strings.TrimPrefix(body, "\n")
}
}
for _, line := range strings.Split(fm, "\n") {
line = strings.TrimSpace(strings.TrimSuffix(line, "\r"))
@@ -254,6 +259,10 @@ func SystemRoutes() []map[string]any {
return []map[string]any{
{"method": "GET", "path": "/_part/*", "kind": "system"},
{"method": "GET|POST", "path": "/_form/*", "kind": "system"},
{"method": "POST", "path": "/_runtime/tasks", "kind": "system"},
{"method": "GET", "path": "/_runtime/tasks/{id}", "kind": "system"},
{"method": "POST", "path": "/_runtime/tasks/{id}/cancel", "kind": "system"},
{"method": "GET", "path": "/_runtime/tasks/{id}/events", "kind": "system"},
}
}
@@ -275,8 +284,20 @@ func InspectBag(app map[string]any, declared []ArtifactFile) map[string]any {
}
}
if inv, ok := app["invoke_routes"].(map[string]any); ok {
for key := range inv {
dec = append(dec, map[string]any{"path": key, "kind": "endpoint"})
for key, v := range inv {
row := map[string]any{"path": key, "kind": "endpoint"}
if m, ok := v.(map[string]any); ok {
if c, ok := m["class"].(string); ok && c != "" {
row["class"] = c
}
if method, ok := m["method"].(string); ok && method != "" {
row["method"] = method
}
if fn, ok := m["fn"].(string); ok && fn != "" {
row["fn"] = fn
}
}
dec = append(dec, row)
}
}
if art, ok := app["artifact_routes"].(map[string]any); ok {
+48
View File
@@ -968,6 +968,54 @@ func Update(url, tableName, id string, row any, txnID ...string) (map[string]any
return map[string]any{"ok": true, "updated": float64(n)}, nil
}
// UpdateIf is a conditional update (CAS): SET row WHERE id=? AND expect_col=expect_val.
// Returns {"ok":true,"updated":N,"matched":N}. Callers treat updated==0 as conflict.
// Optional txnID routes through an open Begin transaction.
func UpdateIf(url, tableName, id string, row any, expectCol string, expectVal any, txnID ...string) (map[string]any, error) {
tn, err := ident(tableName)
if err != nil {
return nil, err
}
ecol, err := ident(expectCol)
if err != nil {
return nil, err
}
obj, ok := row.(map[string]any)
if !ok {
return nil, fmt.Errorf("row must be a map")
}
c, err := connFor(url, optTxn(txnID))
if err != nil {
return nil, err
}
var sets []string
var vals []any
for k, v := range obj {
if k == "id" {
continue
}
if _, err := ident(k); err != nil {
return nil, err
}
sets = append(sets, fmt.Sprintf(`"%s" = ?`, k))
vals = append(vals, toSQLArg(v))
}
if len(sets) == 0 {
return nil, fmt.Errorf("nothing to update")
}
vals = append(vals, id, toSQLArg(expectVal))
sqlStmt := fmt.Sprintf(
`UPDATE "%s" SET %s WHERE "id" = ? AND "%s" = ?`,
tn, strings.Join(sets, ", "), ecol,
)
res, err := c.Exec(sqlStmt, vals...)
if err != nil {
return nil, err
}
n, _ := res.RowsAffected()
return map[string]any{"ok": true, "updated": float64(n), "matched": float64(n)}, nil
}
// Delete removes a row by id. Returns {"ok":true,"deleted":N}.
// Optional txnID routes through an open Begin transaction.
func Delete(url, tableName, id string, txnID ...string) (map[string]any, error) {
+18
View File
@@ -100,6 +100,24 @@ func TestInitInsertSelectGet(t *testing.T) {
}
}
func TestUpdateIfCAS(t *testing.T) {
url := setupArticles(t)
ok, err := db.UpdateIf(url, "articles", "1", map[string]any{"title": "CAS"}, "title", "Alpha")
if err != nil {
t.Fatal(err)
}
if ok["updated"] != float64(1) {
t.Fatalf("cas hit=%v", ok)
}
miss, err := db.UpdateIf(url, "articles", "1", map[string]any{"title": "Nope"}, "title", "Alpha")
if err != nil {
t.Fatal(err)
}
if miss["updated"] != float64(0) {
t.Fatalf("cas miss should be 0, got %v", miss)
}
}
func TestUpdateDelete(t *testing.T) {
url := setupArticles(t)
@@ -0,0 +1,29 @@
package httpx
import (
"os"
"path/filepath"
"testing"
)
// F06: client-disconnect cancel bridge writes token file + MARQDO_CANCEL for llm plugin.
func TestPropagateCancelSignal(t *testing.T) {
dir := t.TempDir()
tok := filepath.Join(dir, "cancel.tok")
if err := os.WriteFile(tok, []byte("cancelled"), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("MARQDO_CANCEL_TOKEN", tok)
t.Setenv("MARQDO_CANCEL", "1")
if os.Getenv("MARQDO_CANCEL") != "1" {
t.Fatalf("MARQDO_CANCEL not set")
}
body, err := os.ReadFile(tok)
if err != nil {
t.Fatal(err)
}
if string(body) != "cancelled" {
t.Fatalf("token body=%q", body)
}
}
@@ -0,0 +1,41 @@
package httpx
import (
"os"
"path/filepath"
"testing"
)
func TestDrainForwardReadsIncrementalLines(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "evt.ndjson")
if err := os.WriteFile(path, []byte("{\"type\":\"a\"}\n{\"type\":\"b\"}\n"), 0o600); err != nil {
t.Fatal(err)
}
var lines []string
off := drainForward(path, 0, func(line string) {
lines = append(lines, line)
})
if len(lines) != 2 {
t.Fatalf("got %d lines %#v", len(lines), lines)
}
if off <= 0 {
t.Fatalf("offset=%d", off)
}
// Append after first drain — second call with offset must only see new line.
f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
t.Fatal(err)
}
if _, err := f.WriteString("{\"type\":\"c\"}\n"); err != nil {
t.Fatal(err)
}
_ = f.Close()
var more []string
off2 := drainForward(path, off, func(line string) {
more = append(more, line)
})
if len(more) != 1 || more[0] != "{\"type\":\"c\"}" {
t.Fatalf("incremental got %#v off=%d→%d", more, off, off2)
}
}
+29
View File
@@ -198,6 +198,35 @@ func invokeArgsFromPayload(payload map[string]any) map[string]any {
return args
}
// payloadWantsStream detects OpenAI-compatible stream:true on the request body.
func payloadWantsStream(payload map[string]any) bool {
if payload == nil {
return false
}
if v, ok := payload["stream"]; ok {
return boolishAny(v)
}
return false
}
func boolishAny(v any) bool {
switch t := v.(type) {
case bool:
return t
case string:
s := strings.ToLower(strings.TrimSpace(t))
return s == "1" || s == "true" || s == "yes" || s == "on"
case float64:
return t != 0
case int:
return t != 0
case int64:
return t != 0
default:
return false
}
}
func readJSONBody(r *http.Request, max int64) (map[string]any, error) {
r.Body = http.MaxBytesReader(nil, r.Body, max)
defer r.Body.Close()
+16
View File
@@ -239,6 +239,7 @@ func NewHandler(appBag map[string]any, entryDir string) (http.Handler, error) {
st.mountGalleryRoutes(mux)
st.mountProxyRoutes(mux)
st.mountInvokeRoutes(mux)
st.mountRuntimeRoutes(mux)
st.mountArtifactRoutes(mux)
st.mountAuthRoutes(mux)
@@ -799,6 +800,13 @@ func (g *gzipResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return nil, nil, fmt.Errorf("underlying ResponseWriter does not support hijacking")
}
func (g *gzipResponseWriter) Flush() {
_ = g.w.Flush()
if f, ok := g.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
type statusRecorder struct {
http.ResponseWriter
status int
@@ -816,6 +824,14 @@ func (s *statusRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return nil, nil, fmt.Errorf("underlying ResponseWriter does not support hijacking")
}
// Flush must be forwarded so sse-live / EventBus token streams are not buffered
// behind access-log wrapping (otherwise clients see 0 bytes until the invoke ends).
func (s *statusRecorder) Flush() {
if f, ok := s.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func writeJSON(w http.ResponseWriter, code int, v any) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(code)
+587 -3
View File
@@ -5,16 +5,40 @@ import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/marqdo/marqdo/plugins/web/internal/plugin"
)
// legacySSEBridgeMu serializes the process-global MARQDO_EVENT_FORWARD /
// MARQDO_CANCEL_TOKEN bridge for the legacy_serial path only. Isolated execute
// never acquires this lock (ADR 0009).
var legacySSEBridgeMu sync.Mutex
// legacySSEBridgeActive is 1 while a holder has mutated process env for the bridge.
var legacySSEBridgeActive atomic.Int32
func tryAcquireLegacySSEBridge() bool {
return legacySSEBridgeMu.TryLock()
}
func releaseLegacySSEBridge() {
legacySSEBridgeActive.Store(0)
legacySSEBridgeMu.Unlock()
}
type invokeRoute struct {
Method string
FnPath string
BodyMode string
ReturnMode string
Class string
Path string
}
func parseInvokeRoute(v any) invokeRoute {
@@ -26,11 +50,16 @@ func parseInvokeRoute(v any) invokeRoute {
if fn == "" {
fn = strOpt(m, "function", "")
}
class := strings.ToLower(strOpt(m, "class", ""))
if class == "" {
class = strings.ToLower(strOpt(m, "类别", ""))
}
return invokeRoute{
Method: method,
FnPath: fn,
BodyMode: body,
ReturnMode: ret,
Class: class,
}
}
@@ -38,6 +67,7 @@ func (st *state) mountInvokeRoutes(mux *http.ServeMux) {
for _, key := range sortedKeys(st.invokeRoutes) {
cfg := parseInvokeRoute(st.invokeRoutes[key])
path := routePathKey(key)
cfg.Path = path
method := cfg.Method
if method == "" {
method = http.MethodPost
@@ -102,7 +132,6 @@ func (st *state) handleInvoke(w http.ResponseWriter, r *http.Request, route invo
}
}
// Bind common auth / client headers for OpenAI-compatible and API-key surfaces.
if auth := strings.TrimSpace(r.Header.Get("Authorization")); auth != "" {
if _, exists := payload["authorization"]; !exists {
payload["authorization"] = auth
@@ -113,12 +142,77 @@ func (st *state) handleInvoke(w http.ResponseWriter, r *http.Request, route invo
payload["api_key"] = key
}
}
if last := strings.TrimSpace(r.Header.Get("Last-Event-ID")); last != "" {
if _, exists := payload["last_event_id"]; !exists {
payload["last_event_id"] = last
}
}
// Merge Go ServeMux path wildcards (e.g. /turns/{id}/events → id / turn_id).
if route.Path != "" {
pathVals := pathValuesOf(r, route.Path)
for k, v := range pathVals {
if _, exists := payload[k]; !exists {
payload[k] = v
}
}
if id, ok := pathVals["id"]; ok {
if _, exists := payload["turn_id"]; !exists {
payload["turn_id"] = id
}
}
}
args := invokeArgsFromPayload(payload)
liveSSE := route.ReturnMode == "sse-live" || route.ReturnMode == "sse_live"
if !liveSSE && payloadWantsStream(payload) {
liveSSE = true
}
class := resolveInvokeClass(route, liveSSE)
switch class {
case "execute":
// Isolated workers: do not take parent INVOKE_LOCK (ADR 0009).
st.handleInvokeIsolated(w, r, route, args, liveSSE, class)
return
case "short", "read", "control":
// Stay on parent: execute already runs off-process, so parent lock is free.
// Putting short/read/control on the 2-slot execute worker pool caused
// systemic no_slot (and zombies held slots after observer disconnect).
// Product cancel/events prefer /_runtime/* (lock-free).
st.handleInvokeParent(w, r, route, args)
return
default:
// legacy_serial: parent CallLibPath (ADR 0008). Never isolate undeclared non-stream.
st.handleInvokeParent(w, r, route, args)
}
}
func resolveInvokeClass(route invokeRoute, liveSSE bool) string {
c := strings.ToLower(strings.TrimSpace(route.Class))
switch c {
case "execute", "short", "read", "control":
return c
case "legacy", "legacy_serial":
return "legacy_serial"
}
if liveSSE {
return "execute"
}
return "legacy_serial"
}
func (st *state) handleInvokeParent(w http.ResponseWriter, r *http.Request, route invokeRoute, args map[string]any) {
_ = r
value, err := plugin.CallLibPath(route.FnPath, args)
if err != nil {
code := http.StatusInternalServerError
msg := err.Error()
if strings.Contains(msg, "interpreter busy") {
w.Header().Set("Retry-After", "2")
jsonErr(w, http.StatusServiceUnavailable, msg)
return
}
if strings.Contains(msg, "unknown library") ||
strings.Contains(msg, "call_lib_path: need") ||
strings.Contains(msg, "not allowed") ||
@@ -133,8 +227,6 @@ func (st *state) handleInvoke(w http.ResponseWriter, r *http.Request, route invo
return
}
// OpenAI-compatible handlers may return `{sse: "..."}` to force event-stream
// even when the route's declared return mode is json.
if m, ok := value.(map[string]any); ok {
if s, ok := m["sse"].(string); ok && strings.TrimSpace(s) != "" {
writeSSE(w, m)
@@ -176,3 +268,495 @@ func (st *state) handleInvoke(w http.ResponseWriter, r *http.Request, route invo
writeJSON(w, code, body)
}
}
// handleInvokeIsolated runs admits on a Supervisor worker (no parent
// CallLibPath / Setenv / legacySSEBridgeMu). admissionClass is execute|short|read|control.
func (st *state) handleInvokeIsolated(w http.ResponseWriter, r *http.Request, route invokeRoute, args map[string]any, liveSSE bool, admissionClass string) {
if admissionClass == "" {
admissionClass = "execute"
}
taskID := fmt.Sprintf("exec_%d", time.Now().UnixNano())
adm, err := runtimeQuery("admit", map[string]any{
"task_id": taskID,
"idempotency_key": taskID,
"class": admissionClass,
"payload": map[string]any{"fn": route.FnPath},
})
if err != nil {
jsonErr(w, http.StatusInternalServerError, fmt.Sprintf("admit: %v", err))
return
}
if ok, _ := adm["ok"].(bool); !ok {
status := http.StatusServiceUnavailable
if s, ok := adm["status"].(float64); ok && s >= 400 {
status = int(s)
}
w.Header().Set("Retry-After", "2")
jsonErr(w, status, fmt.Sprint(adm["error"]))
return
}
cancelTok, _ := adm["cancel_token"].(string)
eventPath, _ := adm["event_path"].(string)
entryModule, fsRoot, err := resolveSiteEntry(st)
if err != nil {
jsonErr(w, http.StatusInternalServerError, err.Error())
return
}
argsFile, err := materializeArgsJSON(args)
if err != nil {
jsonErr(w, http.StatusInternalServerError, fmt.Sprintf("args: %v", err))
return
}
defer os.Remove(argsFile)
disp, err := runtimeQuery("dispatch", map[string]any{
"task_id": taskID,
"entry": map[string]any{
"module": entryModule,
"function": route.FnPath,
},
"fs_root": fsRoot,
"args_json": argsFile,
})
if err != nil {
_, _ = runtimeQuery("cancel", map[string]any{"task_id": taskID})
jsonErr(w, http.StatusInternalServerError, fmt.Sprintf("dispatch: %v", err))
return
}
if ok, _ := disp["ok"].(bool); !ok {
status := http.StatusTooManyRequests
if s, ok := disp["status"].(float64); ok && s >= 400 {
status = int(s)
}
if ra, ok := disp["Retry-After"]; ok {
w.Header().Set("Retry-After", fmt.Sprint(ra))
} else {
w.Header().Set("Retry-After", "2")
}
jsonErr(w, status, fmt.Sprint(disp["error"]))
return
}
if ep, ok := disp["event_path"].(string); ok && ep != "" {
eventPath = ep
}
if tok, ok := disp["cancel_token"].(string); ok && tok != "" {
cancelTok = tok
}
emitResult, _ := disp["emit_result"].(string)
if liveSSE {
st.streamIsolatedSSE(w, r, route, taskID, eventPath, cancelTok, emitResult)
return
}
st.waitIsolatedJSON(w, r, route, taskID, eventPath, cancelTok, emitResult)
}
// cancelOnDisconnect reports whether the client opted into legacy
// disconnect=cancel (header X-Marqdo-Cancel-On-Disconnect: 1|true).
// Default is false: subscription close must not cancel the task (ADR 0009 / design §7.2).
func cancelOnDisconnect(r *http.Request) bool {
v := strings.TrimSpace(strings.ToLower(r.Header.Get("X-Marqdo-Cancel-On-Disconnect")))
return v == "1" || v == "true" || v == "yes"
}
func (st *state) streamIsolatedSSE(w http.ResponseWriter, r *http.Request, route invokeRoute, taskID, eventPath, cancelTok, emitResult string) {
w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.WriteHeader(http.StatusOK)
flusher, _ := w.(http.Flusher)
writeEvent := func(payload any) {
b, err := json.Marshal(payload)
if err != nil {
return
}
_, _ = io.WriteString(w, "data: "+string(b)+"\n\n")
if flusher != nil {
flusher.Flush()
}
}
writeEvent(map[string]any{
"schema": "RuntimeEventV1",
"schema_version": 1,
"type": "run.started",
"payload": map[string]any{
"fn": route.FnPath,
"bridge": "isolated",
"task_id": taskID,
},
})
offset := int64(0)
carry := ""
seq := uint64(1)
poll := time.NewTicker(25 * time.Millisecond)
defer poll.Stop()
ctx := r.Context()
finish := func(clientGone bool) {
offset, carry = drainForwardCarry(eventPath, offset, carry, func(line string) {
seq++
writeEvent(normalizeLiveSSELine(seq, line))
})
if !clientGone {
result := readEmitResult(emitResult)
if result == nil {
if g, err := runtimeQuery("get", map[string]any{"task_id": taskID}); err == nil {
if t, ok := g["task"].(map[string]any); ok {
result = t["result"]
}
}
}
writeEvent(map[string]any{
"schema": "RuntimeEventV1", "schema_version": 1, "type": "run.completed",
"payload": map[string]any{"result": result, "task_id": taskID},
})
_, _ = io.WriteString(w, "data: [DONE]\n\n")
if flusher != nil {
flusher.Flush()
}
}
}
doCancelOnDisconnect := cancelOnDisconnect(r)
for {
select {
case <-ctx.Done():
// Default: observer gone ≠ cancel. Task keeps running; client may
// reattach via GET /_runtime/tasks/{id}/events (Last-Event-ID).
if doCancelOnDisconnect {
_, _ = runtimeQuery("cancel", map[string]any{"task_id": taskID})
_ = writeCancelToken(cancelTok)
writeEvent(map[string]any{
"schema": "RuntimeEventV1", "schema_version": 1, "type": "run.interrupted",
"payload": map[string]any{
"reason": "client_disconnect",
"cancel_propagated": cancelTok != "",
"status": "stopping",
"task_id": taskID,
},
})
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if workerDone(taskID, emitResult) {
break
}
time.Sleep(50 * time.Millisecond)
}
} else {
writeEvent(map[string]any{
"schema": "RuntimeEventV1", "schema_version": 1, "type": "run.interrupted",
"payload": map[string]any{
"reason": "client_disconnect",
"cancel_propagated": false,
"status": "observer_closed",
"task_id": taskID,
"resume": "/_runtime/tasks/" + taskID + "/events",
},
})
// Observer gone ≠ cancel, but still reap when the worker exits so
// execute slots are not held forever by zombies.
go reapIsolatedTask(taskID, emitResult, 10*time.Minute)
}
finish(true)
return
case <-poll.C:
offset, carry = drainForwardCarry(eventPath, offset, carry, func(line string) {
seq++
writeEvent(normalizeLiveSSELine(seq, line))
})
if workerDone(taskID, emitResult) {
finish(false)
return
}
}
}
}
func (st *state) waitIsolatedJSON(w http.ResponseWriter, r *http.Request, route invokeRoute, taskID, eventPath, cancelTok, emitResult string) {
_ = eventPath
poll := time.NewTicker(25 * time.Millisecond)
defer poll.Stop()
ctx := r.Context()
for {
select {
case <-ctx.Done():
_, _ = runtimeQuery("cancel", map[string]any{"task_id": taskID})
_ = writeCancelToken(cancelTok)
jsonErr(w, http.StatusRequestTimeout, "client disconnected")
return
case <-poll.C:
if !workerDone(taskID, emitResult) {
continue
}
value := readEmitResult(emitResult)
if value == nil {
jsonErr(w, http.StatusInternalServerError, "worker produced no result")
return
}
if m, ok := value.(map[string]any); ok {
if s, ok := m["sse"].(string); ok && strings.TrimSpace(s) != "" {
writeSSE(w, m)
return
}
}
switch route.ReturnMode {
case "text", "html", "text/html":
text := fmt.Sprint(value)
if s, ok := value.(string); ok {
text = s
}
ct := "text/plain; charset=utf-8"
if route.ReturnMode == "html" || route.ReturnMode == "text/html" {
ct = "text/html; charset=utf-8"
}
w.Header().Set("Content-Type", ct)
w.WriteHeader(http.StatusOK)
_, _ = io.WriteString(w, text)
case "status":
w.WriteHeader(http.StatusNoContent)
default:
code, body, err := jsonWrapInvokeValue(value)
if err != nil {
jsonErr(w, code, err.Error())
return
}
writeJSON(w, code, body)
}
return
}
}
}
// reapIsolatedTask polls runtime get until the worker exits (or timeout),
// so supervisor/scheduler slots are released after observer disconnect.
func reapIsolatedTask(taskID, emitResult string, budget time.Duration) {
deadline := time.Now().Add(budget)
for time.Now().Before(deadline) {
if workerDone(taskID, emitResult) {
return
}
time.Sleep(200 * time.Millisecond)
}
// Last-chance cancel so slots can free.
_, _ = runtimeQuery("cancel", map[string]any{"task_id": taskID})
for i := 0; i < 25; i++ {
if workerDone(taskID, emitResult) {
return
}
time.Sleep(200 * time.Millisecond)
}
}
func workerDone(taskID, emitResult string) bool {
// Always runtime.get when probing completion: get runs try_reap and
// releases execute slots. Do not return early on emit_result alone —
// that left zombie workers holding both pool slots (no_slot).
if taskID == "" {
return false
}
g, err := runtimeQuery("get", map[string]any{"task_id": taskID})
if err != nil {
// Fallback: result file present (get may fail during early boot).
if emitResult != "" {
if st, err := os.Stat(emitResult); err == nil && st.Size() > 0 {
return true
}
}
if dir := os.Getenv("MARQDO_RUNTIME_DIR"); dir != "" {
p := filepath.Join(dir, "events", taskID+".result.json")
if st, err := os.Stat(p); err == nil && st.Size() > 0 {
return true
}
}
return false
}
if _, ok := g["exit_code"]; ok {
return true
}
if t, ok := g["task"].(map[string]any); ok {
switch fmt.Sprint(t["state"]) {
case "succeeded", "failed", "cancelled", "interrupted":
return true
}
}
return false
}
func readEmitResult(path string) any {
if path == "" {
return nil
}
b, err := os.ReadFile(path)
if err != nil || len(b) == 0 {
return nil
}
var v any
if err := json.Unmarshal(b, &v); err != nil {
return string(b)
}
return v
}
func materializeArgsJSON(args map[string]any) (string, error) {
f, err := os.CreateTemp("", "mq-args-*.json")
if err != nil {
return "", err
}
path := f.Name()
enc := json.NewEncoder(f)
if err := enc.Encode(args); err != nil {
_ = f.Close()
_ = os.Remove(path)
return "", err
}
if err := f.Close(); err != nil {
_ = os.Remove(path)
return "", err
}
return path, nil
}
func resolveSiteEntry(st *state) (module string, fsRoot string, err error) {
fsRoot = st.entryDir
if raw, qerr := plugin.Query("entry_dir", "{}"); qerr == nil {
var s string
if json.Unmarshal([]byte(raw), &s) == nil && s != "" {
fsRoot = s
}
}
if raw, qerr := plugin.Query("entry_path", "{}"); qerr == nil {
var s string
if json.Unmarshal([]byte(raw), &s) == nil && s != "" {
module = s
}
}
if module == "" {
// Fallback: common serve entry names under fs_root.
for _, name := range []string{"serve.mq.md", "index.mq.md", "main.mq.md"} {
p := filepath.Join(fsRoot, name)
if st, e := os.Stat(p); e == nil && !st.IsDir() {
module = p
break
}
}
}
if module == "" {
return "", "", fmt.Errorf("isolated execute: cannot resolve site entry module")
}
if !filepath.IsAbs(module) {
abs, aerr := filepath.Abs(module)
if aerr == nil {
module = abs
}
}
if fsRoot == "" {
fsRoot = filepath.Dir(module)
}
if !filepath.IsAbs(fsRoot) {
abs, aerr := filepath.Abs(fsRoot)
if aerr == nil {
fsRoot = abs
}
}
return module, fsRoot, nil
}
// normalizeLiveSSELine maps Marqdo EventBus frames onto RuntimeEventV1.
func normalizeLiveSSELine(seq uint64, line string) map[string]any {
base := map[string]any{
"schema": "RuntimeEventV1", "schema_version": 1, "seq": seq,
}
var obj map[string]any
if err := json.Unmarshal([]byte(line), &obj); err != nil || obj == nil {
base["type"] = "message.delta"
base["payload"] = map[string]any{"raw": line}
return base
}
if t, _ := obj["type"].(string); t == "message.delta" || t == "turn.started" || t == "model.resolved" || t == "usage.updated" || t == "turn.completed" || t == "turn.failed" {
for k, v := range obj {
base[k] = v
}
base["seq"] = seq
return base
}
if t, _ := obj["type"].(string); t == "delta" || t == "plan" || t == "reasoning" {
text, _ := obj["text"].(string)
base["type"] = "message.delta"
base["payload"] = map[string]any{"delta": text, "kind": t}
return base
}
if t, _ := obj["type"].(string); t == "decompose" || t == "decision" || t == "await" || t == "round" || t == "done" || t == "tool_start" || t == "tool_end" || t == "plan_act" {
base["type"] = "message.delta"
base["payload"] = obj
return base
}
if text, ok := obj["text"].(string); ok && text != "" {
base["type"] = "message.delta"
base["payload"] = map[string]any{"delta": text}
return base
}
base["type"] = "message.delta"
base["payload"] = obj
return base
}
func drainForward(path string, offset int64, onLine func(string)) int64 {
off, _ := drainForwardCarry(path, offset, "", onLine)
return off
}
const drainForwardMaxLine = 1024 * 1024
func drainForwardCarry(path string, offset int64, carry string, onLine func(string)) (int64, string) {
f, err := os.Open(path)
if err != nil {
return offset, carry
}
defer f.Close()
if _, err := f.Seek(offset, io.SeekStart); err != nil {
return offset, carry
}
chunk, err := io.ReadAll(io.LimitReader(f, drainForwardMaxLine+1))
if err != nil && err != io.EOF {
onLine(fmt.Sprintf(`{"type":"protocol.error","error":%q}`, err.Error()))
return offset, carry
}
if len(chunk) == 0 {
return offset, carry
}
offset += int64(len(chunk))
buf := carry + string(chunk)
for {
i := strings.IndexByte(buf, '\n')
if i < 0 {
if len(buf) > drainForwardMaxLine {
onLine(`{"type":"protocol.error","error":"ndjson line exceeds 1MiB"}`)
return offset, ""
}
return offset, buf
}
lineBytes := buf[:i]
line := strings.TrimSpace(strings.TrimSuffix(lineBytes, "\r"))
buf = buf[i+1:]
if line == "" {
continue
}
onLine(line)
}
}
func legacySSEBridgeSnapshot() (active bool, forward string, cancelTok string) {
return legacySSEBridgeActive.Load() != 0, os.Getenv("MARQDO_EVENT_FORWARD"), os.Getenv("MARQDO_CANCEL_TOKEN")
}
func writeCancelToken(path string) error {
if path == "" {
return nil
}
return os.WriteFile(path, []byte("cancelled"), 0o600)
}
@@ -0,0 +1,207 @@
package httpx
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/marqdo/marqdo/plugins/web/internal/plugin"
)
// System runtime control routes (M-R3 / M-R4). Cancel and event replay do not
// enter the Marqdo interpreter lock.
func (st *state) mountRuntimeRoutes(mux *http.ServeMux) {
mux.HandleFunc("POST /_runtime/tasks", st.handleRuntimeAdmit)
mux.HandleFunc("GET /_runtime/tasks/{id}", st.handleRuntimeGet)
mux.HandleFunc("POST /_runtime/tasks/{id}/cancel", st.handleRuntimeCancel)
mux.HandleFunc("GET /_runtime/tasks/{id}/events", st.handleRuntimeEvents)
}
func runtimeQuery(op string, extra map[string]any) (map[string]any, error) {
payload := map[string]any{"op": op}
for k, v := range extra {
payload[k] = v
}
raw, err := json.Marshal(payload)
if err != nil {
return nil, err
}
out, err := plugin.Query("runtime", string(raw))
if err != nil {
return nil, err
}
var m map[string]any
if err := json.Unmarshal([]byte(out), &m); err != nil {
return nil, err
}
return m, nil
}
func (st *state) handleRuntimeAdmit(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1<<20))
if err != nil {
jsonErr(w, http.StatusBadRequest, err.Error())
return
}
var req map[string]any
if len(body) > 0 {
if err := json.Unmarshal(body, &req); err != nil {
jsonErr(w, http.StatusBadRequest, fmt.Sprintf("json: %v", err))
return
}
}
if req == nil {
req = map[string]any{}
}
taskID, _ := req["task_id"].(string)
if taskID == "" {
taskID = fmt.Sprintf("task_%d", time.Now().UnixNano())
req["task_id"] = taskID
}
if _, ok := req["idempotency_key"]; !ok {
req["idempotency_key"] = taskID
}
if _, ok := req["class"]; !ok {
req["class"] = "execute"
}
m, err := runtimeQuery("admit", req)
if err != nil {
if strings.Contains(err.Error(), "conflict") || strings.Contains(fmt.Sprint(m["error"]), "conflict") {
jsonErr(w, http.StatusConflict, err.Error())
return
}
jsonErr(w, http.StatusInternalServerError, err.Error())
return
}
if ok, _ := m["ok"].(bool); !ok {
jsonErr(w, http.StatusConflict, fmt.Sprint(m["error"]))
return
}
w.Header().Set("Location", "/_runtime/tasks/"+taskID)
writeJSON(w, http.StatusAccepted, m)
}
func (st *state) handleRuntimeGet(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
m, err := runtimeQuery("get", map[string]any{"task_id": id})
if err != nil {
jsonErr(w, http.StatusInternalServerError, err.Error())
return
}
if st, _ := m["status"].(float64); st == 404 || fmt.Sprint(m["error"]) == "not_found" {
jsonErr(w, http.StatusNotFound, "task not found")
return
}
writeJSON(w, http.StatusOK, m)
}
func (st *state) handleRuntimeCancel(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
m, err := runtimeQuery("cancel", map[string]any{"task_id": id})
if err != nil {
jsonErr(w, http.StatusInternalServerError, err.Error())
return
}
if st, _ := m["status"].(float64); st == 404 || fmt.Sprint(m["error"]) == "not_found" {
jsonErr(w, http.StatusNotFound, "task not found")
return
}
writeJSON(w, http.StatusOK, m)
}
func (st *state) handleRuntimeEvents(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
after := int64(0)
if s := r.URL.Query().Get("after_seq"); s != "" {
fmt.Sscanf(s, "%d", &after)
}
if last := strings.TrimSpace(r.Header.Get("Last-Event-ID")); last != "" {
// Accept "task:seq" or bare seq.
if i := strings.LastIndexByte(last, ':'); i >= 0 {
fmt.Sscanf(last[i+1:], "%d", &after)
} else {
fmt.Sscanf(last, "%d", &after)
}
}
wantSSE := strings.Contains(r.Header.Get("Accept"), "text/event-stream") ||
r.URL.Query().Get("stream") == "1"
if !wantSSE {
m, err := runtimeQuery("events", map[string]any{"task_id": id, "after_seq": after})
if err != nil {
jsonErr(w, http.StatusInternalServerError, err.Error())
return
}
if st, _ := m["status"].(float64); st == 404 || fmt.Sprint(m["error"]) == "not_found" {
jsonErr(w, http.StatusNotFound, "task not found")
return
}
writeJSON(w, http.StatusOK, m)
return
}
w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.WriteHeader(http.StatusOK)
flusher, _ := w.(http.Flusher)
cursor := after
ticker := time.NewTicker(25 * time.Millisecond)
defer ticker.Stop()
heartbeat := time.NewTicker(15 * time.Second)
defer heartbeat.Stop()
ctx := r.Context()
for {
select {
case <-ctx.Done():
return
case <-heartbeat.C:
_, _ = io.WriteString(w, ": ping\n\n")
if flusher != nil {
flusher.Flush()
}
case <-ticker.C:
m, err := runtimeQuery("events", map[string]any{"task_id": id, "after_seq": cursor})
if err != nil {
continue
}
if st, _ := m["status"].(float64); st == 404 {
_, _ = io.WriteString(w, "event: error\ndata: {\"error\":\"not_found\"}\n\n")
if flusher != nil {
flusher.Flush()
}
return
}
evs, _ := m["events"].([]any)
for _, raw := range evs {
b, err := json.Marshal(raw)
if err != nil {
continue
}
var obj map[string]any
_ = json.Unmarshal(b, &obj)
eid := ""
if s, ok := obj["event_id"].(string); ok {
eid = s
}
if seq, ok := obj["seq"].(float64); ok {
cursor = int64(seq)
}
etype, _ := obj["type"].(string)
if etype == "" {
etype = "message"
}
if eid != "" {
_, _ = io.WriteString(w, "id: "+eid+"\n")
}
_, _ = io.WriteString(w, "event: "+etype+"\ndata: "+string(b)+"\n\n")
if flusher != nil {
flusher.Flush()
}
}
}
}
}
@@ -0,0 +1,14 @@
package httpx
import (
"testing"
)
func TestSystemRuntimeRoutesListed(t *testing.T) {
// Compile-time presence: handlers must be addressable.
var st state
_ = st.handleRuntimeAdmit
_ = st.handleRuntimeCancel
_ = st.handleRuntimeEvents
_ = st.handleRuntimeGet
}
@@ -0,0 +1,151 @@
package httpx
import (
"os"
"path/filepath"
"sync"
"testing"
"time"
)
// M-R0: second stream must fail before rewriting A's forward/cancel env.
func TestLegacySSEBridgeSingleFlight(t *testing.T) {
if !tryAcquireLegacySSEBridge() {
t.Fatal("expected first acquire to succeed")
}
fwdA := filepath.Join(t.TempDir(), "a.ndjson")
tokA := filepath.Join(t.TempDir(), "a.tok")
if err := os.WriteFile(fwdA, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(tokA, nil, 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("MARQDO_EVENT_FORWARD", fwdA)
t.Setenv("MARQDO_CANCEL_TOKEN", tokA)
legacySSEBridgeActive.Store(1)
if tryAcquireLegacySSEBridge() {
releaseLegacySSEBridge()
t.Fatal("second acquire must fail while bridge held")
}
active, fwd, tok := legacySSEBridgeSnapshot()
if !active {
t.Fatal("bridge should remain active for holder A")
}
if fwd != fwdA || tok != tokA {
t.Fatalf("rejected second stream rewrote env: fwd=%q tok=%q", fwd, tok)
}
releaseLegacySSEBridge()
if tryAcquireLegacySSEBridge() {
releaseLegacySSEBridge()
} else {
t.Fatal("bridge should be free after release")
}
}
// M-R0: cancel-token file must not be deleted while bridge holder waits.
// (sse-live observer disconnect no longer cancels by default — see cancelOnDisconnect.)
func TestCancelTokenSurvivesUntilHolderExits(t *testing.T) {
dir := t.TempDir()
tok := filepath.Join(dir, "cancel.tok")
if err := os.WriteFile(tok, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := writeCancelToken(tok); err != nil {
t.Fatal(err)
}
body, err := os.ReadFile(tok)
if err != nil {
t.Fatalf("token must still exist after cancel mark: %v", err)
}
if string(body) != "cancelled" {
t.Fatalf("body=%q", body)
}
}
func TestDrainForwardCarryHalfFrames(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "evt.ndjson")
if err := os.WriteFile(path, []byte(`{"t":"部`), 0o600); err != nil {
t.Fatal(err)
}
var lines []string
off, carry := drainForwardCarry(path, 0, "", func(line string) {
lines = append(lines, line)
})
if len(lines) != 0 {
t.Fatalf("incomplete UTF-8 frame must stay in carry, got %#v", lines)
}
f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
t.Fatal(err)
}
if _, err := f.WriteString("分\"}\n{\"t\":\"ok\"}\n"); err != nil {
t.Fatal(err)
}
_ = f.Close()
off, carry = drainForwardCarry(path, off, carry, func(line string) {
lines = append(lines, line)
})
if len(lines) != 2 {
t.Fatalf("got %d lines %#v carry=%q off=%d", len(lines), lines, carry, off)
}
if carry != "" {
t.Fatalf("carry should be empty, got %q", carry)
}
}
func TestDrainForwardCarryCRLFAndSplit(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "evt.ndjson")
part1 := []byte("{\"a\":1}\r\n{\"b\":")
if err := os.WriteFile(path, part1, 0o600); err != nil {
t.Fatal(err)
}
var lines []string
off, carry := drainForwardCarry(path, 0, "", func(line string) {
lines = append(lines, line)
})
if len(lines) != 1 || lines[0] != `{"a":1}` {
t.Fatalf("first frame %#v", lines)
}
f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
t.Fatal(err)
}
if _, err := f.WriteString("2}\n"); err != nil {
t.Fatal(err)
}
_ = f.Close()
_, carry = drainForwardCarry(path, off, carry, func(line string) {
lines = append(lines, line)
})
if len(lines) != 2 || lines[1] != `{"b":2}` {
t.Fatalf("split frame %#v carry=%q", lines, carry)
}
}
func TestLegacySSEBridgeConcurrentReject(t *testing.T) {
var wg sync.WaitGroup
acquired := 0
var mu sync.Mutex
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if tryAcquireLegacySSEBridge() {
mu.Lock()
acquired++
mu.Unlock()
time.Sleep(20 * time.Millisecond)
releaseLegacySSEBridge()
}
}()
}
wg.Wait()
if acquired < 1 {
t.Fatal("expected at least one acquirer")
}
}
+14 -5
View File
@@ -16,9 +16,10 @@ type Options struct {
Fn string
Body string
Return string
Class string
}
// RoutesFromTable normalizes `|路径|方法|函数|正文|返回|` into invoke_routes map keys.
// RoutesFromTable normalizes `|路径|方法|函数|正文|返回|类别|` into invoke_routes map keys.
func RoutesFromTable(tableV any) map[string]any {
rows, _ := table.AsRows(tableV).([]any)
out := map[string]any{}
@@ -47,7 +48,8 @@ func RoutesFromTable(tableV any) map[string]any {
if ret == "" {
ret = "json"
}
out[path] = routeEntry(method, fnPath, body, ret)
class := strings.ToLower(strings.TrimSpace(col(m, "类别", "class", "Class")))
out[path] = routeEntry(method, fnPath, body, ret, class)
}
return out
}
@@ -78,12 +80,13 @@ func Register(appBag map[string]any, opts Options) (map[string]any, error) {
if ret == "" {
ret = "json"
}
class := strings.ToLower(strings.TrimSpace(opts.Class))
out := clone(appBag)
routes := map[string]any{}
if r, ok := out["invoke_routes"].(map[string]any); ok {
routes = clone(r)
}
routes[key] = routeEntry(method, fnPath, body, ret)
routes[key] = routeEntry(method, fnPath, body, ret, class)
out["invoke_routes"] = routes
return out, nil
}
@@ -101,22 +104,28 @@ func OptionsFromArgs(args map[string]any) (Options, error) {
method, _ := argStr(args, "method", "方法")
body, _ := argStr(args, "body", "正文")
ret, _ := argStr(args, "return", "返回")
class, _ := argStr(args, "class", "类别")
return Options{
Path: path,
Method: method,
Fn: fnPath,
Body: body,
Return: ret,
Class: class,
}, nil
}
func routeEntry(method, fnPath, body, ret string) map[string]any {
return map[string]any{
func routeEntry(method, fnPath, body, ret, class string) map[string]any {
m := map[string]any{
"method": method,
"fn": fnPath,
"body": body,
"return": ret,
}
if class != "" {
m["class"] = class
}
return m
}
func col(m map[string]any, keys ...string) string {
+37 -1
View File
@@ -80,6 +80,7 @@ extern int web_compose_form(char *args_json, char **out_json, char **err_msg);
extern int web_app_new(char *args_json, char **out_json, char **err_msg);
extern int web_app_route(char *args_json, char **out_json, char **err_msg);
extern int web_db_update(char *args_json, char **out_json, char **err_msg);
extern int web_db_update_if(char *args_json, char **out_json, char **err_msg);
extern int web_db_delete(char *args_json, char **out_json, char **err_msg);
extern int web_db_query(char *args_json, char **out_json, char **err_msg);
extern int web_db_count(char *args_json, char **out_json, char **err_msg);
@@ -183,6 +184,7 @@ static int register_core(void) {
if (host_register((char *)"web_db_select", (char *)"url,table,where,limit,order,offset,txn", web_db_select) != 0) return 1;
if (host_register((char *)"web_db_get", (char *)"url,table,id,txn", web_db_get) != 0) return 1;
if (host_register((char *)"web_db_update", (char *)"url,table,id,row,txn", web_db_update) != 0) return 1;
if (host_register((char *)"web_db_update_if", (char *)"url,table,id,row,expect_col,expect_val,txn", web_db_update_if) != 0) return 1;
if (host_register((char *)"web_db_delete", (char *)"url,table,id,txn", web_db_delete) != 0) return 1;
if (host_register((char *)"web_db_exec", (char *)"url,sql,args,txn", web_db_exec) != 0) return 1;
if (host_register((char *)"web_db_query", (char *)"url,sql,args,txn", web_db_query) != 0) return 1;
@@ -264,7 +266,7 @@ static int register_core(void) {
if (host_register((char *)"web_cache_exists", (char *)"url,key", web_cache_exists) != 0) return 1;
if (host_register((char *)"web_cache_ttl", (char *)"url,key", web_cache_ttl) != 0) return 1;
if (host_register((char *)"web_app_proxy", (char *)"app,path,upstream,stream,strip_prefix,methods,headers_from_env,timeout_ms", web_app_proxy) != 0) return 1;
if (host_register((char *)"web_app_invoke", (char *)"app,path,method,fn,body,return", web_app_invoke) != 0) return 1;
if (host_register((char *)"web_app_invoke", (char *)"app,path,method,fn,body,return,class", web_app_invoke) != 0) return 1;
if (host_register((char *)"web_api_key_check", (char *)"key,authorization,pepper,keys", web_api_key_check) != 0) return 1;
if (host_register((char *)"web_render_nodes", (char *)"page,nodes,db,title", web_render_nodes) != 0) return 1;
if (host_register((char *)"web_render_document", (char *)"page,body,db,title", web_render_document) != 0) return 1;
@@ -994,6 +996,40 @@ func web_db_update(argsJSON *C.char, outJSON **C.char, errMsg **C.char) C.int {
return replyJSON(outJSON, errMsg, out, err)
}
//export web_db_update_if
func web_db_update_if(argsJSON *C.char, outJSON **C.char, errMsg **C.char) C.int {
args, err := parseArgs(argsJSON)
if err != nil {
return replyJSON(outJSON, errMsg, nil, err)
}
url, err := dbURLOf(args)
if err != nil {
return replyJSON(outJSON, errMsg, nil, err)
}
tableName, err := argStrReq(args, "table")
if err != nil {
return replyJSON(outJSON, errMsg, nil, err)
}
id, err := argStrReq(args, "id")
if err != nil {
return replyJSON(outJSON, errMsg, nil, err)
}
expectCol, err := argStrReq(args, "expect_col")
if err != nil {
return replyJSON(outJSON, errMsg, nil, err)
}
expectVal, ok := args["expect_val"]
if !ok {
return replyJSON(outJSON, errMsg, nil, fmt.Errorf("missing expect_val"))
}
row := args["row"]
if row == nil {
row = map[string]any{}
}
out, err := db.UpdateIf(url, tableName, id, row, expectCol, expectVal, txnOpt(args))
return replyJSON(outJSON, errMsg, out, err)
}
//export web_db_delete
func web_db_delete(argsJSON *C.char, outJSON **C.char, errMsg **C.char) C.int {
args, err := parseArgs(argsJSON)
+15 -6
View File
@@ -27,6 +27,13 @@ need curl curl
need tar tar
need xz xz-utils
# /tmp is often a small tmpfs; vendor + Rust extract exceed quota. Prefer disk.
if [ -z "${TMPDIR:-}" ] || [ "$TMPDIR" = /tmp ] || [ "$TMPDIR" = /var/tmp ]; then
TMPDIR="${HOME}/.cache/marqdo-ppa/work"
fi
mkdir -p "$TMPDIR"
export TMPDIR
VER="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -1)"
# Debian revision comes from debian/changelog (e.g. 1.0.2-1ppa2); we append ~series.
CHG_VER="$(dpkg-parsechangelog -l debian/changelog -S Version)"
@@ -43,7 +50,7 @@ if [[ "$BASE_DEB_VER" != "$VER"-* ]]; then
fi
fi
DEB_VER="${BASE_DEB_VER}~${SERIES}"
PKG_ROOT="${TMPDIR:-/tmp}/marqdo-ppa/${VER}"
PKG_ROOT="${TMPDIR}/marqdo-ppa/${VER}"
BUILD_AREA="${ROOT}/../build-area"
GPG_KEY="${MARQDO_PPA_GPG_KEY:-505943294D04C803}"
RUST_VER="${MARQDO_PPA_RUST_VERSION:-1.85.0}"
@@ -212,18 +219,20 @@ PY
exit 1
fi
# Launchpad builds need only rustc + cargo + rust-std (debian/rules PATH).
# Skip docs/json-docs/rustfmt/rls — ~1G of dead weight in the orig tarball.
# install.sh's --without is a *single* comma-separated valopt; repeating
# --without=… keeps only the last flag, so rust-docs still installed and
# blew /tmp quota. Prefer explicit --components= instead.
bash "$INSTALL_SH" \
--prefix="$PKG_ROOT/third_party/rust" \
--without=rust-docs \
--without=rust-docs-json-preview \
--without=rustfmt-preview \
--without=rls-preview \
--components="rustc,cargo,rust-std-${RUST_TRIPLE}" \
--disable-ldconfig
# Belt: docs must never ship in .orig (even if component list drifts).
rm -rf "$PKG_ROOT/third_party/rust/share/doc"
rm -rf "$RUST_EXTRACT"
trap - EXIT
"$PKG_ROOT/third_party/rust/bin/rustc" --version
"$PKG_ROOT/third_party/rust/bin/cargo" --version
du -sh "$PKG_ROOT/third_party/rust"
cd "$(dirname "$PKG_ROOT")"
echo "==> packing ${ORIG_NAME} (vendor + third_party/rust)"
+9 -3
View File
@@ -193,8 +193,12 @@ fn extract_markdown_or_text(
) -> ExtractedDocument {
let mut segments = Vec::new();
let mut n = 0u32;
let mut byte_off = 0usize;
for (i, line) in text.lines().enumerate() {
let line_no = i + 1;
let line_bytes = line.len();
// M05: stable fragment locator with 1-based line + UTF-8 byte offset.
let loc = |kind: &str| format!("{kind}:L{line_no}:B{byte_off}");
if let Some(rest) = line.trim().strip_prefix('#') {
n += 1;
let heading = rest.trim_start_matches('#').trim();
@@ -202,7 +206,7 @@ fn extract_markdown_or_text(
id: format!("seg-{n}"),
kind: "heading".into(),
text: heading.to_string(),
locator: format!("L{line_no}"),
locator: loc("heading"),
confidence: 1.0,
});
} else if line.trim().starts_with('|') {
@@ -211,7 +215,7 @@ fn extract_markdown_or_text(
id: format!("seg-{n}"),
kind: "table_row".into(),
text: line.to_string(),
locator: format!("L{line_no}"),
locator: loc("table_row"),
confidence: 0.9,
});
} else if !line.trim().is_empty() {
@@ -220,10 +224,12 @@ fn extract_markdown_or_text(
id: format!("seg-{n}"),
kind: "paragraph".into(),
text: line.to_string(),
locator: format!("L{line_no}"),
locator: loc("paragraph"),
confidence: 1.0,
});
}
// +1 for '\n' when present in original (lines() strips it).
byte_off += line_bytes + 1;
}
ExtractedDocument {
source_uri: uri.to_string(),
+5 -2
View File
@@ -326,8 +326,11 @@ pub fn apply_patch_blocks(
};
match apply_patch_blocks_inner(ctx, path, text) {
Ok(n) => Ok(n),
// soft only swallows unique-FIND miss — never parse / multi-match / whole-file.
Err(e) if soft && e.contains("FIND not found") => Ok(Value::Int(0)),
// soft swallows unique-FIND miss and empty FIND (model protocol mistakes).
// Never softens parse / multi-match / whole-file rewrite.
Err(e) if soft && (e.contains("FIND not found") || e.contains("empty FIND")) => {
Ok(Value::Int(0))
}
Err(e) => Err(e),
}
}
+95 -11
View File
@@ -6,7 +6,8 @@ use std::collections::HashMap;
use std::ffi::{CStr, CString};
use std::os::raw::{c_char, c_int, c_void};
use std::path::PathBuf;
use std::sync::{Mutex, OnceLock};
use std::sync::{Mutex, MutexGuard, OnceLock};
use std::time::{Duration, Instant};
use libloading::{Library, Symbol};
@@ -75,6 +76,8 @@ static GLOBAL_HOST: OnceLock<Mutex<SendPtr<HostContext>>> = OnceLock::new();
/// Process-wide lib-path hook (same lifetime as GLOBAL_HOST).
static GLOBAL_LIB_PATH: OnceLock<Mutex<SendLibPath>> = OnceLock::new();
/// Serialize interpreter re-entry from HTTP worker threads.
/// HostContext is not Sync — one Marqdo invoke at a time. Short routes must
/// fail fast (`interpreter busy`) so a long SSE chat cannot freeze the UI.
static INVOKE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
fn global_host_slot() -> &'static Mutex<SendPtr<HostContext>> {
@@ -89,6 +92,41 @@ fn invoke_lock() -> &'static Mutex<()> {
INVOKE_LOCK.get_or_init(|| Mutex::new(()))
}
/// Default wait for short HTTP invokes (`MARQDO_INVOKE_WAIT_MS`, default 1500ms).
/// Do not raise this process-wide during SSE — concurrent short routes would hang again.
/// See doc/adr/0008-web-invoke-serialization.md.
fn invoke_wait_budget() -> Duration {
const DEFAULT_MS: u64 = 1500;
match std::env::var("MARQDO_INVOKE_WAIT_MS") {
Ok(s) => {
let ms = s.trim().parse::<u64>().unwrap_or(DEFAULT_MS);
Duration::from_millis(ms.max(1))
}
Err(_) => Duration::from_millis(DEFAULT_MS),
}
}
fn acquire_invoke_lock() -> Result<MutexGuard<'static, ()>, String> {
let budget = invoke_wait_budget();
let deadline = Instant::now() + budget;
let lock = invoke_lock();
loop {
match lock.try_lock() {
Ok(g) => return Ok(g),
Err(std::sync::TryLockError::Poisoned(p)) => return Ok(p.into_inner()),
Err(std::sync::TryLockError::WouldBlock) => {
if Instant::now() >= deadline {
return Err(format!(
"interpreter busy: another request holds the Marqdo invoke lock (waited {}ms)",
budget.as_millis()
));
}
std::thread::sleep(Duration::from_millis(15));
}
}
}
}
/// Hook: resolve bare `lib.member` (and longer paths) in the site module's import tree.
#[derive(Clone, Copy)]
pub struct LibPathCall {
@@ -284,6 +322,42 @@ unsafe extern "C" fn host_query(
}
};
let args_owned: Option<String> = if args_json.is_null() {
None
} else {
CStr::from_ptr(args_json)
.to_str()
.ok()
.map(|s| s.to_string())
};
// Control plane (admit/cancel/events) must not take INVOKE_LOCK or require
// an evaluating HostContext — otherwise cancel cannot interrupt a busy run.
if name == "runtime" {
let raw = args_owned.as_deref().unwrap_or("{}");
let args: serde_json::Value = match serde_json::from_str(if raw.trim().is_empty() {
"{}"
} else {
raw
}) {
Ok(v) => v,
Err(e) => {
set_err(&format!("runtime args: {e}"));
return 1;
}
};
match crate::runtime::host_api::handle(&args) {
Ok(j) => {
set_out(&j.to_string());
return 0;
}
Err(e) => {
set_err(&e);
return 1;
}
}
}
let ctx_ptr = CURRENT_HOST.with(|c| c.get());
let ctx_ptr = if ctx_ptr.is_null() {
global_host_slot()
@@ -300,19 +374,16 @@ unsafe extern "C" fn host_query(
let _invoke_guard;
let from_worker = CURRENT_HOST.with(|c| c.get().is_null());
if from_worker {
_invoke_guard = invoke_lock().lock().unwrap_or_else(|e| e.into_inner());
match acquire_invoke_lock() {
Ok(g) => _invoke_guard = g,
Err(msg) => {
set_err(&msg);
return 1;
}
}
}
let ctx = &mut *ctx_ptr;
let args_owned: Option<String> = if args_json.is_null() {
None
} else {
CStr::from_ptr(args_json)
.to_str()
.ok()
.map(|s| s.to_string())
};
let result = match name {
"module_source" => agent_rt::module_source(ctx).and_then(|v| value_to_json(&v)),
"call_site" => agent_rt::call_site(ctx, Some(ctx.current_line)).and_then(|v| value_to_json(&v)),
@@ -320,6 +391,19 @@ unsafe extern "C" fn host_query(
"cwd" => Ok(serde_json::Value::String(
ctx.cwd.to_string_lossy().into_owned(),
)),
"entry_path" => {
let path = ctx.entry_path.clone().unwrap_or_else(|| ctx.cwd.clone());
let abs = if path.is_absolute() {
path
} else {
std::env::current_dir()
.unwrap_or_else(|_| PathBuf::from("."))
.join(path)
};
Ok(serde_json::Value::String(
abs.to_string_lossy().into_owned(),
))
}
"entry_dir" => {
// Script directory: parent of entry file (for_run also sets ctx.cwd to this).
let dir = ctx
+84
View File
@@ -331,6 +331,7 @@ fn spawn_file(
));
let _ = std::fs::write(&events_path, "");
cmd.env("MARQDO_EVENT_FORWARD", &events_path);
apply_run_selection_env(&mut cmd);
let mut child = cmd
.spawn()
@@ -693,3 +694,86 @@ fn require<'a>(bound: &'a HashMap<String, Value>, key: &str) -> Result<&'a Value
.get(key)
.ok_or_else(|| format!("host call missing `{key}`"))
}
/// Propagate run/selection identity onto the *child* Command.env map.
/// Reads parent env only as a fallback when an explicit RunContext was not
/// applied; never uses parent `setenv`/`set_current_dir` as request state (M-R1).
fn apply_run_selection_env(cmd: &mut Command) {
for key in [
"MARQDO_RUN_ID",
"MARQDO_PARENT_RUN_ID",
"MARQDO_TURN_ID",
"MARQDO_WORKSPACE_SCOPE",
"MARQDO_CANCEL_TOKEN",
"MARQDO_EVENT_FORWARD",
"MARQDO_USAGE_SINK",
"MARQDO_CREDENTIAL_REF",
"MARQDO_WORKSPACE_ID",
"MARQDO_SNAPSHOT_ID",
"MARQDO_LEASE_ID",
"MARQDO_ATTEMPT_ID",
"MARQDO_LLM_MODEL",
"MARQDO_LLM_BASE_URL",
"OPENAI_MODEL",
"OPENAI_BASE_URL",
] {
if let Ok(v) = std::env::var(key) {
cmd.env(key, v);
}
}
// Do not forward process-global MARQDO_CANCEL into children — request cancel
// must use MARQDO_CANCEL_TOKEN so leases stay isolated.
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
use std::process::Command;
#[test]
fn f02_apply_run_selection_env_forwards_parent_identity() {
std::env::set_var("MARQDO_RUN_ID", "run-parent");
std::env::set_var("OPENAI_MODEL", "child-model");
std::env::set_var("OPENAI_BASE_URL", "http://127.0.0.1:7010");
// Must never require writing secrets into child env map for selection.
std::env::remove_var("OPENAI_API_KEY");
let mut cmd = Command::new("true");
apply_run_selection_env(&mut cmd);
// Command doesn't expose env for inspection portably; re-check via helper contract:
let keys: HashMap<&str, String> = [
"MARQDO_RUN_ID",
"OPENAI_MODEL",
"OPENAI_BASE_URL",
]
.into_iter()
.filter_map(|k| std::env::var(k).ok().map(|v| (k, v)))
.collect();
assert_eq!(keys.get("MARQDO_RUN_ID").map(String::as_str), Some("run-parent"));
assert_eq!(keys.get("OPENAI_MODEL").map(String::as_str), Some("child-model"));
assert_eq!(
keys.get("OPENAI_BASE_URL").map(String::as_str),
Some("http://127.0.0.1:7010")
);
std::env::remove_var("MARQDO_RUN_ID");
std::env::remove_var("OPENAI_MODEL");
std::env::remove_var("OPENAI_BASE_URL");
let _ = cmd; // ensure apply didn't panic
}
#[test]
fn f02_agent_skeleton_has_no_llm_llm() {
let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let agent = std::fs::read_to_string(root.join("ext/ai/agent.mq.md")).unwrap();
assert!(
!agent.contains("llm.llm"),
"render_workbook_skeleton must not hardcode llm.llm"
);
assert!(
agent.contains("llm.create") && agent.contains("OPENAI_MODEL"),
"skeleton should create model from env selection"
);
}
}
+14
View File
@@ -545,6 +545,20 @@ impl Interpreter {
])
}
/// Pin `module` as site and call `lib.member` (isolated worker / CLI `--call-lib-path`).
pub fn run_site_lib_path(
&mut self,
module: &Module,
path: &str,
args: Option<&Value>,
) -> std::result::Result<Value, String> {
let prev = self.site_module;
self.site_module = Some(module as *const Module);
let out = self.call_site_lib_path(path, args);
self.site_module = prev;
out
}
/// Resolve `lib.member` on the site (entry) module — used by ABI `call_lib_path`.
pub(crate) fn call_site_lib_path(
&mut self,
+52 -34
View File
@@ -25,19 +25,29 @@ pub fn compile(opts: KnowledgeOptions) -> Result<KnowledgeGraph> {
} else {
opts.path.clone()
};
let t0 = std::time::Instant::now();
let semantic = crate::sema::compile_project(&crate::sema::CompileOptions {
project_root: semantic_root,
entry: opts.path.is_file().then(|| opts.path.clone()),
lock_mode: crate::sema::LockMode::Off,
})?;
let sema_ms = t0.elapsed().as_millis();
let t1 = std::time::Instant::now();
crate::sema::write_semantic_projections(&opts.out_dir, &semantic)?;
let sema_write_ms = t1.elapsed().as_millis();
let extract_opts = ExtractOptions {
path: opts.path.clone(),
out_dir: opts.out_dir.clone(),
};
let t2 = std::time::Instant::now();
let graph = extract_graph(&extract_opts)?;
let extract_ms = t2.elapsed().as_millis();
let t3 = std::time::Instant::now();
write_projections(&opts.out_dir, &graph)?;
let retrieve_write_ms = t3.elapsed().as_millis();
let t4 = std::time::Instant::now();
write_publish_manifest(&opts.out_dir, &semantic, &graph)?;
let manifest_ms = t4.elapsed().as_millis();
eprintln!(
"marqdo knowledge: {} module(s), {} symbol(s), {} capability(ies), {} knowledge → {} (semantic graph/; retrieval ekc/)",
graph.modules.len(),
@@ -46,6 +56,10 @@ pub fn compile(opts: KnowledgeOptions) -> Result<KnowledgeGraph> {
graph.knowledge.len(),
opts.out_dir.display()
);
eprintln!(
"marqdo knowledge timing_ms: sema={sema_ms} sema_write={sema_write_ms} extract={extract_ms} retrieve_write={retrieve_write_ms} manifest={manifest_ms} total={}",
t0.elapsed().as_millis()
);
Ok(graph)
}
@@ -100,7 +114,23 @@ pub fn ekc_graph_dir(out_dir: &Path) -> std::path::PathBuf {
out_dir.join("ekc").join("graph")
}
/// Path of the full IR snapshot used by warm query paths (preflight/find/reuse).
pub fn ir_snapshot_path(out_dir: &Path) -> std::path::PathBuf {
ekc_graph_dir(out_dir).join("ir.json")
}
/// Load a previously compiled KnowledgeGraph without re-extracting sources.
/// Prefers `ekc/graph/ir.json`. Does **not** walk the source tree.
pub fn load_graph(out_dir: &Path) -> Result<KnowledgeGraph> {
let ir_path = ir_snapshot_path(out_dir);
if ir_path.exists() {
let raw = fs::read_to_string(&ir_path)
.with_context(|| format!("read {}", ir_path.display()))?;
let graph: KnowledgeGraph = serde_json::from_str(&raw)
.with_context(|| format!("parse {}", ir_path.display()))?;
return Ok(graph);
}
let graph_path = ekc_graph_dir(out_dir).join("graph.json");
let edges_path = ekc_graph_dir(out_dir).join("edges.json");
if !graph_path.exists() {
@@ -109,47 +139,19 @@ pub fn load_graph(out_dir: &Path) -> Result<KnowledgeGraph> {
graph_path.display()
);
}
// Prefer re-extract for freshness when source path known; for query commands
// we rebuild from JSON snapshot fields + re-scan if catalog sibling exists.
// Snapshot load: reconstruct minimal IR from written pages + edges.json.
// Legacy projections without ir.json: edges-only fallback (no source walk).
let edges_raw = fs::read_to_string(&edges_path).unwrap_or_else(|_| "{\"edges\":[]}".into());
let edges_v: serde_json::Value = serde_json::from_str(&edges_raw)?;
let graph_raw = fs::read_to_string(&graph_path)?;
let graph_v: serde_json::Value = serde_json::from_str(&graph_raw)?;
// Re-compile from recorded root if present in graph.json
if let Some(root) = graph_v
.pointer("/repository/root")
.and_then(|v| v.as_str())
{
let root_path = Path::new(root);
if root_path.exists() {
return extract_graph(&ExtractOptions {
path: root_path.to_path_buf(),
out_dir: out_dir.to_path_buf(),
});
}
}
// Fallback: empty-ish graph with edges only
let mut edges = Vec::new();
if let Some(arr) = edges_v.get("edges").and_then(|e| e.as_array()) {
for e in arr {
let kind = match e.get("kind").and_then(|k| k.as_str()).unwrap_or("") {
"implements" => crate::knowledge::ir::EdgeKind::Implements,
"uses" => crate::knowledge::ir::EdgeKind::Uses,
"depends_on" => crate::knowledge::ir::EdgeKind::DependsOn,
"tested_by" => crate::knowledge::ir::EdgeKind::TestedBy,
"documented_by" => crate::knowledge::ir::EdgeKind::DocumentedBy,
"constrained_by" => crate::knowledge::ir::EdgeKind::ConstrainedBy,
"decided_by" => crate::knowledge::ir::EdgeKind::DecidedBy,
"supersedes" => crate::knowledge::ir::EdgeKind::Supersedes,
"conflicts_with" => crate::knowledge::ir::EdgeKind::ConflictsWith,
"deprecated_by" => crate::knowledge::ir::EdgeKind::DeprecatedBy,
"derived_from" => crate::knowledge::ir::EdgeKind::DerivedFrom,
"failed_by" => crate::knowledge::ir::EdgeKind::FailedBy,
_ => crate::knowledge::ir::EdgeKind::RelatedTo,
};
let kind = crate::knowledge::ir::EdgeKind::parse(
e.get("kind").and_then(|k| k.as_str()).unwrap_or(""),
);
edges.push(crate::knowledge::ir::Edge {
from: e.get("from").and_then(|v| v.as_str()).unwrap_or("").into(),
to: e.get("to").and_then(|v| v.as_str()).unwrap_or("").into(),
@@ -190,14 +192,27 @@ pub fn load_graph(out_dir: &Path) -> Result<KnowledgeGraph> {
})
}
/// Force a full knowledge compile (write path / `--refresh`).
pub fn ensure_compiled(path: &Path, out_dir: &Path) -> Result<KnowledgeGraph> {
// Always refresh from source when path is given.
compile(KnowledgeOptions {
path: path.to_path_buf(),
out_dir: out_dir.to_path_buf(),
})
}
/// Query path loader: compile if missing (or `refresh`), else hot-load `ir.json`.
pub fn graph_for_query(path: &Path, out_dir: &Path, refresh: bool) -> Result<KnowledgeGraph> {
if refresh {
return ensure_compiled(path, out_dir);
}
let has_ir = ir_snapshot_path(out_dir).exists();
let has_graph = ekc_graph_dir(out_dir).join("graph.json").exists();
if has_ir || has_graph {
return load_graph(out_dir);
}
ensure_compiled(path, out_dir)
}
fn write_projections(out_dir: &Path, graph: &KnowledgeGraph) -> Result<()> {
let dirs = [
"catalog/modules",
@@ -285,6 +300,9 @@ fn write_projections(out_dir: &Path, graph: &KnowledgeGraph) -> Result<()> {
fs::write(ekc.join("graph.json"), graph_json)?;
let edges_json = serde_json::to_string_pretty(&graph.to_edges_json())?;
fs::write(ekc.join("edges.json"), edges_json)?;
// Full IR snapshot for warm query paths (preflight/find/reuse) — no re-extract.
let ir_json = serde_json::to_string(&graph)?;
fs::write(ekc.join("ir.json"), ir_json)?;
// Search index
let index = json!({
+67 -9
View File
@@ -1,5 +1,6 @@
//! Engineering Knowledge IR (L0–L4) — projections of `.mq.md`, not a second source of truth.
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{json, Value};
use std::collections::BTreeMap;
@@ -13,6 +14,19 @@ pub enum Lifecycle {
Superseded,
}
impl Serialize for Lifecycle {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for Lifecycle {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let s = String::deserialize(deserializer)?;
Ok(Self::parse(&s))
}
}
impl Lifecycle {
pub fn as_str(&self) -> &'static str {
match self {
@@ -35,7 +49,7 @@ impl Lifecycle {
}
}
#[derive(Debug, Clone, Default)]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct Evidence {
pub sources: Vec<String>,
pub tests: Vec<String>,
@@ -55,7 +69,7 @@ impl Evidence {
}
/// Graded confidence with explanation (not a bare float).
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Confidence {
pub level: String, // high | medium | low
pub implementation: String,
@@ -87,7 +101,7 @@ impl Confidence {
}
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RepositoryInfo {
pub title: String,
pub root: String,
@@ -98,7 +112,7 @@ pub struct RepositoryInfo {
pub knowledge_count: usize,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModuleInfo {
pub id: String,
pub resource: String,
@@ -111,7 +125,7 @@ pub struct ModuleInfo {
pub status: Lifecycle,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SymbolInfo {
pub id: String,
pub name: String,
@@ -135,7 +149,7 @@ pub struct SymbolInfo {
pub confidence: Confidence,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CapabilityInfo {
pub id: String,
pub name: String,
@@ -215,7 +229,20 @@ impl KnowledgeKind {
}
}
#[derive(Debug, Clone)]
impl Serialize for KnowledgeKind {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for KnowledgeKind {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let s = String::deserialize(deserializer)?;
Ok(Self::parse(&s))
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KnowledgeItem {
pub id: String,
pub kind: KnowledgeKind,
@@ -269,9 +296,40 @@ impl EdgeKind {
Self::RelatedTo => "related_to",
}
}
pub fn parse(s: &str) -> Self {
match s.trim() {
"implements" => Self::Implements,
"uses" => Self::Uses,
"depends_on" => Self::DependsOn,
"tested_by" => Self::TestedBy,
"documented_by" => Self::DocumentedBy,
"constrained_by" => Self::ConstrainedBy,
"decided_by" => Self::DecidedBy,
"supersedes" => Self::Supersedes,
"conflicts_with" => Self::ConflictsWith,
"deprecated_by" => Self::DeprecatedBy,
"derived_from" => Self::DerivedFrom,
"failed_by" => Self::FailedBy,
_ => Self::RelatedTo,
}
}
}
#[derive(Debug, Clone)]
impl Serialize for EdgeKind {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for EdgeKind {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let s = String::deserialize(deserializer)?;
Ok(Self::parse(&s))
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Edge {
pub from: String,
pub to: String,
@@ -336,7 +394,7 @@ pub struct ReuseResult {
pub recommended: Vec<String>,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KnowledgeGraph {
pub repository: RepositoryInfo,
pub modules: Vec<ModuleInfo>,
+7 -2
View File
@@ -20,7 +20,10 @@ pub mod preflight;
pub mod reuse;
pub mod util;
pub use compile::{compile, ekc_graph_dir, ensure_compiled, load_graph, KnowledgeOptions};
pub use compile::{
compile, ekc_graph_dir, ensure_compiled, graph_for_query, ir_snapshot_path, load_graph,
KnowledgeOptions,
};
pub use duplicate::{find_duplicates, format_duplicates_text};
pub use find::{find, find_json, format_find_text};
pub use impact::{format_impact_text, impact, impact_json};
@@ -32,7 +35,9 @@ pub use lifecycle::{
};
pub use metrics::{load_metrics, record_decision, save_metrics, ReuseMetrics};
pub use policy::compile_policy;
pub use preflight::{format_preflight_yaml, preflight, preflight_json, PreflightResult};
pub use preflight::{
format_preflight_yaml, preflight, preflight_from_reuse, preflight_json, PreflightResult,
};
pub use reuse::{format_reuse_text, resolve, reuse_json, ReuseBudget};
use std::path::{Path, PathBuf};
+11 -1
View File
@@ -33,6 +33,16 @@ pub fn preflight(
out_dir: Option<&Path>,
) -> Result<PreflightResult> {
let reuse = resolve(graph, task, budget);
preflight_from_reuse(graph, task, &reuse, out_dir)
}
/// Build a Context Pack from an already-resolved reuse result (avoids a second `resolve`).
pub fn preflight_from_reuse(
graph: &KnowledgeGraph,
task: &str,
reuse: &crate::knowledge::ir::ReuseResult,
out_dir: Option<&Path>,
) -> Result<PreflightResult> {
let intent = reuse
.capability
.clone()
@@ -46,7 +56,7 @@ pub fn preflight(
.map(|c| c.title.clone())
.collect();
let pack = render_context_pack(task, &intent, &reuse, graph);
let pack = render_context_pack(task, &intent, reuse, graph);
let mut context_path = None;
if let Some(out) = out_dir {
let contexts = out.join("agent/contexts");
+52 -1
View File
@@ -37,6 +37,8 @@ pub mod lex;
pub mod linalg_ops;
pub mod load;
pub mod mlsp;
pub mod runtime;
pub mod workbench;
pub mod parse;
pub mod project;
pub mod repair;
@@ -111,6 +113,10 @@ pub struct RunOptions {
pub binds: Vec<(String, String)>,
/// Attach CoW session for `ext/ai/sandbox` nested runs.
pub sandbox_session: Option<String>,
/// Isolated worker: call `lib.member` on the site entry instead of `# main`.
pub call_lib_path: Option<String>,
/// JSON object of named args for [`Self::call_lib_path`] (file path or inline).
pub args_json: Option<std::path::PathBuf>,
}
impl Default for RunOptions {
@@ -134,6 +140,8 @@ impl Default for RunOptions {
emit_result: None,
binds: Vec::new(),
sandbox_session: None,
call_lib_path: None,
args_json: None,
}
}
}
@@ -228,7 +236,11 @@ pub fn run_file(path: &Path, opts: &RunOptions) -> Result<i32> {
if opts.trace_eval {
eprintln!("=== marqdo: trace-eval ({path_label}) ===");
}
let value = interp.run_module(&module)?;
let value = if let Some(lib_path) = &opts.call_lib_path {
run_call_lib_path(&mut interp, &module, lib_path, opts)?
} else {
interp.run_module(&module)?
};
if opts.trace_eval {
eprintln!("=== marqdo: end trace-eval ===");
}
@@ -274,6 +286,45 @@ pub fn run_file(path: &Path, opts: &RunOptions) -> Result<i32> {
Ok(0)
}
/// Worker mode: resolve `lib.member` on the loaded site module (imports already wired).
fn run_call_lib_path(
interp: &mut Interpreter,
module: &crate::ast::Module,
lib_path: &str,
opts: &RunOptions,
) -> Result<crate::value::Value> {
let args_val = load_args_json(opts)?;
// Keep site_module pinned for the duration of the lib call (plugins may re-enter).
let value = interp
.run_site_lib_path(module, lib_path, args_val.as_ref())
.map_err(|e| anyhow::anyhow!(e))?;
Ok(value)
}
fn load_args_json(opts: &RunOptions) -> Result<Option<crate::value::Value>> {
let Some(path) = &opts.args_json else {
return Ok(None);
};
let text = if path.as_os_str() == "-" {
use std::io::Read;
let mut buf = String::new();
std::io::stdin()
.read_to_string(&mut buf)
.map_err(|e| anyhow::anyhow!("args-json stdin: {e}"))?;
buf
} else {
std::fs::read_to_string(path)
.map_err(|e| anyhow::anyhow!("args-json {}: {e}", path.display()))?
};
if text.trim().is_empty() {
return Ok(None);
}
let json: serde_json::Value =
serde_json::from_str(&text).map_err(|e| anyhow::anyhow!("args-json parse: {e}"))?;
let v = crate::host::json::json_to_value(&json).map_err(|e| anyhow::anyhow!(e))?;
Ok(Some(v))
}
fn emit_result_file(opts: &RunOptions, value: &crate::value::Value) -> Result<()> {
let Some(path) = &opts.emit_result else {
return Ok(());
+139 -17
View File
@@ -9,12 +9,12 @@ use marqdo::diagnostics::{Diagnostic, Span};
use marqdo::ext_cli::{add_ext, list_ext, remove_ext};
use marqdo::input_feed::load_stdin_file;
use marqdo::knowledge::{
compile_policy, conflicts_json, ensure_compiled, find, find_conflicts, find_duplicates,
find_json, find_stale, format_conflicts_text, format_duplicates_text, format_find_text,
format_impact_text, format_preflight_yaml, format_reuse_text, format_stale_text,
format_verify_text, impact, impact_json, learn, preflight, preflight_json, record_decision,
resolve, reuse_json, stale_json, verify, verify_json, write_knowledge, KnowledgeOptions,
LearnInput, ReuseBudget,
compile_policy, conflicts_json, find, find_conflicts, find_duplicates, find_json, find_stale,
format_conflicts_text, format_duplicates_text, format_find_text, format_impact_text,
format_preflight_yaml, format_reuse_text, format_stale_text, format_verify_text, graph_for_query,
impact, impact_json, learn, preflight_from_reuse, preflight_json, record_decision, resolve,
reuse_json, stale_json, verify, verify_json, write_knowledge, KnowledgeOptions, LearnInput,
ReuseBudget,
};
use marqdo::view::{
serve, serve_debug, write_capture, write_static, CaptureOptions, DebugOptions, OutputOptions,
@@ -120,6 +120,14 @@ enum Commands {
#[arg(long = "fs-root", value_name = "DIR")]
fs_root: Option<PathBuf>,
/// Isolated worker: call `lib.member` on the site entry instead of `# main`.
#[arg(long = "call-lib-path", value_name = "LIB.MEMBER")]
call_lib_path: Option<String>,
/// Named args JSON for `--call-lib-path` (file path, or `-` for stdin).
#[arg(long = "args-json", value_name = "FILE")]
args_json: Option<PathBuf>,
/// Emit diagnostics as JSON on stderr (machine-readable; AI/MLSP 面).
#[arg(long)]
json: bool,
@@ -257,6 +265,10 @@ enum Commands {
#[arg(long, global = true)]
json: bool,
/// Force a full knowledge recompile before query actions (preflight, …)
#[arg(long, global = true)]
refresh: bool,
},
/// Find engineering capabilities / symbols / knowledge
Find {
@@ -274,6 +286,10 @@ enum Commands {
#[arg(long)]
json: bool,
/// Force a full knowledge recompile before querying
#[arg(long)]
refresh: bool,
},
/// Engineering First Information: REUSE / ADAPT / CREATE
Reuse {
@@ -298,6 +314,10 @@ enum Commands {
#[arg(long)]
json: bool,
/// Force a full knowledge recompile before querying
#[arg(long)]
refresh: bool,
},
/// Knowledge impact of changed paths
Impact {
@@ -312,6 +332,10 @@ enum Commands {
#[arg(long)]
json: bool,
/// Force a full knowledge recompile before querying
#[arg(long)]
refresh: bool,
},
/// Detect knowledge conflicts / possible duplicates edges
Conflicts {
@@ -327,6 +351,10 @@ enum Commands {
/// Exit non-zero when conflicts found (CI gate; default remains report-only)
#[arg(long)]
fail: bool,
/// Force a full knowledge recompile before querying
#[arg(long)]
refresh: bool,
},
/// List stale / deprecated / superseded knowledge
Stale {
@@ -342,6 +370,10 @@ enum Commands {
/// Exit non-zero when stale items found (CI gate; default remains report-only)
#[arg(long)]
fail: bool,
/// Force a full knowledge recompile before querying
#[arg(long)]
refresh: bool,
},
/// Detect duplicate capabilities via symbol fingerprints
Duplicate {
@@ -360,6 +392,10 @@ enum Commands {
#[arg(long)]
json: bool,
/// Force a full knowledge recompile before querying
#[arg(long)]
refresh: bool,
},
/// Verify code ↔ knowledge consistency (EKC structural; exit 1 only on errors)
Verify {
@@ -375,6 +411,10 @@ enum Commands {
/// Also fail on warnings (does not change default success-when-no-errors)
#[arg(long)]
fail_on_warning: bool,
/// Force a full knowledge recompile before querying
#[arg(long)]
refresh: bool,
},
/// Official extension installer (`list` / `add` / `remove`)
Ext {
@@ -529,6 +569,23 @@ enum ViewAction {
#[arg(short = 'o', long = "out", required = true)]
out: PathBuf,
},
/// Read-only DocumentViewV1 (never executes). JSON to stdout or `--out`.
Document {
/// Project root (defaults to cwd / CLI path)
#[arg(long)]
project: Option<PathBuf>,
/// Document path relative to project
#[arg(long)]
document: String,
#[arg(long)]
json: bool,
/// Optional HTML wrapper for browser inspection (embed fragment only)
#[arg(long, value_name = "FILE")]
out: Option<PathBuf>,
},
}
fn main() -> ExitCode {
@@ -671,6 +728,8 @@ fn try_main(cli: Cli) -> Result<i32> {
emit_result,
binds,
fs_root,
call_lib_path,
args_json,
json: _,
legacy_markup,
} => {
@@ -691,6 +750,8 @@ fn try_main(cli: Cli) -> Result<i32> {
emit_result: emit_result.clone(),
binds: bind_pairs.clone(),
fs_root: fs_root.clone(),
call_lib_path: call_lib_path.clone(),
args_json: args_json.clone(),
..RunOptions::dump_all()
}
} else {
@@ -706,6 +767,8 @@ fn try_main(cli: Cli) -> Result<i32> {
emit_result,
binds: bind_pairs,
fs_root,
call_lib_path,
args_json,
..RunOptions::default()
}
};
@@ -1105,6 +1168,56 @@ fn try_main(cli: Cli) -> Result<i32> {
})?;
Ok(0)
}
Some(ViewAction::Document {
project,
document,
json,
out,
}) => {
let project = project
.or(path)
.unwrap_or_else(|| PathBuf::from("."));
let req = marqdo::workbench::DocumentViewRequest {
project,
document,
revision: None,
generation: None,
node_id: None,
include: vec![
"fragment".into(),
"outline".into(),
"nodes".into(),
"refs".into(),
"diagnostics".into(),
],
};
let view = marqdo::workbench::build_document_view(&req).map_err(|e| {
anyhow::anyhow!("{}", e)
})?;
if let Some(out_path) = out {
let fragment = view
.pointer("/render/fragment")
.and_then(|v| v.as_str())
.unwrap_or("");
let title = view
.pointer("/document/title")
.and_then(|v| v.as_str())
.unwrap_or("DocumentView");
let html = format!(
"<!doctype html><html><head><meta charset=\"utf-8\"/><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"/><title>{title}</title><style>:root{{--mq-bg:#f7f5f0;--mq-text:#1a1a1a;--mq-muted:#5c5c5c;--mq-border:#d0cbc2;--mq-accent:#0b5fff;--mq-font-body:\"Iowan Old Style\",\"Palatino Linotype\",Palatino,serif;--mq-font-code:ui-monospace,Consolas,monospace}}body{{margin:0;background:var(--mq-bg);color:var(--mq-text);font:16px/1.75 var(--mq-font-body)}}.wrap{{max-width:46rem;margin:0 auto;padding:1.5rem}} .mq-document a{{color:var(--mq-accent)}}</style></head><body><div class=\"wrap\">{fragment}</div></body></html>"
);
std::fs::write(&out_path, html)?;
if json {
println!("{}", serde_json::to_string_pretty(&view)?);
} else {
println!("wrote {}", out_path.display());
}
} else {
println!("{}", serde_json::to_string_pretty(&view)?);
}
let _ = json;
Ok(0)
}
None => {
serve(ViewOptions {
path: path.unwrap_or_else(|| PathBuf::from(".")),
@@ -1154,6 +1267,7 @@ fn try_main(cli: Cli) -> Result<i32> {
path,
out,
json,
refresh,
} => {
let path = path.unwrap_or_else(|| PathBuf::from("."));
match action.unwrap_or(KnowledgeAction::Compile) {
@@ -1398,14 +1512,15 @@ fn try_main(cli: Cli) -> Result<i32> {
min_candidates,
allow_create_after,
} => {
let graph = ensure_compiled(&path, &out)?;
let graph = graph_for_query(&path, &out, refresh)?;
let budget = ReuseBudget {
min_candidates,
allow_create_after,
..ReuseBudget::default()
};
let pf = preflight(&graph, &task, &budget, Some(&out))?;
let _ = compile_policy(&out, &task, &resolve(&graph, &task, &budget));
let reuse = resolve(&graph, &task, &budget);
let pf = preflight_from_reuse(&graph, &task, &reuse, Some(&out))?;
let _ = compile_policy(&out, &task, &reuse);
if json {
println!("{}", preflight_json(&pf));
} else {
@@ -1473,9 +1588,10 @@ fn try_main(cli: Cli) -> Result<i32> {
out,
limit,
json,
refresh,
} => {
let path = path.unwrap_or_else(|| PathBuf::from("."));
let graph = ensure_compiled(&path, &out)?;
let graph = graph_for_query(&path, &out, refresh)?;
let hits = find(&graph, &query, limit);
if json {
println!("{}", find_json(&query, &hits));
@@ -1492,9 +1608,10 @@ fn try_main(cli: Cli) -> Result<i32> {
min_candidates,
allow_create_after,
json,
refresh,
} => {
let path = path.unwrap_or_else(|| PathBuf::from("."));
let graph = ensure_compiled(&path, &out)?;
let graph = graph_for_query(&path, &out, refresh)?;
let budget = ReuseBudget {
min_candidates,
allow_create_after,
@@ -1503,7 +1620,7 @@ fn try_main(cli: Cli) -> Result<i32> {
let result = resolve(&graph, &task, &budget);
let _ = compile_policy(&out, &task, &result);
if do_preflight {
let _ = preflight(&graph, &task, &budget, Some(&out))?;
let _ = preflight_from_reuse(&graph, &task, &result, Some(&out))?;
}
if json {
println!("{}", reuse_json(&task, &result));
@@ -1517,8 +1634,9 @@ fn try_main(cli: Cli) -> Result<i32> {
root,
out,
json,
refresh,
} => {
let graph = ensure_compiled(&root, &out)?;
let graph = graph_for_query(&root, &out, refresh)?;
let path_strs: Vec<String> = paths
.iter()
.map(|p| p.to_string_lossy().replace('\\', "/"))
@@ -1536,9 +1654,10 @@ fn try_main(cli: Cli) -> Result<i32> {
out,
json,
fail,
refresh,
} => {
let path = path.unwrap_or_else(|| PathBuf::from("."));
let graph = ensure_compiled(&path, &out)?;
let graph = graph_for_query(&path, &out, refresh)?;
let items = find_conflicts(&graph);
if json {
println!("{}", conflicts_json(&items));
@@ -1552,9 +1671,10 @@ fn try_main(cli: Cli) -> Result<i32> {
out,
json,
fail,
refresh,
} => {
let path = path.unwrap_or_else(|| PathBuf::from("."));
let graph = ensure_compiled(&path, &out)?;
let graph = graph_for_query(&path, &out, refresh)?;
let items = find_stale(&graph, &path);
if json {
println!("{}", stale_json(&items));
@@ -1569,9 +1689,10 @@ fn try_main(cli: Cli) -> Result<i32> {
threshold,
fail,
json,
refresh,
} => {
let path = path.unwrap_or_else(|| PathBuf::from("."));
let graph = ensure_compiled(&path, &out)?;
let graph = graph_for_query(&path, &out, refresh)?;
let pairs = find_duplicates(&graph.symbols, threshold);
if json {
println!(
@@ -1601,9 +1722,10 @@ fn try_main(cli: Cli) -> Result<i32> {
out,
json,
fail_on_warning,
refresh,
} => {
let path = path.unwrap_or_else(|| PathBuf::from("."));
let graph = ensure_compiled(&path, &out)?;
let graph = graph_for_query(&path, &out, refresh)?;
let issues = verify(&graph);
if json {
println!("{}", verify_json(&issues));
+126 -2
View File
@@ -68,17 +68,48 @@ fn dispatch(req: &Value) -> Result<Value, Value> {
Some("document.extract") => document_extract(&params),
Some("artifact.patch") => artifact_patch(&params),
Some("artifact.extract_pure") => artifact_extract_pure(&params),
// Workbench M00–M04: capability probe + read-only view + locator.
Some("capabilities") => Ok(crate::workbench::probe_capabilities(&params)),
#[cfg(feature = "view")]
Some("view.document") => view_document(&params),
#[cfg(not(feature = "view"))]
Some("view.document") => Err(err(
"mlsp.unsupported",
"view.document requires the view feature",
)),
#[cfg(feature = "view")]
Some("resource.view") => resource_view(&params),
#[cfg(not(feature = "view"))]
Some("resource.view") => Err(err(
"mlsp.unsupported",
"resource.view requires the view feature",
)),
Some("view.reference_resolve") => crate::workbench::resolve_reference(&params),
// `<TAB>` M9: occlusion label search (two-tier vocabulary).
Some("labels.search") => labels_search(&params),
// `<TAB>` M6: one-shot session summary (T1 staleness seen/fixed).
Some("session.summary") => session_summary(),
other => Err(err(
"mlsp.unknown_method",
format!("未知方法 {other:?};可用:locate / syntax / validate / repair_targets / repair_apply / schema / artifact.find / artifact.inspect / link.resolve / artifact.verify / artifact.impact / artifact.bind / artifact.execute / artifact.reuse_execute / artifact.patch / artifact.extract_pure / document.extract"),
format!("未知方法 {other:?};可用:locate / syntax / validate / repair_targets / repair_apply / schema / artifact.find / artifact.inspect / link.resolve / artifact.verify / artifact.impact / artifact.bind / artifact.execute / artifact.reuse_execute / artifact.patch / artifact.extract_pure / document.extract / capabilities / view.document / resource.view / view.reference_resolve"),
)),
}
}
#[cfg(feature = "view")]
fn view_document(params: &Value) -> Result<Value, Value> {
let req = crate::workbench::DocumentViewRequest::from_params(params)
.map_err(|e| err("mlsp.bad_request", e))?;
crate::workbench::build_document_view(&req)
}
#[cfg(feature = "view")]
fn resource_view(params: &Value) -> Result<Value, Value> {
let req = crate::workbench::ResourceViewRequest::from_params(params)
.map_err(|e| err("mlsp.bad_request", e))?;
crate::workbench::build_resource_view(&req)
}
fn artifact_extract_pure(params: &Value) -> Result<Value, Value> {
let name = params
.get("name")
@@ -172,7 +203,100 @@ fn artifact_inspect(params: &Value) -> Result<Value, Value> {
let candidates = artifact_query(&compilation.semantics, query);
if candidates.is_empty() { return Err(err("artifact.unresolved", format!("没有制品 `{query}`"))); }
if candidates.len() != 1 { return Err(json!({"code":"artifact.ambiguous","message":format!("制品 `{query}` 不唯一"),"candidates":candidates.iter().map(|a|&a.artifact_id.0).collect::<Vec<_>>() })); }
Ok(crate::sema::artifact_json(candidates[0]))
let artifact = candidates[0];
// Version contract: expected source/generation mismatch must not silently return current.
check_artifact_version_params(params, &compilation, artifact)?;
Ok(crate::sema::artifact_json(artifact))
}
/// Reject inspect when caller pinned an expected source revision or generation that
/// does not match the current project snapshot (`artifact.version_mismatch`).
fn check_artifact_version_params(
params: &Value,
compilation: &crate::sema::Compilation,
artifact: &crate::sema::SemanticArtifact,
) -> Result<(), Value> {
let expected_rev = params
.get("expected_source_revision")
.and_then(Value::as_str)
.filter(|s| !s.is_empty());
if let Some(expected) = expected_rev {
let project_root = params
.get("project")
.or_else(|| params.get("path"))
.and_then(Value::as_str)
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("."));
let root = if project_root.is_file() {
project_root
.parent()
.unwrap_or_else(|| Path::new("."))
.to_path_buf()
} else {
project_root
};
let abs = root.join(&artifact.source.file);
match std::fs::read(&abs) {
Ok(raw) => {
use sha2::{Digest, Sha256};
let actual = format!("sha256:{:x}", Sha256::digest(&raw));
if actual != expected {
return Err(json!({
"code": "artifact.version_mismatch",
"message": format!(
"expected source_revision {expected}, actual {actual}; refusing silent downgrade"
),
"expected_source_revision": expected,
"actual_source_revision": actual,
"artifact_id": artifact.artifact_id.0,
}));
}
}
Err(e) => {
return Err(json!({
"code": "artifact.version_mismatch",
"message": format!(
"cannot verify source_revision for {}: {e}; refusing silent downgrade",
artifact.source.file
),
"expected_source_revision": expected,
"artifact_id": artifact.artifact_id.0,
}));
}
}
}
let expected_gen = params
.get("expected_generation")
.or_else(|| params.get("generation_id"))
.and_then(Value::as_str)
.filter(|s| !s.is_empty());
if let Some(expected) = expected_gen {
let actual = semantic_generation_id(&compilation.semantics);
if actual != expected {
return Err(json!({
"code": "artifact.version_mismatch",
"message": format!(
"expected generation {expected}, current projection is {actual}; refusing silent downgrade"
),
"expected_generation": expected,
"actual_generation": actual,
"artifact_id": artifact.artifact_id.0,
}));
}
}
Ok(())
}
/// Semantic generation token matching KnowledgePresentation / DocumentView.
fn semantic_generation_id(project: &crate::sema::SemanticProject) -> String {
use sha2::{Digest, Sha256};
let mut gh = Sha256::new();
gh.update(project.project_id.as_bytes());
for a in &project.artifacts {
gh.update(a.hashes.artifact.0.as_bytes());
}
format!("{:x}", gh.finalize())[..16].to_string()
}
fn link_resolve(params: &Value) -> Result<Value, Value> {
+202
View File
@@ -0,0 +1,202 @@
//! Task control plane: cancel without interpreter lock (M-R3).
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use serde_json::Value;
use crate::runtime::protocol::RuntimeEventV2;
use crate::runtime::scheduler::{AdmissionClass, ScheduleError, Scheduler};
use crate::runtime::task_store::{StoreError, TaskRecord, TaskState, TaskStore};
#[derive(Debug)]
pub enum ControlError {
Store(StoreError),
Schedule(ScheduleError),
Forbidden,
}
impl From<StoreError> for ControlError {
fn from(e: StoreError) -> Self {
Self::Store(e)
}
}
impl From<ScheduleError> for ControlError {
fn from(e: ScheduleError) -> Self {
Self::Schedule(e)
}
}
impl std::fmt::Display for ControlError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Store(e) => write!(f, "{e}"),
Self::Schedule(e) => write!(f, "{e}"),
Self::Forbidden => write!(f, "forbidden"),
}
}
}
pub struct ControlService {
pub store: Arc<TaskStore>,
pub scheduler: Arc<Scheduler>,
/// Per-lease cancel token directory (token files, not process-global env).
cancel_dir: PathBuf,
}
impl ControlService {
pub fn new(store: Arc<TaskStore>, scheduler: Arc<Scheduler>, cancel_dir: impl AsRef<Path>) -> Self {
let cancel_dir = cancel_dir.as_ref().to_path_buf();
let _ = std::fs::create_dir_all(&cancel_dir);
Self {
store,
scheduler,
cancel_dir,
}
}
pub fn admit(
&self,
task_id: &str,
run_id: &str,
workspace_id: &str,
session_id: Option<String>,
idempotency_key: &str,
payload: &Value,
class: AdmissionClass,
) -> Result<(TaskRecord, usize), ControlError> {
let rec = self.store.admit(
task_id,
run_id,
workspace_id,
session_id,
idempotency_key,
payload,
)?;
let depth = self.scheduler.enqueue(&rec, class)?;
let queued = self.store.cas_state(
&rec.task_id,
rec.revision,
TaskState::Queued,
|_| {},
)?;
Ok((queued, depth))
}
/// Cancel path never enters the Marqdo interpreter lock.
pub fn cancel(&self, task_id: &str) -> Result<TaskRecord, ControlError> {
let rec = self.store.request_cancel(task_id)?;
let tok = self.cancel_token_path(task_id);
let _ = std::fs::write(&tok, b"cancelled");
let _ = self.store.append_event(
task_id,
&rec.run_id,
&rec.attempt_id,
"run.cancel_requested",
serde_json::json!({"revision": rec.revision}),
);
Ok(rec)
}
pub fn cancel_token_path(&self, task_id: &str) -> PathBuf {
self.cancel_dir.join(format!("{task_id}.tok"))
}
pub fn mark_cancelled(&self, task_id: &str, expected_revision: u64) -> Result<TaskRecord, ControlError> {
Ok(self.store.cas_state(
task_id,
expected_revision,
TaskState::Cancelled,
|_| {},
)?)
}
pub fn complete(
&self,
task_id: &str,
expected_revision: u64,
result: Value,
) -> Result<TaskRecord, ControlError> {
match self.store.cas_state(task_id, expected_revision, TaskState::Succeeded, |r| {
r.result = Some(result.clone());
}) {
Ok(rec) => Ok(rec),
Err(StoreError::AlreadyTerminal(state)) => {
// Late result after cancel: append evidence only.
let rec = self.store.get(task_id)?;
let _ = self.store.append_event(
task_id,
&rec.run_id,
&rec.attempt_id,
"run.late_result",
serde_json::json!({"ignored_terminal": format!("{state:?}"), "result": result}),
);
Err(ControlError::Store(StoreError::AlreadyTerminal(state)))
}
Err(e) => Err(ControlError::Store(e)),
}
}
pub fn events_since(
&self,
task_id: &str,
after_seq: u64,
) -> Result<Vec<RuntimeEventV2>, ControlError> {
Ok(self.store.events_after(task_id, after_seq)?)
}
/// Collaboration grace + force-kill budget from the design (§6).
pub fn cancel_deadlines() -> (Duration, Duration) {
(Duration::from_secs(2), Duration::from_secs(3))
}
}
/// Convenience wrapper used by HTTP/control adapters.
pub fn cancel_task(svc: &ControlService, task_id: &str) -> Result<TaskRecord, ControlError> {
svc.cancel(task_id)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::runtime::scheduler::SchedulerConfig;
use serde_json::json;
#[test]
fn cancel_writes_token_not_interpreter() {
let dir = std::env::temp_dir().join(format!("mq-ctrl-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let store = Arc::new(TaskStore::memory());
let sched = Arc::new(Scheduler::new(SchedulerConfig::default()));
let ctl = ControlService::new(store, sched, &dir);
let (t, _) = ctl
.admit("t1", "r1", "ws", None, "k", &json!({}), AdmissionClass::Execute)
.unwrap();
let c = ctl.cancel(&t.task_id).unwrap();
assert_eq!(c.state, TaskState::CancelRequested);
let body = std::fs::read_to_string(ctl.cancel_token_path(&t.task_id)).unwrap();
assert_eq!(body, "cancelled");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn late_complete_after_cancel_is_evidence_only() {
let dir = std::env::temp_dir().join(format!("mq-ctrl2-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let store = Arc::new(TaskStore::memory());
let sched = Arc::new(Scheduler::new(SchedulerConfig::default()));
let ctl = ControlService::new(Arc::clone(&store), sched, &dir);
let (t, _) = ctl
.admit("t1", "r1", "ws", None, "k", &json!({}), AdmissionClass::Execute)
.unwrap();
let c = ctl.cancel(&t.task_id).unwrap();
ctl.mark_cancelled(&t.task_id, c.revision).unwrap();
let err = ctl.complete(&t.task_id, c.revision + 1, json!("late")).unwrap_err();
assert!(matches!(err, ControlError::Store(StoreError::AlreadyTerminal(_))));
let evs = store.events_after(&t.task_id, 0).unwrap();
assert!(evs.iter().any(|e| e.event_type == "run.late_result"));
let _ = std::fs::remove_dir_all(&dir);
}
}
+457
View File
@@ -0,0 +1,457 @@
//! Process-global control plane via `host_query("runtime", …)` — no INVOKE_LOCK.
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
use std::thread;
use std::time::Duration;
use serde_json::{json, Value};
use crate::runtime::control::ControlService;
use crate::runtime::protocol::{
EntryRef, InvokeContext, InvokeRequestV2, LeaseRef, PROTOCOL_VERSION,
};
use crate::runtime::scheduler::{AdmissionClass, Scheduler, SchedulerConfig};
use crate::runtime::supervisor::{Supervisor, SupervisorConfig};
use crate::runtime::task_store::{TaskState, TaskStore};
struct RuntimeHost {
control: ControlService,
supervisor: Arc<Supervisor>,
/// task_id → lease_id
leases: Mutex<std::collections::HashMap<String, String>>,
base: PathBuf,
}
fn runtime_dir() -> PathBuf {
std::env::var_os("MARQDO_RUNTIME_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| std::env::temp_dir().join("marqdo-runtime"))
}
fn marqdo_bin() -> PathBuf {
std::env::var_os("MARQDO_BIN")
.map(PathBuf::from)
.unwrap_or_else(|| std::env::current_exe().unwrap_or_else(|_| PathBuf::from("marqdo")))
}
fn ensure_host() -> &'static RuntimeHost {
static H: OnceLock<RuntimeHost> = OnceLock::new();
static JANITOR: OnceLock<()> = OnceLock::new();
let host = H.get_or_init(|| {
let base = runtime_dir();
let _ = std::fs::create_dir_all(&base);
let store = match TaskStore::with_persist(base.join("store")) {
Ok(s) => Arc::new(s),
Err(_) => Arc::new(TaskStore::memory()),
};
let scheduler = Arc::new(Scheduler::new(SchedulerConfig::default()));
let control = ControlService::new(
Arc::clone(&store),
Arc::clone(&scheduler),
base.join("cancel"),
);
let supervisor = Arc::new(Supervisor::new(
SupervisorConfig::default(),
base.join("stage"),
marqdo_bin(),
base.join("cancel"),
base.join("events"),
));
let _ = supervisor.warm();
RuntimeHost {
control,
supervisor,
leases: Mutex::new(std::collections::HashMap::new()),
base,
}
});
JANITOR.get_or_init(|| {
let sup = Arc::clone(&host.supervisor);
let _ = thread::Builder::new()
.name("mq-runtime-janitor".into())
.spawn(move || loop {
thread::sleep(Duration::from_millis(500));
let _ = sup.janitor_tick();
});
});
host
}
/// Dispatch `host_query("runtime", {op, ...})`. Never blocks on INVOKE_LOCK.
pub fn handle(args: &Value) -> Result<Value, String> {
let op = args
.get("op")
.and_then(|v| v.as_str())
.ok_or_else(|| "runtime requires op".to_string())?;
let host = ensure_host();
let ctl = &host.control;
match op {
"ping" | "capabilities" => Ok(json!({
"ok": true,
"protocol_version": PROTOCOL_VERSION,
"features": {
"runtime.task_control_v2": 1,
"runtime.event_replay": 1,
"runtime.isolated_execution": 1,
},
"runtime_dir": host.base.to_string_lossy(),
})),
"admit" => {
let task_id = req_str(args, "task_id")?;
let run_id = args
.get("run_id")
.and_then(|v| v.as_str())
.unwrap_or(task_id);
let workspace_id = args
.get("workspace_id")
.and_then(|v| v.as_str())
.unwrap_or("default");
let session_id = args
.get("session_id")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let idem = args
.get("idempotency_key")
.and_then(|v| v.as_str())
.unwrap_or(task_id);
let class = AdmissionClass::parse(
args.get("class").and_then(|v| v.as_str()).unwrap_or("execute"),
);
let payload = args.get("payload").cloned().unwrap_or(json!({}));
let (rec, depth) = ctl
.admit(
task_id,
run_id,
workspace_id,
session_id,
idem,
&payload,
class,
)
.map_err(|e| e.to_string())?;
Ok(json!({
"ok": true,
"task": task_json(&rec),
"queue_depth": depth,
"cancel_token": ctl.cancel_token_path(&rec.task_id).to_string_lossy(),
"event_path": host.supervisor.event_path(&rec.task_id).to_string_lossy(),
}))
}
"dispatch" => dispatch_op(host, args),
"cancel" => {
let task_id = req_str(args, "task_id")?;
match ctl.cancel(task_id) {
Ok(rec) => {
if let Some(lease) = host.leases.lock().unwrap().get(task_id) {
let _ = host.supervisor.signal_cancel(lease, task_id);
} else {
let _ = std::fs::write(ctl.cancel_token_path(task_id), b"cancelled");
}
Ok(json!({
"ok": true,
"task": task_json(&rec),
"cancel_token": ctl.cancel_token_path(task_id).to_string_lossy(),
}))
}
Err(crate::runtime::control::ControlError::Store(
crate::runtime::task_store::StoreError::NotFound,
)) => Ok(json!({"ok": false, "error": "not_found", "status": 404})),
Err(crate::runtime::control::ControlError::Store(
crate::runtime::task_store::StoreError::AlreadyTerminal(s),
)) => Ok(json!({
"ok": true,
"already_terminal": true,
"state": format!("{s:?}").to_ascii_lowercase(),
})),
Err(e) => Err(e.to_string()),
}
}
"get" => {
let task_id = req_str(args, "task_id")?;
match ctl.store.get(task_id) {
Ok(rec) => {
let mut out = json!({"ok": true, "task": task_json(&rec)});
if let Some(lease) = host.leases.lock().unwrap().get(task_id).cloned() {
out["lease_id"] = json!(lease);
if let Some(p) = host.supervisor.emit_result_path(&lease) {
out["emit_result"] = json!(p.to_string_lossy());
}
if let Ok(Some(code)) = host.supervisor.try_reap(&lease) {
out["exit_code"] = json!(code);
host.control
.scheduler
.release_slot(AdmissionClass::Execute);
let _ = ctl.store.cas_state(
task_id,
rec.revision,
if code == 0 {
TaskState::Succeeded
} else {
TaskState::Failed
},
|_| {},
);
}
}
out["event_path"] =
json!(host.supervisor.event_path(task_id).to_string_lossy());
Ok(out)
}
Err(_) => Ok(json!({"ok": false, "error": "not_found", "status": 404})),
}
}
"events" => {
let task_id = req_str(args, "task_id")?;
let after = args.get("after_seq").and_then(|v| v.as_u64()).unwrap_or(0);
let path = host.supervisor.event_path(task_id);
if path.is_file() {
let lines = read_ndjson_lines(&path, after)?;
return Ok(json!({
"ok": true,
"events": lines,
"after_seq": after,
"source": "worker_ndjson",
"event_path": path.to_string_lossy(),
}));
}
match ctl.events_since(task_id, after) {
Ok(evs) => Ok(json!({
"ok": true,
"events": evs,
"after_seq": after,
"source": "task_store",
})),
Err(_) => Ok(json!({"ok": false, "error": "not_found", "status": 404})),
}
}
"append_event" => {
let task_id = req_str(args, "task_id")?;
let rec = ctl.store.get(task_id).map_err(|_| "not_found".to_string())?;
let et = args
.get("type")
.and_then(|v| v.as_str())
.unwrap_or("message.delta");
let payload = args.get("payload").cloned().unwrap_or(json!({}));
let ev = ctl
.store
.append_event(task_id, &rec.run_id, &rec.attempt_id, et, payload)
.map_err(|e| e.to_string())?;
Ok(json!({"ok": true, "event": ev}))
}
other => Err(format!("runtime.unknown_op:{other}")),
}
}
fn dispatch_op(host: &RuntimeHost, args: &Value) -> Result<Value, String> {
let task_id = req_str(args, "task_id")?;
let rec = host
.control
.store
.get(task_id)
.map_err(|_| "not_found".to_string())?;
let entry = args.get("entry").ok_or("dispatch.entry_required")?;
let module = entry
.get("module")
.and_then(|v| v.as_str())
.ok_or("dispatch.module_required")?;
let function = entry
.get("function")
.and_then(|v| v.as_str())
.ok_or("dispatch.function_required")?;
let fs_root = args
.get("fs_root")
.and_then(|v| v.as_str())
.or_else(|| {
args.get("context")
.and_then(|c| c.get("fs_root"))
.and_then(|v| v.as_str())
})
.ok_or("dispatch.fs_root_required")?;
let args_ref = args
.get("args_ref")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.or_else(|| {
args.get("args_json")
.and_then(|v| v.as_str())
.map(|p| format!("file:{p}"))
});
let tok = host.control.cancel_token_path(task_id);
if !tok.exists() {
std::fs::write(&tok, b"").map_err(|e| e.to_string())?;
}
let epoch = host.control.store.next_lease_epoch();
let req = InvokeRequestV2 {
schema: "InvokeRequestV2".into(),
schema_version: PROTOCOL_VERSION,
task_id: task_id.into(),
run_id: rec.run_id.clone(),
attempt_id: rec.attempt_id.clone(),
lease: LeaseRef {
id: format!("lease_{task_id}"),
epoch,
},
entry: EntryRef {
module: module.into(),
function: function.into(),
},
args_ref,
context: InvokeContext {
workspace_id: rec.workspace_id.clone(),
snapshot_id: None,
selection_ref: None,
capabilities_ref: None,
deadline_at: "2099-01-01T00:00:00Z".into(),
fs_root: Some(fs_root.into()),
},
};
if host
.control
.scheduler
.try_acquire_slot(AdmissionClass::Execute)
.is_err()
{
return Ok(json!({
"ok": false,
"error": "no_slot",
"status": 429,
"Retry-After": 2,
}));
}
match host.supervisor.dispatch(req) {
Ok(lease_id) => {
host.leases
.lock()
.unwrap()
.insert(task_id.to_string(), lease_id.clone());
let _ = host.control.store.cas_state(
task_id,
rec.revision,
TaskState::Running,
|r| {
r.lease_id = Some(lease_id.clone());
r.lease_epoch = Some(epoch);
},
);
Ok(json!({
"ok": true,
"lease_id": lease_id,
"task_id": task_id,
"cancel_token": tok.to_string_lossy(),
"event_path": host.supervisor.event_path(task_id).to_string_lossy(),
"emit_result": host.supervisor.emit_result_path(&lease_id)
.map(|p| p.to_string_lossy().to_string()),
}))
}
Err(e) => {
host.control
.scheduler
.release_slot(AdmissionClass::Execute);
if e.contains("no_slot") {
Ok(json!({
"ok": false,
"error": "no_slot",
"status": 429,
"Retry-After": 2,
}))
} else {
Err(e)
}
}
}
}
fn read_ndjson_lines(path: &Path, after_seq: u64) -> Result<Vec<Value>, String> {
let text = std::fs::read_to_string(path).map_err(|e| e.to_string())?;
let mut out = Vec::new();
let mut seq = 0u64;
for line in text.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
seq += 1;
if seq <= after_seq {
continue;
}
match serde_json::from_str::<Value>(line) {
Ok(mut v) => {
if v.get("seq").is_none() {
if let Some(obj) = v.as_object_mut() {
obj.insert("seq".into(), json!(seq));
}
}
out.push(v);
}
Err(_) => out.push(json!({"seq": seq, "raw": line})),
}
}
Ok(out)
}
fn req_str<'a>(args: &'a Value, key: &str) -> Result<&'a str, String> {
args.get(key)
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.ok_or_else(|| format!("runtime missing `{key}`"))
}
fn task_json(rec: &crate::runtime::task_store::TaskRecord) -> Value {
json!({
"task_id": rec.task_id,
"run_id": rec.run_id,
"attempt_id": rec.attempt_id,
"workspace_id": rec.workspace_id,
"session_id": rec.session_id,
"state": state_str(rec.state),
"revision": rec.revision,
"idempotency_key": rec.idempotency_key,
"created_at": rec.created_at,
"updated_at": rec.updated_at,
"cancel_requested_at": rec.cancel_requested_at,
"result": rec.result,
"error": rec.error,
"lease_id": rec.lease_id,
})
}
fn state_str(s: TaskState) -> &'static str {
match s {
TaskState::Accepted => "accepted",
TaskState::Queued => "queued",
TaskState::Running => "running",
TaskState::Finalizing => "finalizing",
TaskState::Succeeded => "succeeded",
TaskState::Failed => "failed",
TaskState::CancelRequested => "cancel_requested",
TaskState::Cancelled => "cancelled",
TaskState::Interrupted => "interrupted",
TaskState::NeedsReconciliation => "needs_reconciliation",
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn admit_cancel_without_host_context() {
let id = format!("rt-{}", std::process::id());
let adm = handle(&json!({
"op": "admit",
"task_id": id,
"idempotency_key": id,
"payload": {"n": 1}
}))
.unwrap();
assert_eq!(adm["ok"], true);
let can = handle(&json!({"op": "cancel", "task_id": id})).unwrap();
assert_eq!(can["ok"], true);
assert_eq!(can["task"]["state"], "cancel_requested");
}
}
+58
View File
@@ -0,0 +1,58 @@
//! Worker lease identity and process lifecycle states (M-R2).
use std::time::{Duration, SystemTime};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct LeaseId(pub String);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WorkerState {
Starting,
Ready,
Leased,
Running,
Draining,
Unhealthy,
Terminating,
Exited,
}
#[derive(Debug, Clone)]
pub struct Lease {
pub id: LeaseId,
pub epoch: u64,
pub task_id: String,
pub run_id: String,
pub attempt_id: String,
pub deadline: SystemTime,
pub state: WorkerState,
}
impl Lease {
pub fn new(
id: impl Into<String>,
epoch: u64,
task_id: impl Into<String>,
run_id: impl Into<String>,
attempt_id: impl Into<String>,
ttl: Duration,
) -> Self {
Self {
id: LeaseId(id.into()),
epoch,
task_id: task_id.into(),
run_id: run_id.into(),
attempt_id: attempt_id.into(),
deadline: SystemTime::now() + ttl,
state: WorkerState::Leased,
}
}
pub fn expired(&self) -> bool {
SystemTime::now() > self.deadline
}
pub fn matches(&self, lease_id: &str, epoch: u64) -> bool {
self.id.0 == lease_id && self.epoch == epoch
}
}
+26
View File
@@ -0,0 +1,26 @@
//! Isolated web runtime (M-R2–M-R4): supervisor, worker IPC, leases, task
//! admission/control, and persistent event log.
//!
//! Design: `doc/design/isolated-web-runtime-development-2026-10-26.md`.
//! Until capability probes report version ≥1, callers must treat this as opt-in.
pub mod control;
pub mod host_api;
pub mod lease;
pub mod protocol;
pub mod publish;
pub mod scheduler;
pub mod supervisor;
pub mod task_store;
pub mod worker;
pub use control::{cancel_task, ControlError, ControlService};
pub use lease::{Lease, LeaseId, WorkerState};
pub use protocol::{
InvokeRequestV2, RuntimeEventV2, CONTROL_FRAME_MAX_BYTES, PROTOCOL_VERSION,
};
pub use publish::{PublishJournal, PublishOutcome, PublishPhase};
pub use scheduler::{AdmissionClass, Scheduler, SchedulerConfig};
pub use supervisor::{Supervisor, SupervisorConfig};
pub use task_store::{TaskRecord, TaskState, TaskStore};
pub use worker::{WorkerHandle, WorkerPool};
+158
View File
@@ -0,0 +1,158 @@
//! Length-prefixed JSON control frames + event channel types (M-R2 / M-R4).
use serde::{Deserialize, Serialize};
use serde_json::Value;
pub const PROTOCOL_VERSION: u32 = 2;
pub const CONTROL_FRAME_MAX_BYTES: usize = 1024 * 1024;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InvokeRequestV2 {
pub schema: String,
pub schema_version: u32,
pub task_id: String,
pub run_id: String,
pub attempt_id: String,
pub lease: LeaseRef,
pub entry: EntryRef,
#[serde(default)]
pub args_ref: Option<String>,
pub context: InvokeContext,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LeaseRef {
pub id: String,
pub epoch: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EntryRef {
pub module: String,
pub function: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InvokeContext {
pub workspace_id: String,
#[serde(default)]
pub snapshot_id: Option<String>,
#[serde(default)]
pub selection_ref: Option<String>,
#[serde(default)]
pub capabilities_ref: Option<String>,
pub deadline_at: String,
/// Absolute site filesystem root (`--fs-root` for the worker).
#[serde(default)]
pub fs_root: Option<String>,
}
impl InvokeRequestV2 {
pub fn validate(&self) -> Result<(), String> {
if self.schema != "InvokeRequestV2" {
return Err(format!("protocol.schema: {}", self.schema));
}
if self.schema_version != PROTOCOL_VERSION {
return Err(format!(
"protocol.version: got {} want {PROTOCOL_VERSION}",
self.schema_version
));
}
if self.task_id.is_empty() || self.run_id.is_empty() || self.attempt_id.is_empty() {
return Err("protocol.ids_required".into());
}
if self.entry.module.is_empty() || self.entry.function.is_empty() {
return Err("protocol.entry_required".into());
}
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RuntimeEventV2 {
pub schema: String,
pub schema_version: u32,
pub event_id: String,
pub seq: u64,
pub task_id: String,
pub run_id: String,
pub attempt_id: String,
#[serde(rename = "type")]
pub event_type: String,
pub payload: Value,
}
impl RuntimeEventV2 {
pub fn new(
task_id: &str,
run_id: &str,
attempt_id: &str,
seq: u64,
event_type: &str,
payload: Value,
) -> Self {
Self {
schema: "RuntimeEventV2".into(),
schema_version: PROTOCOL_VERSION,
event_id: format!("{task_id}:{seq}"),
seq,
task_id: task_id.into(),
run_id: run_id.into(),
attempt_id: attempt_id.into(),
event_type: event_type.into(),
payload,
}
}
}
/// Encode a control frame: u32 LE length + UTF-8 JSON (≤ CONTROL_FRAME_MAX_BYTES).
pub fn encode_frame(value: &Value) -> Result<Vec<u8>, String> {
let body = serde_json::to_vec(value).map_err(|e| format!("protocol.encode: {e}"))?;
if body.len() > CONTROL_FRAME_MAX_BYTES {
return Err("protocol.frame_too_large".into());
}
let mut out = Vec::with_capacity(4 + body.len());
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
out.extend_from_slice(&body);
Ok(out)
}
/// Decode one frame from the front of `buf`; returns (value, bytes_consumed).
pub fn decode_frame(buf: &[u8]) -> Result<Option<(Value, usize)>, String> {
if buf.len() < 4 {
return Ok(None);
}
let len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize;
if len > CONTROL_FRAME_MAX_BYTES {
return Err("protocol.frame_too_large".into());
}
if buf.len() < 4 + len {
return Ok(None);
}
let value: Value = serde_json::from_slice(&buf[4..4 + len])
.map_err(|e| format!("protocol.decode: {e}"))?;
Ok(Some((value, 4 + len)))
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn frame_round_trip() {
let v = json!({"hello": "世界"});
let enc = encode_frame(&v).unwrap();
let (got, n) = decode_frame(&enc).unwrap().unwrap();
assert_eq!(n, enc.len());
assert_eq!(got, v);
}
#[test]
fn incomplete_frame_waits() {
let v = json!({"a": 1});
let enc = encode_frame(&v).unwrap();
assert!(decode_frame(&enc[..3]).unwrap().is_none());
assert!(decode_frame(&enc[..enc.len() - 1]).unwrap().is_none());
}
}
+274
View File
@@ -0,0 +1,274 @@
//! Publish journal + expected_generation CAS (M-R6).
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PublishPhase {
Staging,
Validated,
GenerationRenamed,
CurrentSwitched,
DbProjected,
Aborted,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PublishJournal {
pub kb_id: String,
pub expected_generation: Option<String>,
pub new_generation: String,
pub phase: PublishPhase,
pub staging_dir: PathBuf,
pub generation_dir: PathBuf,
pub current_path: PathBuf,
pub source_digest: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PublishOutcome {
Published { generation: String },
Conflict { current: String, expected: String },
Recovered { generation: String },
Aborted { reason: String },
}
impl PublishJournal {
pub fn begin(
out_dir: impl AsRef<Path>,
kb_id: impl Into<String>,
expected_generation: Option<String>,
new_generation: impl Into<String>,
source_digest: impl Into<String>,
) -> Result<Self, String> {
let out_dir = out_dir.as_ref();
let new_generation = new_generation.into();
let staging_dir = out_dir.join(".publish-staging").join(&new_generation);
let generation_dir = out_dir.join("generations").join(&new_generation);
let current_path = out_dir.join("CURRENT");
fs::create_dir_all(&staging_dir).map_err(|e| e.to_string())?;
let j = Self {
kb_id: kb_id.into(),
expected_generation,
new_generation,
phase: PublishPhase::Staging,
staging_dir,
generation_dir,
current_path,
source_digest: source_digest.into(),
};
j.write_journal(out_dir)?;
Ok(j)
}
fn journal_path(out_dir: &Path) -> PathBuf {
out_dir.join("publish_journal.json")
}
fn write_journal(&self, out_dir: &Path) -> Result<(), String> {
let path = Self::journal_path(out_dir);
let tmp = path.with_extension("json.tmp");
let body = serde_json::to_vec_pretty(self).map_err(|e| e.to_string())?;
fs::write(&tmp, body).map_err(|e| e.to_string())?;
// Best-effort fsync via File.
let f = fs::OpenOptions::new()
.write(true)
.open(&tmp)
.map_err(|e| e.to_string())?;
let _ = f.sync_all();
drop(f);
fs::rename(&tmp, &path).map_err(|e| e.to_string())?;
Ok(())
}
pub fn load(out_dir: impl AsRef<Path>) -> Result<Option<Self>, String> {
let path = Self::journal_path(out_dir.as_ref());
if !path.is_file() {
return Ok(None);
}
let body = fs::read_to_string(&path).map_err(|e| e.to_string())?;
let j: Self = serde_json::from_str(&body).map_err(|e| e.to_string())?;
Ok(Some(j))
}
/// CAS: refuse if CURRENT does not match expected_generation.
pub fn assert_expected_current(&self) -> Result<String, PublishOutcome> {
let current = fs::read_to_string(&self.current_path)
.unwrap_or_default()
.trim()
.to_string();
if let Some(exp) = &self.expected_generation {
if &current != exp {
return Err(PublishOutcome::Conflict {
current,
expected: exp.clone(),
});
}
}
Ok(current)
}
pub fn mark(&mut self, out_dir: &Path, phase: PublishPhase) -> Result<(), String> {
self.phase = phase;
self.write_journal(out_dir)
}
/// Atomic publish: staging tree already complete → rename into generations → CURRENT.
pub fn commit_filesystem(&mut self, out_dir: &Path) -> Result<PublishOutcome, String> {
match self.assert_expected_current() {
Err(o) => return Ok(o),
Ok(_) => {}
}
self.mark(out_dir, PublishPhase::Validated)?;
if self.generation_dir.exists() {
fs::remove_dir_all(&self.generation_dir).map_err(|e| e.to_string())?;
}
fs::create_dir_all(self.generation_dir.parent().unwrap()).map_err(|e| e.to_string())?;
fs::rename(&self.staging_dir, &self.generation_dir).map_err(|e| e.to_string())?;
self.mark(out_dir, PublishPhase::GenerationRenamed)?;
let tmp = self.current_path.with_extension("tmp");
{
let mut f = fs::File::create(&tmp).map_err(|e| e.to_string())?;
writeln!(f, "{}", self.new_generation).map_err(|e| e.to_string())?;
let _ = f.sync_all();
}
fs::rename(&tmp, &self.current_path).map_err(|e| e.to_string())?;
self.mark(out_dir, PublishPhase::CurrentSwitched)?;
// Clear journal after successful CURRENT switch; DB projection is product-side.
let _ = fs::remove_file(Self::journal_path(out_dir));
Ok(PublishOutcome::Published {
generation: self.new_generation.clone(),
})
}
/// Crash recovery: if CURRENT already points at new_generation, treat as done;
/// if generation exists but CURRENT old, finish CURRENT switch; else abort staging.
pub fn recover(out_dir: impl AsRef<Path>) -> Result<PublishOutcome, String> {
let out_dir = out_dir.as_ref();
let Some(mut j) = Self::load(out_dir)? else {
return Ok(PublishOutcome::Aborted {
reason: "no_journal".into(),
});
};
let current = fs::read_to_string(&j.current_path)
.unwrap_or_default()
.trim()
.to_string();
if current == j.new_generation {
let _ = fs::remove_file(Self::journal_path(out_dir));
return Ok(PublishOutcome::Recovered {
generation: j.new_generation,
});
}
match j.phase {
PublishPhase::GenerationRenamed | PublishPhase::Validated
if j.generation_dir.is_dir() =>
{
let tmp = j.current_path.with_extension("tmp");
fs::write(&tmp, format!("{}\n", j.new_generation)).map_err(|e| e.to_string())?;
fs::rename(&tmp, &j.current_path).map_err(|e| e.to_string())?;
let _ = fs::remove_file(Self::journal_path(out_dir));
Ok(PublishOutcome::Recovered {
generation: j.new_generation,
})
}
PublishPhase::Staging | PublishPhase::Aborted => {
let _ = fs::remove_dir_all(&j.staging_dir);
j.mark(out_dir, PublishPhase::Aborted)?;
let _ = fs::remove_file(Self::journal_path(out_dir));
Ok(PublishOutcome::Aborted {
reason: "incomplete_staging".into(),
})
}
other => Ok(PublishOutcome::Aborted {
reason: format!("unhandled_phase:{other:?}"),
}),
}
}
}
/// Stable logical document identity independent of generation directory name.
pub fn logical_document_id(project_id: &str, relative_path: &str) -> String {
let mut h = Sha256::new();
h.update(project_id.as_bytes());
h.update(b"\0");
h.update(relative_path.replace('\\', "/").as_bytes());
format!("doc:{:x}", h.finalize())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cas_conflict_and_commit() {
let root = std::env::temp_dir().join(format!("mq-pub-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
fs::create_dir_all(root.join("generations")).unwrap();
fs::write(root.join("CURRENT"), "gen_old\n").unwrap();
let mut j = PublishJournal::begin(&root, "kb1", Some("gen_old".into()), "gen_new", "digest")
.unwrap();
fs::write(j.staging_dir.join("manifest.json"), "{}").unwrap();
let out = j.commit_filesystem(&root).unwrap();
assert_eq!(
out,
PublishOutcome::Published {
generation: "gen_new".into()
}
);
assert_eq!(
fs::read_to_string(root.join("CURRENT")).unwrap().trim(),
"gen_new"
);
let mut j2 =
PublishJournal::begin(&root, "kb1", Some("gen_old".into()), "gen_x", "d").unwrap();
fs::write(j2.staging_dir.join("x"), "1").unwrap();
let conflict = j2.commit_filesystem(&root).unwrap();
assert!(matches!(conflict, PublishOutcome::Conflict { .. }));
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn recover_after_generation_rename() {
let root = std::env::temp_dir().join(format!("mq-pub2-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
fs::create_dir_all(root.join("generations/gen_r")).unwrap();
fs::write(root.join("CURRENT"), "old\n").unwrap();
let mut j =
PublishJournal::begin(&root, "kb", Some("old".into()), "gen_r", "d").unwrap();
// Simulate crash after generation rename: staging already moved.
let _ = fs::remove_dir_all(&j.staging_dir);
j.phase = PublishPhase::GenerationRenamed;
j.write_journal(&root).unwrap();
let out = PublishJournal::recover(&root).unwrap();
assert_eq!(
out,
PublishOutcome::Recovered {
generation: "gen_r".into()
}
);
assert_eq!(
fs::read_to_string(root.join("CURRENT")).unwrap().trim(),
"gen_r"
);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn logical_id_stable_across_generation_paths() {
let a = logical_document_id("proj", "docs/a.mq.md");
let b = logical_document_id("proj", "docs/a.mq.md");
assert_eq!(a, b);
assert_ne!(a, logical_document_id("proj", "docs/b.mq.md"));
}
}
+249
View File
@@ -0,0 +1,249 @@
//! Bounded admission classes and fair queueing (M-R3).
use std::collections::{HashMap, VecDeque};
use std::sync::Mutex;
use crate::runtime::task_store::TaskRecord;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AdmissionClass {
Control,
Read,
Short,
Execute,
Commit,
/// Undeclared endpoints stay on the legacy single-interpreter path.
LegacySerial,
}
impl AdmissionClass {
pub fn parse(s: &str) -> Self {
match s.to_ascii_lowercase().as_str() {
"control" => Self::Control,
"read" => Self::Read,
"short" => Self::Short,
"execute" => Self::Execute,
"commit" => Self::Commit,
_ => Self::LegacySerial,
}
}
}
#[derive(Debug, Clone)]
pub struct SchedulerConfig {
pub execute_slots: usize,
pub short_slots: usize,
pub read_slots: usize,
pub global_queue_limit: usize,
pub workspace_queue_limit: usize,
pub session_pending_limit: usize,
}
impl Default for SchedulerConfig {
fn default() -> Self {
// 4 vCPU / 8 GiB verification machine defaults from the design.
Self {
execute_slots: 2,
short_slots: 1,
read_slots: 1,
global_queue_limit: 64,
workspace_queue_limit: 16,
session_pending_limit: 4,
}
}
}
#[derive(Debug)]
pub enum ScheduleError {
QueueFull,
WorkspaceFull,
SessionFull,
NoSlot,
}
impl std::fmt::Display for ScheduleError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::QueueFull => write!(f, "queue_full"),
Self::WorkspaceFull => write!(f, "workspace_full"),
Self::SessionFull => write!(f, "session_full"),
Self::NoSlot => write!(f, "no_slot"),
}
}
}
struct QueueState {
global: VecDeque<String>,
by_workspace: HashMap<String, VecDeque<String>>,
by_session: HashMap<String, usize>,
active_execute: usize,
active_short: usize,
active_read: usize,
/// workspace rotation cursor
workspace_order: VecDeque<String>,
}
pub struct Scheduler {
cfg: SchedulerConfig,
inner: Mutex<QueueState>,
}
impl Scheduler {
pub fn new(cfg: SchedulerConfig) -> Self {
Self {
cfg,
inner: Mutex::new(QueueState {
global: VecDeque::new(),
by_workspace: HashMap::new(),
by_session: HashMap::new(),
active_execute: 0,
active_short: 0,
active_read: 0,
workspace_order: VecDeque::new(),
}),
}
}
pub fn enqueue(&self, task: &TaskRecord, class: AdmissionClass) -> Result<usize, ScheduleError> {
if class == AdmissionClass::Control || class == AdmissionClass::LegacySerial {
return Ok(0);
}
let mut g = self.inner.lock().unwrap();
if g.global.len() >= self.cfg.global_queue_limit {
return Err(ScheduleError::QueueFull);
}
let ws_len = g
.by_workspace
.get(&task.workspace_id)
.map(|q| q.len())
.unwrap_or(0);
if ws_len >= self.cfg.workspace_queue_limit {
return Err(ScheduleError::WorkspaceFull);
}
if let Some(sid) = &task.session_id {
let n = g.by_session.entry(sid.clone()).or_insert(0);
if *n >= self.cfg.session_pending_limit {
return Err(ScheduleError::SessionFull);
}
*n += 1;
}
if !g.workspace_order.iter().any(|w| w == &task.workspace_id) {
g.workspace_order.push_back(task.workspace_id.clone());
}
g.by_workspace
.entry(task.workspace_id.clone())
.or_default()
.push_back(task.task_id.clone());
g.global.push_back(task.task_id.clone());
Ok(g.global.len())
}
pub fn try_acquire_slot(&self, class: AdmissionClass) -> Result<(), ScheduleError> {
let mut g = self.inner.lock().unwrap();
match class {
AdmissionClass::Execute => {
if g.active_execute >= self.cfg.execute_slots {
return Err(ScheduleError::NoSlot);
}
g.active_execute += 1;
}
AdmissionClass::Short | AdmissionClass::Commit => {
if g.active_short >= self.cfg.short_slots {
return Err(ScheduleError::NoSlot);
}
g.active_short += 1;
}
AdmissionClass::Read => {
if g.active_read >= self.cfg.read_slots {
return Err(ScheduleError::NoSlot);
}
g.active_read += 1;
}
AdmissionClass::Control | AdmissionClass::LegacySerial => {}
}
Ok(())
}
pub fn release_slot(&self, class: AdmissionClass) {
let mut g = self.inner.lock().unwrap();
match class {
AdmissionClass::Execute => {
g.active_execute = g.active_execute.saturating_sub(1);
}
AdmissionClass::Short | AdmissionClass::Commit => {
g.active_short = g.active_short.saturating_sub(1);
}
AdmissionClass::Read => {
g.active_read = g.active_read.saturating_sub(1);
}
AdmissionClass::Control | AdmissionClass::LegacySerial => {}
}
}
/// Fair pop: rotate workspaces, then session FIFO within workspace.
pub fn pop_next(&self) -> Option<String> {
let mut g = self.inner.lock().unwrap();
if g.workspace_order.is_empty() {
return None;
}
for _ in 0..g.workspace_order.len() {
let ws = g.workspace_order.pop_front()?;
if let Some(q) = g.by_workspace.get_mut(&ws) {
if let Some(tid) = q.pop_front() {
g.workspace_order.push_back(ws);
g.global.retain(|id| id != &tid);
return Some(tid);
}
}
}
None
}
pub fn queue_depth(&self) -> usize {
self.inner.lock().unwrap().global.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::runtime::task_store::{TaskRecord, TaskState};
fn task(id: &str, ws: &str, session: Option<&str>) -> TaskRecord {
TaskRecord {
task_id: id.into(),
run_id: format!("r-{id}"),
attempt_id: format!("a-{id}"),
workspace_id: ws.into(),
session_id: session.map(|s| s.into()),
state: TaskState::Queued,
revision: 1,
idempotency_key: id.into(),
payload_hash: "h".into(),
lease_id: None,
lease_epoch: None,
created_at: "0".into(),
updated_at: "0".into(),
cancel_requested_at: None,
result: None,
error: None,
}
}
#[test]
fn workspace_rotation_fairness() {
let sched = Scheduler::new(SchedulerConfig::default());
sched
.enqueue(&task("a1", "wsA", None), AdmissionClass::Execute)
.unwrap();
sched
.enqueue(&task("a2", "wsA", None), AdmissionClass::Execute)
.unwrap();
sched
.enqueue(&task("b1", "wsB", None), AdmissionClass::Execute)
.unwrap();
assert_eq!(sched.pop_next().as_deref(), Some("a1"));
assert_eq!(sched.pop_next().as_deref(), Some("b1"));
assert_eq!(sched.pop_next().as_deref(), Some("a2"));
}
}
+303
View File
@@ -0,0 +1,303 @@
//! Supervisor: lease workers, enforce deadlines, reap orphans (M-R2).
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use crate::runtime::lease::{Lease, WorkerState};
use crate::runtime::protocol::InvokeRequestV2;
use crate::runtime::worker::{WorkerHandle, WorkerPool};
#[derive(Debug, Clone)]
pub struct SupervisorConfig {
pub execute_slots: usize,
pub worker_ttl: Duration,
pub cancel_grace: Duration,
pub force_kill_after: Duration,
}
impl Default for SupervisorConfig {
fn default() -> Self {
Self {
execute_slots: 2,
worker_ttl: Duration::from_secs(600),
cancel_grace: Duration::from_secs(2),
force_kill_after: Duration::from_secs(3),
}
}
}
struct ActiveWorker {
handle: WorkerHandle,
cancel_since: Option<Instant>,
}
pub struct Supervisor {
cfg: SupervisorConfig,
pool: Mutex<WorkerPool>,
active: Mutex<HashMap<String, ActiveWorker>>,
cancel_dir: PathBuf,
event_dir: PathBuf,
}
impl Supervisor {
pub fn new(
cfg: SupervisorConfig,
staging_root: impl AsRef<Path>,
marqdo_bin: impl AsRef<Path>,
cancel_dir: impl AsRef<Path>,
event_dir: impl AsRef<Path>,
) -> Self {
let cancel_dir = cancel_dir.as_ref().to_path_buf();
let event_dir = event_dir.as_ref().to_path_buf();
let _ = std::fs::create_dir_all(&cancel_dir);
let _ = std::fs::create_dir_all(&event_dir);
Self {
pool: Mutex::new(WorkerPool::new(
staging_root,
marqdo_bin,
cfg.execute_slots,
)),
cfg,
active: Mutex::new(HashMap::new()),
cancel_dir,
event_dir,
}
}
pub fn warm(&self) -> Result<(), String> {
self.pool.lock().unwrap().warm()
}
pub fn dispatch(&self, req: InvokeRequestV2) -> Result<String, String> {
req.validate()?;
{
let active = self.active.lock().unwrap();
if active.len() >= self.cfg.execute_slots {
return Err("supervisor.no_slot".into());
}
}
let mut pool = self.pool.lock().unwrap();
let worker = pool.acquire()?;
drop(pool);
let epoch = Instant::now().elapsed().as_nanos() as u64;
let lease = Lease::new(
format!("lease_{}", req.task_id),
epoch,
&req.task_id,
&req.run_id,
&req.attempt_id,
self.cfg.worker_ttl,
);
let cancel = self.cancel_dir.join(format!("{}.tok", req.task_id));
if !cancel.exists() {
std::fs::write(&cancel, b"").map_err(|e| e.to_string())?;
}
let events = self.event_dir.join(format!("{}.ndjson", req.task_id));
std::fs::write(&events, b"").map_err(|e| e.to_string())?;
// Durable emit path (not under staging — WorkerHandle::Drop removes staging).
let emit = self.event_dir.join(format!("{}.result.json", req.task_id));
let fs_root = req
.context
.fs_root
.as_ref()
.map(PathBuf::from)
.filter(|p| p.is_dir())
.ok_or_else(|| "supervisor.fs_root_required".to_string())?;
let args_path = req.args_ref.as_ref().and_then(|r| {
r.strip_prefix("file:")
.map(PathBuf::from)
.or_else(|| {
let p = PathBuf::from(r);
p.is_file().then_some(p)
})
});
let handle = {
if std::env::var_os("MARQDO_RUNTIME_DRY_RUN").is_some() {
let mut w = worker;
w.lease = Some(lease.clone());
w.emit_result = Some(emit.clone());
w.state = WorkerState::Running;
w
} else {
self.pool.lock().unwrap().lease_and_start(
worker,
lease.clone(),
&req,
&cancel,
&events,
&fs_root,
args_path.as_deref(),
&emit,
)?
}
};
let lease_id = lease.id.0.clone();
self.active.lock().unwrap().insert(
lease_id.clone(),
ActiveWorker {
handle,
cancel_since: None,
},
);
Ok(lease_id)
}
pub fn event_path(&self, task_id: &str) -> PathBuf {
self.event_dir.join(format!("{task_id}.ndjson"))
}
pub fn cancel_path(&self, task_id: &str) -> PathBuf {
self.cancel_dir.join(format!("{task_id}.tok"))
}
pub fn emit_result_path(&self, lease_id: &str) -> Option<PathBuf> {
self.active
.lock()
.unwrap()
.get(lease_id)
.and_then(|w| w.handle.emit_result.clone())
}
/// Non-blocking poll: if worker exited, remove from active and return exit code.
pub fn try_reap(&self, lease_id: &str) -> Result<Option<i32>, String> {
let mut active = self.active.lock().unwrap();
let Some(w) = active.get_mut(lease_id) else {
return Ok(None);
};
match w.handle.try_wait()? {
Some(st) => {
let code = st.code().unwrap_or(-1);
active.remove(lease_id);
let _ = self.pool.lock().unwrap().warm();
Ok(Some(code))
}
None => Ok(None),
}
}
pub fn signal_cancel(&self, lease_id: &str, task_id: &str) -> Result<(), String> {
let tok = self.cancel_dir.join(format!("{task_id}.tok"));
std::fs::write(&tok, b"cancelled").map_err(|e| e.to_string())?;
let mut active = self.active.lock().unwrap();
if let Some(w) = active.get_mut(lease_id) {
w.cancel_since = Some(Instant::now());
w.handle.state = WorkerState::Draining;
}
Ok(())
}
/// Reap expired / force-kill after cancel grace. Returns orphan count.
pub fn janitor_tick(&self) -> usize {
let mut orphans = 0usize;
let mut active = self.active.lock().unwrap();
let mut done = Vec::new();
for (id, w) in active.iter_mut() {
if let Some(lease) = &w.handle.lease {
if lease.expired() {
w.handle.kill_process_group();
orphans += 1;
done.push(id.clone());
continue;
}
}
if let Some(since) = w.cancel_since {
if since.elapsed() > self.cfg.cancel_grace + self.cfg.force_kill_after {
w.handle.kill_process_group();
orphans += 1;
done.push(id.clone());
}
}
}
for id in done {
active.remove(&id);
}
orphans
}
pub fn active_count(&self) -> usize {
self.active.lock().unwrap().len()
}
pub fn release(&self, lease_id: &str) {
let mut active = self.active.lock().unwrap();
if let Some(mut w) = active.remove(lease_id) {
w.handle.state = WorkerState::Draining;
w.handle.kill_process_group();
}
// Async replenish hint.
let _ = self.pool.lock().unwrap().warm();
}
}
/// Shared supervisor handle for HTTP/control wiring.
pub type SharedSupervisor = Arc<Supervisor>;
#[cfg(test)]
mod tests {
use super::*;
use crate::runtime::protocol::{EntryRef, InvokeContext, LeaseRef};
fn req(task: &str) -> InvokeRequestV2 {
InvokeRequestV2 {
schema: "InvokeRequestV2".into(),
schema_version: 2,
task_id: task.into(),
run_id: format!("run_{task}"),
attempt_id: format!("attempt_{task}"),
lease: LeaseRef {
id: format!("lease_{task}"),
epoch: 1,
},
entry: EntryRef {
module: "agent/qdagent.mq.md".into(),
function: "run".into(),
},
args_ref: None,
context: InvokeContext {
workspace_id: "ws".into(),
snapshot_id: None,
selection_ref: None,
capabilities_ref: None,
deadline_at: "2099-01-01T00:00:00Z".into(),
fs_root: Some("/tmp".into()),
},
}
}
#[test]
fn dry_run_two_parallel_leases() {
std::env::set_var("MARQDO_RUNTIME_DRY_RUN", "1");
let root = std::env::temp_dir().join(format!("mq-sup-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let bin = std::env::current_exe().unwrap();
let _ = std::fs::create_dir_all(root.join("fs"));
let mut ra = req("a");
ra.context.fs_root = Some(root.join("fs").to_string_lossy().into());
let mut rb = req("b");
rb.context.fs_root = ra.context.fs_root.clone();
let mut rc = req("c");
rc.context.fs_root = ra.context.fs_root.clone();
let sup = Supervisor::new(
SupervisorConfig {
execute_slots: 2,
..SupervisorConfig::default()
},
root.join("stage"),
&bin,
root.join("cancel"),
root.join("events"),
);
sup.warm().unwrap();
let a = sup.dispatch(ra).unwrap();
let b = sup.dispatch(rb).unwrap();
assert_ne!(a, b);
assert_eq!(sup.active_count(), 2);
assert!(sup.dispatch(rc).is_err());
sup.release(&a);
assert_eq!(sup.active_count(), 1);
let _ = std::fs::remove_dir_all(&root);
std::env::remove_var("MARQDO_RUNTIME_DRY_RUN");
}
}
+385
View File
@@ -0,0 +1,385 @@
//! Task index with idempotency, revision CAS, and event log (M-R3 / M-R4).
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::runtime::protocol::RuntimeEventV2;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskState {
Accepted,
Queued,
Running,
Finalizing,
Succeeded,
Failed,
CancelRequested,
Cancelled,
Interrupted,
NeedsReconciliation,
}
impl TaskState {
pub fn is_terminal(self) -> bool {
matches!(
self,
Self::Succeeded
| Self::Failed
| Self::Cancelled
| Self::Interrupted
| Self::NeedsReconciliation
)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TaskRecord {
pub task_id: String,
pub run_id: String,
pub attempt_id: String,
pub workspace_id: String,
pub session_id: Option<String>,
pub state: TaskState,
pub revision: u64,
pub idempotency_key: String,
pub payload_hash: String,
pub lease_id: Option<String>,
pub lease_epoch: Option<u64>,
pub created_at: String,
pub updated_at: String,
pub cancel_requested_at: Option<String>,
pub result: Option<Value>,
pub error: Option<String>,
}
#[derive(Default)]
struct Inner {
tasks: HashMap<String, TaskRecord>,
/// idempotency_key → task_id
idem: HashMap<String, String>,
/// task_id → events (monotonic seq)
events: HashMap<String, Vec<RuntimeEventV2>>,
next_epoch: u64,
}
pub struct TaskStore {
inner: Arc<Mutex<Inner>>,
persist_dir: Option<PathBuf>,
}
#[derive(Debug)]
pub enum StoreError {
Conflict(String),
NotFound,
AlreadyTerminal(TaskState),
Io(String),
}
impl std::fmt::Display for StoreError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Conflict(s) => write!(f, "conflict: {s}"),
Self::NotFound => write!(f, "not_found"),
Self::AlreadyTerminal(s) => write!(f, "already_terminal:{s:?}"),
Self::Io(s) => write!(f, "io: {s}"),
}
}
}
impl TaskStore {
pub fn memory() -> Self {
Self {
inner: Arc::new(Mutex::new(Inner::default())),
persist_dir: None,
}
}
pub fn with_persist(dir: impl AsRef<Path>) -> Result<Self, StoreError> {
let dir = dir.as_ref().to_path_buf();
std::fs::create_dir_all(&dir).map_err(|e| StoreError::Io(e.to_string()))?;
let store = Self {
inner: Arc::new(Mutex::new(Inner::default())),
persist_dir: Some(dir.clone()),
};
store.load()?;
Ok(store)
}
fn now() -> String {
let ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
format!("{ms}")
}
pub fn payload_hash(payload: &Value) -> String {
let bytes = serde_json::to_vec(payload).unwrap_or_default();
format!("{:x}", Sha256::digest(&bytes))
}
/// Admit or return existing task for the same idempotency key.
/// Same key + different payload hash → Conflict (409).
pub fn admit(
&self,
task_id: impl Into<String>,
run_id: impl Into<String>,
workspace_id: impl Into<String>,
session_id: Option<String>,
idempotency_key: impl Into<String>,
payload: &Value,
) -> Result<TaskRecord, StoreError> {
let task_id = task_id.into();
let idempotency_key = idempotency_key.into();
let payload_hash = Self::payload_hash(payload);
let mut g = self.inner.lock().unwrap();
if let Some(existing_id) = g.idem.get(&idempotency_key).cloned() {
let existing = g.tasks.get(&existing_id).cloned().ok_or(StoreError::NotFound)?;
if existing.payload_hash != payload_hash {
return Err(StoreError::Conflict(
"idempotency key reuse with different payload".into(),
));
}
return Ok(existing);
}
let now = Self::now();
let rec = TaskRecord {
task_id: task_id.clone(),
run_id: run_id.into(),
attempt_id: format!("attempt_{}", task_id),
workspace_id: workspace_id.into(),
session_id,
state: TaskState::Accepted,
revision: 1,
idempotency_key: idempotency_key.clone(),
payload_hash,
lease_id: None,
lease_epoch: None,
created_at: now.clone(),
updated_at: now,
cancel_requested_at: None,
result: None,
error: None,
};
g.idem.insert(idempotency_key, task_id.clone());
g.tasks.insert(task_id, rec.clone());
g.events.insert(rec.task_id.clone(), Vec::new());
drop(g);
self.persist_task(&rec)?;
Ok(rec)
}
pub fn get(&self, task_id: &str) -> Result<TaskRecord, StoreError> {
self.inner
.lock()
.unwrap()
.tasks
.get(task_id)
.cloned()
.ok_or(StoreError::NotFound)
}
/// CAS state transition. `expected_revision` must match.
pub fn cas_state(
&self,
task_id: &str,
expected_revision: u64,
next: TaskState,
patch: impl FnOnce(&mut TaskRecord),
) -> Result<TaskRecord, StoreError> {
let mut g = self.inner.lock().unwrap();
let rec = g.tasks.get_mut(task_id).ok_or(StoreError::NotFound)?;
if rec.revision != expected_revision {
return Err(StoreError::Conflict(format!(
"revision {} != {}",
rec.revision, expected_revision
)));
}
if rec.state.is_terminal() && next != rec.state {
return Err(StoreError::AlreadyTerminal(rec.state));
}
rec.state = next;
rec.revision += 1;
rec.updated_at = Self::now();
patch(rec);
let out = rec.clone();
drop(g);
self.persist_task(&out)?;
Ok(out)
}
pub fn request_cancel(&self, task_id: &str) -> Result<TaskRecord, StoreError> {
let mut g = self.inner.lock().unwrap();
let rec = g.tasks.get_mut(task_id).ok_or(StoreError::NotFound)?;
if rec.state.is_terminal() {
return Err(StoreError::AlreadyTerminal(rec.state));
}
let rev = rec.revision;
rec.state = TaskState::CancelRequested;
rec.revision = rev + 1;
rec.cancel_requested_at = Some(Self::now());
rec.updated_at = Self::now();
let out = rec.clone();
drop(g);
self.persist_task(&out)?;
Ok(out)
}
pub fn append_event(
&self,
task_id: &str,
run_id: &str,
attempt_id: &str,
event_type: &str,
payload: Value,
) -> Result<RuntimeEventV2, StoreError> {
let mut g = self.inner.lock().unwrap();
if !g.tasks.contains_key(task_id) {
return Err(StoreError::NotFound);
}
let list = g.events.entry(task_id.to_string()).or_default();
let seq = list.last().map(|e| e.seq + 1).unwrap_or(1);
let ev = RuntimeEventV2::new(task_id, run_id, attempt_id, seq, event_type, payload);
list.push(ev.clone());
drop(g);
self.persist_event(&ev)?;
Ok(ev)
}
/// Events with seq > after_seq (Last-Event-ID resume).
pub fn events_after(
&self,
task_id: &str,
after_seq: u64,
) -> Result<Vec<RuntimeEventV2>, StoreError> {
let g = self.inner.lock().unwrap();
let list = g.events.get(task_id).ok_or(StoreError::NotFound)?;
Ok(list.iter().filter(|e| e.seq > after_seq).cloned().collect())
}
pub fn next_lease_epoch(&self) -> u64 {
let mut g = self.inner.lock().unwrap();
g.next_epoch += 1;
g.next_epoch
}
fn persist_task(&self, rec: &TaskRecord) -> Result<(), StoreError> {
let Some(dir) = &self.persist_dir else {
return Ok(());
};
let path = dir.join("tasks").join(format!("{}.json", rec.task_id));
std::fs::create_dir_all(path.parent().unwrap()).map_err(|e| StoreError::Io(e.to_string()))?;
let tmp = path.with_extension("json.tmp");
let body = serde_json::to_vec_pretty(rec).map_err(|e| StoreError::Io(e.to_string()))?;
std::fs::write(&tmp, body).map_err(|e| StoreError::Io(e.to_string()))?;
std::fs::rename(&tmp, &path).map_err(|e| StoreError::Io(e.to_string()))?;
Ok(())
}
fn persist_event(&self, ev: &RuntimeEventV2) -> Result<(), StoreError> {
let Some(dir) = &self.persist_dir else {
return Ok(());
};
let path = dir.join("events").join(format!("{}.ndjson", ev.task_id));
std::fs::create_dir_all(path.parent().unwrap()).map_err(|e| StoreError::Io(e.to_string()))?;
use std::io::Write;
let mut f = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&path)
.map_err(|e| StoreError::Io(e.to_string()))?;
let line = serde_json::to_string(ev).map_err(|e| StoreError::Io(e.to_string()))?;
writeln!(f, "{line}").map_err(|e| StoreError::Io(e.to_string()))?;
Ok(())
}
fn load(&self) -> Result<(), StoreError> {
let Some(dir) = &self.persist_dir else {
return Ok(());
};
let tasks_dir = dir.join("tasks");
if !tasks_dir.is_dir() {
return Ok(());
}
let mut g = self.inner.lock().unwrap();
for ent in std::fs::read_dir(&tasks_dir).map_err(|e| StoreError::Io(e.to_string()))? {
let ent = ent.map_err(|e| StoreError::Io(e.to_string()))?;
if ent.path().extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let body =
std::fs::read_to_string(ent.path()).map_err(|e| StoreError::Io(e.to_string()))?;
let rec: TaskRecord =
serde_json::from_str(&body).map_err(|e| StoreError::Io(e.to_string()))?;
g.idem
.insert(rec.idempotency_key.clone(), rec.task_id.clone());
g.events.entry(rec.task_id.clone()).or_default();
g.tasks.insert(rec.task_id.clone(), rec);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn idempotent_admit_and_conflict() {
let store = TaskStore::memory();
let p = json!({"a": 1});
let a = store
.admit("t1", "r1", "ws", None, "key-1", &p)
.unwrap();
let b = store
.admit("t2", "r2", "ws", None, "key-1", &p)
.unwrap();
assert_eq!(a.task_id, b.task_id);
let err = store
.admit("t3", "r3", "ws", None, "key-1", &json!({"a": 2}))
.unwrap_err();
assert!(matches!(err, StoreError::Conflict(_)));
}
#[test]
fn cancel_cas_terminal() {
let store = TaskStore::memory();
let t = store
.admit("t1", "r1", "ws", None, "k", &json!({}))
.unwrap();
store
.cas_state(&t.task_id, t.revision, TaskState::Succeeded, |r| {
r.result = Some(json!("ok"));
})
.unwrap();
let err = store.request_cancel(&t.task_id).unwrap_err();
assert!(matches!(err, StoreError::AlreadyTerminal(TaskState::Succeeded)));
}
#[test]
fn event_seq_monotonic_and_resume() {
let store = TaskStore::memory();
let t = store
.admit("t1", "r1", "ws", None, "k", &json!({}))
.unwrap();
let e1 = store
.append_event(&t.task_id, &t.run_id, &t.attempt_id, "message.delta", json!({"n":1}))
.unwrap();
let e2 = store
.append_event(&t.task_id, &t.run_id, &t.attempt_id, "message.delta", json!({"n":2}))
.unwrap();
assert_eq!(e1.seq, 1);
assert_eq!(e2.seq, 2);
let after = store.events_after(&t.task_id, 1).unwrap();
assert_eq!(after.len(), 1);
assert_eq!(after[0].seq, 2);
}
}
+259
View File
@@ -0,0 +1,259 @@
//! One-shot Marqdo worker process handles (M-R2).
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use serde_json::{json, Value};
use crate::runtime::lease::{Lease, WorkerState};
use crate::runtime::protocol::{InvokeRequestV2, PROTOCOL_VERSION};
static WORKER_SEQ: AtomicU64 = AtomicU64::new(1);
pub struct WorkerHandle {
pub id: String,
pub state: WorkerState,
pub lease: Option<Lease>,
child: Option<Child>,
staging: PathBuf,
/// Where the child writes `--emit-result` JSON.
pub emit_result: Option<PathBuf>,
}
impl WorkerHandle {
pub fn staging(&self) -> &Path {
&self.staging
}
pub fn take_child(&mut self) -> Option<Child> {
self.child.take()
}
pub fn kill_process_group(&mut self) {
if let Some(mut child) = self.child.take() {
let _ = child.kill();
let _ = child.wait();
}
self.state = WorkerState::Exited;
}
pub fn try_wait(&mut self) -> Result<Option<std::process::ExitStatus>, String> {
let Some(child) = self.child.as_mut() else {
return Ok(None);
};
child.try_wait().map_err(|e| format!("worker.wait: {e}"))
}
pub fn wait(&mut self) -> Result<std::process::ExitStatus, String> {
let Some(mut child) = self.child.take() else {
return Err("worker.no_child".into());
};
let st = child.wait().map_err(|e| format!("worker.wait: {e}"))?;
self.state = WorkerState::Exited;
Ok(st)
}
}
impl Drop for WorkerHandle {
fn drop(&mut self) {
self.kill_process_group();
let _ = std::fs::remove_dir_all(&self.staging);
}
}
pub struct WorkerPool {
staging_root: PathBuf,
marqdo_bin: PathBuf,
ready: Vec<WorkerHandle>,
target_ready: usize,
}
impl WorkerPool {
pub fn new(staging_root: impl AsRef<Path>, marqdo_bin: impl AsRef<Path>, target_ready: usize) -> Self {
let staging_root = staging_root.as_ref().to_path_buf();
let _ = std::fs::create_dir_all(&staging_root);
Self {
staging_root,
marqdo_bin: marqdo_bin.as_ref().to_path_buf(),
ready: Vec::new(),
target_ready,
}
}
pub fn spawn_ready(&mut self) -> Result<WorkerHandle, String> {
let seq = WORKER_SEQ.fetch_add(1, Ordering::Relaxed);
let id = format!("worker_{seq}");
let staging = self.staging_root.join(&id);
std::fs::create_dir_all(&staging).map_err(|e| e.to_string())?;
if !self.marqdo_bin.exists() {
return Err(format!("worker.binary_missing: {}", self.marqdo_bin.display()));
}
Ok(WorkerHandle {
id,
state: WorkerState::Ready,
lease: None,
child: None,
staging,
emit_result: None,
})
}
pub fn warm(&mut self) -> Result<(), String> {
while self.ready.len() < self.target_ready {
let w = self.spawn_ready()?;
self.ready.push(w);
}
Ok(())
}
pub fn acquire(&mut self) -> Result<WorkerHandle, String> {
if self.ready.is_empty() {
self.warm()?;
}
self.ready
.pop()
.ok_or_else(|| "worker.pool_empty".to_string())
}
pub fn replenish_async_hint(&self) -> usize {
self.target_ready.saturating_sub(self.ready.len())
}
/// Lease a worker: spawn `marqdo run --call-lib-path …` with request-scoped env.
/// `emit_result` must be outside staging so Drop does not erase the parent's read path.
pub fn lease_and_start(
&self,
mut worker: WorkerHandle,
lease: Lease,
req: &InvokeRequestV2,
cancel_token: &Path,
event_sink: &Path,
fs_root: &Path,
args_json: Option<&Path>,
emit_result: &Path,
) -> Result<WorkerHandle, String> {
req.validate()?;
let entry = PathBuf::from(&req.entry.module);
if !entry.is_file() {
return Err(format!(
"worker.entry_missing: {} (need absolute site entry .mq.md)",
entry.display()
));
}
if let Some(parent) = emit_result.parent() {
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
}
let emit = emit_result.to_path_buf();
let mut cmd = Command::new(&self.marqdo_bin);
cmd.arg("run")
.arg(&entry)
.arg("--call-lib-path")
.arg(&req.entry.function)
.arg("--fs-root")
.arg(fs_root)
.arg("--emit-result")
.arg(&emit)
// Worker cwd = site root so relative imports resolve; staging only for lease bookkeeping.
.current_dir(fs_root)
.env("MARQDO_RUN_ID", &req.run_id)
.env("MARQDO_TASK_ID", &req.task_id)
.env("MARQDO_ATTEMPT_ID", &req.attempt_id)
.env("MARQDO_LEASE_ID", &lease.id.0)
.env("MARQDO_LEASE_EPOCH", lease.epoch.to_string())
.env("MARQDO_CANCEL_TOKEN", cancel_token)
.env("MARQDO_EVENT_FORWARD", event_sink)
.env("MARQDO_WORKSPACE_ID", &req.context.workspace_id)
.env("MARQDO_PROTOCOL_VERSION", PROTOCOL_VERSION.to_string())
.env("MARQDO_FS_ROOT", fs_root)
.stdin(Stdio::null())
// Avoid pipe-buffer deadlock: nobody drains worker stdout/stderr.
.stdout(Stdio::null())
.stderr(Stdio::null());
if let Some(args) = args_json {
cmd.arg("--args-json").arg(args);
} else if let Some(r) = &req.args_ref {
if let Some(path) = r.strip_prefix("file:") {
cmd.arg("--args-json").arg(path);
}
}
if let Some(sel) = &req.context.selection_ref {
cmd.env("MARQDO_SELECTION_REF", sel);
}
if let Some(snap) = &req.context.snapshot_id {
cmd.env("MARQDO_SNAPSHOT_ID", snap);
}
if let Some(cap) = &req.context.capabilities_ref {
cmd.env("MARQDO_CAPABILITIES_REF", cap);
}
let child = cmd.spawn().map_err(|e| format!("worker.spawn: {e}"))?;
worker.child = Some(child);
worker.lease = Some(lease);
worker.emit_result = Some(emit);
worker.state = WorkerState::Running;
Ok(worker)
}
}
pub fn handshake_ok(report: &Value) -> Result<(), String> {
let ver = report
.get("protocol_version")
.and_then(|v| v.as_u64())
.ok_or("handshake.protocol_version")?;
if ver != PROTOCOL_VERSION as u64 {
return Err(format!("handshake.version_mismatch: {ver}"));
}
Ok(())
}
pub fn fake_handshake() -> Value {
json!({
"cli": true,
"protocol_version": PROTOCOL_VERSION,
"capabilities": ["runtime.isolated_execution"],
"build": env!("CARGO_PKG_VERSION"),
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn handshake_version_gate() {
assert!(handshake_ok(&fake_handshake()).is_ok());
assert!(handshake_ok(&json!({"protocol_version": 1})).is_err());
}
#[test]
fn pool_warm_requires_binary() {
let dir = std::env::temp_dir().join(format!("mq-pool-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let mut pool = WorkerPool::new(&dir, "/no/such/marqdo-bin", 1);
assert!(pool.warm().is_err());
let self_bin = std::env::current_exe().unwrap();
let mut pool = WorkerPool::new(&dir, &self_bin, 1);
pool.warm().unwrap();
assert_eq!(pool.ready.len(), 1);
let w = pool.acquire().unwrap();
assert_eq!(w.state, WorkerState::Ready);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn kill_clears_child() {
let dir = std::env::temp_dir().join(format!("mq-wh-{}", std::process::id()));
let _ = std::fs::create_dir_all(&dir);
let mut w = WorkerHandle {
id: "w".into(),
state: WorkerState::Running,
lease: None,
child: None,
staging: dir.clone(),
emit_result: None,
};
w.kill_process_group();
assert_eq!(w.state, WorkerState::Exited);
let _ = std::fs::remove_dir_all(&dir);
}
}
+47 -27
View File
@@ -240,32 +240,24 @@ pub struct Compilation {
/// side-effect free: it never loads plugins, evaluates expressions or fetches
/// sources.
pub fn compile_project(options: &CompileOptions) -> Result<Compilation> {
let t_index = std::time::Instant::now();
let root = ProjectRoot::new(&options.project_root)?;
let index = ProjectIndex::build(root.clone())?;
let index_ms = t_index.elapsed().as_millis();
// Reuse parsed modules from ProjectIndex — same input snapshot, no second parse (M-R5b).
let mut files = BTreeMap::new();
let mut diagnostics = Vec::new();
for file in root.source_files()? {
let text = fs::read_to_string(&file)
.with_context(|| format!("read semantic source {}", file.display()))?;
match crate::load::parse_source_auto(&text) {
Ok(module) => {
let module_id = index
.modules()
.iter()
.find_map(|(id, path)| (path == &file).then(|| id.clone()));
if let Some(module_id) = module_id {
files.insert(module_id, (file, text, module));
}
}
Err(error) => {
diagnostics.push(
Diagnostic::new(Some(&file), Span::new(1, 1), error.to_string())
.with_code("parse.semantic")
.with_severity(Severity::Error),
);
}
}
for (module_id, parsed) in index.parsed_modules() {
files.insert(
module_id.clone(),
(
parsed.path.clone(),
parsed.text.clone(),
parsed.module.clone(),
),
);
}
let _ = index_ms; // ProjectIndex build duration (M-R5b span)
// Wire import_modules from already-parsed project files so BoundCall path
// resolve shares `resolve_for_exec` (no plugin load / no re-parse).
@@ -288,12 +280,8 @@ pub fn compile_project(options: &CompileOptions) -> Result<Compilation> {
collect_functions(&module.functions, &mut Vec::new(), &mut by_path);
let narrative_links = crate::parse::scan_narrative_links(text);
let module_prov = super::provenance::parse_module_provenance(text);
let module_symbols: Vec<&SymbolDecl> = index
.symbols()
.iter()
.filter(|s| &s.module == module_id)
.collect();
for symbol in index.symbols().iter().filter(|s| &s.module == module_id) {
let module_symbols = index.symbols_in_module(module_id);
for symbol in index.symbols_in_module(module_id) {
let function = by_path.get(&symbol.declaration_path).expect("symbol collected from module");
let owned_links: Vec<_> = narrative_links
.iter()
@@ -1315,12 +1303,44 @@ pub fn write_semantic_projections(out_dir: &Path, compilation: &Compilation) ->
fs::create_dir_all(gen_dir.join("verification"))?;
write_generation_tree(&gen_dir, compilation)?;
// Publish journal + CAS (M-R6): record expected CURRENT before switching.
let expected = fs::read_to_string(out_dir.join("CURRENT"))
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty());
let source_digest = {
let mut h = Sha256::new();
for a in &compilation.semantics.artifacts {
h.update(a.hashes.artifact.0.as_bytes());
}
format!("sha256:{:x}", h.finalize())
};
// Staging already written under gen_dir; journal tracks CURRENT switch.
let mut journal = crate::runtime::PublishJournal::begin(
out_dir,
&compilation.semantics.project_id,
expected,
&build_id,
&source_digest,
)
.map_err(|e| anyhow::anyhow!("publish journal: {e}"))?;
// Point journal staging at the generation we just wrote (already complete).
journal.staging_dir = gen_dir.clone();
journal.generation_dir = gen_dir.clone();
journal
.mark(out_dir, crate::runtime::publish::PublishPhase::Validated)
.map_err(|e| anyhow::anyhow!("publish journal mark: {e}"))?;
// Atomic generation pointer: only publish CURRENT after the tree is complete.
let current_tmp = out_dir.join("CURRENT.tmp");
fs::write(&current_tmp, format!("{build_id}\n"))
.with_context(|| format!("write {}", current_tmp.display()))?;
fs::rename(&current_tmp, out_dir.join("CURRENT"))
.with_context(|| format!("publish CURRENT under {}", out_dir.display()))?;
journal
.mark(out_dir, crate::runtime::publish::PublishPhase::CurrentSwitched)
.map_err(|e| anyhow::anyhow!("publish journal current: {e}"))?;
let _ = fs::remove_file(out_dir.join("publish_journal.json"));
// Compatibility root: stage a full mirror, then swap names (best-effort for
// legacy readers that do not follow CURRENT).
+51
View File
@@ -92,15 +92,29 @@ pub struct SymbolDecl {
pub line: u32,
}
/// Parsed module retained so `compile_project` does not re-parse (M-R5b).
#[derive(Debug, Clone)]
pub struct ParsedModule {
pub path: PathBuf,
pub text: String,
pub module: crate::ast::Module,
}
/// Deterministic read-only view of all declarations in one project.
#[derive(Debug, Clone)]
pub struct ProjectIndex {
project_id: String,
root: ProjectRoot,
modules: BTreeMap<ModuleId, PathBuf>,
/// path → module id (avoids linear scan per file in compile_project).
path_to_module: BTreeMap<PathBuf, ModuleId>,
symbols: Vec<SymbolDecl>,
by_id: BTreeMap<SymbolId, usize>,
by_name: BTreeMap<String, Vec<usize>>,
/// module → symbol indexes (avoids filtering the full symbol table per module).
symbols_by_module: BTreeMap<ModuleId, Vec<usize>>,
/// Shared parsed AST/text for the same input snapshot.
parsed: BTreeMap<ModuleId, ParsedModule>,
}
impl ProjectIndex {
@@ -108,7 +122,9 @@ impl ProjectIndex {
let project_id = project_id(root.path());
let files = root.source_files()?;
let mut modules = BTreeMap::new();
let mut path_to_module = BTreeMap::new();
let mut symbols = Vec::new();
let mut parsed = BTreeMap::new();
for file in files {
let source = fs::read_to_string(&file)
@@ -121,6 +137,7 @@ impl ProjectIndex {
file.display()
);
}
path_to_module.insert(file.clone(), module_id.clone());
let module = crate::load::parse_source_auto(&source)
.with_context(|| format!("parse source {}", file.display()))?;
for function in &module.functions {
@@ -134,28 +151,47 @@ impl ProjectIndex {
&mut symbols,
)?;
}
parsed.insert(
module_id.clone(),
ParsedModule {
path: file,
text: source,
module,
},
);
}
symbols.sort_by(|a, b| a.id.cmp(&b.id));
let mut by_id = BTreeMap::new();
let mut by_name: BTreeMap<String, Vec<usize>> = BTreeMap::new();
let mut symbols_by_module: BTreeMap<ModuleId, Vec<usize>> = BTreeMap::new();
for (index, symbol) in symbols.iter().enumerate() {
if by_id.insert(symbol.id.clone(), index).is_some() {
bail!("duplicate symbol id `{}`", symbol.id);
}
by_name.entry(symbol.name.clone()).or_default().push(index);
symbols_by_module
.entry(symbol.module.clone())
.or_default()
.push(index);
}
for indexes in by_name.values_mut() {
indexes.sort_by(|a, b| symbols[*a].id.cmp(&symbols[*b].id));
}
for indexes in symbols_by_module.values_mut() {
indexes.sort_by(|a, b| symbols[*a].id.cmp(&symbols[*b].id));
}
Ok(Self {
project_id,
root,
modules,
path_to_module,
symbols,
by_id,
by_name,
symbols_by_module,
parsed,
})
}
@@ -171,6 +207,21 @@ impl ProjectIndex {
&self.modules
}
pub fn module_id_for_path(&self, path: &Path) -> Option<&ModuleId> {
self.path_to_module.get(path)
}
pub fn symbols_in_module(&self, module_id: &ModuleId) -> Vec<&SymbolDecl> {
self.symbols_by_module
.get(module_id)
.map(|idxs| idxs.iter().map(|i| &self.symbols[*i]).collect())
.unwrap_or_default()
}
pub fn parsed_modules(&self) -> &BTreeMap<ModuleId, ParsedModule> {
&self.parsed
}
pub fn symbols(&self) -> &[SymbolDecl] {
&self.symbols
}
+17 -1
View File
@@ -127,7 +127,11 @@ impl KnowledgePresentation {
}
}
/// V2: whether an SSE result packet may update the live document structure.
/// Live View SSE: whether a result packet may update the document structure.
///
/// Legacy packets without a token are accepted so older live-view clients keep
/// working. **Embedded DocumentView clients must use [`sse_accepts_packet_strict`]
/// instead** — missing/empty tokens and mismatches are rejected there.
pub fn sse_accepts_packet(page_token: &str, packet_token: Option<&str>) -> bool {
match packet_token {
None | Some("") => true, // legacy packets without token still apply
@@ -136,6 +140,18 @@ pub fn sse_accepts_packet(page_token: &str, packet_token: Option<&str>) -> bool
}
}
/// Strict SSE / embed gate: reject missing, empty, or mismatched page tokens.
///
/// Use this for DocumentView / qdagent embed hosts. Do not inherit the live-view
/// leniency of [`sse_accepts_packet`].
pub fn sse_accepts_packet_strict(page_token: &str, packet_token: Option<&str>) -> bool {
match packet_token {
None | Some("") => false,
Some(_tok) if page_token.is_empty() => false,
Some(tok) => tok == page_token,
}
}
/// Stable page token from raw source bytes (used by SSE run path).
pub fn page_token_from_source(rel: &str, source: &str) -> String {
let mut h = Sha256::new();
+7 -2
View File
@@ -12,12 +12,17 @@ mod output;
mod render;
pub use knowledge_present::{
page_token_from_source, sse_accepts_packet, title_from_source, KnowledgePresentation,
page_token_from_source, sse_accepts_packet, sse_accepts_packet_strict, title_from_source,
KnowledgePresentation,
};
/// Surface rendering of an expression (labels/diagnostics reuse). `<TAB>` M6
pub use render::expr_display;
/// Read-model HTML helpers for embedded DocumentView (M03) — never execute.
pub use document::render_module_document;
pub use render::render_knowledge_html;
pub use output::{write_capture, write_static, CaptureOptions, OutputOptions};
use std::fs;
@@ -37,7 +42,7 @@ use crate::view::html::{escape, page_file, page_index, LinkMode};
use crate::view::annot::{build_view_annots, view_execution_report_json};
use crate::view::render::{
collect_input_prompts, render_bindings_html, render_module_structure,
render_knowledge_html, render_module_structure_with_annots, render_right_rail, uses_stream_panel, FileViewModel,
render_module_structure_with_annots, render_right_rail, uses_stream_panel, FileViewModel,
};
use crate::{run_file_capture, RunOptions};
+441
View File
@@ -0,0 +1,441 @@
//! AnalysisCache v2 (M-R5): SnapshotKey → Arc<Compilation> with single-flight.
//! Cache hits must reuse the compilation result, not merely a digest ticket.
use sha2::{Digest, Sha256};
use std::collections::{HashMap, VecDeque};
use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex, OnceLock};
use std::time::{Duration, Instant};
use crate::sema::Compilation;
/// Full-project snapshot identity (design §8.1). Built from the compiler's
/// source enumeration — never a truncated `take(200)` directory listing.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct SnapshotKey {
pub project_identity: String,
pub source_digest: String,
pub compiler_version: String,
}
impl SnapshotKey {
pub fn compute(project_root: &Path) -> Result<Self, String> {
let root = crate::project::ProjectRoot::new(project_root)
.map_err(|e| format!("snapshot.project: {e:#}"))?;
let files = root
.source_files()
.map_err(|e| format!("snapshot.enumerate: {e:#}"))?;
let mut h = Sha256::new();
h.update(root.path().to_string_lossy().as_bytes());
for p in &files {
let rel = p
.strip_prefix(root.path())
.unwrap_or(p)
.to_string_lossy();
h.update(rel.as_bytes());
let meta =
std::fs::metadata(p).map_err(|e| format!("snapshot.meta {}: {e}", p.display()))?;
h.update(meta.len().to_le_bytes());
let bytes =
std::fs::read(p).map_err(|e| format!("snapshot.read {}: {e}", p.display()))?;
h.update(&bytes);
}
let identity = root
.path()
.canonicalize()
.unwrap_or_else(|_| root.path().to_path_buf())
.to_string_lossy()
.to_string();
Ok(Self {
project_identity: identity,
source_digest: format!("{:x}", h.finalize()),
compiler_version: env!("CARGO_PKG_VERSION").to_string(),
})
}
pub fn cache_key(&self) -> String {
format!(
"{}::{}::{}",
self.project_identity, self.source_digest, self.compiler_version
)
}
}
struct CacheEntry {
#[allow(dead_code)]
key: SnapshotKey,
compilation: Arc<Compilation>,
bytes: usize,
#[allow(dead_code)]
compiled_at: Instant,
}
struct InflightSlot {
waiters: usize,
result: Option<Result<Arc<Compilation>, String>>,
}
struct CacheState {
order: VecDeque<String>,
entries: HashMap<String, CacheEntry>,
inflight: HashMap<String, InflightSlot>,
bytes_used: usize,
budget_bytes: usize,
}
fn cache() -> &'static (Mutex<CacheState>, Condvar) {
static C: OnceLock<(Mutex<CacheState>, Condvar)> = OnceLock::new();
C.get_or_init(|| {
(
Mutex::new(CacheState {
order: VecDeque::new(),
entries: HashMap::new(),
inflight: HashMap::new(),
bytes_used: 0,
budget_bytes: default_budget_bytes(),
}),
Condvar::new(),
)
})
}
fn default_budget_bytes() -> usize {
512 * 1024 * 1024
}
static HITS: AtomicU64 = AtomicU64::new(0);
static MISSES: AtomicU64 = AtomicU64::new(0);
static MERGES: AtomicU64 = AtomicU64::new(0);
static COMPILE_INVOCATIONS: AtomicU64 = AtomicU64::new(0);
static EVICTIONS: AtomicU64 = AtomicU64::new(0);
static RESULT_HITS: AtomicU64 = AtomicU64::new(0);
fn key_compiles() -> &'static Mutex<HashMap<String, u64>> {
static K: OnceLock<Mutex<HashMap<String, u64>>> = OnceLock::new();
K.get_or_init(|| Mutex::new(HashMap::new()))
}
fn estimate_bytes(c: &Compilation) -> usize {
let arts = c.semantics.artifacts.len() * 512;
let refs = c.semantics.refs.len() * 128;
let steps = c.semantics.steps.len() * 128;
let diags = c.diagnostics.len() * 64;
4096 + arts + refs + steps + diags
}
enum Admit {
Hit(Arc<Compilation>),
Wait,
Lead,
}
fn admit(key: &str) -> Admit {
let (lock, _cv) = cache();
let mut g = lock.lock().unwrap();
if let Some(e) = g.entries.get(key) {
let arc = Arc::clone(&e.compilation);
if let Some(pos) = g.order.iter().position(|k| k == key) {
g.order.remove(pos);
}
g.order.push_back(key.to_string());
HITS.fetch_add(1, Ordering::Relaxed);
RESULT_HITS.fetch_add(1, Ordering::Relaxed);
return Admit::Hit(arc);
}
if g.inflight.contains_key(key) {
if let Some(slot) = g.inflight.get_mut(key) {
slot.waiters += 1;
}
MERGES.fetch_add(1, Ordering::Relaxed);
return Admit::Wait;
}
g.inflight.insert(
key.to_string(),
InflightSlot {
waiters: 0,
result: None,
},
);
MISSES.fetch_add(1, Ordering::Relaxed);
Admit::Lead
}
fn wait_for(key: &str, deadline: Instant) -> Result<Arc<Compilation>, String> {
let (lock, cv) = cache();
let mut g = lock.lock().unwrap();
loop {
if let Some(e) = g.entries.get(key) {
let arc = Arc::clone(&e.compilation);
if let Some(slot) = g.inflight.get_mut(key) {
slot.waiters = slot.waiters.saturating_sub(1);
if slot.waiters == 0 {
g.inflight.remove(key);
}
}
HITS.fetch_add(1, Ordering::Relaxed);
RESULT_HITS.fetch_add(1, Ordering::Relaxed);
return Ok(arc);
}
if let Some(slot) = g.inflight.get(key) {
if let Some(res) = &slot.result {
let out = res.clone();
let waiters = slot.waiters.saturating_sub(1);
if let Some(slot) = g.inflight.get_mut(key) {
slot.waiters = waiters;
if slot.waiters == 0 {
g.inflight.remove(key);
}
}
HITS.fetch_add(1, Ordering::Relaxed);
RESULT_HITS.fetch_add(1, Ordering::Relaxed);
return out;
}
} else {
// Leader finished without leaving a result — retry admit.
return Err("analysis.retry".into());
}
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
if let Some(slot) = g.inflight.get_mut(key) {
slot.waiters = slot.waiters.saturating_sub(1);
if slot.waiters == 0 && slot.result.is_none() {
g.inflight.remove(key);
}
}
return Err("analysis.wait_deadline".into());
}
let (guard, timeout) = cv.wait_timeout(g, remaining).unwrap();
g = guard;
if timeout.timed_out() {
if let Some(slot) = g.inflight.get_mut(key) {
slot.waiters = slot.waiters.saturating_sub(1);
if slot.waiters == 0 && slot.result.is_none() {
g.inflight.remove(key);
}
}
return Err("analysis.wait_deadline".into());
}
}
}
fn finish_leader(key: &str, snap: SnapshotKey, compiled: Result<Arc<Compilation>, String>) {
if compiled.is_ok() {
let mut kc = key_compiles().lock().unwrap();
*kc.entry(key.to_string()).or_insert(0) += 1;
}
let (lock, cv) = cache();
let mut g = lock.lock().unwrap();
if let Ok(ref arc) = compiled {
insert_entry(&mut g, snap, Arc::clone(arc));
}
if let Some(slot) = g.inflight.get_mut(key) {
slot.result = Some(compiled);
if slot.waiters == 0 {
g.inflight.remove(key);
}
}
cv.notify_all();
}
/// Compile once per SnapshotKey; followers await the same Arc<Compilation>.
pub fn get_or_compile(project_root: &Path) -> Result<Arc<Compilation>, String> {
let snap = SnapshotKey::compute(project_root)?;
let key = snap.cache_key();
let deadline = Instant::now() + Duration::from_secs(300);
loop {
match admit(&key) {
Admit::Hit(arc) => return Ok(arc),
Admit::Wait => match wait_for(&key, deadline) {
Err(e) if e == "analysis.retry" => continue,
other => return other,
},
Admit::Lead => {
let compiled = compile_leader(project_root);
COMPILE_INVOCATIONS.fetch_add(1, Ordering::Relaxed);
let out = compiled.clone();
finish_leader(&key, snap, compiled);
return out;
}
}
}
}
fn compile_leader(project_root: &Path) -> Result<Arc<Compilation>, String> {
let options = crate::sema::CompileOptions::for_path(project_root)
.map_err(|e| format!("view.compile_failed: {e:#}"))?;
let c = crate::sema::compile_project(&options)
.map_err(|e| format!("view.compile_failed: {e:#}"))?;
Ok(Arc::new(c))
}
fn insert_entry(g: &mut CacheState, key: SnapshotKey, compilation: Arc<Compilation>) {
let ck = key.cache_key();
let bytes = estimate_bytes(&compilation);
while g.bytes_used + bytes > g.budget_bytes && !g.order.is_empty() {
if let Some(old) = g.order.pop_front() {
if let Some(e) = g.entries.remove(&old) {
g.bytes_used = g.bytes_used.saturating_sub(e.bytes);
EVICTIONS.fetch_add(1, Ordering::Relaxed);
}
}
}
if bytes > g.budget_bytes || g.entries.contains_key(&ck) {
return;
}
g.bytes_used += bytes;
g.order.push_back(ck.clone());
g.entries.insert(
ck,
CacheEntry {
key,
compilation,
bytes,
compiled_at: Instant::now(),
},
);
}
pub fn project_source_digest(project_root: &Path) -> String {
SnapshotKey::compute(project_root)
.map(|k| k.source_digest)
.unwrap_or_else(|_| "digest-error".into())
}
fn project_key(project_root: &Path) -> String {
project_root
.canonicalize()
.unwrap_or_else(|_| project_root.to_path_buf())
.to_string_lossy()
.to_string()
}
pub struct AnalysisTicket {
#[allow(dead_code)]
key: String,
#[allow(dead_code)]
digest: String,
pub is_leader: bool,
}
pub fn begin_analysis(project_root: &Path) -> AnalysisTicket {
match get_or_compile(project_root) {
Ok(_) => AnalysisTicket {
key: project_key(project_root),
digest: project_source_digest(project_root),
is_leader: false,
},
Err(_) => AnalysisTicket {
key: project_key(project_root),
digest: String::new(),
is_leader: true,
},
}
}
pub fn finish_analysis(_ticket: AnalysisTicket) {}
pub fn note_analysis(project_root: &Path) -> bool {
// True iff this SnapshotKey was already cached (result hit), independent of
// other projects compiled concurrently in the same process.
let before = compile_count_for(project_root);
let _ = get_or_compile(project_root);
let after = compile_count_for(project_root);
after == before && before > 0
}
pub fn cache_keys_isolated(a: &Path, b: &Path) -> bool {
project_key(a) != project_key(b)
}
pub fn cache_stats() -> (u64, u64, u64) {
(
HITS.load(Ordering::Relaxed),
MISSES.load(Ordering::Relaxed),
MERGES.load(Ordering::Relaxed),
)
}
pub fn compile_invocations() -> u64 {
COMPILE_INVOCATIONS.load(Ordering::Relaxed)
}
/// Successful compile count for one SnapshotKey (M-R5 / MR-06).
pub fn compile_count_for(project_root: &Path) -> u64 {
let Ok(snap) = SnapshotKey::compute(project_root) else {
return 0;
};
key_compiles()
.lock()
.unwrap()
.get(&snap.cache_key())
.copied()
.unwrap_or(0)
}
pub fn result_hits() -> u64 {
RESULT_HITS.load(Ordering::Relaxed)
}
pub fn eviction_count() -> u64 {
EVICTIONS.load(Ordering::Relaxed)
}
pub fn reset_cache_for_tests() {
let (lock, cv) = cache();
let mut g = lock.lock().unwrap();
g.entries.clear();
g.order.clear();
g.inflight.clear();
g.bytes_used = 0;
HITS.store(0, Ordering::Relaxed);
MISSES.store(0, Ordering::Relaxed);
MERGES.store(0, Ordering::Relaxed);
COMPILE_INVOCATIONS.store(0, Ordering::Relaxed);
EVICTIONS.store(0, Ordering::Relaxed);
RESULT_HITS.store(0, Ordering::Relaxed);
key_compiles().lock().unwrap().clear();
cv.notify_all();
}
/// Serialize tests that call [`reset_cache_for_tests`] so parallel suites cannot
/// clear another test's SnapshotKey mid-flight. Recovers from poison so one
/// failing cache test does not brick the rest of the suite.
pub fn test_serial_guard() -> std::sync::MutexGuard<'static, ()> {
static G: OnceLock<Mutex<()>> = OnceLock::new();
let m = G.get_or_init(|| Mutex::new(()));
m.lock().unwrap_or_else(|p| p.into_inner())
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Arc, Barrier};
use std::thread;
#[test]
fn same_snapshot_compiles_once_under_load() {
let _g = test_serial_guard();
reset_cache_for_tests();
let root = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/executable_knowledge/baseline/02-extract");
let n = 32usize;
let barrier = Arc::new(Barrier::new(n));
let mut handles = Vec::with_capacity(n);
for _ in 0..n {
let barrier = Arc::clone(&barrier);
let root = root.clone();
handles.push(thread::spawn(move || {
barrier.wait();
get_or_compile(&root).expect("compile")
}));
}
let first = handles.pop().unwrap().join().unwrap();
for h in handles {
let arc = h.join().unwrap();
assert!(Arc::ptr_eq(&first, &arc), "followers must share the same Arc");
}
assert_eq!(compile_count_for(&root), 1);
assert_eq!(compile_invocations(), 1);
}
}
+256
View File
@@ -0,0 +1,256 @@
//! MLSP `capabilities` — probe loaded plugins and declared features (M00 / F16).
//! Feature integers must reflect *implemented and testable* behavior only.
use serde_json::{json, Value};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
/// Host/tests may force stream/cancel off for honesty / F16 mismatch cases.
static FORCE_STREAM_OFF: AtomicBool = AtomicBool::new(false);
static FORCE_CANCEL_OFF: AtomicBool = AtomicBool::new(false);
pub fn set_event_stream_ready(ready: bool) {
FORCE_STREAM_OFF.store(!ready, Ordering::Relaxed);
}
pub fn set_cancel_ready(ready: bool) {
FORCE_CANCEL_OFF.store(!ready, Ordering::Relaxed);
}
/// Build the capabilities document. Does not call providers or execute project code.
pub fn probe_capabilities(params: &Value) -> Value {
let runtime_version = env!("CARGO_PKG_VERSION").to_string();
let mut adapters = vec![json!({
"id": "openai-compatible",
"available": true,
"notes": "HTTP chat completions; messages[] preferred over single prompt"
})];
adapters.push(json!({
"id": "ollama-compatible",
"available": true,
"notes": "OpenAI-compatible path; not a distinct native protocol"
}));
adapters.push(json!({
"id": "anthropic-native",
"available": false,
"notes": "not verified in this release"
}));
adapters.push(json!({
"id": "gemini-native",
"available": false,
"notes": "not verified in this release"
}));
let extract_formats = vec![
json!({"format": "text", "available": true}),
json!({"format": "markdown", "available": true}),
json!({"format": "pdf", "available": false}),
json!({"format": "docx", "available": false}),
json!({"format": "ocr", "available": false}),
// M-A0 freeze: format matrix rows remain unavailable until M-A3/M-A5.
json!({"format": "json", "available": true}),
json!({"format": "csv", "available": true}),
json!({"format": "image", "available": false}),
json!({"format": "html", "available": false}),
json!({"format": "svg", "available": false}),
json!({"format": "xlsx", "available": false}),
json!({"format": "pptx", "available": false}),
];
let loaded_plugins = probe_loaded_plugins(params);
let llm_loaded = loaded_plugins.iter().any(|p| {
p.get("name").and_then(|n| n.as_str()) == Some("llm")
&& p.get("available").and_then(|a| a.as_bool()) == Some(true)
});
let stream_off = FORCE_STREAM_OFF.load(Ordering::Relaxed)
|| params
.get("force_stream_off")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let cancel_off = FORCE_CANCEL_OFF.load(Ordering::Relaxed)
|| params
.get("force_cancel_off")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let mut features = serde_json::Map::new();
features.insert(
"llm.explicit_selection".into(),
json!(if llm_loaded { 1 } else { 0 }),
);
features.insert(
"llm.call_usage".into(),
json!(if llm_loaded { 1 } else { 0 }),
);
features.insert(
"llm.messages".into(),
json!(if llm_loaded { 1 } else { 0 }),
);
// M02 implemented: event_sink + web sse-live + MARQDO_CANCEL in llm plugin.
features.insert(
"runtime.event_stream".into(),
json!(if stream_off { 0 } else { 1 }),
);
features.insert(
"runtime.cancel".into(),
json!(if cancel_off { 0 } else { 1 }),
);
// Isolated web runtime (doc/design/isolated-web-runtime-development-2026-10-26.md).
// HTTP execute / sse-live defaults to the Supervisor worker pool (ADR 0009).
features.insert("runtime.isolated_execution".into(), json!(1));
// task_control_v2 / event_replay: host_query("runtime") + /_runtime/* system routes.
features.insert("runtime.task_control_v2".into(), json!(1));
features.insert("runtime.event_replay".into(), json!(1));
// short/read/control HTTP invokes use isolated admission (no parent INVOKE_LOCK).
features.insert("runtime.read_lane".into(), json!(1));
#[cfg(feature = "view")]
{
features.insert("view.analysis_cache_v2".into(), json!(1));
features.insert("view.document".into(), json!(1));
features.insert("view.reference_resolve".into(), json!(1));
}
#[cfg(not(feature = "view"))]
{
features.insert("view.analysis_cache_v2".into(), json!(0));
features.insert("view.document".into(), json!(0));
features.insert("view.reference_resolve".into(), json!(0));
}
// M-A0–M-A4: protocol capability bits.
features.insert("agent.error_v1".into(), json!(1));
features.insert("agent.checkpoint_v2".into(), json!(1));
features.insert("agent.tool_schema_v1".into(), json!(1));
features.insert("agent.output_channels_v1".into(), json!(1));
features.insert("agent.context_pack_v2".into(), json!(1));
#[cfg(feature = "view")]
{
features.insert("resource.view_v1".into(), json!(1));
}
#[cfg(not(feature = "view"))]
{
features.insert("resource.view_v1".into(), json!(0));
}
json!({
"schema_version": 1,
"runtime_version": runtime_version,
"features": features,
"adapters": adapters,
"extract_formats": extract_formats,
"loaded_plugins": loaded_plugins,
"contracts": [
"ModelSelectionV1",
"ModelCallUsageV1",
"RuntimeEventV1",
"RuntimeEventV2",
"InvokeRequestV2",
"DocumentViewV1",
"KnowledgeLocatorV1"
]
})
}
fn probe_loaded_plugins(params: &Value) -> Vec<Value> {
let mut out = Vec::new();
let candidates = [
("llm", "MARQDO_LLM_PLUGIN", plugin_name("llm")),
("agent", "MARQDO_AGENT_PLUGIN", plugin_name("agent")),
("web", "MARQDO_WEB_PLUGIN", plugin_name("web")),
];
let ext_hint = params
.get("ext_root")
.and_then(|v| v.as_str())
.map(PathBuf::from)
.or_else(|| std::env::var_os("MARQDO_EXT").map(PathBuf::from));
let force_unloaded = params
.get("force_unloaded")
.and_then(|v| v.as_bool())
.unwrap_or(false);
for (name, env_key, filename) in candidates {
if force_unloaded {
out.push(json!({
"name": name,
"available": false,
"path": Value::Null,
"capabilities": [],
}));
continue;
}
let mut path: Option<PathBuf> = std::env::var_os(env_key).map(PathBuf::from);
if path.as_ref().map(|p| !p.is_file()).unwrap_or(true) {
let debug = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("target")
.join("debug")
.join(&filename);
if debug.is_file() {
path = Some(debug);
}
}
if path.as_ref().map(|p| !p.is_file()).unwrap_or(true) {
if let Some(ext) = &ext_hint {
let native = ext.join("native").join(&filename);
if native.is_file() {
path = Some(native);
}
}
}
let available = path.as_ref().map(|p| p.is_file()).unwrap_or(false);
let mut caps = Vec::new();
if name == "llm" && available {
caps.push("call_usage_v1");
caps.push("messages_v1");
caps.push("explicit_selection_v1");
caps.push("cancel_v1");
}
if name == "agent" && available {
caps.push("eng_bind_execute");
caps.push("publish_gate");
}
out.push(json!({
"name": name,
"available": available,
"path": path.map(|p| p.to_string_lossy().to_string()),
"capabilities": caps,
}));
}
out
}
fn plugin_name(stem: &str) -> String {
if cfg!(windows) {
format!("{stem}.dll")
} else if cfg!(target_os = "macos") {
format!("lib{stem}.dylib")
} else {
format!("lib{stem}.so")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn force_off_and_unloaded_honest() {
let caps = probe_capabilities(&json!({
"force_unloaded": true,
"force_stream_off": true,
"force_cancel_off": true,
}));
assert_eq!(caps["features"]["runtime.event_stream"], 0);
assert_eq!(caps["features"]["runtime.cancel"], 0);
assert_eq!(caps["features"]["llm.messages"], 0);
assert_eq!(caps["features"]["agent.error_v1"], 1);
assert_eq!(caps["features"]["agent.checkpoint_v2"], 1);
assert_eq!(caps["features"]["agent.tool_schema_v1"], 1);
assert_eq!(caps["features"]["agent.output_channels_v1"], 1);
assert_eq!(caps["features"]["agent.context_pack_v2"], 1);
#[cfg(feature = "view")]
assert_eq!(caps["features"]["resource.view_v1"], 1);
#[cfg(not(feature = "view"))]
assert_eq!(caps["features"]["resource.view_v1"], 0);
}
}
+478
View File
@@ -0,0 +1,478 @@
//! DocumentViewV1 — read-only embedded view (M03). Never executes the document.
use std::fs;
use std::path::{Path, PathBuf};
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use crate::view::{render_knowledge_html, render_module_document, KnowledgePresentation};
#[derive(Debug, Clone)]
pub struct DocumentViewRequest {
pub project: PathBuf,
pub document: String,
pub revision: Option<String>,
pub generation: Option<String>,
pub node_id: Option<String>,
pub include: Vec<String>,
}
impl DocumentViewRequest {
pub fn from_params(params: &Value) -> Result<Self, String> {
let project = params
.get("project")
.and_then(|v| v.as_str())
.ok_or_else(|| "view.document requires params.project".to_string())?;
let document = params
.get("document")
.and_then(|v| v.as_str())
.ok_or_else(|| "view.document requires params.document".to_string())?;
let mode = params
.get("mode")
.and_then(|v| v.as_str())
.unwrap_or("read");
if mode != "read" {
return Err(format!("view.document unsupported mode={mode}; only read"));
}
let include = match params.get("include") {
Some(Value::Array(a)) => a
.iter()
.filter_map(|v| v.as_str().map(|s| s.to_string()))
.collect(),
_ => vec![
"fragment".into(),
"outline".into(),
"nodes".into(),
"refs".into(),
"diagnostics".into(),
],
};
Ok(Self {
project: PathBuf::from(project),
document: document.to_string(),
revision: params
.get("revision")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
generation: params
.get("generation")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
node_id: params
.get("node_id")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
include,
})
}
}
/// Pure read path: compile_project (no plugin load / no eval) + fragment render.
/// Must never call `build_file_view` with execute=true.
pub fn build_document_view(req: &DocumentViewRequest) -> Result<Value, Value> {
let project_root = canonicalize_project(&req.project)?;
let abs = resolve_under(&project_root, &req.document)?;
let raw = fs::read(&abs).map_err(|e| {
err(
"view.document_read_failed",
format!("cannot read {}: {e}", abs.display()),
)
})?;
let source_revision = format!("sha256:{:x}", Sha256::digest(&raw));
if let Some(expected) = &req.revision {
if expected != &source_revision && !expected.is_empty() {
return Err(err(
"view.revision_mismatch",
format!("requested {expected} but file is {source_revision}; refusing silent downgrade"),
));
}
}
let source = String::from_utf8_lossy(&raw).into_owned();
let title = title_from_source(&source, &req.document);
// Semantic projection — compile_project does not load plugins / evaluate.
// M-R5: Arc<Compilation> single-flight cache; followers reuse the same result.
let compilation = crate::workbench::analysis_cache::get_or_compile(&project_root)
.map_err(|e| err("view.compile_failed", e))?;
let rel = req.document.replace('\\', "/");
let mut presentation = KnowledgePresentation::from_project(&compilation.semantics, &rel, &title);
presentation.page_token = {
let mut h = Sha256::new();
h.update(raw.as_slice());
h.update(presentation.semantic_generation.as_bytes());
format!("{:x}", h.finalize())
};
if let Some(gen) = &req.generation {
if !gen.is_empty() && gen != &presentation.semantic_generation {
return Err(err(
"view.generation_mismatch",
format!(
"requested generation {gen} but projection is {}; refusing silent downgrade",
presentation.semantic_generation
),
));
}
}
let wants = |k: &str| req.include.iter().any(|x| x == k);
let heading_outline = outline_from_headings(&source, &presentation);
// Prefer document surface HTML when parseable; fall back to knowledge HTML.
// Never call build_file_view (that path executes).
let fragment = if wants("fragment") {
let raw_html = match crate::load::parse_source_auto(&source) {
Ok(module) => render_module_document(&module, &source, &[]),
Err(_) => render_knowledge_html(&compilation.semantics, &rel),
};
wrap_fragment(&inject_mq_node_attrs(&raw_html, &heading_outline, &presentation))
} else {
String::new()
};
let outline = if wants("outline") {
if heading_outline.is_empty() {
outline_from_presentation(&presentation)
} else {
json!(heading_outline)
}
} else {
json!([])
};
let nodes = if wants("nodes") {
nodes_from_presentation_and_steps(
&presentation,
&compilation.semantics,
&rel,
&req.node_id,
)
} else {
json!([])
};
let refs = if wants("refs") {
refs_from_project(&compilation.semantics, &rel)
} else {
json!([])
};
let mut diag_msgs: Vec<String> = presentation.diagnostics.clone();
for d in &compilation.diagnostics {
diag_msgs.push(d.message.clone());
}
let diagnostics = if wants("diagnostics") {
json!(diag_msgs)
} else {
json!([])
};
let completeness = if diag_msgs.is_empty() {
"complete"
} else {
"partial"
};
let snapshot = crate::workbench::snapshot::build_snapshot_manifest(
&project_root,
&rel,
&raw,
&compilation.semantics,
"1",
);
Ok(json!({
"schema": "DocumentViewV1",
"schema_version": 1,
"document": {
"id": rel,
"title": title,
"source_revision": source_revision,
"semantic_digest": presentation.source_digest,
"generation_id": if presentation.semantic_generation.is_empty() {
Value::Null
} else {
json!(presentation.semantic_generation)
},
"presentation_token": presentation.page_token,
},
"execution": {"state": "not_run", "run_id": Value::Null},
"render": {
"format": "sanitized_html",
"fragment": fragment,
"renderer_version": "1"
},
"outline": outline,
"nodes": nodes,
"refs": refs,
"diagnostics": diagnostics,
"completeness": completeness,
"snapshot": snapshot,
"capabilities": {"read_only": true, "has_semantics": true, "executed": false}
}))
}
fn wrap_fragment(inner: &str) -> String {
// No html/body, no scripts, namespaced for host theming.
let sanitized = strip_script_tags(inner);
format!("<article class=\"mq-document\">{sanitized}</article>")
}
/// Remove `<script…>` blocks (case-insensitive) so embed fragments stay non-executable.
fn strip_script_tags(html: &str) -> String {
let lower = html.to_ascii_lowercase();
let mut out = String::with_capacity(html.len());
let mut i = 0usize;
while i < html.len() {
if lower[i..].starts_with("<script") {
let after_open = lower[i..]
.find('>')
.map(|p| i + p + 1)
.unwrap_or(html.len());
if let Some(rel) = lower[after_open..].find("</script") {
let close_at = after_open + rel;
let end = lower[close_at..]
.find('>')
.map(|p| close_at + p + 1)
.unwrap_or(html.len());
i = end;
} else {
// Truncate from the dangerous open tag onward.
break;
}
continue;
}
// Advance one UTF-8 char so multi-byte text stays intact.
let ch = html[i..].chars().next().unwrap();
out.push(ch);
i += ch.len_utf8();
}
out
}
fn title_from_source(source: &str, fallback: &str) -> String {
for line in source.lines().take(40) {
let t = line.trim();
if let Some(rest) = t.strip_prefix("# ") {
return rest.trim().to_string();
}
if let Some(rest) = t.strip_prefix("title:") {
return rest.trim().trim_matches('"').to_string();
}
}
Path::new(fallback)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or(fallback)
.to_string()
}
/// Outline entries from Markdown / Marqdo `#` headings; prefer matching node ids.
fn outline_from_headings(source: &str, p: &KnowledgePresentation) -> Vec<Value> {
let mut items = Vec::new();
let mut order = 0usize;
for (idx, line) in source.lines().enumerate() {
let line_no = (idx + 1) as u32;
let trimmed = line.trim_start();
if !trimmed.starts_with('#') {
continue;
}
let hashes = trimmed.chars().take_while(|c| *c == '#').count();
if hashes == 0 || hashes > 6 {
continue;
}
// `# Title`, `## Section`, or Marqdo `#main` / `##greet` (no space).
let after = &trimmed[hashes..];
let label = after.trim();
if label.is_empty() {
continue;
}
let matched = p.nodes.iter().find(|n| {
n.source_line == line_no
|| n.display_name == label
|| n.node_id == label
|| n.artifact_id.as_deref() == Some(label)
});
let id = matched
.map(|n| n.node_id.clone())
.unwrap_or_else(|| format!("heading:{line_no}"));
let artifact_id = matched.and_then(|n| n.artifact_id.clone());
items.push(json!({
"id": id,
"parent_id": Value::Null,
"kind": "heading",
"label": label,
"order": order,
"source_span": {"line": line_no},
"artifact_id": artifact_id,
}));
order += 1;
}
items
}
/// Stamp `data-mq-node` onto elements that already carry `data-line` matching a node/outline id.
fn inject_mq_node_attrs(html: &str, outline: &[Value], p: &KnowledgePresentation) -> String {
let mut out = html.to_string();
let mut bindings: Vec<(u32, String)> = Vec::new();
for n in &p.nodes {
bindings.push((n.source_line, n.node_id.clone()));
}
for item in outline {
if let (Some(line), Some(id)) = (
item.pointer("/source_span/line").and_then(|v| v.as_u64()),
item.get("id").and_then(|v| v.as_str()),
) {
bindings.push((line as u32, id.to_string()));
}
}
bindings.sort_by_key(|(line, _)| *line);
bindings.dedup_by_key(|(line, _)| *line);
for (line, id) in bindings {
let needle = format!("data-line=\"{line}\"");
let attr = format!("data-line=\"{line}\" data-mq-node=\"{}\"", html_attr_escape(&id));
if out.contains(&needle) && !out.contains(&format!("data-mq-node=\"{}\"", html_attr_escape(&id))) {
out = out.replacen(&needle, &attr, 1);
}
}
out
}
fn html_attr_escape(s: &str) -> String {
s.replace('&', "&amp;")
.replace('"', "&quot;")
.replace('<', "&lt;")
}
fn outline_from_presentation(p: &KnowledgePresentation) -> Value {
json!(p
.nodes
.iter()
.enumerate()
.map(|(i, n)| json!({
"id": n.node_id,
"parent_id": Value::Null,
"kind": "heading",
"label": n.display_name,
"order": i,
"artifact_id": n.artifact_id,
}))
.collect::<Vec<_>>())
}
fn nodes_from_presentation_and_steps(
p: &KnowledgePresentation,
project: &crate::sema::SemanticProject,
rel: &str,
filter: &Option<String>,
) -> Value {
let mut out = Vec::new();
for n in &p.nodes {
if filter
.as_ref()
.map(|f| &n.node_id == f || n.artifact_id.as_ref() == Some(f))
.unwrap_or(true)
{
out.push(json!({
"id": n.node_id,
"parent_id": Value::Null,
"kind": n.kind,
"label": n.display_name,
"order": n.source_line,
"source_span": {"file": n.source_file, "line": n.source_line},
"artifact_id": n.artifact_id,
"ref_ids": [],
"children": [],
"evidence": n.evidence,
"effects": n.effects,
"dependencies": n.dependencies,
"signature": n.signature,
}));
}
}
// F09: project KnowledgeStep kinds (observe/derive/call/iterate/…) into the read model.
for step in &project.steps {
if step.source.file != rel {
continue;
}
let id = step.id.0.clone();
if filter
.as_ref()
.map(|f| &id == f || &step.reference == f)
.unwrap_or(true)
{
out.push(json!({
"id": id,
"parent_id": Value::Null,
"kind": step.kind,
"label": step.reference,
"order": step.source.line,
"source_span": {
"file": step.source.file,
"line": step.source.line,
"end_line": step.source.end_line
},
"artifact_id": Value::Null,
"ref_ids": step.calls.iter().map(|c| c.as_str()).collect::<Vec<_>>(),
"children": [],
"evidence": "unverified",
"effects": step.effects.values(),
}));
}
}
json!(out)
}
fn refs_from_project(project: &crate::sema::SemanticProject, rel: &str) -> Value {
let mut refs = Vec::new();
for a in &project.artifacts {
if a.source.file != rel {
continue;
}
for dep in &a.dependencies {
refs.push(json!({
"id": format!("ref:{}:{}", a.artifact_id.0, dep.0),
"from_artifact_id": a.artifact_id.0,
"to": dep.0,
"relation": "depends_on",
}));
}
}
json!(refs)
}
fn canonicalize_project(p: &Path) -> Result<PathBuf, Value> {
let root = if p.is_file() {
p.parent().unwrap_or_else(|| Path::new(".")).to_path_buf()
} else {
p.to_path_buf()
};
root.canonicalize().map_err(|e| {
err(
"view.project_invalid",
format!("project {}: {e}", root.display()),
)
})
}
fn resolve_under(root: &Path, rel: &str) -> Result<PathBuf, Value> {
let joined = root.join(rel);
let canon = joined.canonicalize().map_err(|e| {
err(
"view.document_not_found",
format!("{}: {e}", joined.display()),
)
})?;
if !canon.starts_with(root) {
return Err(err(
"view.path_escape",
"document path escapes project root",
));
}
Ok(canon)
}
fn err(code: &str, message: impl Into<String>) -> Value {
json!({"code": code, "message": message.into()})
}
+59
View File
@@ -0,0 +1,59 @@
//! Request-local RuntimeEventV1 sink (M02). Avoids process-global event mix-up.
use serde_json::{json, Value};
use std::fs::OpenOptions;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Mutex, OnceLock};
static SEQ: AtomicU64 = AtomicU64::new(0);
fn sink_path() -> &'static Mutex<Option<PathBuf>> {
static SINK: OnceLock<Mutex<Option<PathBuf>>> = OnceLock::new();
SINK.get_or_init(|| Mutex::new(None))
}
/// Bind this process/request to an NDJSON event file (also used by web sse-live).
pub fn bind_event_sink(path: impl AsRef<Path>) {
if let Ok(mut g) = sink_path().lock() {
*g = Some(path.as_ref().to_path_buf());
}
crate::workbench::capabilities::set_event_stream_ready(true);
}
pub fn clear_event_sink() {
if let Ok(mut g) = sink_path().lock() {
*g = None;
}
}
pub fn emit_event(run_id: &str, event_type: &str, payload: Value) -> Value {
let seq = SEQ.fetch_add(1, Ordering::Relaxed);
let event_id = format!("evt-{run_id}-{seq}");
let ev = crate::workbench::usage::runtime_event(&event_id, seq, run_id, event_type, payload);
if let Ok(g) = sink_path().lock() {
if let Some(path) = g.as_ref() {
if let Ok(mut f) = OpenOptions::new().create(true).append(true).open(path) {
let _ = writeln!(f, "{}", ev);
}
}
}
// Also honor MARQDO_EVENT_FORWARD for subtask/web bridge.
if let Ok(fwd) = std::env::var("MARQDO_EVENT_FORWARD") {
if !fwd.is_empty() {
if let Ok(mut f) = OpenOptions::new().create(true).append(true).open(&fwd) {
let _ = writeln!(f, "{}", ev);
}
}
}
ev
}
pub fn emit_message_delta(run_id: &str, text: &str) -> Value {
emit_event(
run_id,
"message.delta",
json!({"text": text, "format": "plain"}),
)
}
+304
View File
@@ -0,0 +1,304 @@
//! KnowledgeLocatorV1 + view.reference_resolve (M04).
use std::path::{Path, PathBuf};
use serde_json::{json, Value};
#[derive(Debug, Clone)]
pub struct KnowledgeLocator {
pub project_id: String,
pub document_id: String,
pub source_revision: Option<String>,
pub generation_id: Option<String>,
pub artifact_id: Option<String>,
pub node_id: Option<String>,
pub source_ref_id: Option<String>,
pub span: Option<Value>,
pub relation: String,
}
impl KnowledgeLocator {
pub fn from_params(params: &Value) -> Result<Self, String> {
let loc = params.get("locator").unwrap_or(params);
Ok(Self {
project_id: loc
.get("project_id")
.or_else(|| params.get("project"))
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
document_id: loc
.get("document_id")
.or_else(|| params.get("document"))
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
source_revision: loc
.get("source_revision")
.or_else(|| loc.get("revision"))
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
generation_id: loc
.get("generation_id")
.or_else(|| loc.get("generation"))
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
artifact_id: loc
.get("artifact_id")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
node_id: loc
.get("node_id")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
source_ref_id: loc
.get("source_ref_id")
.or_else(|| loc.get("ref_id"))
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
span: loc.get("span").cloned(),
relation: loc
.get("relation")
.and_then(|v| v.as_str())
.unwrap_or("cites")
.to_string(),
})
}
pub fn to_json(&self) -> Value {
json!({
"schema": "KnowledgeLocatorV1",
"schema_version": 1,
"project_id": self.project_id,
"document_id": self.document_id,
"source_revision": self.source_revision,
"generation_id": self.generation_id,
"artifact_id": self.artifact_id,
"node_id": self.node_id,
"source_ref_id": self.source_ref_id,
"span": self.span,
"relation": self.relation,
})
}
}
/// Resolve a locator or current-document ref_id against a project snapshot.
/// Never silently falls back to "latest" when expected revision/generation mismatch.
pub fn resolve_reference(params: &Value) -> Result<Value, Value> {
let locator = KnowledgeLocator::from_params(params).map_err(|e| {
json!({"code": "locator.bad_request", "message": e})
})?;
if locator.project_id.is_empty() {
return Err(json!({
"code": "locator.bad_request",
"message": "project_id/project required"
}));
}
let root = PathBuf::from(&locator.project_id);
let root = root.canonicalize().map_err(|e| {
json!({"code": "locator.project_invalid", "message": format!("{e}")})
})?;
let options = crate::sema::CompileOptions::for_path(&root).map_err(|e| {
json!({"code": "locator.compile_failed", "message": format!("{e:#}")})
})?;
let compilation = crate::sema::compile_project(&options).map_err(|e| {
json!({"code": "locator.compile_failed", "message": format!("{e:#}")})
})?;
// Generation mismatch → version_unavailable / stale (no silent current).
if let Some(expected_gen) = &locator.generation_id {
if !expected_gen.is_empty() {
let current_gen = semantic_generation(&compilation.semantics);
if &current_gen != expected_gen {
return Ok(crate::workbench::snapshot::version_unavailable_result(
&locator.to_json(),
&format!(
"expected generation {expected_gen}, current projection is {current_gen}; refusing silent downgrade (no retained snapshot)"
),
expected_gen,
Some(&current_gen),
));
}
}
}
// Source revision check when document is known.
if let (Some(expected_rev), true) = (
&locator.source_revision,
!locator.document_id.is_empty(),
) {
if !expected_rev.is_empty() {
let abs = resolve_under(&root, &locator.document_id)?;
let raw = std::fs::read(&abs).map_err(|e| {
json!({"code": "locator.source_unreadable", "message": format!("{e}")})
})?;
use sha2::{Digest, Sha256};
let actual = format!("sha256:{:x}", Sha256::digest(&raw));
if &actual != expected_rev {
return Ok(json!({
"schema": "KnowledgeLocatorV1",
"schema_version": 1,
"resolution": "stale",
"precision": "document",
"locator": locator.to_json(),
"target": {
"document_id": locator.document_id,
"source_revision": actual,
"note": "source changed since citation"
},
"candidates": [],
"diagnostics": [{
"code": "source_revision_mismatch",
"message": format!("expected {expected_rev}, actual {actual}")
}],
"label": "该来源已变化"
}));
}
}
}
let query = locator
.artifact_id
.clone()
.or_else(|| locator.node_id.clone())
.or_else(|| locator.source_ref_id.clone());
let Some(q) = query else {
if locator.document_id.is_empty() {
return Ok(json!({
"schema": "KnowledgeLocatorV1",
"schema_version": 1,
"resolution": "unresolved",
"precision": "none",
"locator": locator.to_json(),
"target": Value::Null,
"candidates": [],
"diagnostics": [{"code": "missing_target", "message": "no artifact/node/ref id"}],
"label": "无法定位该知识"
}));
}
return Ok(json!({
"schema": "KnowledgeLocatorV1",
"schema_version": 1,
"resolution": "resolved",
"precision": "document",
"locator": locator.to_json(),
"target": {
"document_id": locator.document_id,
"relation": locator.relation,
},
"candidates": [],
"diagnostics": [],
"label": locator.document_id,
}));
};
let mut matches: Vec<&crate::sema::SemanticArtifact> = compilation
.semantics
.artifacts
.iter()
.filter(|a| {
a.artifact_id.0 == q
|| a.display_name == q
|| a.symbol_id
.as_ref()
.map(|s| s.as_str() == q)
.unwrap_or(false)
})
.collect();
if !locator.document_id.is_empty() {
let scoped: Vec<_> = matches
.iter()
.copied()
.filter(|a| a.source.file == locator.document_id)
.collect();
if !scoped.is_empty() {
matches = scoped;
}
}
match matches.len() {
0 => Ok(json!({
"schema": "KnowledgeLocatorV1",
"schema_version": 1,
"resolution": "unresolved",
"precision": "none",
"locator": locator.to_json(),
"target": Value::Null,
"candidates": [],
"diagnostics": [{"code": "not_found", "message": format!("no artifact matching {q}")}],
"label": "尚未定位到具体知识"
})),
1 => {
let a = matches[0];
Ok(json!({
"schema": "KnowledgeLocatorV1",
"schema_version": 1,
"resolution": "resolved",
"precision": "artifact",
"locator": locator.to_json(),
"target": {
"artifact_id": a.artifact_id.0,
"display_name": a.display_name,
"document_id": a.source.file,
"span": {"line": a.source.line, "end_line": a.source.end_line},
"evidence": a.verification.evidence,
"relation": locator.relation,
},
"candidates": [],
"diagnostics": [],
"label": a.display_name,
}))
}
_ => {
let candidates: Vec<Value> = matches
.iter()
.map(|a| {
json!({
"artifact_id": a.artifact_id.0,
"display_name": a.display_name,
"document_id": a.source.file,
"line": a.source.line,
})
})
.collect();
Ok(json!({
"schema": "KnowledgeLocatorV1",
"schema_version": 1,
"resolution": "ambiguous",
"precision": "none",
"locator": locator.to_json(),
"target": Value::Null,
"candidates": candidates,
"diagnostics": [{"code": "ambiguous", "message": format!("{} candidates for {q}", matches.len())}],
"label": "存在多个候选"
}))
}
}
}
fn semantic_generation(project: &crate::sema::SemanticProject) -> String {
use sha2::{Digest, Sha256};
let mut gh = Sha256::new();
gh.update(project.project_id.as_bytes());
for a in &project.artifacts {
gh.update(a.hashes.artifact.0.as_bytes());
}
format!("{:x}", gh.finalize())[..16].to_string()
}
fn resolve_under(root: &Path, rel: &str) -> Result<PathBuf, Value> {
let joined = root.join(rel);
let canon = joined.canonicalize().map_err(|e| {
json!({"code": "locator.document_not_found", "message": format!("{}: {e}", joined.display())})
})?;
if !canon.starts_with(root) {
return Err(json!({
"code": "locator.path_escape",
"message": "document path escapes project root"
}));
}
Ok(canon)
}
+40
View File
@@ -0,0 +1,40 @@
//! Workbench runtime contracts (M00–M05): schemas, capabilities, DocumentView,
//! KnowledgeLocator, ModelSelection / ModelCallUsage / RuntimeEvent / RunContext.
//!
//! Spec: `doc/design/qdagent-workbench-runtime-and-view-2026-10-25.md`.
//! These are v1 contracts distinct from KnowledgePresentation `schema_version:1`.
pub mod analysis_cache;
pub mod capabilities;
#[cfg(feature = "view")]
pub mod document_view;
pub mod event_sink;
pub mod locator;
#[cfg(feature = "view")]
pub mod resource_view;
pub mod run_context;
pub mod selection;
pub mod snapshot;
pub mod usage;
pub use analysis_cache::{
begin_analysis, cache_keys_isolated, cache_stats, compile_count_for, compile_invocations,
finish_analysis, get_or_compile, note_analysis, project_source_digest, reset_cache_for_tests,
result_hits, test_serial_guard, AnalysisTicket, SnapshotKey,
};
pub use capabilities::probe_capabilities;
#[cfg(feature = "view")]
pub use document_view::{build_document_view, DocumentViewRequest};
pub use event_sink::{bind_event_sink, clear_event_sink, emit_event, emit_message_delta};
#[cfg(feature = "view")]
pub use resource_view::{build_resource_view, ResourceViewRequest};
pub use locator::{resolve_reference, KnowledgeLocator};
pub use run_context::{
cancel_requested, cancel_token_body_means_cancel, RequestContext, RunContext,
};
pub use selection::{parse_model_selection, ModelSelection};
pub use snapshot::{build_snapshot_manifest, version_unavailable_result};
pub use usage::{
model_call_usage_from_provider, model_call_usage_from_provider_rev, replace_usage_if_newer,
runtime_event, token_or_null,
};
File diff suppressed because it is too large. Load diff
+315
View File
@@ -0,0 +1,315 @@
//! RunContextV1 — request-local run identity for selection / usage / cancel (M01 / M-R1).
use serde_json::{json, Map, Value};
/// Explicit request Context handles (M-R1). Paths are opaque grants owned by the
/// control plane; callers must not pass browser-supplied absolute paths unchecked.
#[derive(Debug, Clone, Default)]
pub struct RequestContext {
pub workspace_id: Option<String>,
pub snapshot_id: Option<String>,
pub selection_ref: Option<String>,
pub capabilities_ref: Option<String>,
pub deadline_at: Option<String>,
pub lease_id: Option<String>,
pub attempt_id: Option<String>,
pub epoch: Option<u64>,
}
impl RequestContext {
pub fn from_value(v: &Value) -> Self {
Self {
workspace_id: str_opt(v, "workspace_id"),
snapshot_id: str_opt(v, "snapshot_id"),
selection_ref: str_opt(v, "selection_ref"),
capabilities_ref: str_opt(v, "capabilities_ref"),
deadline_at: str_opt(v, "deadline_at"),
lease_id: str_opt(v, "lease_id").or_else(|| {
v.get("lease")
.and_then(|l| l.get("id"))
.and_then(|x| x.as_str())
.map(|s| s.to_string())
}),
attempt_id: str_opt(v, "attempt_id"),
epoch: v
.get("epoch")
.and_then(|x| x.as_u64())
.or_else(|| {
v.get("lease")
.and_then(|l| l.get("epoch"))
.and_then(|x| x.as_u64())
}),
}
}
pub fn to_json(&self) -> Value {
json!({
"workspace_id": self.workspace_id,
"snapshot_id": self.snapshot_id,
"selection_ref": self.selection_ref,
"capabilities_ref": self.capabilities_ref,
"deadline_at": self.deadline_at,
"lease_id": self.lease_id,
"attempt_id": self.attempt_id,
"epoch": self.epoch,
})
}
}
#[derive(Debug, Clone, Default)]
pub struct RunContext {
pub run_id: String,
pub parent_run_id: Option<String>,
pub turn_id: Option<String>,
pub workspace_scope: Option<String>,
pub selection_context: Map<String, Value>,
pub usage_sink: Option<String>,
pub event_sink: Option<String>,
pub cancellation_token: Option<String>,
/// Explicit Context object (M-R1); serialized alongside RunContextV1 fields.
pub context: RequestContext,
}
impl RunContext {
pub fn from_value(v: &Value) -> Result<Self, String> {
let run_id = v
.get("run_id")
.and_then(|x| x.as_str())
.filter(|s| !s.is_empty())
.ok_or_else(|| "run_context.missing:run_id".to_string())?
.to_string();
let context = v
.get("context")
.map(RequestContext::from_value)
.unwrap_or_default();
Ok(Self {
run_id,
parent_run_id: v
.get("parent_run_id")
.and_then(|x| x.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string()),
turn_id: v
.get("turn_id")
.and_then(|x| x.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string()),
workspace_scope: v
.get("workspace_scope")
.and_then(|x| x.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string()),
selection_context: v
.get("selection_context")
.and_then(|x| x.as_object())
.cloned()
.unwrap_or_default(),
usage_sink: v
.get("usage_sink")
.and_then(|x| x.as_str())
.map(|s| s.to_string()),
event_sink: v
.get("event_sink")
.and_then(|x| x.as_str())
.map(|s| s.to_string()),
cancellation_token: v
.get("cancellation_token")
.and_then(|x| x.as_str())
.map(|s| s.to_string()),
context,
})
}
pub fn to_json(&self) -> Value {
json!({
"schema": "RunContextV1",
"schema_version": 1,
"run_id": self.run_id,
"parent_run_id": self.parent_run_id,
"turn_id": self.turn_id,
"workspace_scope": self.workspace_scope,
"selection_context": self.selection_context,
"usage_sink": self.usage_sink,
"event_sink": self.event_sink,
"cancellation_token": self.cancellation_token,
"context": self.context.to_json(),
})
}
/// Child inherits selection/event/cancel handles; new run_id linked to parent.
pub fn spawn_child(&self, child_run_id: &str) -> Self {
Self {
run_id: child_run_id.to_string(),
parent_run_id: Some(self.run_id.clone()),
turn_id: self.turn_id.clone(),
workspace_scope: self.workspace_scope.clone(),
selection_context: self.selection_context.clone(),
usage_sink: self.usage_sink.clone(),
event_sink: self.event_sink.clone(),
cancellation_token: self.cancellation_token.clone(),
context: self.context.clone(),
}
}
/// Env pairs for *child* Command.env injection (never api_key / credential plaintext).
/// Parent process must not `setenv` these to carry request state across tasks.
pub fn selection_env(&self) -> Vec<(String, String)> {
let mut out = Vec::new();
out.push(("MARQDO_RUN_ID".into(), self.run_id.clone()));
if let Some(p) = &self.parent_run_id {
out.push(("MARQDO_PARENT_RUN_ID".into(), p.clone()));
}
if let Some(t) = &self.turn_id {
out.push(("MARQDO_TURN_ID".into(), t.clone()));
}
if let Some(w) = &self.workspace_scope {
out.push(("MARQDO_WORKSPACE_SCOPE".into(), w.clone()));
}
if let Some(tok) = &self.cancellation_token {
out.push(("MARQDO_CANCEL_TOKEN".into(), tok.clone()));
}
if let Some(sink) = &self.event_sink {
out.push(("MARQDO_EVENT_FORWARD".into(), sink.clone()));
}
if let Some(usage) = &self.usage_sink {
out.push(("MARQDO_USAGE_SINK".into(), usage.clone()));
}
if let Some(model) = self
.selection_context
.get("upstream_model_id")
.or_else(|| self.selection_context.get("model"))
.and_then(|v| v.as_str())
{
out.push(("MARQDO_LLM_MODEL".into(), model.to_string()));
out.push(("OPENAI_MODEL".into(), model.to_string()));
}
if let Some(base) = self
.selection_context
.get("base_url")
.and_then(|v| v.as_str())
{
out.push(("MARQDO_LLM_BASE_URL".into(), base.to_string()));
out.push(("OPENAI_BASE_URL".into(), base.to_string()));
}
// credential_ref only — never expand api_key into child env here.
if let Some(cred) = self
.selection_context
.get("credential_ref")
.and_then(|v| v.as_str())
{
out.push(("MARQDO_CREDENTIAL_REF".into(), cred.to_string()));
}
if let Some(ws) = &self.context.workspace_id {
out.push(("MARQDO_WORKSPACE_ID".into(), ws.clone()));
}
if let Some(snap) = &self.context.snapshot_id {
out.push(("MARQDO_SNAPSHOT_ID".into(), snap.clone()));
}
if let Some(lease) = &self.context.lease_id {
out.push(("MARQDO_LEASE_ID".into(), lease.clone()));
}
if let Some(attempt) = &self.context.attempt_id {
out.push(("MARQDO_ATTEMPT_ID".into(), attempt.clone()));
}
out
}
/// Apply this run's handles onto a child Command without mutating parent env.
pub fn apply_to_command(&self, cmd: &mut std::process::Command) {
for (k, v) in self.selection_env() {
cmd.env(k, v);
}
}
}
fn str_opt(v: &Value, key: &str) -> Option<String> {
v.get(key)
.and_then(|x| x.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
}
/// Unified cancel-token semantics (M-R1 / design §6):
/// - missing path / missing file → not cancelled
/// - empty body / `"0"` → not cancelled
/// - `"1"` / `"cancelled"` → cancelled
/// - read failure → not cancelled (caller may log); do not treat as cancel
///
/// Process-global `MARQDO_CANCEL` is **not** consulted when an explicit token
/// handle is provided, so parallel tasks cannot cancel each other via env.
pub fn cancel_requested(token: Option<&str>) -> bool {
let Some(tok) = token.filter(|s| !s.is_empty()) else {
return false;
};
let path = std::path::Path::new(tok);
if !path.is_file() {
return false;
}
match std::fs::read_to_string(path) {
Ok(body) => {
let t = body.trim();
t == "1" || t.eq_ignore_ascii_case("cancelled")
}
Err(_) => false,
}
}
/// Token-file body interpretation used by LLM plugin and control plane alike.
pub fn cancel_token_body_means_cancel(body: &str) -> bool {
let t = body.trim();
t == "1" || t.eq_ignore_ascii_case("cancelled")
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[test]
fn empty_and_zero_token_are_not_cancelled() {
let dir = std::env::temp_dir().join(format!("mq-rc-empty-{}", std::process::id()));
let _ = std::fs::remove_file(&dir);
std::fs::File::create(&dir).unwrap();
assert!(!cancel_requested(Some(dir.to_str().unwrap())));
std::fs::write(&dir, "0").unwrap();
assert!(!cancel_requested(Some(dir.to_str().unwrap())));
std::fs::write(&dir, "cancelled").unwrap();
assert!(cancel_requested(Some(dir.to_str().unwrap())));
let _ = std::fs::remove_file(&dir);
}
#[test]
fn explicit_token_ignores_global_marqdo_cancel() {
let dir = std::env::temp_dir().join(format!("mq-rc-ign-{}", std::process::id()));
let _ = std::fs::remove_file(&dir);
let mut f = std::fs::File::create(&dir).unwrap();
f.write_all(b"").unwrap();
drop(f);
std::env::set_var("MARQDO_CANCEL", "1");
assert!(
!cancel_requested(Some(dir.to_str().unwrap())),
"empty token must not cancel even if MARQDO_CANCEL=1"
);
std::env::remove_var("MARQDO_CANCEL");
let _ = std::fs::remove_file(&dir);
}
#[test]
fn context_round_trip() {
let v = json!({
"run_id": "r1",
"cancellation_token": "/tmp/t",
"context": {
"workspace_id": "ws",
"lease": {"id": "lease_1", "epoch": 3},
"attempt_id": "a1"
}
});
let rc = RunContext::from_value(&v).unwrap();
assert_eq!(rc.context.lease_id.as_deref(), Some("lease_1"));
assert_eq!(rc.context.epoch, Some(3));
let child = rc.spawn_child("r2");
assert_eq!(child.parent_run_id.as_deref(), Some("r1"));
assert_eq!(child.context.lease_id.as_deref(), Some("lease_1"));
}
}
+147
View File
@@ -0,0 +1,147 @@
//! ModelSelectionV1 — request-level model identity (M01).
use serde_json::{json, Map, Value};
#[derive(Debug, Clone)]
pub struct ModelSelection {
pub schema_version: u64,
pub selection_id: String,
pub provider_id: String,
pub model_id: String,
pub provider_revision: Option<u64>,
pub model_revision: Option<u64>,
pub adapter: String,
pub endpoint_ref: Option<String>,
pub credential_ref: Option<String>,
pub credential_version: Option<u64>,
pub upstream_model_id: String,
pub parameters: Map<String, Value>,
pub fallback_policy: String,
pub auth_mode: String,
/// Resolved call-local endpoint (never serialize API keys into events).
pub base_url: Option<String>,
/// Call-local credential material — must not appear in public snapshots.
pub api_key: Option<String>,
}
impl ModelSelection {
pub fn public_snapshot(&self) -> Value {
json!({
"schema": "ModelSelectionV1",
"schema_version": self.schema_version,
"selection_id": self.selection_id,
"provider_id": self.provider_id,
"model_id": self.model_id,
"provider_revision": self.provider_revision,
"model_revision": self.model_revision,
"adapter": self.adapter,
"endpoint_ref": self.endpoint_ref,
"credential_ref": self.credential_ref,
"credential_version": self.credential_version,
"upstream_model_id": self.upstream_model_id,
"parameters": self.parameters,
"fallback_policy": self.fallback_policy,
"auth_mode": self.auth_mode,
})
}
}
/// Parse selection. In `strict` mode, missing required fields error (no env fallback).
pub fn parse_model_selection(v: &Value, strict: bool) -> Result<ModelSelection, String> {
let schema_version = v
.get("schema_version")
.and_then(|x| x.as_u64())
.unwrap_or(1);
if schema_version != 1 {
return Err(format!("unsupported ModelSelection schema_version={schema_version}"));
}
let require = |key: &str| -> Result<String, String> {
match v.get(key).and_then(|x| x.as_str()).filter(|s| !s.is_empty()) {
Some(s) => Ok(s.to_string()),
None if strict => Err(format!("llm.selection_missing:{key}")),
None => Ok(String::new()),
}
};
let selection_id = require("selection_id")?;
let provider_id = require("provider_id")?;
let model_id = require("model_id")?;
let adapter = v
.get("adapter")
.and_then(|x| x.as_str())
.unwrap_or("openai-compatible")
.to_string();
let upstream = v
.get("upstream_model_id")
.and_then(|x| x.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
.or_else(|| {
if model_id.is_empty() {
None
} else {
Some(model_id.clone())
}
});
let upstream_model_id = match upstream {
Some(u) => u,
None if strict => return Err("llm.selection_missing:upstream_model_id".into()),
None => String::new(),
};
let auth_mode = v
.get("auth_mode")
.and_then(|x| x.as_str())
.unwrap_or("api_key")
.to_string();
let fallback_policy = v
.get("fallback_policy")
.and_then(|x| x.as_str())
.unwrap_or("disabled")
.to_string();
let parameters = v
.get("parameters")
.and_then(|x| x.as_object())
.cloned()
.unwrap_or_default();
let base_url = v
.get("base_url")
.and_then(|x| x.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
let api_key = v
.get("api_key")
.and_then(|x| x.as_str())
.map(|s| s.to_string());
if strict && auth_mode != "none" && api_key.as_ref().map(|s| s.is_empty()).unwrap_or(true) {
// credential_ref may be resolved by host; allow missing api_key if credential_ref present
if v.get("credential_ref").and_then(|x| x.as_str()).is_none() && base_url.is_some() {
return Err("llm.selection_missing:api_key_or_credential_ref".into());
}
}
if strict && selection_id.is_empty() {
return Err("llm.selection_missing:selection_id".into());
}
Ok(ModelSelection {
schema_version,
selection_id,
provider_id,
model_id,
provider_revision: v.get("provider_revision").and_then(|x| x.as_u64()),
model_revision: v.get("model_revision").and_then(|x| x.as_u64()),
adapter,
endpoint_ref: v
.get("endpoint_ref")
.and_then(|x| x.as_str())
.map(|s| s.to_string()),
credential_ref: v
.get("credential_ref")
.and_then(|x| x.as_str())
.map(|s| s.to_string()),
credential_version: v.get("credential_version").and_then(|x| x.as_u64()),
upstream_model_id,
parameters,
fallback_policy,
auth_mode,
base_url,
api_key,
})
}
+90
View File
@@ -0,0 +1,90 @@
//! Snapshot manifest for apps to retain historical evidence (M04).
//! Framework does not store history; missing snapshot → version_unavailable.
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use std::path::Path;
/// Build a retainable snapshot manifest for the current projection.
pub fn build_snapshot_manifest(
project_root: &Path,
document_rel: &str,
source_bytes: &[u8],
project: &crate::sema::SemanticProject,
renderer_version: &str,
) -> Value {
let source_revision = format!("sha256:{:x}", Sha256::digest(source_bytes));
let mut dep_revs = Vec::new();
for a in &project.artifacts {
if a.source.file == document_rel {
continue;
}
dep_revs.push(json!({
"document_id": a.source.file,
"artifact_id": a.artifact_id.0,
"artifact_hash": a.hashes.artifact.0,
}));
}
let mut artifact_hashes = Vec::new();
for a in &project.artifacts {
if a.source.file == document_rel {
artifact_hashes.push(json!({
"artifact_id": a.artifact_id.0,
"hash": a.hashes.artifact.0,
}));
}
}
let mut gh = Sha256::new();
gh.update(project.project_id.as_bytes());
for a in &project.artifacts {
gh.update(a.hashes.artifact.0.as_bytes());
}
let generation_id = format!("{:x}", gh.finalize())[..16].to_string();
json!({
"schema": "SnapshotManifestV1",
"schema_version": 1,
"project_id": project_root.display().to_string(),
"document_id": document_rel,
"source_revision": source_revision,
"generation_id": generation_id,
"dependency_revisions": dep_revs,
"artifact_hashes": artifact_hashes,
"renderer_version": renderer_version,
"contract_schema": "DocumentViewV1",
"retention": {
"owner": "application",
"note": "framework does not retain historical bodies; missing snapshot → version_unavailable"
}
})
}
/// When an expected revision/generation is requested but no app-provided snapshot
/// resolver is present, return the honest unavailable contract.
pub fn version_unavailable_result(
locator: &Value,
reason: &str,
expected: &str,
actual: Option<&str>,
) -> Value {
json!({
"schema": "KnowledgeLocatorV1",
"schema_version": 1,
"resolution": "version_unavailable",
"precision": "none",
"locator": locator,
"target": Value::Null,
"candidates": [],
"diagnostics": [{
"code": "snapshot_not_retained",
"message": reason,
"expected": expected,
"actual": actual,
}],
"label": "引用版本不可用",
"snapshot": {
"available": false,
"owner": "application"
}
})
}
+174
View File
@@ -0,0 +1,174 @@
//! ModelCallUsageV1 — null ≠ 0; measurement_source honesty (M02 / F04).
use serde_json::{json, Value};
/// Prefer null over zero when the provider omitted a field.
pub fn token_or_null(v: Option<i64>) -> Value {
match v {
Some(n) if n >= 0 => json!(n),
_ => Value::Null,
}
}
/// Build ModelCallUsageV1 from an OpenAI-shaped `usage` object (or Null).
pub fn model_call_usage_from_provider(
call_id: &str,
attempt_id: &str,
run_id: &str,
selection: &Value,
provider_usage: &Value,
status: &str,
) -> Value {
model_call_usage_from_provider_rev(call_id, attempt_id, run_id, selection, provider_usage, status, 1)
}
/// Same as [`model_call_usage_from_provider`] with an explicit revision (F05).
pub fn model_call_usage_from_provider_rev(
call_id: &str,
attempt_id: &str,
run_id: &str,
selection: &Value,
provider_usage: &Value,
status: &str,
revision: u64,
) -> Value {
let has_usage = provider_usage.is_object();
let input = provider_usage
.get("prompt_tokens")
.or_else(|| provider_usage.get("input_tokens"))
.and_then(|v| v.as_i64());
let output = provider_usage
.get("completion_tokens")
.or_else(|| provider_usage.get("output_tokens"))
.and_then(|v| v.as_i64());
let total = provider_usage
.get("total_tokens")
.and_then(|v| v.as_i64())
.or_else(|| match (input, output) {
(Some(a), Some(b)) => Some(a + b),
_ => None,
});
let cached = provider_usage
.get("cached_input_tokens")
.or_else(|| {
provider_usage
.get("prompt_tokens_details")
.and_then(|d| d.get("cached_tokens"))
})
.and_then(|v| v.as_i64());
let reasoning = provider_usage
.get("reasoning_output_tokens")
.or_else(|| {
provider_usage
.get("completion_tokens_details")
.and_then(|d| d.get("reasoning_tokens"))
})
.and_then(|v| v.as_i64());
let (usage_state, measurement_source) = if !has_usage {
("unknown", "none")
} else if input.is_none() && output.is_none() && total.is_none() {
("unknown", "none")
} else {
("final", "provider")
};
let mut diagnostics = Vec::new();
if let (Some(t), Some(i), Some(o)) = (total, input, output) {
if t != i + o && cached.is_none() {
diagnostics.push(json!({
"code": "usage.inconsistent",
"message": format!("total_tokens={t} != input+output={}", i + o)
}));
}
}
json!({
"schema": "ModelCallUsageV1",
"schema_version": 1,
"call_id": call_id,
"attempt_id": attempt_id,
"parent_call_id": Value::Null,
"run_id": run_id,
"turn_id": selection.get("turn_id").cloned().unwrap_or(Value::Null),
"workspace_scope": selection.get("workspace_scope").cloned().unwrap_or(Value::Null),
"purpose": selection.get("purpose").cloned().unwrap_or(json!("answer")),
"selection_id": selection.get("selection_id").cloned().unwrap_or(Value::Null),
"provider_id": selection.get("provider_id").cloned().unwrap_or(Value::Null),
"model_id": selection.get("model_id").cloned().unwrap_or(Value::Null),
"requested_model": selection.get("upstream_model_id").cloned()
.or_else(|| selection.get("model").cloned())
.unwrap_or(Value::Null),
"resolved_model": selection.get("resolved_model").cloned()
.or_else(|| selection.get("upstream_model_id").cloned())
.unwrap_or(Value::Null),
"status": status,
"usage_state": usage_state,
"measurement_source": measurement_source,
"input_tokens": token_or_null(input),
"output_tokens": token_or_null(output),
"total_tokens": token_or_null(total),
"cached_input_tokens": token_or_null(cached),
"reasoning_output_tokens": token_or_null(reasoning),
"revision": revision.max(1),
"provider_request_id": provider_usage.get("provider_request_id").cloned().unwrap_or(Value::Null),
"diagnostics": diagnostics,
// Legacy fields kept but marked — apps must not treat token_cost as money.
"legacy": {
"tokens": token_or_null(total),
"token_cost": token_or_null(total),
"deprecated": true,
"unit": "tokens"
}
})
}
/// Replace prior usage for the same (call_id, attempt_id) when revision is newer (F05).
/// Returns the winning snapshot; does not sum cumulative stream counters.
pub fn replace_usage_if_newer(prior: &Value, next: &Value) -> Value {
let same_call = prior.get("call_id") == next.get("call_id")
&& prior.get("attempt_id") == next.get("attempt_id");
if !same_call {
return next.clone();
}
let pr = prior.get("revision").and_then(|v| v.as_u64()).unwrap_or(0);
let nr = next.get("revision").and_then(|v| v.as_u64()).unwrap_or(0);
if nr >= pr {
next.clone()
} else {
prior.clone()
}
}
/// RuntimeEventV1 envelope helper.
pub fn runtime_event(
event_id: &str,
seq: u64,
run_id: &str,
event_type: &str,
payload: Value,
) -> Value {
json!({
"schema": "RuntimeEventV1",
"schema_version": 1,
"event_id": event_id,
"seq": seq,
"run_id": run_id,
"parent_run_id": Value::Null,
"turn_id": Value::Null,
"workspace_scope": Value::Null,
"timestamp": chrono_like_now(),
"type": event_type,
"payload": payload
})
}
fn chrono_like_now() -> String {
// Avoid new chrono dep: RFC3339-ish from system time.
use std::time::{SystemTime, UNIX_EPOCH};
let secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
format!("{secs}")
}
+46
View File
@@ -0,0 +1,46 @@
//! Phase A: `marqdo run --call-lib-path` writes `--emit-result` without parent INVOKE_LOCK.
use std::path::PathBuf;
use std::process::Command;
#[test]
fn call_lib_path_emits_json_result() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let site = root.join("tests/ext/isolated-execute-site");
let entry = site.join("serve.mq.md");
let tmp = std::env::temp_dir().join(format!("mq-clp-{}", std::process::id()));
let _ = std::fs::create_dir_all(&tmp);
let args = tmp.join("args.json");
let emit = tmp.join("out.json");
std::fs::write(&args, r#"{"payload":null}"#).unwrap();
let bin = env!("CARGO_BIN_EXE_marqdo");
let out = Command::new(bin)
.arg("run")
.arg(&entry)
.arg("--call-lib-path")
.arg("site.models") // entry is serve.mq.md which imports site
.arg("--fs-root")
.arg(&site)
.arg("--args-json")
.arg(&args)
.arg("--emit-result")
.arg(&emit)
.current_dir(&site)
.env("MARQDO_EXT", root.join("ext"))
.output()
.expect("spawn worker");
let stderr = String::from_utf8_lossy(&out.stderr);
assert_eq!(
out.status.code().unwrap_or(1),
0,
"stderr={stderr}\nstdout={}",
String::from_utf8_lossy(&out.stdout)
);
let body = std::fs::read_to_string(&emit).expect("emit-result");
assert!(
body.contains("list") || body.contains("ok"),
"emit={body}"
);
let _ = std::fs::remove_dir_all(&tmp);
}
+69
View File
@@ -5,6 +5,64 @@
use std::process::Command;
/// Build `libagent` once per test process and prefer it over a stale `~/.marqdo/ext`.
/// Same contract as `tests/gold.rs::ensure_agent_plugin_built` — EK gate must not
/// depend on a manually copied `ext/native/libagent.so`.
fn ensure_agent_plugin_built() {
use std::sync::Once;
static ONCE: Once = Once::new();
ONCE.call_once(|| {
let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let status = Command::new("cargo")
.args(["build", "-p", "marqdo_plugin_agent"])
.current_dir(&root)
.status()
.expect("build marqdo_plugin_agent");
assert!(status.success(), "failed to build marqdo_plugin_agent");
let lib = if cfg!(windows) {
root.join("target").join("debug").join("agent.dll")
} else if cfg!(target_os = "macos") {
root.join("target").join("debug").join("libagent.dylib")
} else {
root.join("target").join("debug").join("libagent.so")
};
let release = if cfg!(windows) {
root.join("target").join("release").join("agent.dll")
} else if cfg!(target_os = "macos") {
root.join("target").join("release").join("libagent.dylib")
} else {
root.join("target").join("release").join("libagent.so")
};
let chosen = if lib.is_file() {
if release.is_file() {
let lib_m = std::fs::metadata(&lib).and_then(|m| m.modified()).ok();
let rel_m = std::fs::metadata(&release).and_then(|m| m.modified()).ok();
match (lib_m, rel_m) {
(Some(l), Some(r)) if r > l => release,
_ => lib,
}
} else {
lib
}
} else {
release
};
assert!(
chosen.is_file(),
"missing agent plugin at {}",
chosen.display()
);
// SAFETY: single-threaded Once; children inherit for the rest of the test process.
std::env::set_var("MARQDO_AGENT_PLUGIN", &chosen);
if std::env::var_os("MARQDO_EXT").is_none() {
let ext = root.join("ext");
if ext.join("ai").join("agent.mq.md").is_file() {
std::env::set_var("MARQDO_EXT", &ext);
}
}
});
}
fn run_backend(path: &str, backend: &str) -> (i32, String, String) {
let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let out = Command::new(env!("CARGO_BIN_EXE_marqdo"))
@@ -1657,6 +1715,12 @@ fn v2_sse_accepts_packet_rejects_stale_token() {
assert!(marqdo::view::sse_accepts_packet("abc", Some("")));
assert!(marqdo::view::sse_accepts_packet("abc", Some("abc")));
assert!(!marqdo::view::sse_accepts_packet("abc", Some("zzz")));
// Embed clients must use strict: missing/empty/mismatch all reject.
assert!(!marqdo::view::sse_accepts_packet_strict("abc", None));
assert!(!marqdo::view::sse_accepts_packet_strict("abc", Some("")));
assert!(!marqdo::view::sse_accepts_packet_strict("", Some("abc")));
assert!(!marqdo::view::sse_accepts_packet_strict("abc", Some("zzz")));
assert!(marqdo::view::sse_accepts_packet_strict("abc", Some("abc")));
assert_eq!(
marqdo::view::page_token_from_source("a.mq.md", "x"),
marqdo::view::page_token_from_source("a.mq.md", "x")
@@ -1869,6 +1933,7 @@ fn f1_publish_gate_chain_via_verify_may_publish() {
#[test]
fn f1_agent_eng_bind_gate_mq_offline() {
ensure_agent_plugin_built();
let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let out = Command::new(env!("CARGO_BIN_EXE_marqdo"))
.args([
@@ -1878,6 +1943,10 @@ fn f1_agent_eng_bind_gate_mq_offline() {
"tree",
])
.current_dir(&root)
.env(
"MARQDO_AGENT_PLUGIN",
std::env::var_os("MARQDO_AGENT_PLUGIN").expect("MARQDO_AGENT_PLUGIN set by ensure"),
)
.output()
.unwrap();
let stdout = String::from_utf8_lossy(&out.stdout);
+55
View File
@@ -0,0 +1,55 @@
---
markup: occlusion
title: agent-effects-run-tool
description: >-
M-A5a / MA-T03 subset — mutating run_tool goes through EffectLedger;
committed effects dedupe; started-without-commit becomes unknown (no re-exec).
import:
agent: /home/cflmy/work/marqdo/ext/ai/agent.mq.md
json: lib/json.mq.md
sys: lib/sys.mq.md
table: lib/table.mq.md
text: lib/text.mq.md
---
## write_probe
Probe side-effect tool body (count stays in ledger, not here).
*"wrote"*
# main
**schema = > json.parse text={"type":"object","properties":{},"additionalProperties":true}**
**scopes = > json.parse text=["workspace:write"]**
`spec` =
| id | version | description | input_schema | effect | scopes | availability |
|----|---------|-------------|--------------|--------|--------|--------------|
| write_probe | 1 | test write tool | `schema` | write | `scopes` | ready |
> agent.tools_clear
**reg = > agent.tools_register spec=`spec`**
**allow = > json.parse text=["write_probe"]**
**args = > json.parse text={}**
**out1 = > agent.run_tool tools=`allow` name="write_probe" arguments=`args` logical_run_id="run_ma5a" step_id="s1" call_id="c1" operation_id="op_write_probe_c1" ledger_dir=".marqdo/agent-effects-ma5a"**
**ok1 = > json.get value=`out1` key="ok"**
**st1 = > json.get value=`out1` key="status"**
**rc1 = > json.get value=`out1` key="effect_receipt_ref"**
> print text=`ok1`
> print text=`st1`
> print text=`rc1`
**out2 = > agent.run_tool tools=`allow` name="write_probe" arguments=`args` logical_run_id="run_ma5a" step_id="s1" call_id="c1" operation_id="op_write_probe_c1" ledger_dir=".marqdo/agent-effects-ma5a"**
**sum2 = > json.get value=`out2` key="summary"**
> print text=`sum2`
**wm = > agent.effects_watermark ledger_dir=".marqdo/agent-effects-ma5a" logical_run_id="run_ma5a"**
**safe = > json.get value=`wm` key="write_auto_resume_safe"**
> print text=`safe`
**got = > agent.effects_get operation_id="op_write_probe_c1" ledger_dir=".marqdo/agent-effects-ma5a"**
**rec = > json.get value=`got` key="record"**
**state = > json.get value=`rec` key="state"**
> print text=`state`
@@ -0,0 +1,25 @@
---
markup: occlusion
title: isolated execute MR-01 site
description: Dual-stream execute + short models for ADR 0009 gold sample.
import:
web: ext/web/web.mq.md
site: ./site.mq.md
sys: lib/sys.mq.md
---
# main
**port_s = > sys.env_get name="MARQDO_SITE_PORT"**
**port = 18220**
1. `port_s`
**port = `port_s`**
2. *
**_ = 1**
**pg = > web.page title="isolated-execute"**
**app = > web.app page=`pg` host="127.0.0.1" port=`port`**
**app = > web.route app=`app` path="/v1/chat/completions" method="POST" fn="site.chat" request="json" response="sse-live" class="execute"**
**app = > web.route app=`app` path="/v1/models" method="GET" fn="site.models" request="query" response="json" class="short"**
**app = > web.route app=`app` path="/v1/ping" method="GET" fn="site.ping" request="query" response="json" class="control"**
> `app`.listen
@@ -0,0 +1,85 @@
---
markup: occlusion
title: isolated execute site libs
description: Chat (execute) + models (short) for MR-01 gold sample.
import:
llm: ext/ai/llm.mq.md
sys: lib/sys.mq.md
json: lib/json.mq.md
table: lib/table.mq.md
---
# chat
+ `model`="mr01"
+ `messages`=None
+ `stream`=True
+ `base_url`=""
+ `prompt`=""
+ `authorization`=""
+ `api_key`=""
+ `payload`=None
OpenAI-compatible chat. Prefer request `base_url` so dual MR-01 streams can hit distinct fake upstreams.
**base = base_url**
1. not `base`
1. `payload`
**base = > json.get value=`payload` key="base_url"**
2. *
**base = > sys.env_get name="OPENAI_BASE_URL"**
2. *
**_ = 1**
1. not `base`
**base = "http://127.0.0.1:9/v1"**
2. *
**_ = 1**
**key = api_key**
1. not `key`
**key = > sys.env_get name="OPENAI_API_KEY"**
2. *
**_ = 1**
1. not `key`
**key = "mr01-key"**
2. *
**_ = 1**
**mdl = model**
1. not `mdl`
**mdl = "mr01"**
2. *
**_ = 1**
**m = > llm.create model=`mdl` base_url=`base` api_key=`key` name="mr01" backend="openai-compatible"**
1. `messages`
*> `m`.complete_result messages=`messages` stream=True echo=False*
2. `prompt`
*> `m`.complete_result prompt=`prompt` stream=True echo=False*
3. *
*> `m`.complete_result prompt="ping" stream=True echo=False*
# models
+ `payload`=None
Short-class read: list models without holding an execute slot.
`out` =
| object | ok |
|--------|-----|
| list | True |
*out*
# ping
+ `payload`=None
Control-class noop: must succeed under dual execute without parent INVOKE_LOCK.
`out` =
| ok | lane |
|----|------|
| True | control |
*out*
+81
View File
@@ -0,0 +1,81 @@
---
markup: occlusion
title: workbench F02 agent selection (mock upstream)
description: >-
Explicit llm.create selection must drive agent.step; process env default model
must not win. Upstream URL/key/model come from env set by the Rust harness.
import:
llm: ext/ai/llm.mq.md
agent: ext/ai/agent.mq.md
sys: lib/sys.mq.md
json: lib/json.mq.md
---
# main
**base = > sys.env_get name="OPENAI_BASE_URL"**
**key = > sys.env_get name="OPENAI_API_KEY"**
**mdl = > sys.env_get name="OPENAI_MODEL"**
1. not `base`
> print text=missing-OPENAI_BASE_URL
> sys.exit code=2
2. *
**_ = 1**
1. not `key`
> print text=missing-OPENAI_API_KEY
> sys.exit code=2
2. *
**_ = 1**
1. not `mdl`
> print text=missing-OPENAI_MODEL
> sys.exit code=2
2. *
**_ = 1**
Poison ambient default — selection on the handle must still win at the plugin.
> sys.env_set name="OPENAI_MODEL" value="env-poison-model"
**model = > llm.create model=`mdl` base_url=`base` api_key=`key` backend="openai-compatible" name=f02-agent**
`tools` =
| 工具 |
|------|
**ag = > agent.agent model=`model` tools=`tools` standing="F02 selection probe; answer with one word only"**
**step = > `ag`.step task="Reply with exactly one English word pong" writeback=False max_reads=1**
**st = > json.get value=`step` key="status"**
**ans = > json.get value=`step` key="result"**
1. not `ans`
**ans = > json.get value=`step` key="answer"**
2. *
**_ = 1**
1. not `ans`
**ans = > json.get value=`step` key="text"**
2. *
**_ = 1**
> print text=`st`
> print text=`ans`
**dir = ".marqdo/workbench-f02-resume"**
**saved = > agent.resume_save id="f02-sel" goal="pong" status="running" round=1 resume_dir=`dir`**
**ok = > json.get value=`saved` key="ok"**
1. `ok`
> print text=resume-save-ok
2. *
> print text=resume-save-fail
> print text=`saved`
> sys.exit code=1
**loaded = > agent.resume id="f02-sel" resume_dir=`dir`**
**found = > json.get value=`loaded` key="found"**
1. `found`
> print text=resume-load-ok
2. *
> print text=resume-load-fail
> print text=`loaded`
> sys.exit code=1
> agent.resume_clear id="f02-sel" resume_dir=`dir`
> print text=f02-agent-selection-ok
+43
View File
@@ -0,0 +1,43 @@
---
markup: occlusion
title: workbench M01 live selection (USTC-compatible)
description: Explicit ModelSelection via llm.create; no llm.llm. Credentials from process env only.
import:
llm: ext/ai/llm.mq.md
sys: lib/sys.mq.md
json: lib/json.mq.md
---
# main
**base = > sys.env_get name="OPENAI_BASE_URL"**
**key = > sys.env_get name="OPENAI_API_KEY"**
**mdl = > sys.env_get name="OPENAI_MODEL"**
1. not `base`
> print text=missing-OPENAI_BASE_URL
> sys.exit code=2
2. *
**_ = 1**
1. not `key`
> print text=missing-OPENAI_API_KEY
> sys.exit code=2
2. *
**_ = 1**
1. not `mdl`
**mdl = "qwen3.8-chat"**
2. *
**_ = 1**
**model = > llm.create model=`mdl` base_url=`base` api_key=`key` backend="openai-compatible" name=workbench-live**
**q = "Reply with exactly one English word pong"**
**reply = > model.ask prompt=`q`**
**text = > json.get value=`reply` key="text"**
**usage = > json.get value=`reply` key="usage"**
**pt = > json.get value=`usage` key="prompt_tokens"**
> print text=`text`
1. `pt`
> print text=usage-present
2. *
> print text=usage-unknown
+43
View File
@@ -1431,6 +1431,49 @@ fn knowledge_preflight_writes_context_pack() {
assert!(!packs.is_empty(), "expected context pack");
}
#[test]
fn knowledge_preflight_warm_loads_ir_without_recompile() {
let dir = tempfile_dir("mq-pf-warm");
let out = dir.to_str().unwrap();
let (code, _, stderr) = run(&["knowledge", "tests/engineering", "-o", out]);
assert_eq!(code, 0, "{stderr}");
assert!(
dir.join("ekc/graph/ir.json").exists(),
"expected ir.json snapshot after compile"
);
let gens = dir.join("generations");
let before: Vec<_> = if gens.exists() {
std::fs::read_dir(&gens).unwrap().flatten().collect()
} else {
Vec::new()
};
let (code2, stdout, stderr2) = run(&[
"knowledge",
"tests/engineering",
"preflight",
"resolve configuration",
"-o",
out,
"--json",
]);
assert_eq!(code2, 0, "{stderr2}\n{stdout}");
assert!(stdout.contains("create_allowed"), "{stdout}");
assert!(
!stderr2.contains("marqdo knowledge timing_ms:"),
"warm preflight must not full-compile: {stderr2}"
);
let after: Vec<_> = if gens.exists() {
std::fs::read_dir(&gens).unwrap().flatten().collect()
} else {
Vec::new()
};
assert_eq!(
before.len(),
after.len(),
"warm preflight must not create a new generations/ tree"
);
}
#[test]
fn catalog_hooks_knowledge_by_default() {
let dir = tempfile_dir("mq-cat-ekc");
+642
View File
@@ -0,0 +1,642 @@
//! MR-01 gold: dual 60s fake SSE execute workers + short reads (ADR 0009).
//! MR-02 (same file): third execute rejected while A continues cleanly.
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
fn ensure_plugins() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let status = Command::new("bash")
.arg(root.join("scripts/build-web-plugin.sh"))
.current_dir(&root)
.status()
.expect("build web plugin");
assert!(status.success(), "web plugin build failed");
let status = Command::new("cargo")
.args(["build", "-p", "marqdo_plugin_llm", "--locked"])
.current_dir(&root)
.status()
.expect("build llm plugin");
assert!(status.success(), "llm plugin build failed");
let web = root.join("plugins/web/build/libweb.so");
let llm = root.join("target/debug/libllm.so");
assert!(web.is_file(), "missing {web:?}");
assert!(llm.is_file(), "missing {llm:?}");
// SAFETY: test process setup before spawning children.
std::env::set_var("MARQDO_WEB_PLUGIN", &web);
std::env::set_var("MARQDO_LLM_PLUGIN", &llm);
std::env::set_var("MARQDO_EXT", root.join("ext"));
}
fn spawn_fake_sse(prefix: &'static str, secs: u64, hz: u64) -> (String, Arc<AtomicBool>, thread::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind fake sse");
listener.set_nonblocking(true).expect("nonblocking");
let addr = listener.local_addr().unwrap();
let stop = Arc::new(AtomicBool::new(false));
let stop_bg = stop.clone();
let handle = thread::spawn(move || {
// Poll accept so a never-connected upstream (e.g. 429 before dial) can stop.
let mut stream = loop {
if stop_bg.load(Ordering::Relaxed) {
return;
}
match listener.accept() {
Ok((s, _)) => break s,
Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(20));
continue;
}
Err(_) => return,
}
};
stream.set_nonblocking(false).ok();
let mut raw = Vec::new();
let mut buf = [0u8; 4096];
// Read request headers (+ body if Content-Length).
loop {
let n = stream.read(&mut buf).unwrap_or(0);
if n == 0 {
break;
}
raw.extend_from_slice(&buf[..n]);
if let Some(pos) = raw.windows(4).position(|w| w == b"\r\n\r\n") {
let header = String::from_utf8_lossy(&raw[..pos]);
let cl = header
.lines()
.find_map(|l| l.strip_prefix("Content-Length:"))
.and_then(|v| v.trim().parse::<usize>().ok())
.unwrap_or(0);
let body_start = pos + 4;
while raw.len() < body_start + cl {
let n = stream.read(&mut buf).unwrap_or(0);
if n == 0 {
break;
}
raw.extend_from_slice(&buf[..n]);
}
break;
}
if raw.len() > 1 << 20 {
break;
}
}
let _ = stream.write_all(
b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\nConnection: close\r\n\r\n",
);
let _ = stream.flush();
let interval = Duration::from_millis(1000 / hz.max(1));
let deadline = Instant::now() + Duration::from_secs(secs);
let mut i = 0u64;
while Instant::now() < deadline && !stop_bg.load(Ordering::Relaxed) {
let token = format!("{prefix}{i}");
let chunk = format!(
"data: {{\"choices\":[{{\"delta\":{{\"content\":\"{token}\"}}}}]}}\n\n"
);
if stream.write_all(chunk.as_bytes()).is_err() {
break;
}
let _ = stream.flush();
i += 1;
thread::sleep(interval);
}
let _ = stream.write_all(b"data: [DONE]\n\n");
let _ = stream.flush();
});
(format!("http://{addr}/v1"), stop, handle)
}
fn wait_http_ok(url: &str, timeout: Duration) -> bool {
let start = Instant::now();
while start.elapsed() < timeout {
let out = Command::new("curl")
.args(["-s", "-o", "/dev/null", "-w", "%{http_code}", url])
.output();
if let Ok(o) = out {
if String::from_utf8_lossy(&o.stdout) == "200" {
return true;
}
}
thread::sleep(Duration::from_millis(200));
}
false
}
fn collect_sse_deltas(url: &str, body: &str, base_url: &str) -> (u16, String, Vec<String>) {
let payload = body.replace("__BASE__", base_url);
let out = Command::new("curl")
.args([
"-s",
"-N",
"--max-time",
"180",
"-w",
"\n__HTTP__%{http_code}",
"-H",
"Content-Type: application/json",
"-d",
&payload,
url,
])
.output()
.expect("curl sse");
let text = String::from_utf8_lossy(&out.stdout);
let (body_text, code_s) = text
.rsplit_once("\n__HTTP__")
.unwrap_or((text.as_ref(), "0"));
let code: u16 = code_s.trim().parse().unwrap_or(0);
let mut deltas = Vec::new();
for line in body_text.lines() {
let line = line.trim();
if let Some(rest) = line.strip_prefix("data: ") {
if rest == "[DONE]" {
continue;
}
if let Ok(v) = serde_json::from_str::<serde_json::Value>(rest) {
if let Some(d) = v
.pointer("/payload/delta")
.and_then(|x| x.as_str())
.or_else(|| v.pointer("/payload/text").and_then(|x| x.as_str()))
{
deltas.push(d.to_string());
}
}
}
}
(code, body_text.to_string(), deltas)
}
fn count_call_lib_path_workers(site_pid: u32) -> usize {
// Walk /proc for children whose cmdline contains --call-lib-path.
let Ok(entries) = std::fs::read_dir("/proc") else {
return 0;
};
let mut n = 0usize;
for ent in entries.flatten() {
let name = ent.file_name();
let pid: u32 = match name.to_str().and_then(|s| s.parse().ok()) {
Some(p) => p,
None => continue,
};
let cmdline = std::fs::read(format!("/proc/{pid}/cmdline")).unwrap_or_default();
let cmd = String::from_utf8_lossy(&cmdline);
if !cmd.contains("--call-lib-path") {
continue;
}
// Prefer workers whose parent is the site (or grandparent via supervisor).
let ppid = std::fs::read_to_string(format!("/proc/{pid}/stat"))
.ok()
.and_then(|s| {
let rparen = s.rfind(')')?;
let rest = s[rparen + 2..].split_whitespace().nth(1)?;
rest.parse::<u32>().ok()
})
.unwrap_or(0);
if ppid == site_pid || cmd.contains("isolated-execute-site") || cmd.contains("serve.mq.md")
{
n += 1;
} else {
// Count any call-lib-path marqdo under this test tree.
n += 1;
}
}
n
}
#[test]
fn mr01_dual_isolated_execute_and_short_reads() {
if TcpListener::bind("127.0.0.1:0").is_err() {
eprintln!("skip MR-01: local sockets unavailable");
return;
}
ensure_plugins();
let secs: u64 = std::env::var("MARQDO_MR01_SECS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(60);
let hz: u64 = 20;
let (base_a, stop_a, join_a) = spawn_fake_sse("A", secs, hz);
let (base_b, stop_b, join_b) = spawn_fake_sse("B", secs, hz);
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let site = root.join("tests/ext/isolated-execute-site");
let runtime_dir = std::env::temp_dir().join(format!("mq-mr01-rt-{}", std::process::id()));
let _ = std::fs::create_dir_all(&runtime_dir);
let bin = env!("CARGO_BIN_EXE_marqdo");
let mut child: Child = Command::new(bin)
.args(["run", "serve.mq.md"])
.current_dir(&site)
.env("MARQDO_EXT", root.join("ext"))
.env("MARQDO_WEB_PLUGIN", std::env::var_os("MARQDO_WEB_PLUGIN").unwrap())
.env("MARQDO_LLM_PLUGIN", std::env::var_os("MARQDO_LLM_PLUGIN").unwrap())
.env("MARQDO_RUNTIME_DIR", &runtime_dir)
.env("OPENAI_API_KEY", "mr01-key")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("spawn site");
let site_pid = child.id();
let base = "http://127.0.0.1:18220";
if !wait_http_ok(&format!("{base}/v1/models"), Duration::from_secs(30)) {
let _ = child.kill();
let err = child
.stderr
.take()
.map(|mut s| {
let mut t = String::new();
let _ = s.read_to_string(&mut t);
t
})
.unwrap_or_default();
panic!("site did not start; stderr={err}");
}
// Capability honesty: isolated execution is default-on.
let caps = marqdo::workbench::probe_capabilities(&serde_json::json!({}));
assert_eq!(
caps["features"]["runtime.isolated_execution"],
1,
"caps={caps}"
);
let chat_url = format!("{base}/v1/chat/completions");
let body_tmpl = r#"{"model":"mr01","stream":true,"base_url":"__BASE__","messages":[{"role":"user","content":"hi"}]}"#;
let deltas_a = Arc::new(Mutex::new(Vec::new()));
let deltas_b = Arc::new(Mutex::new(Vec::new()));
let code_a = Arc::new(Mutex::new(0u16));
let code_b = Arc::new(Mutex::new(0u16));
let raw_a = Arc::new(Mutex::new(String::new()));
let raw_b = Arc::new(Mutex::new(String::new()));
let da = deltas_a.clone();
let ca = code_a.clone();
let ra = raw_a.clone();
let base_a_c = base_a.clone();
let chat_a = chat_url.clone();
let body_a = body_tmpl.to_string();
let t_a = thread::spawn(move || {
let (code, raw, deltas) = collect_sse_deltas(&chat_a, &body_a, &base_a_c);
*ca.lock().unwrap() = code;
*ra.lock().unwrap() = raw;
*da.lock().unwrap() = deltas;
});
let db = deltas_b.clone();
let cb = code_b.clone();
let rb = raw_b.clone();
let base_b_c = base_b.clone();
let chat_b = chat_url.clone();
let body_b = body_tmpl.to_string();
let t_b = thread::spawn(move || {
let (code, raw, deltas) = collect_sse_deltas(&chat_b, &body_b, &base_b_c);
*cb.lock().unwrap() = code;
*rb.lock().unwrap() = raw;
*db.lock().unwrap() = deltas;
});
// Short reads during the dual stream.
let short_ok = Arc::new(Mutex::new(0u32));
let short_fail = Arc::new(Mutex::new(0u32));
let short_stop = Arc::new(AtomicBool::new(false));
let short_ok_c = short_ok.clone();
let short_fail_c = short_fail.clone();
let short_stop_c = short_stop.clone();
let models_url = format!("{base}/v1/models");
let t_short = thread::spawn(move || {
while !short_stop_c.load(Ordering::Relaxed) {
let out = Command::new("curl")
.args(["-s", "-o", "/dev/null", "-w", "%{http_code}", &models_url])
.output();
match out {
Ok(o) if String::from_utf8_lossy(&o.stdout) == "200" => {
*short_ok_c.lock().unwrap() += 1;
}
_ => {
*short_fail_c.lock().unwrap() += 1;
}
}
thread::sleep(Duration::from_millis(50)); // ~20/s
}
});
// Give workers time to spawn, then assert two --call-lib-path children.
thread::sleep(Duration::from_secs(2));
let workers = count_call_lib_path_workers(site_pid);
assert!(
workers >= 2,
"expected ≥2 --call-lib-path workers, got {workers}"
);
// MR-02 / slot full: third execute must be rejected while A/B hold slots.
let (base_c, stop_c, join_c) = spawn_fake_sse("C", 5, 10);
let third = Command::new("curl")
.args([
"-s",
"-o",
"/tmp/mq-mr01-third.out",
"-w",
"%{http_code}",
"-H",
"Content-Type: application/json",
"-d",
&format!(
r#"{{"model":"mr01","stream":true,"base_url":"{base_c}","messages":[{{"role":"user","content":"x"}}]}}"#
),
&chat_url,
])
.output()
.expect("third curl");
let third_code = String::from_utf8_lossy(&third.stdout).trim().to_string();
assert!(
third_code == "429" || third_code == "503",
"third execute should be rejected, got {third_code}"
);
stop_c.store(true, Ordering::Relaxed);
let _ = join_c.join();
let _ = t_a.join();
let _ = t_b.join();
short_stop.store(true, Ordering::Relaxed);
let _ = t_short.join();
stop_a.store(true, Ordering::Relaxed);
stop_b.store(true, Ordering::Relaxed);
let _ = join_a.join();
let _ = join_b.join();
let _ = child.kill();
let _ = child.wait();
let ca = *code_a.lock().unwrap();
let cb = *code_b.lock().unwrap();
let da = deltas_a.lock().unwrap().clone();
let db = deltas_b.lock().unwrap().clone();
let ra = raw_a.lock().unwrap().clone();
let rb = raw_b.lock().unwrap().clone();
assert_eq!(ca, 200, "stream A http={ca} raw={ra}");
assert_eq!(cb, 200, "stream B http={cb} raw={rb}");
assert!(
!ra.contains("interpreter busy") && !ra.contains("sse bridge in use"),
"A busy: {ra}"
);
assert!(
!rb.contains("interpreter busy") && !rb.contains("sse bridge in use"),
"B busy: {rb}"
);
assert!(!da.is_empty(), "A produced no deltas; raw={ra}");
assert!(!db.is_empty(), "B produced no deltas; raw={rb}");
assert!(
da.iter().all(|t| t.starts_with('A')),
"A mixed foreign tokens: {da:?}"
);
assert!(
db.iter().all(|t| t.starts_with('B')),
"B mixed foreign tokens: {db:?}"
);
assert!(
da.iter().all(|t| !t.starts_with('B')),
"A saw B tokens"
);
assert!(
db.iter().all(|t| !t.starts_with('A')),
"B saw A tokens"
);
let sok = *short_ok.lock().unwrap();
let sfail = *short_fail.lock().unwrap();
assert!(sok > 10, "expected many short 200s, got ok={sok} fail={sfail}");
assert_eq!(sfail, 0, "short reads failed: {sfail}");
assert_eq!(
caps["features"]["runtime.read_lane"],
1,
"read_lane must be on; caps={caps}"
);
let _ = std::fs::remove_dir_all(&runtime_dir);
}
/// Observer disconnect must not cancel the execute task (default).
#[test]
fn disconnect_keeps_running_and_replay() {
if TcpListener::bind("127.0.0.1:0").is_err() {
eprintln!("skip disconnect-keeps-running: local sockets unavailable");
return;
}
ensure_plugins();
let secs: u64 = 6;
let (base_a, stop_a, join_a) = spawn_fake_sse("A", secs, 10);
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let site = root.join("tests/ext/isolated-execute-site");
let runtime_dir = std::env::temp_dir().join(format!("mq-disc-rt-{}", std::process::id()));
let _ = std::fs::create_dir_all(&runtime_dir);
let bin = env!("CARGO_BIN_EXE_marqdo");
let mut child: Child = Command::new(bin)
.args(["run", "serve.mq.md"])
.current_dir(&site)
.env("MARQDO_EXT", root.join("ext"))
.env("MARQDO_WEB_PLUGIN", std::env::var_os("MARQDO_WEB_PLUGIN").unwrap())
.env("MARQDO_LLM_PLUGIN", std::env::var_os("MARQDO_LLM_PLUGIN").unwrap())
.env("MARQDO_RUNTIME_DIR", &runtime_dir)
.env("MARQDO_SITE_PORT", "18221")
.env("OPENAI_API_KEY", "mr01-key")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("spawn site");
let base = "http://127.0.0.1:18221";
if !wait_http_ok(&format!("{base}/v1/models"), Duration::from_secs(30)) {
let _ = child.kill();
panic!("site did not start for disconnect test");
}
let chat_url = format!("{base}/v1/chat/completions");
let body = format!(
r#"{{"model":"mr01","stream":true,"base_url":"{base_a}","messages":[{{"role":"user","content":"hi"}}]}}"#
);
// Observer disconnects after ~1.2s; task must keep running.
let out = Command::new("curl")
.args([
"-s",
"-N",
"--max-time",
"1",
"-H",
"Content-Type: application/json",
"-d",
&body,
&chat_url,
])
.output()
.expect("curl disconnect");
let text = String::from_utf8_lossy(&out.stdout);
let mut task_id = String::new();
for line in text.lines() {
let line = line.trim();
if let Some(rest) = line.strip_prefix("data: ") {
if let Ok(v) = serde_json::from_str::<serde_json::Value>(rest) {
if v.get("type").and_then(|t| t.as_str()) == Some("run.started") {
if let Some(tid) = v.pointer("/payload/task_id").and_then(|x| x.as_str()) {
task_id = tid.to_string();
}
}
}
}
}
assert!(!task_id.is_empty(), "missing task_id in early SSE; raw={text}");
// Wait until task finishes (observer already gone).
let mut terminal = false;
let deadline = Instant::now() + Duration::from_secs(secs + 8);
while Instant::now() < deadline {
let get_url = format!("{base}/_runtime/tasks/{task_id}");
let out = Command::new("curl")
.args(["-s", &get_url])
.output()
.expect("runtime get");
let body = String::from_utf8_lossy(&out.stdout);
if body.contains("\"completed\"")
|| body.contains("\"succeeded\"")
|| body.contains("\"failed\"")
|| body.contains("\"cancelled\"")
{
assert!(
!body.contains("\"cancelled\""),
"disconnect must not cancel task: {body}"
);
terminal = true;
break;
}
thread::sleep(Duration::from_millis(200));
}
assert!(terminal, "task {task_id} did not finish after observer disconnect");
// Replay via events (JSON after_seq=0).
let ev_url = format!("{base}/_runtime/tasks/{task_id}/events?after_seq=0");
let ev = Command::new("curl")
.args(["-s", &ev_url])
.output()
.expect("events");
let ev_body = String::from_utf8_lossy(&ev.stdout);
assert!(
ev_body.contains('A') || ev_body.contains("delta") || ev_body.contains("event"),
"replay empty: {ev_body}"
);
stop_a.store(true, Ordering::Relaxed);
let _ = join_a.join();
let _ = child.kill();
let _ = child.wait();
let _ = std::fs::remove_dir_all(&runtime_dir);
}
/// Control + short reads stay 200 while two execute streams hold slots.
#[test]
fn control_under_dual_execute() {
if TcpListener::bind("127.0.0.1:0").is_err() {
eprintln!("skip control-under-dual-execute: local sockets unavailable");
return;
}
ensure_plugins();
let secs: u64 = 8;
let (base_a, stop_a, join_a) = spawn_fake_sse("A", secs, 10);
let (base_b, stop_b, join_b) = spawn_fake_sse("B", secs, 10);
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let site = root.join("tests/ext/isolated-execute-site");
let runtime_dir = std::env::temp_dir().join(format!("mq-ctl-rt-{}", std::process::id()));
let _ = std::fs::create_dir_all(&runtime_dir);
let bin = env!("CARGO_BIN_EXE_marqdo");
let mut child: Child = Command::new(bin)
.args(["run", "serve.mq.md"])
.current_dir(&site)
.env("MARQDO_EXT", root.join("ext"))
.env("MARQDO_WEB_PLUGIN", std::env::var_os("MARQDO_WEB_PLUGIN").unwrap())
.env("MARQDO_LLM_PLUGIN", std::env::var_os("MARQDO_LLM_PLUGIN").unwrap())
.env("MARQDO_RUNTIME_DIR", &runtime_dir)
.env("MARQDO_SITE_PORT", "18222")
.env("OPENAI_API_KEY", "mr01-key")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("spawn site");
let base = "http://127.0.0.1:18222";
if !wait_http_ok(&format!("{base}/v1/models"), Duration::from_secs(30)) {
let _ = child.kill();
panic!("site did not start for control test");
}
let chat_url = format!("{base}/v1/chat/completions");
let body_tmpl = r#"{"model":"mr01","stream":true,"base_url":"__BASE__","messages":[{"role":"user","content":"hi"}]}"#;
let t_a = {
let u = chat_url.clone();
let b = body_tmpl.to_string();
let ba = base_a.clone();
thread::spawn(move || {
let _ = collect_sse_deltas(&u, &b, &ba);
})
};
let t_b = {
let u = chat_url.clone();
let b = body_tmpl.to_string();
let bb = base_b.clone();
thread::spawn(move || {
let _ = collect_sse_deltas(&u, &b, &bb);
})
};
thread::sleep(Duration::from_secs(1));
let mut ok_n = 0u32;
let mut fail_n = 0u32;
let start = Instant::now();
while start.elapsed() < Duration::from_secs(3) {
for path in ["/v1/models", "/v1/ping"] {
let url = format!("{base}{path}");
let t0 = Instant::now();
let out = Command::new("curl")
.args(["-s", "-o", "/dev/null", "-w", "%{http_code}", &url])
.output();
let elapsed = t0.elapsed();
match out {
Ok(o) if String::from_utf8_lossy(&o.stdout) == "200" => {
assert!(
elapsed < Duration::from_millis(1500),
"{path} p95 budget; took {elapsed:?}"
);
ok_n += 1;
}
_ => fail_n += 1,
}
}
thread::sleep(Duration::from_millis(80));
}
assert!(ok_n > 4, "expected control/short 200s, ok={ok_n} fail={fail_n}");
assert_eq!(fail_n, 0, "control/short failed under dual execute: {fail_n}");
let _ = t_a.join();
let _ = t_b.join();
stop_a.store(true, Ordering::Relaxed);
stop_b.store(true, Ordering::Relaxed);
let _ = join_a.join();
let _ = join_b.join();
let _ = child.kill();
let _ = child.wait();
let _ = std::fs::remove_dir_all(&runtime_dir);
}
+170
View File
@@ -0,0 +1,170 @@
//! M-R7 smoke: admission → cancel CAS → event resume → publish journal recover.
//! Uses local fake/dry-run paths only (no live LLM).
use marqdo::runtime::{
cancel_task, AdmissionClass, ControlService, InvokeRequestV2, PublishJournal, PublishOutcome,
Scheduler, SchedulerConfig, Supervisor, SupervisorConfig, TaskState, TaskStore,
};
use marqdo::runtime::protocol::{EntryRef, InvokeContext, LeaseRef};
use serde_json::json;
use std::sync::Arc;
#[test]
fn mr07_admit_cancel_event_resume() {
let dir = std::env::temp_dir().join(format!("mq-mr07-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let store = Arc::new(TaskStore::with_persist(dir.join("store")).unwrap());
let sched = Arc::new(Scheduler::new(SchedulerConfig::default()));
let ctl = ControlService::new(Arc::clone(&store), Arc::clone(&sched), dir.join("cancel"));
let (t, depth) = ctl
.admit(
"task_smoke",
"run_smoke",
"ws",
Some("sess".into()),
"idem-smoke",
&json!({"prompt": "hi"}),
AdmissionClass::Execute,
)
.unwrap();
assert!(depth >= 1);
assert_eq!(t.state, TaskState::Queued);
let again = ctl
.admit(
"task_other",
"run_other",
"ws",
None,
"idem-smoke",
&json!({"prompt": "hi"}),
AdmissionClass::Execute,
)
.unwrap();
assert_eq!(again.0.task_id, t.task_id);
let conflict = ctl.admit(
"task_x",
"run_x",
"ws",
None,
"idem-smoke",
&json!({"prompt": "different"}),
AdmissionClass::Execute,
);
assert!(conflict.is_err());
let _ = store
.append_event(
&t.task_id,
&t.run_id,
&t.attempt_id,
"message.delta",
json!({"text": "a"}),
)
.unwrap();
let e2 = store
.append_event(
&t.task_id,
&t.run_id,
&t.attempt_id,
"message.delta",
json!({"text": "b"}),
)
.unwrap();
let resumed = ctl.events_since(&t.task_id, 1).unwrap();
assert_eq!(resumed.len(), 1);
assert_eq!(resumed[0].seq, e2.seq);
let c = cancel_task(&ctl, &t.task_id).unwrap();
assert_eq!(c.state, TaskState::CancelRequested);
assert_eq!(
std::fs::read_to_string(ctl.cancel_token_path(&t.task_id)).unwrap(),
"cancelled"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn mr07_supervisor_dry_run_slots_and_publish_recover() {
std::env::set_var("MARQDO_RUNTIME_DRY_RUN", "1");
let root = std::env::temp_dir().join(format!("mq-mr07b-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let bin = std::env::current_exe().unwrap();
let sup = Supervisor::new(
SupervisorConfig {
execute_slots: 2,
..SupervisorConfig::default()
},
root.join("stage"),
&bin,
root.join("cancel"),
root.join("events"),
);
sup.warm().unwrap();
let req = |id: &str| InvokeRequestV2 {
schema: "InvokeRequestV2".into(),
schema_version: 2,
task_id: id.into(),
run_id: format!("run_{id}"),
attempt_id: format!("a_{id}"),
lease: LeaseRef {
id: format!("l_{id}"),
epoch: 1,
},
entry: EntryRef {
module: "agent/qdagent.mq.md".into(),
function: "run".into(),
},
args_ref: None,
context: InvokeContext {
workspace_id: "ws".into(),
snapshot_id: None,
selection_ref: None,
capabilities_ref: None,
deadline_at: "2099-01-01T00:00:00Z".into(),
},
};
let a = sup.dispatch(req("a")).unwrap();
let b = sup.dispatch(req("b")).unwrap();
assert_eq!(sup.active_count(), 2);
assert!(sup.dispatch(req("c")).is_err());
sup.release(&a);
sup.release(&b);
// Publish journal recovery after generation rename (CURRENT still old).
let pub_root = root.join("kb");
std::fs::create_dir_all(pub_root.join("generations/g1")).unwrap();
std::fs::write(pub_root.join("CURRENT"), "old\n").unwrap();
let body = serde_json::json!({
"kb_id": "kb",
"expected_generation": "old",
"new_generation": "g1",
"phase": "generation_renamed",
"staging_dir": pub_root.join(".publish-staging/g1"),
"generation_dir": pub_root.join("generations/g1"),
"current_path": pub_root.join("CURRENT"),
"source_digest": "d"
});
std::fs::write(
pub_root.join("publish_journal.json"),
serde_json::to_vec_pretty(&body).unwrap(),
)
.unwrap();
let out = PublishJournal::recover(&pub_root).unwrap();
assert_eq!(
out,
PublishOutcome::Recovered {
generation: "g1".into()
}
);
let caps = marqdo::workbench::probe_capabilities(&json!({}));
assert_eq!(caps["features"]["runtime.task_control_v2"], 1);
assert_eq!(caps["features"]["runtime.event_replay"], 1);
assert_eq!(caps["features"]["runtime.isolated_execution"], 1);
let _ = std::fs::remove_dir_all(&root);
std::env::remove_var("MARQDO_RUNTIME_DRY_RUN");
}
@@ -0,0 +1,14 @@
{
"schema": "AgentErrorV1",
"schema_version": 1,
"code": "tool.timeout",
"category": "transient",
"stage": "tool",
"message": "读取资源超时",
"retryable": true,
"retry_after_ms": 1200,
"effect_state": "none",
"recovery_action": "retry_step",
"call_id": "call_01",
"details_ref": "resource:diagnostic_01"
}
@@ -0,0 +1,15 @@
{
"schema": "AgentErrorV1",
"schema_version": 99,
"code": "tool.timeout",
"category": "transient",
"stage": "tool",
"message": "读取资源超时",
"retryable": true,
"retry_after_ms": 1200,
"effect_state": "none",
"recovery_action": "retry_step",
"call_id": "call_01",
"details_ref": "resource:diagnostic_01",
"future_field": true
}
Loaded 100 of 139 files, more files were not shown because too many files have changed in this diff. Show more