diff --git a/CHANGELOG.md b/CHANGELOG.md index c0d8600..60389f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,11 @@ ## Unreleased ### Added +- **宿主集成 H1–H4**:`ext/web` `app.proxy`(同域 SSE/HTTP 流式中继)与 `app.invoke`(HTTP→`lib.member`);跨模块 `#` 对象方法查找;`agent.corpus_search` 自动加载插件;`agent.mcp_server` stdio。金样 `web-proxy-invoke-smoke` · `web-hosting-live` · `web-db-cross-module-smoke` · `agent-mcp-server-smoke`。设计 [ext-hosting-fill.md](doc/design/ext-hosting-fill.md)。 +- **文档**:宿主集成缺口盘点与补齐设计——[ext-hosting-gaps.md](doc/research/ext-hosting-gaps.md) · [ext-hosting.md](doc/roadmap/ext-hosting.md)。 ### Fixed +- **方法分发**:`find_object_type` 沿导入树传递查找,跨模块拿到的 `web.db` 等句柄可再调方法。 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 453e366..b5e2b88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,6 +317,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.2" @@ -631,6 +637,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + [[package]] name = "futures-macro" version = "0.3.33" @@ -661,9 +673,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -694,8 +708,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -717,9 +733,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -848,6 +866,23 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls 0.23.44", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.9", ] [[package]] @@ -856,13 +891,21 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64", "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", + "socket2 0.6.5", "tokio", "tower-service", + "tracing", ] [[package]] @@ -939,6 +982,12 @@ dependencies = [ "quote", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1022,9 +1071,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "marqdo" -version = "0.3.5" +version = "0.3.6" dependencies = [ "anyhow", "assert_cmd", @@ -1055,7 +1110,7 @@ dependencies = [ [[package]] name = "marqdo-wasm" -version = "0.3.5" +version = "0.3.6" dependencies = [ "marqdo", "serde_json", @@ -1098,6 +1153,7 @@ version = "0.1.0" dependencies = [ "argon2", "axum", + "bytes", "futures-util", "getrandom 0.2.17", "hex", @@ -1105,6 +1161,7 @@ dependencies = [ "postgres", "pulldown-cmark", "redis", + "reqwest", "rusqlite", "serde", "serde_json", @@ -1448,6 +1505,62 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.44", + "socket2 0.6.5", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls 0.23.44", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.5", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -1545,6 +1658,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redis" version = "0.27.6" @@ -1601,6 +1723,47 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.44", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.9", +] + [[package]] name = "ring" version = "0.17.14" @@ -1629,6 +1792,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustls" version = "0.22.4" @@ -1638,7 +1807,21 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls" +version = "0.23.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.103.15", "subtle", "zeroize", ] @@ -1649,6 +1832,7 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ + "web-time", "zeroize", ] @@ -1663,6 +1847,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -1902,6 +2097,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "termtree" @@ -2029,6 +2227,16 @@ dependencies = [ "whoami", ] +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls 0.23.44", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.24.0" @@ -2105,9 +2313,11 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", + "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -2142,6 +2352,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "tungstenite" version = "0.24.0" @@ -2243,9 +2459,9 @@ dependencies = [ "flate2", "log", "once_cell", - "rustls", + "rustls 0.22.4", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "url", "webpki-roots 0.26.11", ] @@ -2301,6 +2517,15 @@ dependencies = [ "libc", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2347,6 +2572,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.127" @@ -2379,6 +2614,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.104" @@ -2389,6 +2637,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.11" diff --git a/doc/README.md b/doc/README.md index 906ce0f..0fbce99 100644 --- a/doc/README.md +++ b/doc/README.md @@ -71,6 +71,9 @@ | [research/agent-frameworks-and-marqdo.md](research/agent-frameworks-and-marqdo.md) | 主流智能体框架 vs Marqdo 优势 | | [research/agent-framework-gaps-after-a4.md](research/agent-framework-gaps-after-a4.md) | **`ext/agent` A0–A4 之后缺口盘点** | | [research/agent-framework-2026-09.md](research/agent-framework-2026-09.md) | **2026-09 刷新:宪法优先 Wave B** | +| [research/ext-hosting-gaps.md](research/ext-hosting-gaps.md) | **宿主集成缺口**:同域 SSE 中继 · HTTP→`##` · MCP Server · 副端口 · 跨模块 db/corpus | +| [design/ext-hosting-fill.md](design/ext-hosting-fill.md) | **宿主缺口补齐设计**(`ext/web` proxy/invoke · `ext/ai` mcp_server) | +| [roadmap/ext-hosting.md](roadmap/ext-hosting.md) | **宿主集成 H0–H4**(规划) | | [roadmap/ext-agent-optimize.md](roadmap/ext-agent-optimize.md) | **`ext/agent` A0–A4 done · Wave B Active** | 金样例与集成测试:[`tests/`](../tests/)。面向访客的可执行文档:[`public/`](../public/)。Spike 存档:[`spike/`](../spike/)。 diff --git a/doc/design/ext-hosting-fill.md b/doc/design/ext-hosting-fill.md new file mode 100644 index 0000000..7829bc1 --- /dev/null +++ b/doc/design/ext-hosting-fill.md @@ -0,0 +1,306 @@ +# 设计:宿主集成缺口补齐(`ext/web` · `ext/ai`) + +| | | +|---|---| +| 状态 | **Accepted · H1–H4 stdio 已落地**(H4b HTTP 可选) | +| 日期 | 2026-09-07 | +| 缺口盘点 | [ext-hosting-gaps.md](../research/ext-hosting-gaps.md) | +| 路线图 | [ext-hosting.md](../roadmap/ext-hosting.md) | +| 边界 | [ext-web-net.md](ext-web-net.md) · [ext-abi.md](ext-abi.md) · [ext-agent.md](ext-agent.md) | +| 目标 | 用**扩展库一等 API**关闭 GAP-01–05,取代下游 Python 旁路 | + +--- + +## 0. 一句话 + +**服务器中继与「请求→`##`」进 `ext/web`;MCP Server 与 corpus 可发现进 `ext/ai`;标准库只保留已有客户端 SSE。** 作者用 GFM 表声明路由/工具,不写 JS、不启第二语言运行时。 + +--- + +## 1. 分层与禁止项 + +| 层 | 职责 | 落点 | +|----|------|------| +| 作者面 | 声明路径、上游、函数名、工具表 | `ext/web/web.mq.md` · `ext/ai/agent.mq.md`(+ 中文对偶) | +| ABI | axum 流式代理、每请求 interp 调用、MCP 传输 | `plugins/web` · `plugins/agent` | +| 标准库 | 不变:缓冲客户端 SSE / 解析 | `lib/net`(**不**加中继) | +| 核心 host | **禁止**新增 agent/web 领域 `host_*` | 见 Skill 硬规则 9 | + +**禁止:** + +1. 为中继在浏览器手写业务 JS。 +2. `json=` 继续塞「任意 `##`」语义(保持 DB-only;新能力用独立装配名)。 +3. MCP Server 把向量库升为权威真相(工具结果可带 `authority=workbook`)。 +4. 无白名单的「任意文件系统路径 + 任意 `##`」远程触发。 + +--- + +## 2. GAP-01 · `app.proxy` 流式反向代理 + +### 2.1 作者 API + +```markdown +*应用 = > `应用`.proxy path="/v1/chat/completions" upstream=$OPENAI_BASE_URL* +*应用 = > `应用`.proxy path="/llm/stream" upstream="https://api.example.com" strip_prefix="/llm" stream=True* +``` + +或装配表(一次挂多条): + +```markdown +*应用 = > `应用`.configure proxy=`代理表`* + +| path | upstream | stream | strip_prefix | headers_from_env | +|------|----------|--------|--------------|------------------| +| /proxy/chat | $OPENAI_BASE_URL | True | | OPENAI_API_KEY=Authorization | +``` + +| 参数 | 默认 | 含义 | +|------|------|------| +| `path` | 必填 | 浏览器同域路径 | +| `upstream` | 必填 | 上游根或完整 URL;支持 `$ENV` | +| `stream` | `True` | `True`:响应按上游 `Content-Type` 管道转发(含 `text/event-stream`) | +| `strip_prefix` | 空 | 转发前去掉本站前缀 | +| `methods` | `POST` | 允许方法 | +| `headers_from_env` | 空 | `ENV=Header-Name` 注入(密钥不进前端) | +| `timeout_ms` | 插件默认 | 上游超时 | + +中文:`## 代理` / `流式=` / `上游=` / `去前缀=`。 + +### 2.2 ABI / 插件行为 + +| 步骤 | 行为 | +|------|------| +| 注册 | `web_app_proxy` 写入 `AppState` 路由表 | +| 请求 | 读 body + 过滤危险 hop-by-hop 头;合并 env 注入头 | +| 上游 | `reqwest`/`hyper` 流式客户端(插件内,不进核心) | +| 响应 | 复制状态码与 `content-type`;`body` 为 stream;SSE 不整段缓冲 | +| CORS | 仍走既有 `configure cors=`(同域调用时浏览器不依赖上游 CORS) | + +### 2.3 与 `lib/net.http_post_sse` 关系 + +| API | 角色 | +|-----|------| +| `http_post_sse` | **出站消费者**(CLI/agent 侧收齐 events) | +| `app.proxy` | **入站中继**(浏览器 ← Marqdo ← 上游) | + +二者互补,不互相替代。 + +### 2.4 验收 + +- 金样:mock 上游发两帧 SSE,浏览器侧(或 `curl -N`)收到两帧且中间无全缓冲延迟门槛。 +- 密钥仅出现在服务端 env 注入头。 +- 金样名建议:`tests/ext/web-proxy-sse-smoke.mq.md`。 + +--- + +## 3. GAP-02 · `app.invoke` / `configure invoke=`(HTTP → `##`) + +### 3.1 作者 API + +```markdown +*应用 = > `应用`.invoke path="/api/沉淀" method=POST fn="run.沉淀"* +``` + +表形式: + +```markdown +*应用 = > `应用`.configure invoke=`调用表`* + +| path | method | fn | body | return | +|------|--------|-----|------|--------| +| /api/note | POST | run.写出 | json | json | +| /api/search | GET | db.recent | query | json | +``` + +| 列 / 参数 | 含义 | +|-----------|------| +| `path` / `method` | HTTP 路由 | +| `fn` | `库.成员`(入口模块已 `import`)或 `模块路径##标题`(白名单前缀内) | +| `body` | `json` \| `form` \| `query` \| `raw` → 映射到 `##` 形参(同名优先;其余进 `payload` 袋) | +| `return` | `json`(默认)\| `text` \| `status` | + +中文:`## 调用` / `函数=` / `正文=` / `返回=`。 + +### 3.2 安全默认(锁定) + +1. **仅**解析入口 workbook 已导入的 `bind` 路径,或 `invoke_allow=` 显式前缀列表。 +2. 单请求超时 + 可选并发上限(防 listen 线程饿死)。 +3. 默认不启;作者显式 `invoke` / `configure invoke=`。 +4. 错误:`{ok:false, error}` + HTTP 4xx/5xx;不把内部路径栈默认回传生产模式。 + +### 3.3 ABI / 插件行为 + +| 步骤 | 行为 | +|------|------| +| 注册 | `web_app_invoke` 存 `(path, method, FnRef)` | +| 命中 | 解析 body → Value 袋;`call_lib_path` / 等价「同步调用已加载模块的 `##`」 | +| 返回 | Value → JSON(复用现有 host JSON 通路);侧效(写盘)在同一解释器事务语义内尽最大努力 | + +**不**把 `json=` DB 路由改语义;`invoke` 是平行装配。 + +### 3.4 与 compose 期 `call_lib_path` 关系 + +| 时机 | API | +|------|-----| +| 装配页 | 现有 compose / `call_lib_path` | +| **每个 HTTP 请求** | 本设计 `invoke` | + +### 3.5 验收 + +- `POST` JSON → 用户 `##` 写入临时文件 + 返回 `{ok:true,…}`。 +- 未导入路径 → 4xx,不执行。 +- 金样:`tests/ext/web-invoke-smoke.mq.md`。 + +--- + +## 4. GAP-03 · MCP Server(`ext/ai`) + +### 4.1 作者 API + +```markdown +*服务 = > agent.mcp_server name="marqdo-qd"* +*服务 = > `服务`.tool name="qd_search" fn="求道.搜索" description="关键词检索"* +*服务 = > `服务`.tool name="qd_capture" fn="求道.捕捉"* +*服务 = > `服务`.serve transport=stdio* +# 或 +*服务 = > `服务`.serve transport=http host=127.0.0.1 port=7433* +``` + +工具表: + +| name | fn | description | +|------|-----|-------------| +| qd_search | 求道.搜索 | … | +| qd_list | 求道.列出 | … | + +### 4.2 传输 + +| transport | 用途 | +|-----------|------| +| `stdio` | Cursor / Claude Desktop 拉起 | +| `http` | Streamable HTTP(与官方 MCP 演进对齐;首版可先 SSE+POST 子集) | + +### 4.3 与 A4 fixture 客户端关系 + +| API | 方向 | +|-----|------| +| `mcp_list_tools` / `mcp_call` | **出站**(Agent 调外部 MCP;现 fixture) | +| `mcp_server` + `serve` | **入站**(外部调 Marqdo) | + +客户端「真 MCP」仍见 agent 缺口文;本设计先交 Server,fixture 可保留测客户端。 + +### 4.4 ABI + +- `plugins/agent`:`agent_mcp_server_*`(注册工具、stdio loop、可选 HTTP)。 +- 工具实现:复用 GAP-02 同款「调用 `##`」内核(可抽共享 `invoke_fn`),避免两套分发。 +- 返回包装:可含 `authority=workbook`。 + +### 4.5 验收 + +- 金样:stdio 夹具发 `tools/list` + `tools/call`,断言 JSON-RPC 形结果。 +- 可选:`tests/ext/agent-mcp-server-smoke.mq.md`。 + +--- + +## 5. GAP-04 · 同寿副端口 + +### 5.1 关闭策略(优先) + +GAP-01 + GAP-02 挂在**同一** `app.listen` → **不实现副端口**,文档宣布 GAP-04 关闭。 + +### 5.2 备选 API(仅当必须隔离) + +```markdown +*应用 = > `应用`.listen host=127.0.0.1 port=18081 also_port=7432 also_routes=`中继表`* +``` + +`also_routes` 仅允许 `proxy` / `invoke` 已注册子集。实现成本高,**默认不做**。 + +--- + +## 6. GAP-05 · 跨模块 `db` 与 corpus 可发现 + +### 6.1 `db` 句柄(扩展约定 + 运行时) + +**作者约定(立即文档化):** + +1. 跨模块优先传 **`url` 字符串**,调用方 `*store = > web.db url=…*` 再 `select`。 +2. 若传对象:必须保留 `_type` 为插件注册名(与 `web.db` 构造一致)。 + +**运行时修补(H 波次):** + +| 选项 | 做法 | 选用 | +|------|------|------| +| A | 跨模块返回插件对象时规范 `_type` + 全局方法表查找 | **首选**(一次修好所有插件对象) | +| B | `db.open` 只返回 `{url}` 朴素 map,禁止「裸 select」跨文件 | 文档契约;体验较差 | + +金样:模块 A `open` → 返回 → 模块 B `select` 成功。 +`tests/ext/web-db-cross-module-smoke.mq.md`。 + +### 6.2 `corpus_search` 可发现 + +```markdown +## corpus_search ++ `query` ++ `root`=".marqdo/agent-kb" + +* > plugin.load name=agent * +** > agent_corpus_search query=`query` root=`root` ** +``` + +或在 `# agent` / `## corpus_search` 入口统一 `ensure_plugin`(与现 `agent` 构造对齐),保证 **纯 `marqdo run` + import agent** 即可用。 + +**不**把 `agent_corpus_search` 搬进 `src/host/`。 + +### 6.3 验收 + +- 无显式手工 ABI 名的情况下,`marqdo run` 金样调用 `agent.corpus_search` 成功。 +- 跨模块 `db.select` 金样绿。 + +--- + +## 7. 扩展库表面清单(落地时改这些文件) + +| 文件 | 增补 | +|------|------| +| `ext/web/web.mq.md` · `网页.mq.md` | `## proxy` / `## invoke`;`configure` 增 `proxy`/`invoke` 列 | +| `plugins/web/src/http.rs` · `middleware.rs` · 新 `proxy.rs` / `invoke.rs` | 路由注册与流式/调用 | +| `ext/ai/agent.mq.md` · `智能体.mq.md` | `# mcp_server` · `## tool` · `## serve`;`corpus_search` ensure_plugin | +| `plugins/agent/` | MCP Server 传输 + 共享 invoke | +| `tests/ext/web-proxy-sse-smoke.mq.md` 等 | 见上 | +| `doc/roadmap/ext-web.md` | 链到 H 波次(不重开 W0–W7) | +| `public/` / Skill | 发版时补「同域 LLM 代理 / HTTP 调用 ##」短节 | + +**不**新增 `lib/run` 作为核心依赖:用户业务 `##` 仍在其自己的 `lib/*.mq.md`;Marqdo 只提供 **invoke 管道**。 + +--- + +## 8. 迁移:下游旁路 → 扩展 API + +| 旁路(下游) | 替换为 | +|--------------|--------| +| `llm_proxy.py` `/proxy/stream` | `app.proxy` / `configure proxy=` | +| `llm_proxy.py` `/store/*` + `marqdo run` | `app.invoke` → 用户 `##` | +| `mq_bridge.py` 仅过滤 | 用户 `##` 内 `table`/`re` 过滤,或修复后的 `corpus_search` | +| `qdagent_mcp.py` | `agent.mcp_server` + `serve` | +| `mq.sh` 双进程 | 单 `marqdo run index.mq.md`(proxy+invoke 同端口) | + +--- + +## 9. 开放点(实现前拍板) + +| # | 问题 | 默认倾向 | +|---|------|----------| +| 1 | `invoke` 的 `fn` 是否允许 `path.mq.md##标题` 字面量 | 允许但必须 `invoke_allow` 前缀 | +| 2 | MCP HTTP 首版范围 | stdio 先交;HTTP 跟一波 | +| 3 | 代理是否缓冲「非 SSE」大 body | 非流式可限长缓冲;SSE 禁止整段缓冲 | +| 4 | GAP-05 选项 A vs B | **A** | + +--- + +## 10. 完成定义(整包) + +1. 无 Python 旁路即可:浏览器同域 SSE 对话 + HTTP 触发沉淀 `##` +(可选)MCP stdio 暴露同一批 `##`。 +2. 金样覆盖 proxy / invoke / cross-db / corpus / mcp-server(stdio)。 +3. `doc/research/ext-hosting-gaps.md` 各 GAP 标为关闭或降级为「备选副端口不做」。 +4. 发版说明指向 `ext/web` · `ext/ai` 新作者面,不指向 `scripts/*.py`。 diff --git a/doc/design/web-net-capabilities.md b/doc/design/web-net-capabilities.md index 4a4f740..8706781 100644 --- a/doc/design/web-net-capabilities.md +++ b/doc/design/web-net-capabilities.md @@ -13,6 +13,8 @@ **`ext/web` + `plugins/web` 已完成 W0–W7 + P3 路线图**:中间件/JSON API、数据层(连接池/事务/分页/FTS/迁移/外键/审计时间戳)、安全硬化(argon2/CSRF/会话持久化/限速/RBAC)、内容站点标配(SEO/RSS/Markdown/分页 UI)、上传下载/相册/ETag、sitemap/robots/错误页/重定向、WebSocket 广播与访问日志。足以在 Marqdo 上实现博客/CMS/中小型 API 站点;**未内置**的仅剩标签页模板(D6,可用路由+`db.count` 自建)与应用层反垃圾。边界不变:**纯解析进 `lib/net`,HTTP 服务器与领域能力进 `plugins/web` + `ext/web` 作者面**。 +**宿主集成下一刀**(同域 LLM SSE 中继、HTTP→用户 `##`、与 MCP Server 协作)不在本 W 表内,见 [ext-hosting-gaps.md](../research/ext-hosting-gaps.md) · [ext-hosting-fill.md](ext-hosting-fill.md) · [ext-hosting.md](../roadmap/ext-hosting.md)。 + --- diff --git a/doc/research/agent-framework-gaps-after-a4.md b/doc/research/agent-framework-gaps-after-a4.md index b17fc4a..2488291 100644 --- a/doc/research/agent-framework-gaps-after-a4.md +++ b/doc/research/agent-framework-gaps-after-a4.md @@ -70,6 +70,7 @@ | **高** | 官方端到端示例 + 文档新鲜度 | 无独立 `examples/agent-*`;Skill/参考里仍有「layout / roadmap」陈旧表述;外人难「照抄即跑通 plan→固化→二次命中」 | `examples/agent-pong`(或同类)+ README;Skill 专节与 A0–A4 对齐 | | **高** | Agent 评测 / 回归 harness | 有离线金样,缺成本·恢复·命中率·补丁质量的系统评测;业界强调 evaluation 先于上线 | `tests/` 或脚本:命中率、llm_free 占比、失败恢复、可选 live 预算 | | **高** | 真 MCP(HTTP/SSE)与凭证 | A4 仅 JSON fixture;缺 streamable HTTP、鉴权、resources/prompts | 金样打 mock MCP server;返回仍带 `authority=workbook` | +| **高** | **MCP Server 宿主**(stdio / HTTP) | Agent 只能当客户端 fixture;IDE 无法把 `.mq.md` `##` 暴露为 MCP 工具 | 见 [ext-hosting-gaps.md](ext-hosting-gaps.md) GAP-03 · [ext-hosting-fill.md](../design/ext-hosting-fill.md) | | **中** | Plan 断点续跑 | 工作簿在盘上,缺显式 checkpoint / resume(对标 LangGraph checkpointer) | `plan resume=` 从第 N 轮 DECISION 继续;进程崩溃可恢复 | | **中** | 人机协同(HITL)加深 | 仅有 `confirm=True` 建簿即停;缺审批卡、拒绝重开、view 一键继续 | view 审批控件 + 金样 | | **中** | 飞轮可观测指标 | `hits` / `improve_every` / `llm_free` 有字段,缺汇总与 CLI/view 报表 | `agent_kb_stats` 或 view 面板:命中率、探索浪费 | diff --git a/doc/research/ext-hosting-gaps.md b/doc/research/ext-hosting-gaps.md new file mode 100644 index 0000000..ffc3498 --- /dev/null +++ b/doc/research/ext-hosting-gaps.md @@ -0,0 +1,156 @@ +# 调研:宿主集成缺口(浏览器中继 · HTTP→`##` · MCP Server · 同寿副端口 · 跨模块 DB/检索) + +| | | +|---|---| +| 状态 | **调研笔记 · 缺口盘点(H1–H4 stdio 已落地;H4b HTTP 可选)** | +| 日期 | 2026-09-07 | +| 触发 | 下游应用(对话站 / 求道类 Agent)在真实集成中被迫旁路 Python 脚本 | +| 基线版本 | **v0.3.6+**(宿主波次 H1–H4 见 [ext-hosting.md](../roadmap/ext-hosting.md)) | +| 相关设计 | [ext-web.md](../design/ext-web.md) · [ext-web-net.md](../design/ext-web-net.md) · [web-net-capabilities.md](../design/web-net-capabilities.md) · [ext-agent.md](../design/ext-agent.md) · [ext-abi.md](../design/ext-abi.md) | +| 邻近缺口 | [agent-framework-gaps-after-a4.md](agent-framework-gaps-after-a4.md)(真 MCP **客户端**) | +| 补齐设计 | [ext-hosting-fill.md](../design/ext-hosting-fill.md) | +| 实现路线 | [ext-hosting.md](../roadmap/ext-hosting.md) | + +--- + +## 0. 一句话结论 + +`ext/web` 已能做**内容站 / CRUD / JSON-DB API / WS**;`ext/ai` 已能做**工作簿 Agent + 客户端 SSE 消费**。下游要把「浏览器同域流式 LLM」「请求触发任意 `##`」「MCP Server 被 IDE 拉起」合进**单一 Marqdo 进程**时,现有扩展**没有一等装配**——只能外挂缓冲代理 / CLI 子进程。缺口集中在 **扩展库作者面 + `plugins/web|agent` ABI**,不是语法层,也不是再开一套标准库。 + +**原则:** 旁路脚本只证明需求;关闭缺口必须把能力沉进 `ext/*` + `plugins/*`(可选极薄 `lib/` 编排),禁止把领域逻辑塞进 `src/host/`。 + +--- + +## 1. 已有基线(不是缺口) + +| 能力 | 现状 | 文档 / 金样 | +|------|------|-------------| +| CORS / 安全头 / gzip / body limit | `app.configure` | `tests/ext/web-middleware-smoke.mq.md` | +| JSON **DB select** 路由 | `configure json=` | `plugins/web/src/middleware.rs` | +| `/_form/*` SQL CRUD | `mount_form` | `plugins/web/src/form.rs` | +| 客户端缓冲 SSE | `lib/net.http_post_sse` → `ext/llm.complete stream=` | `tests/lib/net-openai-sse.mq.md` | +| 执行过程 SSE(非 LLM 中继) | `marqdo view` `/api/run` | [view.md](../design/view.md) §5 | +| Agent 证据工具(fixture MCP) | `corpus_search` / `mcp_list_tools` / `mcp_call` | A4;[agent-framework-gaps-after-a4.md](agent-framework-gaps-after-a4.md) | +| 同模块 `# db` CRUD | `ext/web` `# db` | `examples/web-site/db/` | + +--- + +## 2. 缺口总表 + +| ID | 缺口 | 需要 | Marqdo 现状 | 典型旁路 | 解除条件 | +|----|------|------|-------------|----------|----------| +| **GAP-01** | 浏览器 → 上游 LLM **同域流式中继** | 绕过对方 CORS;**SSE 原样转发** | **`app.proxy` / `configure proxy=`**(H1) | ~~`llm_proxy.py`~~ | ✅ 已关闭 | +| **GAP-02** | HTTP 处理函数可调用用户 `##` | 请求内执行用户 `##`,返回 JSON | **`app.invoke` / `configure invoke=`**(H2) | ~~外挂 `marqdo run`~~ | ✅ 已关闭 | +| **GAP-03** | **MCP Server** 宿主 | IDE 拉起暴露工具 | **`agent.mcp_server` + `serve` stdio**(H4);HTTP=H4b | ~~`qdagent_mcp.py`~~ | ✅ stdio;HTTP 可选 | +| **GAP-04** | 同寿「第二监听端口」 | 与 `listen` 同寿中继 | 并入主端口(H1+H2) | ~~`mq.sh` 双进程~~ | ✅ won't | +| **GAP-05** | 跨模块 `db`;纯 run 用 corpus | 跨文件 `select`;CLI 检索 | 传递归查找 + `corpus_search` ensure_plugin(H3) | ~~db/index 导出~~ | ✅ 已关闭 | + +旁路脚本名来自下游集成仓库;**本仓库 `scripts/` 不含这些文件**——它们是需求证据,不是上游实现。 + +--- + +## 3. 分项说明 + +### 3.1 GAP-01 · 同域 SSE 中继 + +**问题** +浏览器页面跑在 `ext/web` 源上时,直连 OpenAI 兼容上游会撞 CORS;`lib/net.http_post_sse` 只在 **CLI/宿主侧缓冲整段流**,不能把字节边收边推回浏览器。 + +**现状边界** + +| 已有 | 缺什么 | +|------|--------| +| `app.configure cors=` | 解决的是**本站对外** CORS,不是替上游开 CORS | +| `http_post_sse` → `{status, events}` | **客户端消费**,非服务器中继 | +| `app.download` 对象流 | 存储下载,非上游 SSE | +| view `/api/run` SSE | Marqdo **执行事件**,非 LLM token 流 | + +**解除形态(产品)** +作者声明:`path` + `upstream`(或 env)+ 可选 header 白名单;监听端对浏览器返回 `Content-Type: text/event-stream`,chunk 原样或按 OpenAI SSE 帧转发。 + +### 3.2 GAP-02 · 路由 → `##` + +**问题** +对话沉淀、表单后写 `data/runs/*.mq.md`、触发 `lib/run.沉淀` 等,需要**同进程**执行用户函数。今天只有: + +1. 页面装配期 `call_lib_path`(compose,非每请求); +2. `/_form` / `json=` 的 **DB 专用** handler。 + +**解除形态(产品)** +声明式:`path` · `method` · `fn=lib.member` 或 `file=… ## name` · body→形参映射 · 返回值序列化为 JSON(错误码约定)。安全默认:仅允许入口模块已导入路径;禁止任意路径注入。 + +### 3.3 GAP-03 · MCP Server + +**问题** +A4 解决的是 Agent **调用** MCP 工具(且仍是 fixture)。下游要把 Marqdo 工作簿 **暴露为** MCP Server(stdio / Streamable HTTP),IDE 拉起后调 `qd_*`。 + +与 [agent-framework-gaps-after-a4.md](agent-framework-gaps-after-a4.md)「真 MCP 客户端」正交:本缺口是 **Server 宿主**。 + +**解除形态(产品)** +`ext/ai`(或 `ext/mcp`)提供:`server tools=` 表(name → `.mq.md`/`##`);`serve stdio=` / `serve http port=`;工具结果仍可带 `authority=workbook`。 + +### 3.4 GAP-04 · 同寿副端口 + +**问题** +在 GAP-01/02 未并入主端口前,下游用第二端口跑代理。`app.listen` 同步阻塞,无「同进程再绑一个 TcpListener」的作者 API;只能 shell 并起另一运行时。 + +**解除形态(产品)** +优先:GAP-01/02 挂在同一 `listen` → **本缺口关闭**。备选:`app.attach` / `listen also=` 同 tokio 运行时附加服务(仅当必须隔离路径树)。 + +### 3.5 GAP-05 · 跨模块 DB 句柄与 corpus 可发现性 + +**问题** + +1. **类型分发:** 插件对象依赖 `_type` / 方法表;跨模块传回的 store 若丢类型或未注册方法,报 `unknown object type db`。同文件 `db/index.mq.md` 内 `## recent` 再导出 JSON 是现行绕过。 +2. **`agent_corpus_search`:** 注册在 `plugins/agent`;普通 `marqdo run` 未 `plugin.load` / 未构造 `# agent` 时不可用。导入 `ext/ai/agent.mq.md` 文本本身不保证原生插件已加载。 + +**解除形态(产品)** +句柄:跨模块返回值保留 `_type=db` 且方法分发查全局/插件表(修运行时或约定「只返回 url,调用方再 `web.db`」二选一,见补齐设计)。corpus:`plugin.load` 自动路径,或 `ext/ai` 导出独立 `## corpus_search` 并在调用前 `ensure_plugin`。 + +--- + +## 4. 归属判定(标准库 vs 扩展) + +沿用 [ext-web-net.md](../design/ext-web-net.md) / [web-net-capabilities.md](../design/web-net-capabilities.md) 边界: + +| 能力 | 归属 | 理由 | +|------|------|------| +| SSE **中继**路由、HTTP→`##`、副端口 | **`plugins/web` + `ext/web`** | 服务器结构 / 网站领域;axum 已在依赖链 | +| MCP **Server**、corpus 可发现 | **`plugins/agent` + `ext/ai`** | Agent/工具宿主;禁止塞 `src/host/` | +| 缓冲 SSE 客户端、解析 | 已有 **`lib/net`** | 不新增中继到 stdlib | +| 任意路径 shell 出去跑 `marqdo` | **刻意不做进核心** | 旁路脚本可留在下游;上游给一等 API 后废弃 | + +--- + +## 5. 优先级建议 + +| 优先级 | ID | 理由 | +|--------|-----|------| +| **高** | GAP-01 + GAP-02 | 去掉 Python 中继的主路径;浏览器站 + 沉淀闭环 | +| **高** | GAP-03 | IDE 生态入口;与「真 MCP 客户端」可并行设计、分波交付 | +| **中** | GAP-05 | 减少 `db/index` 与 bridge 胶水;影响面偏语言/插件加载 | +| **低(可随 01/02 关闭)** | GAP-04 | 并入主端口后自然消失 | + +--- + +## 6. 明确「不是缺口」 + +| 项 | 说明 | +|----|------| +| 内容站 W0–W7+P3 | 已完结;本清单不重开上传/RSS/RBAC | +| Mid M1–M6 | 通用原语已够;本清单不是再开 `lib/re` 一类 | +| 隐藏 JSON TOOL 主循环 | 与 Agent 宪法冲突 | +| Dense embedding 默认化 | 仍见 [okf-near-match.md](../roadmap/okf-near-match.md) | +| 浏览器手写业务 JS 代理 | 与 WASM 路线「作者零 JS」冲突;中继应在服务端 `ext/web` | + +--- + +## 7. 与其它文档 + +| 文档 | 关系 | +|------|------| +| [ext-hosting-fill.md](../design/ext-hosting-fill.md) | **补齐设计(作者 API + ABI)** | +| [ext-hosting.md](../roadmap/ext-hosting.md) | 实现波次 H0–H4 | +| [ext-web.md](../roadmap/ext-web.md) | 站点主线完结;本缺口为后续宿主波次 | +| [agent-framework-gaps-after-a4.md](agent-framework-gaps-after-a4.md) | 客户端真 MCP;与 GAP-03 Server 互补 | +| [web-asgi-servers-and-marqdo.md](../design/web-asgi-servers-and-marqdo.md) | 反代模型;中继仍在 Marqdo `listen` 内 | diff --git a/doc/roadmap/ext-hosting.md b/doc/roadmap/ext-hosting.md new file mode 100644 index 0000000..06c1965 --- /dev/null +++ b/doc/roadmap/ext-hosting.md @@ -0,0 +1,102 @@ +# 路线图:宿主集成补齐(H0–H4) + +| | | +|---|---| +| 状态 | **Active · 规划** | +| 日期 | 2026-09-07 | +| 缺口 | [ext-hosting-gaps.md](../research/ext-hosting-gaps.md) | +| 设计 | [ext-hosting-fill.md](../design/ext-hosting-fill.md) | +| 相关 | [ext-web.md](ext-web.md)(W/C 完结)· [ext-agent-optimize.md](ext-agent-optimize.md) · [agent-streaming.md](agent-streaming.md) | +| 基线 | v0.3.6+ | + +本文只跟踪**实现波次**。作者 API、ABI 边界、禁止项以设计文为准。 + +--- + +## 0. 目标回顾 + +让 `ext/web` + `ext/ai` 具备:同域 SSE 中继、HTTP→用户 `##`、MCP Server(至少 stdio)、跨模块 db/corpus 可发现——从而下游无需 Python 旁路进程。 + +--- + +## 1. 波次总表 + +| 波次 | 内容 | 关闭缺口 | 状态 | +|------|------|----------|------| +| **H0** | 文档契约:跨模块 `db` 传 `url`;`corpus_search` 调用前 `ensure_plugin` 说明;缺口索引进 `doc/README` | GAP-05 文档侧 | **done** | +| **H1** | `app.proxy` / `configure proxy=`:上游 SSE/HTTP **流式**转发 | GAP-01 | **done** | +| **H2** | `app.invoke` / `configure invoke=`:请求 → 白名单 `##` → JSON | GAP-02 | **done** | +| **H3** | 跨模块插件对象 `_type` 分发修补 + `corpus_search` 自动 `ensure_plugin` | GAP-05 运行时 | **done** | +| **H4** | `agent.mcp_server` + `serve transport=stdio`(HTTP 可选 H4b) | GAP-03 | **done**(stdio) | +| **H4b** | MCP Streamable HTTP | GAP-03 余量 | **todo**(可选) | +| **—** | 副端口 API | GAP-04 | **won't**:H1+H2 同端口关闭 | + +建议顺序:**H1 → H2 → H3 → H4**(H2 的 invoke 内核可被 H4 工具复用,故 H2 不宜晚于 H4)。 + +--- + +## 2. 分波工作项 + +### H1 — 流式代理 + +1. `plugins/web`:`proxy.rs` + `web_app_proxy`;listen 挂流式 handler。 +2. `ext/web`:`## proxy` + `configure proxy=` 表。 +3. 金样 `web-proxy-sse-smoke`(mock 上游)。 +4. 教程/public 短节:「浏览器同域调 LLM」。 + +**完成定义:** `curl -N` 同域 path 收到多帧 SSE;密钥仅 env 注入。 + +### H2 — HTTP invoke + +1. 共享或插件内 `invoke_fn`(解析 `库.成员`、映射 body)。 +2. `web_app_invoke` + 白名单。 +3. 金样:临时 `##` 写文件 + JSON 响应;拒绝未允许路径。 +4. 迁移说明:替换 `/store/*` + 外挂 `marqdo run`。 + +**完成定义:** 单进程 `listen` 下表单/fetch 可触发用户 `##`。 + +### H3 — db / corpus + +1. 复现并修跨模块 `unknown object type db`(设计选项 A)。 +2. `ext/ai` `corpus_search` 路径 `ensure_plugin`。 +3. 金样 `web-db-cross-module-smoke` + corpus CLI run 金样。 + +### H4 — MCP Server + +1. `plugins/agent` stdio JSON-RPC 最小子集:`initialize` / `tools/list` / `tools/call`。 +2. `ext/ai` `# mcp_server` · `## tool` · `## serve`。 +3. 工具调用复用 H2 `invoke_fn`。 +4. 金样夹具;文档对接 Cursor `mcp.json` 示例。 + +**H4b:** Streamable HTTP 与鉴权;不阻塞 H4 stdio 发版。 + +--- + +## 3. 刻意不做 + +| 项 | 原因 | +|----|------| +| 默认开启任意 `##` 远程执行 | 安全 | +| 中继进 `lib/net` / `src/host` | 边界 | +| 重开 W0–W7 内容站清单 | 已完结 | +| 以副端口为推荐架构 | H1+H2 同端口即可 | +| Dense embedding 默认 | OKF 路线不变 | + +--- + +## 4. 与发版 + +- 每波可随小版本发布;H1+H2 建议同一版本对外宣传「可去掉 llm_proxy」。 +- H4 可标 `ext/ai` 能力点,与「真 MCP 客户端」分列 Release notes。 + +--- + +## 5. 进度勾选(落地时改) + +- [x] H0 缺口 + 补齐文档 +- [x] H1 proxy SSE +- [x] H2 invoke +- [x] H3 db/corpus +- [x] H4 mcp_server stdio +- [ ] H4b mcp http(可选) +- [x] 研究文 GAP 状态改为 closed(H4b 除外) diff --git a/doc/roadmap/ext-web.md b/doc/roadmap/ext-web.md index bb795ca..d72836d 100644 --- a/doc/roadmap/ext-web.md +++ b/doc/roadmap/ext-web.md @@ -69,3 +69,4 @@ | [design/ext-web.md](../design/ext-web.md) | **真理** | | [user-site.md](../design/user-site.md) | 静态站互补 | | [ext-quantum.md](ext-quantum.md) | 同属官方 ext + ABI + `ext add` | +| [ext-hosting.md](ext-hosting.md) · [ext-hosting-gaps.md](../research/ext-hosting-gaps.md) | **W/C 之后**:同域 SSE 中继、HTTP→`##`(不重开本表 W0–W7) | diff --git a/ext/ai/agent.mq.md b/ext/ai/agent.mq.md index 04ad84e..b7b8cb1 100644 --- a/ext/ai/agent.mq.md +++ b/ext/ai/agent.mq.md @@ -388,6 +388,15 @@ Callable dotted names (whitelist for `CALL:lib…`) plus static module file list Local keyword search over `.md` / `.mq.md` / `.txt` under `root`. Returns ranked excerpts. **Evidence only** — runnable workbook / OKF skill stays authority (`authority=workbook`). +Ensures the native agent plugin is loaded so plain `marqdo run` (without constructing `# agent`) can call this. + +*p = > plugin.native_path name="agent"* +1. `p` + > plugin.load path=`p` +2. * + > print text=ext/ai/agent: native agent plugin not found (build marqdo_plugin_agent or marqdo ext add agent) + > sys.exit code=1 + **> agent_corpus_search query=`query` root=`root` limit=`limit`** --- @@ -2099,3 +2108,46 @@ Deterministic success stop (loop engineering): when the child already returned a *out = > plan_finish_stream out=`out` events=`events` stream=`stream` trace=`trace` result=`plan_result`* **out** + +--- + +# mcp_server + + `name`=marqdo + +Minimal MCP **Server** host (stdio JSON-RPC). Tools point at entry-module `lib.member` functions (same bridge as `ext/web` invoke). Evidence/results may carry `authority=workbook`. + +*p = > plugin.native_path name="agent"* +1. `p` + > plugin.load path=`p` +2. * + > print text=ext/ai/agent: native agent plugin not found (build marqdo_plugin_agent or marqdo ext add agent) + > sys.exit code=1 + +*h = > json.parse text={"_type":"mcp_server","tools":[]}* +*h = > json.set map=`h` key="name" value=`name`* +**h** + +## tool + + `name` + + `fn` + + `description`="" + +Register one tool. `fn` must be `lib.member` imported on the entry workbook. + +*tools = > json.get value=`self` key="tools"* +*row = > json.parse text={"name":"","fn":"","description":""}* +*row = > json.set map=`row` key="name" value=`name`* +*row = > json.set map=`row` key="fn" value=`fn`* +*row = > json.set map=`row` key="description" value=`description`* +*tools = > json.append list=`tools` item=`row`* +*h = > json.set map=`self` key="tools" value=`tools`* +**h** + +## serve + + `transport`=stdio + +Block on stdio MCP (`initialize` / `tools/list` / `tools/call`). HTTP transport is H4b. + +*name = > json.get value=`self` key="name"* +*tools = > json.get value=`self` key="tools"* +**> agent_mcp_serve name=`name` tools=`tools` transport=`transport`** diff --git a/ext/web/web.mq.md b/ext/web/web.mq.md index 9e34bbe..c23a9c4 100644 --- a/ext/web/web.mq.md +++ b/ext/web/web.mq.md @@ -653,16 +653,42 @@ Register site icons from a GFM table (`|path|rel|type|sizes|url|` or ZH + `json`=None + `access_log`=None + `cache_control`=None + + `proxy`=None + + `invoke`=None -Each capability is declared as a data table and assembled at listen time. The `cors` parameter takes a `|允许来源|方法|头|暴露头|凭证|` table (one row per origin; an empty `允许来源` column means any origin). The `security` parameter takes a `|头|值|` response-header table (e.g. `X-Frame-Options`, `Content-Security-Policy`, `X-Content-Type-Options`, `Referrer-Policy`, `Strict-Transport-Security`). Set `compress` to `True` to gzip response bodies. `body_limit` is the max request body bytes (e.g. `1048576`). The `json` parameter takes a `|路径|方法|表|条件|排序|上限|` table of JSON API endpoints backed by DB queries (each returns `application/json`). Set `access_log` to `True` to log `METHOD path status duration_ms` on stderr. `cache_control` sets a global `Cache-Control` header (e.g. `public, max-age=3600`). +Each capability is declared as a data table and assembled at listen time. The `cors` parameter takes a `|允许来源|方法|头|暴露头|凭证|` table (one row per origin; an empty `允许来源` column means any origin). The `security` parameter takes a `|头|值|` response-header table (e.g. `X-Frame-Options`, `Content-Security-Policy`, `X-Content-Type-Options`, `Referrer-Policy`, `Strict-Transport-Security`). Set `compress` to `True` to gzip response bodies. `body_limit` is the max request body bytes (e.g. `1048576`). The `json` parameter takes a `|路径|方法|表|条件|排序|上限|` table of JSON API endpoints backed by DB queries (each returns `application/json`). Set `access_log` to `True` to log `METHOD path status duration_ms` on stderr. `cache_control` sets a global `Cache-Control` header (e.g. `public, max-age=3600`). `proxy` takes a `|路径|上游|流式|去前缀|方法|环境头|超时|` table of streaming reverse-proxy routes. `invoke` takes a `|路径|方法|函数|正文|返回|` table that calls entry-module `lib.member` handlers per request. Tables stay as data; `configure` assembles them. 配置即数据、装配即函数. -**> web_app_middleware app=`self` cors=`cors` security=`security` compress=`compress` body_limit=`body_limit` json_routes=`json` access_log=`access_log` cache_control=`cache_control`** +**> web_app_middleware app=`self` cors=`cors` security=`security` compress=`compress` body_limit=`body_limit` json_routes=`json` access_log=`access_log` cache_control=`cache_control` proxy=`proxy` invoke=`invoke`** + +## proxy + + `path` + + `upstream` + + `stream`=True + + `strip_prefix`=None + + `methods`=None + + `headers_from_env`=None + + `timeout_ms`=120000 + +Same-origin reverse proxy to an upstream HTTP(S) URL. When `stream` is True (default), the response body — including `text/event-stream` SSE — is piped without buffering the full payload. `upstream` may contain `$ENV` / `${ENV}`. `headers_from_env` is `ENV=Header-Name` (comma-separated); secrets stay on the server. Prefer this over a side Python LLM proxy. + +**> web_app_proxy app=`self` path=`path` upstream=`upstream` stream=`stream` strip_prefix=`strip_prefix` methods=`methods` headers_from_env=`headers_from_env` timeout_ms=`timeout_ms`** + +## invoke + + `path` + + `fn` + + `method`=POST + + `body`=json + + `return`=json + +Call a user `##` on each HTTP request. `fn` must be `lib.member` imported on the entry module. `body` is `json` / `form` / `query` / `raw`; object keys map to named params (plus `payload`). Returns JSON by default. + +**> web_app_invoke app=`self` path=`path` method=`method` fn=`fn` body=`body` return=`return`** ## listen -Serve `/`, routed pages, `/_part/{id}` (home) and `{path}/_part/{id}` (routes), `/_form/{id}` (from mounts + page embeds), optional `/static` (or custom mount), optional `/admin`, upload/download/WebSocket/RSS/sitemap/robots routes, redirects, and a custom 404 fallback. Production HTTPS should terminate at a reverse proxy; set `cookie_secure=True` when serving over TLS. +Serve `/`, routed pages, `/_part/{id}` (home) and `{path}/_part/{id}` (routes), `/_form/{id}` (from mounts + page embeds), optional `/static` (or custom mount), optional `/admin`, upload/download/WebSocket/RSS/sitemap/robots routes, redirects, proxy/invoke routes, and a custom 404 fallback. Production HTTPS should terminate at a reverse proxy; set `cookie_secure=True` when serving over TLS. **> web_listen app=`self`** diff --git a/ext/web/网页.mq.md b/ext/web/网页.mq.md index e210eda..1b070f3 100644 --- a/ext/web/网页.mq.md +++ b/ext/web/网页.mq.md @@ -623,16 +623,42 @@ SEO / OpenGraph 元数据表(`|键|值|`)。如 `title`、`description`、`c + `接口`=None + `访问日志`=None + `缓存控制`=None + + `代理`=None + + `调用`=None -每项能力用数据表声明,`监听` 时统一装配。`跨域` 参数接受 `|允许来源|方法|头|暴露头|凭证|` 表(每来源一行;`允许来源` 留空表示允许任意来源)。`安全` 参数接受 `|头|值|` 响应头表(如 `X-Frame-Options`、`Content-Security-Policy`、`X-Content-Type-Options`、`Referrer-Policy`、`Strict-Transport-Security`)。`压缩` 为 `真` 时对响应体做 gzip 压缩。`请求体上限` 是请求体最大字节数(如 `1048576`)。`接口` 参数接受 `|路径|方法|表|条件|排序|上限|` JSON API 端点表(每个返回 `application/json`)。`访问日志` 为 `真` 时在 stderr 打印 `METHOD path status duration_ms`。`缓存控制` 设置全局 `Cache-Control`(如 `public, max-age=3600`)。 +每项能力用数据表声明,`监听` 时统一装配。`跨域` 参数接受 `|允许来源|方法|头|暴露头|凭证|` 表(每来源一行;`允许来源` 留空表示允许任意来源)。`安全` 参数接受 `|头|值|` 响应头表(如 `X-Frame-Options`、`Content-Security-Policy`、`X-Content-Type-Options`、`Referrer-Policy`、`Strict-Transport-Security`)。`压缩` 为 `真` 时对响应体做 gzip 压缩。`请求体上限` 是请求体最大字节数(如 `1048576`)。`接口` 参数接受 `|路径|方法|表|条件|排序|上限|` JSON API 端点表(每个返回 `application/json`)。`访问日志` 为 `真` 时在 stderr 打印 `METHOD path status duration_ms`。`缓存控制` 设置全局 `Cache-Control`(如 `public, max-age=3600`)。`代理` 接受 `|路径|上游|流式|去前缀|方法|环境头|超时|` 流式反向代理表。`调用` 接受 `|路径|方法|函数|正文|返回|` 表,按请求调用入口模块已导入的 `库.成员`。 表仍是数据,`装配` 把它们变成能力。配置即数据、装配即函数。 -**> web_app_middleware app=`self` cors=`跨域` security=`安全` compress=`压缩` body_limit=`请求体上限` json_routes=`接口` access_log=`访问日志` cache_control=`缓存控制`** +**> web_app_middleware app=`self` cors=`跨域` security=`安全` compress=`压缩` body_limit=`请求体上限` json_routes=`接口` access_log=`访问日志` cache_control=`缓存控制` proxy=`代理` invoke=`调用`** + +## 代理 + + `路径` + + `上游` + + `流式`=真 + + `去前缀`=None + + `方法`=None + + `环境头`=None + + `超时`=120000 + +同域反向代理到上游 HTTP(S)。`流式` 为真(默认)时管道转发响应体(含 `text/event-stream`),不整段缓冲。`上游` 可含 `$ENV` / `${ENV}`。`环境头` 为 `ENV=Header-Name`(逗号分隔),密钥留在服务端。 + +**> web_app_proxy app=`self` path=`路径` upstream=`上游` stream=`流式` strip_prefix=`去前缀` methods=`方法` headers_from_env=`环境头` timeout_ms=`超时`** + +## 调用 + + `路径` + + `函数` + + `方法`=POST + + `正文`=json + + `返回`=json + +每个 HTTP 请求调用用户 `##`。`函数` 必须是入口模块已 `导入` 的 `库.成员`。`正文` 为 `json` / `form` / `query` / `raw`;对象键映射为具名形参(另有 `payload`)。默认返回 JSON。 + +**> web_app_invoke app=`self` path=`路径` method=`方法` fn=`函数` body=`正文` return=`返回`** ## 监听 -提供 `/`、路由页、首页 `/_part/{id}`、路由 `{路径}/_part/{id}`、`/_form/{id}`、可选静态目录、可选 `/admin`,上传/下载/WebSocket/RSS/sitemap/robots、重定向,以及自定义 404。生产 HTTPS 请在反向代理终止;启用 TLS 时设 `cookie_secure`。 +提供 `/`、路由页、首页 `/_part/{id}`、路由 `{路径}/_part/{id}`、`/_form/{id}`、可选静态目录、可选 `/admin`,上传/下载/WebSocket/RSS/sitemap/robots、重定向、代理/调用路由,以及自定义 404。生产 HTTPS 请在反向代理终止;启用 TLS 时设 `cookie_secure`。 **> web_listen app=`self`** diff --git a/plugins/agent/src/lib.rs b/plugins/agent/src/lib.rs index 369daec..2bef508 100644 --- a/plugins/agent/src/lib.rs +++ b/plugins/agent/src/lib.rs @@ -2,6 +2,7 @@ mod corpus; mod kb; +mod mcp_server; use std::collections::HashMap; use std::ffi::{CStr, CString}; @@ -548,10 +549,18 @@ fn register(host: &MarqdoHostApi, name: &str, params: &str, fn_ptr: PluginFn) -> } pub(crate) fn host_query_json(name: &str) -> Result { + host_query_json_args(name, &serde_json::json!({})) +} + +pub(crate) fn host_query_json_args( + name: &str, + args: &serde_json::Value, +) -> Result { let query = unsafe { HOST_QUERY }.ok_or_else(|| "host_query not available".to_string())?; let userdata = unsafe { HOST_USERDATA }; let c_name = CString::new(name).map_err(|e| e.to_string())?; - let c_args = CString::new("{}").unwrap(); + let args_s = serde_json::to_string(args).map_err(|e| e.to_string())?; + let c_args = CString::new(args_s).map_err(|e| e.to_string())?; let mut out_ptr: *mut c_char = ptr::null_mut(); let mut err_ptr: *mut c_char = ptr::null_mut(); let rc = unsafe { @@ -987,6 +996,11 @@ pub unsafe extern "C" fn marqdo_plugin_init(host: *const MarqdoHostApi) -> c_int if register(host, "agent_mcp_fixture", "action,fixture", agent_mcp_fixture) != 0 { return 1; } + if register(host, "agent_mcp_serve", "name,tools,transport", mcp_server::agent_mcp_serve) + != 0 + { + return 1; + } if register(host, "agent_kb_add_alias", "kb_dir,slug,alias", agent_kb_add_alias) != 0 { return 1; } diff --git a/plugins/agent/src/mcp_server.rs b/plugins/agent/src/mcp_server.rs new file mode 100644 index 0000000..9b8502f --- /dev/null +++ b/plugins/agent/src/mcp_server.rs @@ -0,0 +1,212 @@ +//! Minimal MCP Server (stdio JSON-RPC) — tools map to `lib.member` via host `call_lib_path`. + +use serde_json::{json, Map, Value}; +use std::ffi::CStr; +use std::io::{BufRead, BufReader, Write}; +use std::os::raw::{c_char, c_int}; + +use crate::{host_query_json_args, parse_args, set_err, set_out}; + +fn read_line_message(reader: &mut impl BufRead) -> Result, String> { + let mut line = String::new(); + let n = reader + .read_line(&mut line) + .map_err(|e| format!("stdin: {e}"))?; + if n == 0 { + return Ok(None); + } + let line = line.trim(); + if line.is_empty() { + return read_line_message(reader); + } + let v: Value = serde_json::from_str(line).map_err(|e| format!("json-rpc parse: {e}"))?; + Ok(Some(v)) +} + +fn write_message(msg: &Value) -> Result<(), String> { + let mut out = std::io::stdout().lock(); + let s = serde_json::to_string(msg).map_err(|e| e.to_string())?; + writeln!(out, "{s}").map_err(|e| format!("stdout: {e}"))?; + out.flush().map_err(|e| format!("flush: {e}"))?; + Ok(()) +} + +fn rpc_result(id: &Value, result: Value) -> Value { + json!({ + "jsonrpc": "2.0", + "id": id, + "result": result, + }) +} + +fn rpc_error(id: &Value, code: i64, message: &str) -> Value { + json!({ + "jsonrpc": "2.0", + "id": id, + "error": { "code": code, "message": message }, + }) +} + +fn tools_list(tools: &[Value]) -> Value { + let mut arr = Vec::new(); + for t in tools { + let name = t.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let desc = t + .get("description") + .and_then(|v| v.as_str()) + .unwrap_or(""); + if name.is_empty() { + continue; + } + arr.push(json!({ + "name": name, + "description": desc, + "inputSchema": { + "type": "object", + "properties": {}, + "additionalProperties": true + } + })); + } + json!({ "tools": arr }) +} + +fn call_tool(tools: &[Value], name: &str, arguments: &Value) -> Result { + let tool = tools + .iter() + .find(|t| t.get("name").and_then(|v| v.as_str()) == Some(name)) + .ok_or_else(|| format!("unknown tool `{name}`"))?; + let fn_path = tool + .get("fn") + .and_then(|v| v.as_str()) + .ok_or_else(|| format!("tool `{name}` missing fn"))? + .to_string(); + let result = host_query_json_args( + "call_lib_path", + &json!({ + "path": fn_path, + "args": arguments, + }), + )?; + Ok(json!({ + "content": [{ + "type": "text", + "text": serde_json::to_string(&result).unwrap_or_else(|_| "{}".into()) + }], + "isError": false, + "authority": "workbook" + })) +} + +/// Blocking stdio MCP loop. `tools` is a JSON array of `{name, fn, description}`. +pub fn serve_stdio(name: &str, tools: Vec) -> Result { + let stdin = std::io::stdin(); + let mut reader = BufReader::new(stdin.lock()); + loop { + let Some(msg) = read_line_message(&mut reader)? else { + break; + }; + let id = msg.get("id").cloned().unwrap_or(Value::Null); + let method = msg.get("method").and_then(|v| v.as_str()).unwrap_or(""); + let params = msg.get("params").cloned().unwrap_or(json!({})); + + if matches!(id, Value::Null) && method.starts_with("notifications/") { + continue; + } + + let reply_msg = match method { + "initialize" => rpc_result( + &id, + json!({ + "protocolVersion": "2024-11-05", + "capabilities": { "tools": {} }, + "serverInfo": { "name": name, "version": "0.1.0" } + }), + ), + "tools/list" | "list_tools" => rpc_result(&id, tools_list(&tools)), + "tools/call" | "call_tool" => { + let tname = params.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let args = params.get("arguments").cloned().unwrap_or(json!({})); + match call_tool(&tools, tname, &args) { + Ok(r) => rpc_result(&id, r), + Err(e) => rpc_error(&id, -32000, &e), + } + } + "ping" => rpc_result(&id, json!({})), + "" => rpc_error(&id, -32600, "missing method"), + other => rpc_error(&id, -32601, &format!("method not found: {other}")), + }; + write_message(&reply_msg)?; + } + Ok(json!({ "ok": true, "transport": "stdio" })) +} + +pub fn tools_from_value(v: &Value) -> Vec { + if let Some(arr) = v.as_array() { + return arr.clone(); + } + if let Value::Object(map) = v { + let mut out = Vec::new(); + for (name, spec) in map { + let mut m = Map::new(); + m.insert("name".into(), json!(name)); + if let Some(obj) = spec.as_object() { + for (k, val) in obj { + m.insert(k.clone(), val.clone()); + } + } else if let Some(s) = spec.as_str() { + m.insert("fn".into(), json!(s)); + } + out.push(Value::Object(m)); + } + return out; + } + Vec::new() +} + +pub unsafe extern "C" fn agent_mcp_serve( + args_json: *const c_char, + out_json: *mut *mut c_char, + err_msg: *mut *mut c_char, +) -> c_int { + let args = match parse_args(args_json) { + Ok(v) => v, + Err(e) => { + set_err(err_msg, &e); + return 1; + } + }; + let transport = args + .get("transport") + .and_then(|v| v.as_str()) + .unwrap_or("stdio"); + if transport != "stdio" { + set_err( + err_msg, + &format!("mcp serve: transport `{transport}` not implemented yet (use stdio)"), + ); + return 1; + } + let name = args + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or("marqdo"); + let tools = args + .get("tools") + .map(tools_from_value) + .unwrap_or_default(); + match serve_stdio(name, tools) { + Ok(v) => { + set_out(out_json, &v.to_string()); + 0 + } + Err(e) => { + set_err(err_msg, &e); + 1 + } + } +} + +// Silence unused import if CStr unused in this module. +#[allow(dead_code)] +fn _unused(_: &CStr) {} diff --git a/plugins/web/Cargo.toml b/plugins/web/Cargo.toml index f3c4b84..c232ed3 100644 --- a/plugins/web/Cargo.toml +++ b/plugins/web/Cargo.toml @@ -25,6 +25,8 @@ pulldown-cmark = "0.12" postgres = { version = "0.19", features = ["with-serde_json-1"] } redis = { version = "0.27", default-features = true } ureq = { version = "=2.9.7", default-features = false, features = ["tls"] } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "json"] } +bytes = "1" hmac = "0.12" sha2 = "0.10" hex = "0.4" diff --git a/plugins/web/src/http.rs b/plugins/web/src/http.rs index 3e7743b..88f40d0 100644 --- a/plugins/web/src/http.rs +++ b/plugins/web/src/http.rs @@ -281,6 +281,8 @@ pub fn listen( middleware: &Middleware, mut site_head: Vec, mut icon_routes: Vec, + proxy_routes: Vec<(String, crate::proxy::ProxyRoute)>, + invoke_routes: Vec<(String, crate::invoke::InvokeRoute)>, ) -> Result { let mut form_owners = HashMap::new(); collect_page_forms(page, &mut forms, &mut form_owners); @@ -601,6 +603,10 @@ pub fn listen( ); } + // Streaming proxy + HTTP→## invoke (before fallback so paths are explicit). + app = crate::proxy::mount_all(app, &proxy_routes); + app = crate::invoke::mount_all(app, &invoke_routes); + app = app.fallback(get(fallback_404)); let app = crate::middleware::apply(app, middleware); diff --git a/plugins/web/src/invoke.rs b/plugins/web/src/invoke.rs new file mode 100644 index 0000000..9181839 --- /dev/null +++ b/plugins/web/src/invoke.rs @@ -0,0 +1,332 @@ +//! HTTP → user `##` invoke routes for `app.invoke` / `configure invoke=`. + +use std::sync::Arc; + +use axum::extract::{Request, State}; +use axum::http::{header, HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::Router; +use serde_json::{json, Map, Value}; + +use crate::middleware::{cell_text, JsonBody}; +use crate::table; +use crate::call_lib_with_args; + +/// One declared invoke mount. +#[derive(Clone, Debug)] +pub struct InvokeRoute { + pub method: String, + pub fn_path: String, + pub body_mode: String, + pub return_mode: String, +} + +impl InvokeRoute { + pub fn from_json(v: &Value) -> Option { + let obj = v.as_object()?; + let fn_path = obj + .get("fn") + .or_else(|| obj.get("function")) + .and_then(|x| x.as_str())? + .trim() + .to_string(); + if fn_path.is_empty() { + return None; + } + let method = obj + .get("method") + .and_then(|x| x.as_str()) + .unwrap_or("POST") + .to_uppercase(); + let body_mode = obj + .get("body") + .and_then(|x| x.as_str()) + .unwrap_or("json") + .to_ascii_lowercase(); + let return_mode = obj + .get("return") + .and_then(|x| x.as_str()) + .unwrap_or("json") + .to_ascii_lowercase(); + Some(Self { + method, + fn_path, + body_mode, + return_mode, + }) + } +} + +/// Normalize invoke-routes GFM table into `app.invoke_routes` map. +pub fn invoke_routes_from_table(table_v: &Value) -> Value { + let rows = table::as_rows(table_v); + let mut routes = Map::new(); + if let Some(arr) = rows.as_array() { + for row in arr { + let col = |keys: &[&str]| -> String { + keys.iter() + .find_map(|k| row.get(*k).map(cell_text)) + .unwrap_or_default() + }; + let path = col(&["路径", "path", "Path"]) + .trim() + .trim_start_matches('/') + .to_string(); + if path.is_empty() { + continue; + } + let fn_path = col(&["函数", "fn", "function", "Function"]); + if fn_path.is_empty() { + continue; + } + let method = { + let m = col(&["方法", "method", "Method"]).to_uppercase(); + if m.is_empty() { + "POST".into() + } else { + m + } + }; + let body = { + let b = col(&["正文", "body", "Body"]).to_ascii_lowercase(); + if b.is_empty() { + "json".into() + } else { + b + } + }; + let ret = { + let r = col(&["返回", "return", "Return"]).to_ascii_lowercase(); + if r.is_empty() { + "json".into() + } else { + r + } + }; + routes.insert( + path, + json!({ + "method": method, + "fn": fn_path, + "body": body, + "return": ret, + }), + ); + } + } + Value::Object(routes) +} + +pub fn mount_all( + mut app: Router>, + routes: &[(String, InvokeRoute)], +) -> Router> { + for (path, route) in routes { + app = mount_one(app, path, route.clone()); + } + app +} + +fn mount_one( + app: Router>, + path: &str, + route: InvokeRoute, +) -> Router> { + let r = route.clone(); + match route.method.as_str() { + "GET" => app.route( + path, + get(move |State(st): State>, req: Request| { + let r = r.clone(); + async move { invoke_handler(st, r, req).await } + }), + ), + _ => app.route( + path, + post(move |State(st): State>, req: Request| { + let r = r.clone(); + async move { invoke_handler(st, r, req).await } + }), + ), + } +} + +async fn invoke_handler( + _st: Arc, + route: InvokeRoute, + req: Request, +) -> Response { + let (parts, body) = req.into_parts(); + let query_map = query_to_map(parts.uri.query()); + + let payload = match route.body_mode.as_str() { + "query" => Value::Object(query_map.clone()), + "raw" => { + let bytes = match axum::body::to_bytes(body, 8 * 1024 * 1024).await { + Ok(b) => b, + Err(e) => { + return err_json(StatusCode::BAD_REQUEST, &format!("body: {e}")); + } + }; + json!({ "payload": String::from_utf8_lossy(&bytes).to_string() }) + } + "form" => { + let bytes = match axum::body::to_bytes(body, 8 * 1024 * 1024).await { + Ok(b) => b, + Err(e) => { + return err_json(StatusCode::BAD_REQUEST, &format!("body: {e}")); + } + }; + form_to_map(&bytes) + } + _ => { + // json (default) + let bytes = match axum::body::to_bytes(body, 8 * 1024 * 1024).await { + Ok(b) => b, + Err(e) => { + return err_json(StatusCode::BAD_REQUEST, &format!("body: {e}")); + } + }; + if bytes.is_empty() { + Value::Object(query_map.clone()) + } else { + match serde_json::from_slice::(&bytes) { + Ok(Value::Object(mut m)) => { + for (k, v) in query_map { + m.entry(k).or_insert(v); + } + Value::Object(m) + } + Ok(other) => json!({ "payload": other }), + Err(e) => { + return err_json(StatusCode::BAD_REQUEST, &format!("json: {e}")); + } + } + } + } + }; + + // Bind JSON object keys as named args; always offer `payload` too. + let mut args = Map::new(); + if let Value::Object(ref m) = payload { + for (k, v) in m { + args.insert(k.clone(), v.clone()); + } + } + if !args.contains_key("payload") { + args.insert("payload".to_string(), payload.clone()); + } + + let fn_path = route.fn_path.clone(); + let result = tokio::task::spawn_blocking(move || call_lib_with_args(&fn_path, &Value::Object(args))) + .await; + + let value = match result { + Ok(Ok(v)) => v, + Ok(Err(e)) => { + let status = if e.contains("unknown library") + || e.contains("call_lib_path: need") + || e.contains("not allowed") + || e.contains("no site") + { + StatusCode::BAD_REQUEST + } else { + StatusCode::INTERNAL_SERVER_ERROR + }; + return err_json(status, &e); + } + Err(e) => { + return err_json(StatusCode::INTERNAL_SERVER_ERROR, &format!("join: {e}")); + } + }; + + match route.return_mode.as_str() { + "text" => { + let text = match &value { + Value::String(s) => s.clone(), + other => other.to_string(), + }; + ( + StatusCode::OK, + [( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + )], + text, + ) + .into_response() + } + "status" => StatusCode::NO_CONTENT.into_response(), + _ => { + // Wrap non-objects so clients always get JSON object when possible. + let body = match value { + Value::Null => json!({ "ok": true }), + Value::Object(_) => value, + other => json!({ "ok": true, "result": other }), + }; + JsonBody(body).into_response() + } + } +} + +fn query_to_map(q: Option<&str>) -> Map { + let mut m = Map::new(); + let Some(q) = q else { + return m; + }; + for pair in q.split('&') { + if pair.is_empty() { + continue; + } + let (k, v) = pair.split_once('=').unwrap_or((pair, "")); + let k = urlencoding_decode(k); + let v = urlencoding_decode(v); + if !k.is_empty() { + m.insert(k, json!(v)); + } + } + m +} + +fn form_to_map(bytes: &[u8]) -> Value { + let s = String::from_utf8_lossy(bytes); + Value::Object(query_to_map(Some(&s))) +} + +fn urlencoding_decode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() { + match b[i] { + b'+' => { + out.push(' '); + i += 1; + } + b'%' if i + 2 < b.len() => { + let h = u8::from_str_radix(std::str::from_utf8(&b[i + 1..i + 3]).unwrap_or(""), 16); + if let Ok(c) = h { + out.push(c as char); + i += 3; + } else { + out.push('%'); + i += 1; + } + } + c => { + out.push(c as char); + i += 1; + } + } + } + out +} + +fn err_json(status: StatusCode, msg: &str) -> Response { + ( + status, + JsonBody(json!({ "ok": false, "error": msg })), + ) + .into_response() +} diff --git a/plugins/web/src/lib.rs b/plugins/web/src/lib.rs index 2cbec04..f2697b5 100644 --- a/plugins/web/src/lib.rs +++ b/plugins/web/src/lib.rs @@ -8,9 +8,11 @@ mod driver; mod form; mod http; mod cache; +mod invoke; mod markdown; mod middleware; mod password; +mod proxy; mod rate_limit; mod render; mod rss; @@ -156,6 +158,17 @@ fn call_lib(path: &str) -> Result { host_query_json("call_lib_path", &json!({ "path": path })) } +/// Call `lib.member` with a JSON object of named arguments (for HTTP invoke). +pub(crate) fn call_lib_with_args(path: &str, args: &Value) -> Result { + host_query_json( + "call_lib_path", + &json!({ + "path": path, + "args": args, + }), + ) +} + /// Absolute directory of the entry `.mq.md` (falls back to process cwd). fn entry_dir() -> PathBuf { host_query_json("entry_dir", &json!({})) @@ -1507,74 +1520,245 @@ web_ffi!(web_app_static, |args: &Value| { // backed by db queries (method GET/POST, `表`=table name). web_ffi!(web_app_middleware, |args: &Value| { let mut app = args.get("app").cloned().unwrap_or(json!({})); + { + let obj = app + .as_object_mut() + .ok_or_else(|| "app must be a map".to_string())?; + let mw = obj + .entry("middleware".to_string()) + .or_insert_with(|| json!({})); + let m = mw + .as_object_mut() + .ok_or_else(|| "app.middleware must be a map".to_string())?; + + if let Some(cors) = args.get("cors") { + m.insert("cors".into(), middleware::cors_from_table(cors)); + } + if let Some(security) = args.get("security") { + m.insert( + "security".into(), + middleware::security_from_table(security), + ); + } + if let Some(compress) = args.get("compress") { + let on = match compress { + Value::Bool(b) => *b, + Value::String(s) => matches!(s.as_str(), "true" | "True" | "1" | "yes" | "on"), + Value::Number(n) => n.as_i64().unwrap_or(0) != 0, + _ => false, + }; + m.insert("compress".into(), json!(on)); + } + if let Some(al) = args.get("access_log").or_else(|| args.get("访问日志")) { + let on = match al { + Value::Bool(b) => *b, + Value::String(s) => { + matches!(s.as_str(), "true" | "True" | "1" | "yes" | "on" | "真") + } + Value::Number(n) => n.as_i64().unwrap_or(0) != 0, + _ => false, + }; + m.insert("access_log".into(), json!(on)); + } + if let Some(cc) = args + .get("cache_control") + .or_else(|| args.get("缓存控制")) + .and_then(|v| v.as_str()) + { + let s = cc.trim(); + if !s.is_empty() { + m.insert("cache_control".into(), json!(s)); + } + } + if let Some(bl) = args.get("body_limit") { + let n = bl + .as_u64() + .or_else(|| bl.as_i64().map(|i| i as u64)) + .unwrap_or(0); + if n > 0 { + m.insert("body_limit".into(), json!(n)); + } + } + if let Some(routes) = args.get("json_routes") { + m.insert( + "json_routes".into(), + middleware::json_routes_from_table(routes), + ); + } + } + + if let Some(proxy) = args + .get("proxy") + .or_else(|| args.get("proxy_routes")) + .or_else(|| args.get("代理")) + .filter(|v| !v.is_null()) + { + let incoming = proxy::proxy_routes_from_table(proxy); + let obj_app = app + .as_object_mut() + .ok_or_else(|| "app must be a map".to_string())?; + let mut existing = obj_app + .get("proxy_routes") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + if let Some(map) = incoming.as_object() { + for (k, v) in map { + existing.insert(k.clone(), v.clone()); + } + } + obj_app.insert("proxy_routes".into(), Value::Object(existing)); + } + if let Some(inv) = args + .get("invoke") + .or_else(|| args.get("invoke_routes")) + .or_else(|| args.get("调用")) + .filter(|v| !v.is_null()) + { + let incoming = invoke::invoke_routes_from_table(inv); + let obj_app = app + .as_object_mut() + .ok_or_else(|| "app must be a map".to_string())?; + let mut existing = obj_app + .get("invoke_routes") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + if let Some(map) = incoming.as_object() { + for (k, v) in map { + existing.insert(k.clone(), v.clone()); + } + } + obj_app.insert("invoke_routes".into(), Value::Object(existing)); + } + Ok(app) +}); + +web_ffi!(web_app_proxy, |args: &Value| { + let mut app = args.get("app").cloned().unwrap_or(json!({})); + let path = normalize_route_path_in_app( + arg_str(args, "path").or_else(|_| arg_str(args, "路径"))?, + &app, + )?; + let upstream = arg_str(args, "upstream") + .or_else(|_| arg_str(args, "上游"))? + .to_string(); + if upstream.trim().is_empty() { + return Err("proxy requires `upstream`".into()); + } + let stream = match args + .get("stream") + .or_else(|| args.get("流式")) + .cloned() + .unwrap_or(json!(true)) + { + Value::Bool(b) => b, + Value::String(s) => matches!( + s.as_str(), + "true" | "True" | "1" | "yes" | "on" | "真" + ), + Value::Number(n) => n.as_i64().unwrap_or(0) != 0, + _ => true, + }; + let strip_prefix = arg_str_opt(args, "strip_prefix") + .or_else(|| arg_str_opt(args, "去前缀")) + .unwrap_or("") + .to_string(); + let methods = match args.get("methods").or_else(|| args.get("方法")) { + Some(Value::Array(arr)) => arr + .iter() + .filter_map(|v| v.as_str().map(|s| s.to_uppercase())) + .collect::>(), + Some(Value::String(s)) => s + .split(',') + .map(|m| m.trim().to_uppercase()) + .filter(|m| !m.is_empty()) + .collect(), + _ => vec!["POST".into()], + }; + let headers_from_env = arg_str_opt(args, "headers_from_env") + .or_else(|| arg_str_opt(args, "环境头")) + .unwrap_or("") + .to_string(); + let timeout_ms = args + .get("timeout_ms") + .or_else(|| args.get("超时")) + .and_then(|v| v.as_u64().or_else(|| v.as_i64().map(|i| i as u64))) + .unwrap_or(120_000); let obj = app .as_object_mut() .ok_or_else(|| "app must be a map".to_string())?; - let mw = obj - .entry("middleware".to_string()) - .or_insert_with(|| json!({})); - let m = mw - .as_object_mut() - .ok_or_else(|| "app.middleware must be a map".to_string())?; + let mut routes = obj + .get("proxy_routes") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + let key = path.trim_start_matches('/').to_string(); + routes.insert( + key, + json!({ + "upstream": upstream, + "stream": stream, + "strip_prefix": strip_prefix, + "methods": methods, + "headers_from_env": headers_from_env, + "timeout_ms": timeout_ms, + }), + ); + obj.insert("proxy_routes".into(), Value::Object(routes)); + Ok(app) +}); - if let Some(cors) = args.get("cors") { - m.insert( - "cors".into(), - middleware::cors_from_table(cors), - ); +web_ffi!(web_app_invoke, |args: &Value| { + let mut app = args.get("app").cloned().unwrap_or(json!({})); + let path = normalize_route_path_in_app( + arg_str(args, "path").or_else(|_| arg_str(args, "路径"))?, + &app, + )?; + let fn_path = arg_str(args, "fn") + .or_else(|_| arg_str(args, "function")) + .or_else(|_| arg_str(args, "函数"))? + .to_string(); + if fn_path.trim().is_empty() { + return Err("invoke requires `fn`".into()); } - if let Some(security) = args.get("security") { - m.insert( - "security".into(), - middleware::security_from_table(security), - ); - } - if let Some(compress) = args.get("compress") { - let on = match compress { - Value::Bool(b) => *b, - Value::String(s) => matches!(s.as_str(), "true" | "True" | "1" | "yes" | "on"), - Value::Number(n) => n.as_i64().unwrap_or(0) != 0, - _ => false, - }; - m.insert("compress".into(), json!(on)); - } - if let Some(al) = args - .get("access_log") - .or_else(|| args.get("访问日志")) - { - let on = match al { - Value::Bool(b) => *b, - Value::String(s) => matches!(s.as_str(), "true" | "True" | "1" | "yes" | "on" | "真"), - Value::Number(n) => n.as_i64().unwrap_or(0) != 0, - _ => false, - }; - m.insert("access_log".into(), json!(on)); - } - if let Some(cc) = args - .get("cache_control") - .or_else(|| args.get("缓存控制")) - .and_then(|v| v.as_str()) - { - let s = cc.trim(); - if !s.is_empty() { - m.insert("cache_control".into(), json!(s)); - } - } - if let Some(bl) = args.get("body_limit") { - let n = bl - .as_u64() - .or_else(|| bl.as_i64().map(|i| i as u64)) - .unwrap_or(0); - if n > 0 { - m.insert("body_limit".into(), json!(n)); - } - } - if let Some(routes) = args.get("json_routes") { - m.insert( - "json_routes".into(), - middleware::json_routes_from_table(routes), + // Optional allowlist: only `lib.member` form for now (no arbitrary file paths). + if !fn_path.contains('.') || fn_path.contains('/') || fn_path.contains('#') { + return Err( + "invoke `fn` must be `lib.member` (imported on the entry module)".into(), ); } + let method = arg_str_opt(args, "method") + .or_else(|| arg_str_opt(args, "方法")) + .unwrap_or("POST") + .to_uppercase(); + let body = arg_str_opt(args, "body") + .or_else(|| arg_str_opt(args, "正文")) + .unwrap_or("json") + .to_ascii_lowercase(); + let ret = arg_str_opt(args, "return") + .or_else(|| arg_str_opt(args, "返回")) + .unwrap_or("json") + .to_ascii_lowercase(); + let obj = app + .as_object_mut() + .ok_or_else(|| "app must be a map".to_string())?; + let mut routes = obj + .get("invoke_routes") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + let key = path.trim_start_matches('/').to_string(); + routes.insert( + key, + json!({ + "method": method, + "fn": fn_path, + "body": body, + "return": ret, + }), + ); + obj.insert("invoke_routes".into(), Value::Object(routes)); Ok(app) }); @@ -1737,6 +1921,8 @@ web_ffi!(web_listen, |args: &Value| { page_500, gates, gallery_routes, + proxy_routes, + invoke_routes, middleware, cookie_secure, site_head, @@ -1793,6 +1979,8 @@ web_ffi!(web_listen, |args: &Value| { None, Vec::::new(), HashMap::new(), + Vec::<(String, crate::proxy::ProxyRoute)>::new(), + Vec::<(String, crate::invoke::InvokeRoute)>::new(), middleware::Middleware::default(), cookie_secure, Vec::::new(), @@ -1943,6 +2131,34 @@ web_ffi!(web_listen, |args: &Value| { gallery_routes.insert(k.clone(), v.clone()); } } + let mut proxy_routes: Vec<(String, proxy::ProxyRoute)> = Vec::new(); + if let Some(obj) = app.get("proxy_routes").and_then(|v| v.as_object()) { + for (k, v) in obj { + if let Some(r) = proxy::ProxyRoute::from_json(v) { + let path = if k.starts_with('/') { + k.clone() + } else { + format!("/{k}") + }; + proxy_routes.push((path, r)); + } + } + proxy_routes.sort_by(|a, b| a.0.cmp(&b.0)); + } + let mut invoke_routes: Vec<(String, invoke::InvokeRoute)> = Vec::new(); + if let Some(obj) = app.get("invoke_routes").and_then(|v| v.as_object()) { + for (k, v) in obj { + if let Some(r) = invoke::InvokeRoute::from_json(v) { + let path = if k.starts_with('/') { + k.clone() + } else { + format!("/{k}") + }; + invoke_routes.push((path, r)); + } + } + invoke_routes.sort_by(|a, b| a.0.cmp(&b.0)); + } if app.get("tls_cert").is_some() || app.get("tls_key").is_some() { eprintln!( "marqdo web: in-process TLS is not enabled; terminate HTTPS at a reverse proxy (nginx/caddy) and set cookie_secure=True" @@ -2026,6 +2242,8 @@ web_ffi!(web_listen, |args: &Value| { page_500, gates, gallery_routes, + proxy_routes, + invoke_routes, middleware, cookie_secure, site_head, @@ -2141,6 +2359,8 @@ web_ffi!(web_listen, |args: &Value| { &middleware, site_head, icon_routes, + proxy_routes, + invoke_routes, ) }); @@ -2369,9 +2589,19 @@ pub unsafe extern "C" fn marqdo_plugin_init(host: *const MarqdoHostApi) -> c_int ), ( "web_app_middleware", - "app,cors,security,compress,body_limit,json_routes,access_log,cache_control", + "app,cors,security,compress,body_limit,json_routes,access_log,cache_control,proxy,invoke", web_app_middleware as PluginFn, ), + ( + "web_app_proxy", + "app,path,upstream,stream,strip_prefix,methods,headers_from_env,timeout_ms", + web_app_proxy as PluginFn, + ), + ( + "web_app_invoke", + "app,path,method,fn,body,return", + web_app_invoke as PluginFn, + ), ("web_form_new", "table,action,id", web_form_new as PluginFn), ("web_form_fields", "form,fields", web_form_fields as PluginFn), ("web_form_rules", "form,rules", web_form_rules as PluginFn), diff --git a/plugins/web/src/middleware.rs b/plugins/web/src/middleware.rs index 11ebced..d519cfc 100644 --- a/plugins/web/src/middleware.rs +++ b/plugins/web/src/middleware.rs @@ -24,7 +24,7 @@ use tower_http::set_header::SetResponseHeaderLayer; use crate::db; /// Render a cell value as text (strings stay, numbers/bools stringify). -fn cell_text(v: &Value) -> String { +pub fn cell_text(v: &Value) -> String { match v { Value::Null => String::new(), Value::String(s) => s.clone(), diff --git a/plugins/web/src/proxy.rs b/plugins/web/src/proxy.rs new file mode 100644 index 0000000..e38e2e9 --- /dev/null +++ b/plugins/web/src/proxy.rs @@ -0,0 +1,462 @@ +//! Streaming reverse proxy routes for `app.proxy` / `configure proxy=`. +//! +//! Browser → Marqdo (same origin) → upstream, piping response bodies +//! (including `text/event-stream`) without buffering the full stream. + +use std::sync::Arc; +use std::time::Duration; + +use axum::body::Body; +use axum::extract::{Request, State}; +use axum::http::{header, HeaderMap, HeaderName, HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{MethodFilter, MethodRouter}; +use axum::Router; +use futures_util::TryStreamExt; +use serde_json::{json, Map, Value}; + +use crate::middleware::cell_text; +use crate::table; + +/// One declared proxy mount. +#[derive(Clone, Debug)] +pub struct ProxyRoute { + pub upstream: String, + pub stream: bool, + pub strip_prefix: String, + pub methods: Vec, + /// `ENV_NAME=Header-Name` pairs injected on the upstream request. + pub headers_from_env: Vec<(String, String)>, + pub timeout_ms: u64, +} + +impl ProxyRoute { + pub fn from_json(v: &Value) -> Option { + let obj = v.as_object()?; + let upstream = obj.get("upstream")?.as_str()?.trim().to_string(); + if upstream.is_empty() { + return None; + } + let stream = obj + .get("stream") + .and_then(|x| x.as_bool()) + .unwrap_or(true); + let strip_prefix = obj + .get("strip_prefix") + .and_then(|x| x.as_str()) + .unwrap_or("") + .to_string(); + let methods = obj + .get("methods") + .and_then(|x| x.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|m| m.as_str().map(|s| s.to_uppercase())) + .filter(|s| !s.is_empty()) + .collect::>() + }) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| vec!["POST".into()]); + let headers_from_env = parse_headers_from_env_value(obj.get("headers_from_env")); + let timeout_ms = obj + .get("timeout_ms") + .and_then(|x| x.as_u64().or_else(|| x.as_i64().map(|i| i as u64))) + .unwrap_or(120_000) + .max(1); + Some(Self { + upstream, + stream, + strip_prefix, + methods, + headers_from_env, + timeout_ms, + }) + } +} + +fn parse_headers_from_env_value(v: Option<&Value>) -> Vec<(String, String)> { + let Some(v) = v else { + return Vec::new(); + }; + match v { + Value::String(s) => parse_headers_from_env_str(s), + Value::Array(arr) => arr + .iter() + .filter_map(|x| x.as_str()) + .flat_map(parse_headers_from_env_str) + .collect(), + _ => Vec::new(), + } +} + +fn parse_headers_from_env_str(s: &str) -> Vec<(String, String)> { + s.split(|c| c == ',' || c == ';') + .filter_map(|part| { + let part = part.trim(); + if part.is_empty() { + return None; + } + let (env, hdr) = part.split_once('=')?; + let env = env.trim(); + let hdr = hdr.trim(); + if env.is_empty() || hdr.is_empty() { + return None; + } + Some((env.to_string(), hdr.to_string())) + }) + .collect() +} + +/// Expand `$VAR` / `${VAR}` in upstream URL strings. +pub fn expand_env(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let bytes = s.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'$' { + if i + 1 < bytes.len() && bytes[i + 1] == b'{' { + if let Some(end) = s[i + 2..].find('}') { + let name = &s[i + 2..i + 2 + end]; + out.push_str(&std::env::var(name).unwrap_or_default()); + i += 3 + end; + continue; + } + } else { + let start = i + 1; + let mut end = start; + while end < bytes.len() + && (bytes[end].is_ascii_alphanumeric() || bytes[end] == b'_') + { + end += 1; + } + if end > start { + let name = &s[start..end]; + out.push_str(&std::env::var(name).unwrap_or_default()); + i = end; + continue; + } + } + } + out.push(bytes[i] as char); + i += 1; + } + out +} + +/// Normalize a proxy-routes GFM table into `app.proxy_routes` map shape. +pub fn proxy_routes_from_table(table_v: &Value) -> Value { + let rows = table::as_rows(table_v); + let mut routes = Map::new(); + if let Some(arr) = rows.as_array() { + for row in arr { + let col = |keys: &[&str]| -> String { + keys.iter() + .find_map(|k| row.get(*k).map(cell_text)) + .unwrap_or_default() + }; + let path = normalize_path(&col(&["路径", "path", "Path"])); + if path.is_empty() || path == "/" { + continue; + } + let upstream = col(&["上游", "upstream", "Upstream"]); + if upstream.is_empty() { + continue; + } + let stream = { + let s = col(&["流式", "stream", "Stream"]); + if s.is_empty() { + true + } else { + matches!( + s.to_ascii_lowercase().as_str(), + "true" | "1" | "yes" | "on" | "真" + ) + } + }; + let strip_prefix = col(&["去前缀", "strip_prefix", "Strip-Prefix"]); + let methods_raw = col(&["方法", "methods", "Methods"]); + let methods: Vec = if methods_raw.is_empty() { + vec!["POST".into()] + } else { + methods_raw + .split(',') + .map(|m| m.trim().to_uppercase()) + .filter(|m| !m.is_empty()) + .collect() + }; + let headers_from_env = col(&[ + "环境头", + "headers_from_env", + "Headers-From-Env", + ]); + let timeout_ms = col(&["超时", "timeout_ms", "Timeout"]) + .parse::() + .unwrap_or(120_000); + routes.insert( + path.trim_start_matches('/').to_string(), + json!({ + "upstream": upstream, + "stream": stream, + "strip_prefix": strip_prefix, + "methods": methods, + "headers_from_env": headers_from_env, + "timeout_ms": timeout_ms, + }), + ); + } + } + Value::Object(routes) +} + +fn normalize_path(raw: &str) -> String { + let s = raw.trim(); + if s.is_empty() { + return String::new(); + } + if s.starts_with('/') { + s.to_string() + } else { + format!("/{s}") + } +} + +/// Mount all proxy routes onto the router. +pub fn mount_all( + mut app: Router>, + routes: &[(String, ProxyRoute)], +) -> Router> { + for (path, route) in routes { + app = mount_one(app, path, route.clone()); + } + app +} + +fn mount_one( + app: Router>, + path: &str, + route: ProxyRoute, +) -> Router> { + let mut filter: Option = None; + for m in &route.methods { + let f = match m.as_str() { + "GET" => MethodFilter::GET, + "POST" => MethodFilter::POST, + "PUT" => MethodFilter::PUT, + "PATCH" => MethodFilter::PATCH, + "DELETE" => MethodFilter::DELETE, + "HEAD" => MethodFilter::HEAD, + "OPTIONS" => MethodFilter::OPTIONS, + _ => continue, + }; + filter = Some(match filter { + None => f, + Some(acc) => acc.or(f), + }); + } + let filter = filter.unwrap_or(MethodFilter::POST); + let r = route.clone(); + let handler = MethodRouter::new().on(filter, move |State(_st): State>, req: Request| { + let r = r.clone(); + async move { proxy_handler(r, req).await } + }); + app.route(path, handler) +} + +async fn proxy_handler(route: ProxyRoute, req: Request) -> Response { + let method = req.method().clone(); + let uri = req.uri().clone(); + let in_headers = req.headers().clone(); + let body_bytes = match axum::body::to_bytes(req.into_body(), 64 * 1024 * 1024).await { + Ok(b) => b, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + format!("read body: {e}"), + ) + .into_response(); + } + }; + + let target = match build_upstream_url(&route, uri.path(), uri.query()) { + Ok(u) => u, + Err(e) => return (StatusCode::BAD_GATEWAY, e).into_response(), + }; + + let client = match reqwest::Client::builder() + .timeout(Duration::from_millis(route.timeout_ms)) + .build() + { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("client: {e}"), + ) + .into_response(); + } + }; + + let mut builder = client.request( + reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap_or(reqwest::Method::POST), + &target, + ); + + // Forward selected inbound headers (skip hop-by-hop). + for (name, value) in in_headers.iter() { + let lname = name.as_str().to_ascii_lowercase(); + if is_hop_by_hop(&lname) || lname == "host" { + continue; + } + if let Ok(v) = value.to_str() { + builder = builder.header(name.as_str(), v); + } + } + + // Inject secrets from env (never from the browser). + for (env_name, header_name) in &route.headers_from_env { + if let Ok(val) = std::env::var(env_name) { + if !val.is_empty() { + // Common pattern: Authorization Bearer + let hdr_val = if header_name.eq_ignore_ascii_case("authorization") + && !val.to_ascii_lowercase().starts_with("bearer ") + { + format!("Bearer {val}") + } else { + val + }; + builder = builder.header(header_name.as_str(), hdr_val); + } + } + } + + if !body_bytes.is_empty() { + builder = builder.body(body_bytes); + } + + let upstream = match builder.send().await { + Ok(r) => r, + Err(e) => { + return (StatusCode::BAD_GATEWAY, format!("upstream: {e}")).into_response(); + } + }; + + let status = StatusCode::from_u16(upstream.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let mut out_headers = HeaderMap::new(); + for (name, value) in upstream.headers().iter() { + let lname = name.as_str().to_ascii_lowercase(); + if is_hop_by_hop(&lname) { + continue; + } + if let (Ok(n), Ok(v)) = ( + HeaderName::from_bytes(name.as_str().as_bytes()), + HeaderValue::from_bytes(value.as_bytes()), + ) { + out_headers.append(n, v); + } + } + // Ensure SSE content-type is preserved if present. + if !out_headers.contains_key(header::CONTENT_TYPE) { + if let Some(ct) = upstream.headers().get(reqwest::header::CONTENT_TYPE) { + if let Ok(v) = HeaderValue::from_bytes(ct.as_bytes()) { + out_headers.insert(header::CONTENT_TYPE, v); + } + } + } + + if route.stream { + let stream = upstream.bytes_stream().map_err(|e| { + std::io::Error::new(std::io::ErrorKind::Other, e) + }); + let body = Body::from_stream(stream); + let mut resp = Response::new(body); + *resp.status_mut() = status; + *resp.headers_mut() = out_headers; + resp + } else { + let bytes = match upstream.bytes().await { + Ok(b) => b, + Err(e) => { + return (StatusCode::BAD_GATEWAY, format!("upstream body: {e}")).into_response(); + } + }; + let mut resp = Response::new(Body::from(bytes)); + *resp.status_mut() = status; + *resp.headers_mut() = out_headers; + resp + } +} + +fn build_upstream_url( + route: &ProxyRoute, + req_path: &str, + query: Option<&str>, +) -> Result { + let base = expand_env(&route.upstream); + if base.is_empty() { + return Err("proxy upstream empty after env expand".into()); + } + // If upstream already looks like a full URL with path (contains scheme and + // more than origin), and strip_prefix empties the remainder, use base as-is. + let mut path = req_path.to_string(); + let strip = route.strip_prefix.trim(); + if !strip.is_empty() { + let strip_n = normalize_path(strip); + if path.starts_with(&strip_n) { + path = path[strip_n.len()..].to_string(); + if path.is_empty() { + path = "/".into(); + } else if !path.starts_with('/') { + path = format!("/{path}"); + } + } + } + + let url = if base.contains("://") { + let trimmed = base.trim_end_matches('/'); + // If base has no path beyond origin, append path; if base includes a + // path suffix, append remaining path. + if let Some(idx) = trimmed.find("://") { + let after = &trimmed[idx + 3..]; + if after.contains('/') { + // base includes path — append only if remaining path is not "/" + if path == "/" { + trimmed.to_string() + } else { + format!("{trimmed}{path}") + } + } else { + format!("{trimmed}{path}") + } + } else { + format!("{trimmed}{path}") + } + } else { + return Err(format!("proxy upstream must be absolute URL, got `{base}`")); + }; + + Ok(match query { + Some(q) if !q.is_empty() => { + if url.contains('?') { + format!("{url}&{q}") + } else { + format!("{url}?{q}") + } + } + _ => url, + }) +} + +fn is_hop_by_hop(name: &str) -> bool { + matches!( + name, + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailers" + | "transfer-encoding" + | "upgrade" + | "content-length" + ) +} diff --git a/src/host/plugin.rs b/src/host/plugin.rs index 6c8688f..f6ee736 100644 --- a/src/host/plugin.rs +++ b/src/host/plugin.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_int, c_void}; use std::path::PathBuf; +use std::sync::{Mutex, OnceLock}; use libloading::{Library, Symbol}; @@ -59,9 +60,38 @@ thread_local! { static LIB_PATH_CALL: RefCell> = const { RefCell::new(None) }; } +/// Process-wide host pointer while a long-lived plugin call (e.g. `web_listen`) is active. +/// Lets Tokio worker threads invoke `host_query` during HTTP handlers. +struct SendPtr(*mut T); +unsafe impl Send for SendPtr {} +unsafe impl Sync for SendPtr {} + +struct SendLibPath(Option); +unsafe impl Send for SendLibPath {} +unsafe impl Sync for SendLibPath {} + +static GLOBAL_HOST: OnceLock>> = OnceLock::new(); +/// Process-wide lib-path hook (same lifetime as GLOBAL_HOST). +static GLOBAL_LIB_PATH: OnceLock> = OnceLock::new(); +/// Serialize interpreter re-entry from HTTP worker threads. +static INVOKE_LOCK: OnceLock> = OnceLock::new(); + +fn global_host_slot() -> &'static Mutex> { + GLOBAL_HOST.get_or_init(|| Mutex::new(SendPtr(std::ptr::null_mut()))) +} + +fn global_lib_slot() -> &'static Mutex { + GLOBAL_LIB_PATH.get_or_init(|| Mutex::new(SendLibPath(None))) +} + +fn invoke_lock() -> &'static Mutex<()> { + INVOKE_LOCK.get_or_init(|| Mutex::new(())) +} + /// Hook: resolve bare `lib.member` (and longer paths) in the site module's import tree. +#[derive(Clone, Copy)] pub struct LibPathCall { - pub call: fn(*mut (), &str) -> Result, + pub call: fn(*mut (), &str, Option<&Value>) -> Result, pub data: *mut (), } @@ -69,7 +99,15 @@ pub struct LibPathCall { pub fn with_lib_path_call(hook: LibPathCall, body: impl FnOnce() -> R) -> R { LIB_PATH_CALL.with(|slot| { let prev = slot.replace(Some(hook)); + let prev_g = { + let mut g = global_lib_slot().lock().unwrap_or_else(|e| e.into_inner()); + std::mem::replace(&mut g.0, Some(hook)) + }; let out = body(); + { + let mut g = global_lib_slot().lock().unwrap_or_else(|e| e.into_inner()); + g.0 = prev_g; + } slot.replace(prev); out }) @@ -252,10 +290,23 @@ unsafe extern "C" fn host_query( }; let ctx_ptr = CURRENT_HOST.with(|c| c.get()); + let ctx_ptr = if ctx_ptr.is_null() { + global_host_slot() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .0 + } else { + ctx_ptr + }; if ctx_ptr.is_null() { set_err("host_query: no active host context (call during plugin fn)"); return 1; } + let _invoke_guard; + let from_worker = CURRENT_HOST.with(|c| c.get().is_null()); + if from_worker { + _invoke_guard = invoke_lock().lock().unwrap_or_else(|e| e.into_inner()); + } let ctx = &mut *ctx_ptr; let args_owned: Option = if args_json.is_null() { @@ -314,14 +365,26 @@ unsafe extern "C" fn host_query( .and_then(|v| v.as_str()) .ok_or_else(|| "call_lib_path requires `path`".to_string())? .to_string(); - LIB_PATH_CALL.with(|slot| { - let hook = slot.borrow(); - let hook = hook - .as_ref() - .ok_or_else(|| "call_lib_path: no site lib resolver".to_string())?; - let v = (hook.call)(hook.data, &path)?; - value_to_json(&v) - }) + let call_args = args.get("args").and_then(|v| { + if v.is_null() { + None + } else { + json_to_value(v).ok() + } + }); + let call_args_ref = call_args.as_ref(); + let from_tls = LIB_PATH_CALL.with(|slot| slot.borrow().clone()); + let hook = if let Some(h) = from_tls { + h + } else { + global_lib_slot() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .0 + .ok_or_else(|| "call_lib_path: no site lib resolver".to_string())? + }; + let v = (hook.call)(hook.data, &path, call_args_ref)?; + value_to_json(&v) })(), "record_plot" => (|| { let raw = args_owned.as_deref().unwrap_or("{}"); @@ -509,7 +572,15 @@ pub fn call_registered( let mut err_ptr: *mut c_char = std::ptr::null_mut(); CURRENT_HOST.with(|c| c.set(ctx as *mut HostContext)); + { + let mut g = global_host_slot().lock().unwrap_or_else(|e| e.into_inner()); + g.0 = ctx as *mut HostContext; + } let rc = unsafe { (reg.fn_ptr)(c_args.as_ptr(), &mut out_ptr, &mut err_ptr) }; + { + let mut g = global_host_slot().lock().unwrap_or_else(|e| e.into_inner()); + g.0 = std::ptr::null_mut(); + } CURRENT_HOST.with(|c| c.set(std::ptr::null_mut())); let err_s = take_c_string(err_ptr); diff --git a/src/host/plugin_stub.rs b/src/host/plugin_stub.rs index ea8d084..128b9dc 100644 --- a/src/host/plugin_stub.rs +++ b/src/host/plugin_stub.rs @@ -9,8 +9,9 @@ pub const ABI_VERSION: u32 = 2; pub const ABI_VERSION_MIN: u32 = 1; /// Hook: resolve bare `lib.member` (unused without native plugins). +#[derive(Clone, Copy)] pub struct LibPathCall { - pub call: fn(*mut (), &str) -> Result, + pub call: fn(*mut (), &str, Option<&Value>) -> Result, pub data: *mut (), } diff --git a/src/interp.rs b/src/interp.rs index 999ec81..8b9fb22 100644 --- a/src/interp.rs +++ b/src/interp.rs @@ -195,7 +195,11 @@ impl Interpreter { } /// Resolve `lib.member` on the site (entry) module — used by ABI `call_lib_path`. - pub(crate) fn call_site_lib_path(&mut self, path: &str) -> std::result::Result { + pub(crate) fn call_site_lib_path( + &mut self, + path: &str, + args: Option<&Value>, + ) -> std::result::Result { let site = self .site_module .ok_or_else(|| "call_lib_path: no site module".to_string())?; @@ -210,7 +214,13 @@ impl Interpreter { "call_lib_path: need `lib.member`, got `{path}`" )); } - self.eval_path_call(site, &parts, &[]) + let mut ev_args: Vec = Vec::new(); + if let Some(Value::Map(pairs)) = args { + for (k, v) in pairs { + ev_args.push(EvArg::Named(k.clone(), v.clone())); + } + } + self.eval_path_call(site, &parts, &ev_args) .map_err(|e| e.to_string()) } @@ -222,10 +232,14 @@ impl Interpreter { struct HookData { interp: *mut Interpreter, } - fn trampoline(data: *mut (), path: &str) -> std::result::Result { + fn trampoline( + data: *mut (), + path: &str, + args: Option<&Value>, + ) -> std::result::Result { let data = unsafe { &mut *(data as *mut HookData) }; let interp = unsafe { &mut *data.interp }; - interp.call_site_lib_path(path) + interp.call_site_lib_path(path, args) } let mut data = HookData { interp: self as *mut Interpreter, @@ -1042,19 +1056,31 @@ fn find_top<'a>(module: &'a Module, name: &str) -> Option<&'a Function> { /// Resolve `#` object type by name in the entry module or any imported library. fn find_object_type<'a>(module: &'a Module, name: &str) -> Option<(&'a Module, &'a Function)> { - module - .functions - .iter() - .find(|f| f.name == name && f.is_object()) - .map(|f| (module, f)) - .or_else(|| { - module.import_modules.values().find_map(|lib| { - lib.functions - .iter() - .find(|f| f.name == name && f.is_object()) - .map(|f| (lib, f)) - }) - }) + fn search<'a>( + module: &'a Module, + name: &str, + seen: &mut std::collections::HashSet<*const Module>, + ) -> Option<(&'a Module, &'a Function)> { + let key = module as *const Module; + if !seen.insert(key) { + return None; + } + if let Some(f) = module + .functions + .iter() + .find(|f| f.name == name && f.is_object()) + { + return Some((module, f)); + } + for lib in module.import_modules.values() { + if let Some(hit) = search(lib, name, seen) { + return Some(hit); + } + } + None + } + let mut seen = std::collections::HashSet::new(); + search(module, name, &mut seen) } /// Resolve `` `obj`.method `` along the inheritance chain (`_type` first, then bases). diff --git a/tests/ext/agent-mcp-server-helpers.mq.md b/tests/ext/agent-mcp-server-helpers.mq.md new file mode 100644 index 0000000..9fb51e3 --- /dev/null +++ b/tests/ext/agent-mcp-server-helpers.mq.md @@ -0,0 +1,15 @@ +--- +title: agent mcp server smoke helpers +description: Tool target for MCP server smoke. +--- + +## ping ++ `payload`=None + +`out` = + +| ok | via | +|----|-----| +| true | mcp | + +**out** diff --git a/tests/ext/agent-mcp-server-smoke.mq.md b/tests/ext/agent-mcp-server-smoke.mq.md new file mode 100644 index 0000000..8cccb92 --- /dev/null +++ b/tests/ext/agent-mcp-server-smoke.mq.md @@ -0,0 +1,32 @@ +--- +title: agent mcp server smoke +description: Offline assemble mcp_server + one tool (no serve). +import agent:ext/ai/agent.mq.md +import plugin:lib/plugin.mq.md +import sys:lib/sys.mq.md +import demo:agent-mcp-server-helpers.mq.md +--- + +# main + +*p = > plugin.native_path name="agent"* +1. `p` + > plugin.load path=`p` +2. * + > sys.exit code=1 + +*srv = > agent.mcp_server name="marqdo-test"* +*srv = > `srv`.tool name="ping" fn="demo.ping" description="ping tool"* +*tools = srv[^tools]* +*n = > len value=`tools`* +1. `n` == 1 + > print text=mcp-tool-ok +2. * + > print text=mcp-tool-fail + +*t0 = tools[^1]* +*fn = t0[^fn]* +1. `fn` == "demo.ping" + > print text=mcp-fn-ok +2. * + > print text=mcp-fn-fail diff --git a/tests/ext/web-db-cross-module-lib.mq.md b/tests/ext/web-db-cross-module-lib.mq.md new file mode 100644 index 0000000..837ab93 --- /dev/null +++ b/tests/ext/web-db-cross-module-lib.mq.md @@ -0,0 +1,26 @@ +--- +title: web db cross-module lib +description: Opens sqlite, inits a tiny table, returns the db handle. +import web:ext/web/web.mq.md +--- + +## open + +*store = > web.db url="sqlite::memory:"* + +`fields` = + +| name | type | +|------|------| +| title | text | + +> `store`.init name=items fields=`fields` + +`seed` = + +| @ | title | +|---|-------| +| 1 | hello | + +> `store`.insert table=items rows=`seed` +**store** diff --git a/tests/ext/web-db-cross-module-smoke.mq.md b/tests/ext/web-db-cross-module-smoke.mq.md new file mode 100644 index 0000000..2f7e41e --- /dev/null +++ b/tests/ext/web-db-cross-module-smoke.mq.md @@ -0,0 +1,24 @@ +--- +title: web db cross-module smoke +description: Return a web.db handle from an imported module and call select in the entry module. +import web:ext/web/web.mq.md +import plugin:lib/plugin.mq.md +import sys:lib/sys.mq.md +import storemod:web-db-cross-module-lib.mq.md +--- + +# main + +*p = > plugin.native_path name="web"* +1. `p` + > plugin.load path=`p` +2. * + > sys.exit code=1 + +*store = > storemod.open* +*rows = > `store`.select table="items" limit=10* +*n = > len value=`rows`* +1. `n` >= 1 + > print text=cross-db-ok +2. * + > print text=cross-db-fail diff --git a/tests/ext/web-hosting-helpers.mq.md b/tests/ext/web-hosting-helpers.mq.md new file mode 100644 index 0000000..17aa70d --- /dev/null +++ b/tests/ext/web-hosting-helpers.mq.md @@ -0,0 +1,16 @@ +--- +title: web hosting live helpers +description: Tiny lib used by hosting live-server invoke route. +--- + +## echo ++ `msg`="" ++ `payload`=None + +`out` = + +| ok | msg | +|----|-----| +| true | `msg` | + +**out** diff --git a/tests/ext/web-hosting-live-server.mq.md b/tests/ext/web-hosting-live-server.mq.md new file mode 100644 index 0000000..76a03c7 --- /dev/null +++ b/tests/ext/web-hosting-live-server.mq.md @@ -0,0 +1,22 @@ +--- +title: web hosting live server +description: Listen with proxy (→ mock upstream) and invoke routes for gold.rs. +import web:ext/web/web.mq.md +import plugin:lib/plugin.mq.md +import sys:lib/sys.mq.md +import demo:web-hosting-helpers.mq.md +--- + +# main + +*p = > plugin.native_path name="web"* +1. `p` + > plugin.load path=`p` +2. * + > sys.exit code=1 + +*pg = > web.page title="hosting-live"* +*app = > web.app page=`pg` host=127.0.0.1 port=18141* +*app = > `app`.proxy path="/proxy/sse" upstream="http://127.0.0.1:18142/sse" stream=True methods="POST"* +*app = > `app`.invoke path="/api/echo" fn="demo.echo" method="POST" body="json"* +> `app`.listen diff --git a/tests/ext/web-proxy-invoke-smoke.mq.md b/tests/ext/web-proxy-invoke-smoke.mq.md new file mode 100644 index 0000000..3a41b87 --- /dev/null +++ b/tests/ext/web-proxy-invoke-smoke.mq.md @@ -0,0 +1,75 @@ +--- +title: web proxy/invoke smoke +description: Offline assemble of app.proxy and app.invoke routes (no listen). +import web:ext/web/web.mq.md +import plugin:lib/plugin.mq.md +import sys:lib/sys.mq.md +--- + +# main + +*p = > plugin.native_path name="web"* +1. `p` + > plugin.load path=`p` +2. * + > sys.exit code=1 + +`proxy_tbl` = + +| path | upstream | stream | strip_prefix | methods | headers_from_env | timeout_ms | +|------|----------|--------|--------------|---------|------------------|------------| +| /proxy/chat | $OPENAI_BASE_URL | True | /proxy | POST | OPENAI_API_KEY=Authorization | 60000 | + +`invoke_tbl` = + +| path | method | fn | body | return | +|------|--------|-----|------|--------| +| /api/echo | POST | demo.echo | json | json | + +*pg = > web.page title="proxy-invoke"* +*app = > web.app page=`pg`* +*app = > `app`.configure proxy=`proxy_tbl` invoke=`invoke_tbl`* +*app = > `app`.proxy path="/v1/stream" upstream="https://example.com/v1/chat" stream=True headers_from_env="TOKEN=X-Token"* +*app = > `app`.invoke path="/api/ping" fn="demo.ping" method="GET" body="query"* + +*proutes = app[^proxy_routes]* +*chat = proutes[^proxy/chat]* +*up = chat[^upstream]* +1. `up` == "$OPENAI_BASE_URL" + > print text=proxy-table-ok +2. * + > print text=proxy-table-fail + +*stream = chat[^stream]* +1. `stream` + > print text=proxy-stream-ok +2. * + > print text=proxy-stream-fail + +*hdr = chat[^headers_from_env]* +1. `hdr` == "OPENAI_API_KEY=Authorization" + > print text=proxy-env-ok +2. * + > print text=proxy-env-fail + +*v1 = proutes[^v1/stream]* +*v1u = v1[^upstream]* +1. `v1u` == "https://example.com/v1/chat" + > print text=proxy-method-ok +2. * + > print text=proxy-method-fail + +*iroutes = app[^invoke_routes]* +*echo = iroutes[^api/echo]* +*efn = echo[^fn]* +1. `efn` == "demo.echo" + > print text=invoke-table-ok +2. * + > print text=invoke-table-fail + +*ping = iroutes[^api/ping]* +*pm = ping[^method]* +1. `pm` == "GET" + > print text=invoke-method-ok +2. * + > print text=invoke-method-fail diff --git a/tests/gold.rs b/tests/gold.rs index 7557cfc..dd0fec5 100644 --- a/tests/gold.rs +++ b/tests/gold.rs @@ -1952,6 +1952,167 @@ fn ext_web_middleware_live() { let _ = child.wait(); } +#[test] +fn ext_web_proxy_invoke_smoke() { + let status = Command::new("cargo") + .args(["build", "-p", "marqdo_plugin_web"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .status() + .expect("build web plugin"); + assert!(status.success(), "failed to build marqdo_plugin_web"); + assert_out( + "tests/ext/web-proxy-invoke-smoke.mq.md", + "proxy-table-ok +proxy-stream-ok +proxy-env-ok +proxy-method-ok +invoke-table-ok +invoke-method-ok", + ); +} + +#[test] +fn ext_web_hosting_live() { + // Mock upstream SSE on 18142 + Marqdo web proxy/invoke on 18141. + let status = Command::new("cargo") + .args(["build", "-p", "marqdo_plugin_web"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .status() + .expect("build web plugin"); + assert!(status.success(), "failed to build marqdo_plugin_web"); + + use std::io::{Read, Write}; + use std::net::TcpListener; + + let upstream = TcpListener::bind("127.0.0.1:18142").expect("bind mock upstream"); + upstream.set_nonblocking(true).ok(); + let upstream_thread = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + if std::time::Instant::now() > deadline { + break; + } + match upstream.accept() { + Ok((mut sock, _)) => { + let mut buf = [0u8; 4096]; + let _ = sock.set_read_timeout(Some(std::time::Duration::from_secs(2))); + let _ = sock.read(&mut buf); + let body = "data: {\"id\":1}\n\ndata: {\"id\":2}\n\n"; + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = sock.write_all(resp.as_bytes()); + let _ = sock.flush(); + } + Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(std::time::Duration::from_millis(50)); + } + Err(_) => break, + } + } + }); + + let script = "tests/ext/web-hosting-live-server.mq.md"; + let bin = env!("CARGO_BIN_EXE_marqdo"); + let mut child = Command::new(bin) + .args(["run", script]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn hosting live server"); + + let base = "http://127.0.0.1:18141"; + let ready = std::time::Instant::now(); + loop { + if ready.elapsed().as_secs() > 15 { + let _ = child.kill(); + panic!("hosting live server did not start in time"); + } + let out = Command::new("curl") + .args(["-s", "-o", "/dev/null", "-w", "%{http_code}", &format!("{base}/")]) + .output() + .expect("curl probe"); + if String::from_utf8_lossy(&out.stdout) == "200" { + break; + } + std::thread::sleep(std::time::Duration::from_millis(300)); + } + + // Proxy SSE: expect both data frames. + let out = Command::new("curl") + .args([ + "-s", + "-N", + "-X", + "POST", + &format!("{base}/proxy/sse"), + "-H", + "Content-Type: application/json", + "-d", + "{}", + ]) + .output() + .expect("curl proxy"); + let body = String::from_utf8_lossy(&out.stdout); + assert!( + body.contains("data: {\"id\":1}") && body.contains("data: {\"id\":2}"), + "proxy sse body={body}" + ); + + // Invoke: POST JSON → demo.echo + let out = Command::new("curl") + .args([ + "-s", + "-X", + "POST", + &format!("{base}/api/echo"), + "-H", + "Content-Type: application/json", + "-d", + r#"{"msg":"hi"}"#, + ]) + .output() + .expect("curl invoke"); + let body = String::from_utf8_lossy(&out.stdout); + assert!( + body.contains("\"ok\"") && body.contains("\"msg\":\"hi\""), + "invoke body={body}" + ); + + let _ = child.kill(); + let _ = child.wait(); + let _ = upstream_thread.join(); +} + +#[test] +fn ext_web_db_cross_module_smoke() { + let status = Command::new("cargo") + .args(["build", "-p", "marqdo_plugin_web"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .status() + .expect("build web plugin"); + assert!(status.success(), "failed to build marqdo_plugin_web"); + assert_out("tests/ext/web-db-cross-module-smoke.mq.md", "cross-db-ok"); +} + +#[test] +fn ext_agent_mcp_server_smoke() { + let status = Command::new("cargo") + .args(["build", "-p", "marqdo_plugin_agent"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .status() + .expect("build agent plugin"); + assert!(status.success(), "failed to build marqdo_plugin_agent"); + assert_out( + "tests/ext/agent-mcp-server-smoke.mq.md", + "mcp-tool-ok +mcp-fn-ok", + ); +} + #[test] fn ext_web_security_smoke() { let status = Command::new("cargo")