From 88dc6d5a54cccce67a2f60d8dc675fc768b62c13 Mon Sep 17 00:00:00 2001 From: cflmy Date: Thu, 24 Sep 2026 19:51:24 +0800 Subject: [PATCH] feat(core): Artifact Metadata Binding (${env|arg|sys|secret}) Load-time resolve of frontmatter bindings into Module.metadata; Secret masking; sys.meta / --bind; llm ctor prefers meta over env_get. Co-authored-by: Cursor --- doc/adr/0006-artifact-metadata-binding.md | 40 + doc/design/binding.md | 163 +++ doc/design/core-surface.md | 2 +- doc/design/markdown-mapping-v0.3.md | 1 + doc/design/markdown-mapping.md | 2 +- doc/next/006.md | 1334 +++++++++++++++++++++ doc/roadmap/ai-stack.md | 1 + ext/ai/llm.mq.md | 70 +- lib/sys.mq.md | 13 + lib/系统.mq.md | 13 + src/aliases.rs | 1 + src/ast.rs | 6 + src/binding/mod.rs | 531 ++++++++ src/builtin.rs | 1 + src/bytecode/vm.rs | 3 + src/contract.rs | 1 + src/host/dispatch.rs | 13 +- src/host/json.rs | 1 + src/host/mod.rs | 3 + src/host/subtask.rs | 1 + src/host/sys.rs | 16 + src/interp.rs | 3 + src/lib.rs | 20 +- src/load.rs | 44 +- src/main.rs | 11 + src/parse/mod.rs | 127 +- src/value.rs | 12 + src/view/debug_api.rs | 1 + src/view/render.rs | 1 + tests/gold.rs | 52 + tests/structure/meta-binding.mq.md | 28 + 31 files changed, 2466 insertions(+), 49 deletions(-) create mode 100644 doc/adr/0006-artifact-metadata-binding.md create mode 100644 doc/design/binding.md create mode 100644 doc/next/006.md create mode 100644 src/binding/mod.rs create mode 100644 tests/structure/meta-binding.mq.md diff --git a/doc/adr/0006-artifact-metadata-binding.md b/doc/adr/0006-artifact-metadata-binding.md new file mode 100644 index 0000000..ce6902b --- /dev/null +++ b/doc/adr/0006-artifact-metadata-binding.md @@ -0,0 +1,40 @@ +# ADR 0006:Artifact Metadata Binding(`${…}` 进核心 · Metadata-only) + +| | | +|---|---| +| Status | **Accepted** | +| Date | 2026-09-24 | +| Related | [binding.md](../design/binding.md) · [core-surface.md](../design/core-surface.md) · [layers.md](../design/layers.md) · [doc/next/006.md](../next/006.md) | + +## Context + +- Executable Document 需要声明「依赖哪些外部运行时资源」(模型名、API 密钥、base URL),而不是在 `ext/*` 里各自 `sys.env_get`。 +- [doc/next/006.md](../next/006.md) 提出 **Marqdo Binding**(`${namespace.name}`)与 Artifact Metadata 统一解析。 +- [layers.md](../design/layers.md):Language 层变更须 ADR;核心构造计数只许持平或下降;默认进 ext。 +- 正文插值会与 `$$…$$` 公式面冲突,且会新增内联构造压力。 + +## Decision + +1. 采纳 **Artifact Metadata Binding** 为 Language 层能力;正式规范见 [binding.md](../design/binding.md)。 +2. **Phase 1**:`${…}` **仅**在文件头 `---` Metadata 中生效;正文 / 粗体 / 调用实参不做。 +3. **不**把 Binding 记入 [core-surface.md](../design/core-surface.md) 的 19 构造守卫表——与 `import` 同属 §4「其它语言面」。 +4. 核心只做声明与解析;扩展(如 `ext/ai/llm`)消费已绑定的 `Module.metadata` / `sys.meta*`。 +5. 第一版 namespace:`env` · `arg` · `sys` · `secret`。`config` / `file` 与正文插值推迟。 +6. 加载期一次 resolve → Bound Artifact;`Value::Secret` 遮掩展示;`sys.env_get` 保留为逃逸口。 + +## Consequences + +### Positive + +- 配置成为文档的一部分;LLM / agent / 其它 ext 共用同一套 Binding,避免各写一套 getenv。 +- 缺失 `!` 在加载期失败,而不是运行到 HTTP 401。 +- 不膨胀 CORE_CONSTRUCTS;与公式 `$$` 无冲突。 + +### Negative / Risks + +- Metadata-only 时,作者仍须用 `sys.meta_get`(或 ctor 约定)把配置读进程序——不如正文 `${}` 直观(Phase 2 再议)。 +- Secret 在 `Text + Secret` 拼接时会揭示明文(供 Authorization);须约束日志面。 + +### Neutral + +- `#` / `##` / `**` / `*` 等核心标记不变;frontmatter `import` 句法不变。 diff --git a/doc/design/binding.md b/doc/design/binding.md new file mode 100644 index 0000000..abcbbcc --- /dev/null +++ b/doc/design/binding.md @@ -0,0 +1,163 @@ +# Marqdo Binding · Artifact Metadata(Phase 1) + +| | | +|---|---| +| 状态 | **Accepted · Phase 1(Metadata-only)** | +| 日期 | 2026-09-24 | +| ADR | [0006-artifact-metadata-binding.md](../adr/0006-artifact-metadata-binding.md) | +| 讨论稿 | [doc/next/006.md](../next/006.md) | +| 实现 | `src/binding/` · `Module.metadata` · `sys.meta` / `sys.meta_get` · `marqdo run --bind` | + +> **边界**:核心负责「声明与解析配置」;扩展负责「解释配置的语义」。 +> **本波**:`${…}` 只出现在文件头 `---` Artifact Metadata 中。正文插值见 §8。 + +--- + +## 1. Artifact Metadata + +文件开头成对 `---` 之间为 **Artifact Metadata**(声明式元信息环境),不是「随便贴的 YAML」。 + +```text +.mq.md +├── Metadata ← Identity / Configuration / Dependencies / Bindings +└── Executable Body +``` + +- `import bind:…` / `导入` 行:模块依赖(既有语义不变)。 +- 其它 `key: value` 行:元信息;值可为字面量或 **Binding Expression**。 +- Phase 1 仅支持 **flat** 键(同级 `key: value`);块标量 `>` / `|` 收成 Text。 + +例: + +```yaml +--- +title: demo +type: llm +model: ${env.OPENAI_MODEL ?? "gpt-4o-mini"} +api_key: ${secret.OPENAI_API_KEY!} +base_url: ${env.OPENAI_BASE_URL ?? "https://api.openai.com/v1"} +import sys:lib/sys.mq.md +--- +``` + +加载后内部为已绑定的 `Module.metadata`(确定性快照),执行期不再对同一键反复 `getenv`。 + +--- + +## 2. Binding Expression + +外形: + +```text +${expression} +``` + +### 2.1 路径 + +```text +${namespace.name} +``` + +| Namespace | 意义 | Phase 1 | +|-----------|------|---------| +| `env` | 进程环境变量 | ✅ | +| `arg` | CLI / Runtime 绑定(`marqdo run --bind KEY=VALUE`) | ✅ | +| `sys` | 系统信息:`cwd` · `platform` | ✅ | +| `secret` | 敏感环境变量 → `Value::Secret` | ✅ | +| `config` | 配置 Artifact | 推迟 | +| `file` | 外部文件内容 | 推迟 | + +### 2.2 默认与必需 + +```text +${env.OPENAI_MODEL ?? "gpt-4o-mini"} +${secret.OPENAI_API_KEY!} +``` + +- `A ?? B`:A 缺失(未设置 / 空)则用 B。 +- `A!`:A 必须存在,否则加载失败(`config.required_binding_missing`)。 + +### 2.3 类型转换 + +```text +${int(env.MAX_TOKENS ?? "4096")} +${float(env.TEMPERATURE ?? "0.2")} +${bool(env.STREAM ?? "false")} +``` + +### 2.4 独占 value vs 插值 + +| 形态 | 语义 | +|------|------| +| `model: ${env.OPENAI_MODEL}` | **Value binding** — 保留解析后的类型(含 Secret) | +| `title: "Hello ${env.USER}"` | **Interpolation** — 结果为 Text | +| `model: gpt-4o-mini` | **Literal** — 字面量;**不会**被环境变量静默覆盖 | + +--- + +## 3. Resolution Policy + +对每个 metadata 键: + +1. 解析该键的字面量 / Binding / 插值(字面量定稿后不受 env 覆盖)。 +2. 若 CLI 提供了同名 `--bind KEY=VALUE`,**覆盖**该键的最终值(Text)。 +3. Binding 内部查找顺序(仅当表达式写出时):`arg` / `env` / `secret` / `sys` 按表达式 namespace;`??` 链从左到右。 + +推荐作者意图: + +```text +CLI --bind + → Artifact 字面量 / 已写死的 Binding 结果 + → Binding 源(env / secret / …) + → ?? 默认 +``` + +--- + +## 4. Secret + +`${secret.NAME}` → `Value::Secret`。 + +- `print` / `str` / Display → ``(不泄露)。 +- `type` → `secret`。 +- 与 Text 拼接(如 `bearer + api_key`)时**揭示**明文,供 HTTP Authorization。 +- 不得进入普通 episode / 错误信息明文(扩展作者责任;核心 Display 已遮掩)。 + +--- + +## 5. Runtime API + +| API | 语义 | +|-----|------| +| `sys.meta` / `系统.元信息` | 入口模块已绑定 metadata(map) | +| `sys.meta_get` / `系统.取元信息` | `key=` → 值或 `None` | +| `marqdo run FILE --bind KEY=VALUE` | 可重复;写入 `arg` 并覆盖同名 metadata 键 | +| `sys.env_get` | **保留**逃逸口;新代码优先 Metadata Binding | + +--- + +## 6. 错误 + +| code | 何时 | +|------|------| +| `config.required_binding_missing` | `${ns.name!}` 且源缺失 | +| `config.unknown_binding_ns` | 未知 namespace | +| `config.bad_binding` | 表达式非法 / cast 失败 | + +诊断尽量带 frontmatter 行号。 + +--- + +## 7. 与扩展的分工 + +- **核心**:解析 Metadata、resolve Binding、暴露 `Module.metadata` / `sys.meta*`。 +- **`ext/ai/llm`**:构造次序为 **显式实参 → `sys.meta_get`(model / base_url / api_key)→ `sys.env_get` 回退 → 默认**。 + +--- + +## 8. Phase 2(不做于本波) + +- 正文 / 粗体 / 表 / 调用实参中的 `${…}` +- `config` / `file` namespace、嵌套 YAML map +- 显式 `override:` 糖 +- 计入 `CORE_CONSTRUCTS` diff --git a/doc/design/core-surface.md b/doc/design/core-surface.md index 680cacc..4ad885b 100644 --- a/doc/design/core-surface.md +++ b/doc/design/core-surface.md @@ -44,4 +44,4 @@ ## 4. 范围外(进 Language 层,但不在本守卫) -frontmatter YAML(元信息)与 `import` 行(模块装载)属于 Language 层的其它语言面,随 [layers.md](layers.md) 准入流程登记,暂不进本清单守卫;如需一并守卫,走 ADR 扩表。 +frontmatter / Artifact Metadata(元信息)、`import` 行(模块装载)、以及 Metadata 内的 **Binding Expression**(`${ns.name}`,见 [binding.md](binding.md) · [ADR 0006](../adr/0006-artifact-metadata-binding.md))属于 Language 层的其它语言面,随 [layers.md](layers.md) 准入流程登记,**暂不进本清单守卫**(不增 CORE_CONSTRUCTS 计数)。如需一并守卫,走 ADR 扩表。 diff --git a/doc/design/markdown-mapping-v0.3.md b/doc/design/markdown-mapping-v0.3.md index eed873c..d5666da 100644 --- a/doc/design/markdown-mapping-v0.3.md +++ b/doc/design/markdown-mapping-v0.3.md @@ -616,3 +616,4 @@ FnDef.params: | 2026-09-13 | 决议空返回(§7.2)与提及性反引号(§4.4);形参改为「升参」与绑定图分离(§8.2) | | 2026-09-14 | §7.2:`---` 不再收束;返回语句结束函数体解析;空返回首选 `*None*` | | 2026-09-14 | 取元主语法改为 `[键](集合)`;脚注取元过渡保留 | +| 2026-09-24 | Artifact Metadata Binding(`${…}` Phase 1,仅 frontmatter):见 [binding.md](binding.md) · ADR 0006 | diff --git a/doc/design/markdown-mapping.md b/doc/design/markdown-mapping.md index 9db4704..d54dfbe 100644 --- a/doc/design/markdown-mapping.md +++ b/doc/design/markdown-mapping.md @@ -130,7 +130,7 @@ Markdown **标记即语法**。输出**不是**语言架构原语,而是内置 ## 5. Frontmatter 与跨文件 -文件头 `---` 对为元信息。其中 `import bind:path.mq.md` / `导入` 为**文件导入**;`import bind:lib.member` 为**短名**(废除独立 `use` 与 frontmatter `>` 导入)。调用经裸点号路径(`time.parse`、`agent.agent`);实例方法在独立 `>` 调用须 `` > `var`.method ``(`*…*` 段内接收者裸名)。详见 [module-namespace.md](module-namespace.md)。 +文件头 `---` 对为 **Artifact Metadata**。其中 `import bind:path.mq.md` / `导入` 为**文件导入**;`import bind:lib.member` 为**短名**(废除独立 `use` 与 frontmatter `>` 导入)。其它 `key: value` 可为字面量或 **Binding**(`${env.NAME}` 等,加载期解析);见 [binding.md](binding.md)。调用经裸点号路径(`time.parse`、`agent.agent`);实例方法在独立 `>` 调用须 `` > `var`.method ``(`*…*` 段内接收者裸名)。详见 [module-namespace.md](module-namespace.md)。 **函数体内的** `---` / `***` 不解释为 frontmatter(见 §11)。 diff --git a/doc/next/006.md b/doc/next/006.md new file mode 100644 index 0000000..32f6319 --- /dev/null +++ b/doc/next/006.md @@ -0,0 +1,1334 @@ +--- +type: discussion +status: superseded-by-design +formal: doc/design/binding.md +adr: doc/adr/0006-artifact-metadata-binding.md +--- + +> **正式规范已落地**:讨论收敛见 [doc/design/binding.md](../design/binding.md)(Phase 1 · Metadata-only)与 [ADR 0006](../adr/0006-artifact-metadata-binding.md)。下文为原始讨论稿,保留备查。 + +你的这个设计我觉得不错,配置信息写进元信息,能够自动的处理一些从环境变量中得到的变量,我觉得这个可以写进核心的语法之中,你认为呢?如果可以给我详细的设计方案 + +我认为**非常可以,而且我会进一步建议:把“环境变量 / 外部配置解析”提升为 Marqdo 核心语法的一部分,而不是 `ext/ai/llm` 自己实现的功能。** + +但有一个关键边界: + +> **核心语法负责“声明与解析配置”,具体扩展负责“解释配置的语义”。** + +这样它不会变成“为了 LLM 发明的配置语法”,而会成为 Marqdo 作为 **Executable Document** 的基础能力。 + +--- + +# 1. 我建议把它定义成:Artifact Metadata Binding + +你现在有: + +```yaml +--- +type: llm +backend: openai-compatible +model: gpt-5.6 +--- +``` + +我建议正式把 `---` 这一块定义为: + +> **Artifact Metadata** + +它不只是 YAML front matter,而是 Marqdo 文档的**声明式元信息环境**。 + +完整模型: + +```text +.mq.md +│ +├── Metadata +│ ├── Identity +│ ├── Configuration +│ ├── Dependencies +│ └── Bindings +│ +└── Executable Body +``` + +于是: + +```yaml +--- +type: llm +backend: openai-compatible +model: gpt-5.6 +--- +``` + +实际上是在声明: + +```text +这个 Document 是一个 LLM Artifact +它使用 openai-compatible backend +默认 model 是 gpt-5.6 +``` + +--- + +# 2. 最重要的设计:Metadata 可以引用外部环境 + +我建议引入一个核心语法: + +```text +${...} +``` + +例如: + +```yaml +--- +type: llm +backend: openai-compatible +model: ${OPENAI_MODEL} +api_key: ${OPENAI_API_KEY} +base_url: ${OPENAI_BASE_URL} +--- +``` + +这样: + +```text +${OPENAI_MODEL} +``` + +不是 YAML 的语法,而是: + +> **Marqdo Binding Expression** + +也就是说: + +```text +Metadata + ↓ +Binding + ↓ +Environment / Secret / File / Argument / System +``` + +--- + +# 3. 但是不要只设计 `${ENV_VAR}` + +如果现在只支持: + +```text +${OPENAI_API_KEY} +``` + +几年以后你一定会发现不够。 + +我建议一开始就设计成**命名空间式 Binding**。 + +例如: + +```text +${env.OPENAI_API_KEY} +${arg.model} +${sys.cwd} +${file.config.model} +${secret.OPENAI_API_KEY} +``` + +这样 Marqdo 的配置系统可以形成: + +```text +binding +├── env +├── arg +├── sys +├── file +├── secret +└── artifact +``` + +--- + +# 4. 最基础的语法 + +我建议: + +```text +${namespace.name} +``` + +例如: + +```yaml +--- +type: llm +model: ${env.OPENAI_MODEL} +api_key: ${env.OPENAI_API_KEY} +base_url: ${env.OPENAI_BASE_URL} +--- +``` + +这比: + +```yaml +model: ${OPENAI_MODEL} +``` + +更值得作为长期语言设计。 + +因为它避免了未来命名冲突。 + +--- + +# 5. 默认值必须支持 + +这是非常重要的。 + +例如: + +```yaml +--- +model: ${env.OPENAI_MODEL | "gpt-5.6"} +base_url: ${env.OPENAI_BASE_URL | "https://api.openai.com/v1"} +--- +``` + +意思是: + +```text +如果 OPENAI_MODEL 存在 + 使用它 + +否则 + 使用 gpt-5.6 +``` + +我甚至建议允许更明确的: + +```text +${env.OPENAI_MODEL ?? "gpt-5.6"} +``` + +也就是: + +```text +A ?? B +``` + +代表: + +> 如果 A 不存在,则使用 B。 + +--- + +# 6. Secret 应该和普通 Environment 区分 + +这是我非常建议你现在就设计进去的。 + +不要让: + +```yaml +api_key: ${env.OPENAI_API_KEY} +``` + +成为唯一方式。 + +应该支持: + +```yaml +api_key: ${secret.OPENAI_API_KEY} +``` + +语义上: + +```text +env + 普通运行环境 + +secret + 敏感运行环境 +``` + +这样以后 Marqdo Runtime 可以控制: + +```text +secret +├── 不写入 episode +├── 不进入普通 trace +├── 不显示在 echo +├── 不进入错误信息 +└── 不进入生成的 knowledge artifact +``` + +这对你未来 Agent Framework 非常重要。 + +--- + +# 7. 我建议增加“必需配置”的语法 + +比如: + +```yaml +--- +type: llm +backend: openai-compatible +model: ${env.OPENAI_MODEL ?? "gpt-5.6"} +api_key: ${secret.OPENAI_API_KEY!} +base_url: ${env.OPENAI_BASE_URL ?? "https://api.openai.com/v1"} +--- +``` + +这里: + +```text +${secret.OPENAI_API_KEY!} +``` + +表示: + +> 这个值必须存在。 + +如果没有: + +```text +ConfigurationError: +required binding secret.OPENAI_API_KEY is missing +``` + +而不是运行到 HTTP 请求才发现: + +```text +401 Unauthorized +``` + +这会让 Marqdo 的错误体验好很多。 + +--- + +# 8. Binding 应该在执行前完成 + +这是一个非常重要的架构决定。 + +不要: + +```text +执行程序 + ↓ +遇到 ${env.X} + ↓ +临时解析 +``` + +而应该: + +```text +Load Artifact + ↓ +Parse Metadata + ↓ +Resolve Bindings + ↓ +Validate Metadata + ↓ +Create Runtime Object + ↓ +Execute +``` + +也就是说: + +```text +.mq.md + ↓ +Artifact + ↓ +Bound Artifact +``` + +例如: + +```yaml +--- +type: llm +model: ${env.OPENAI_MODEL ?? "gpt-5.6"} +--- +``` + +加载以后内部应该变成: + +```text +model = "gpt-5.6" +``` + +而不是每次访问: + +```text +model + ↓ +去环境变量查询 +``` + +这样有三个好处: + +1. 配置确定性更强 +2. 性能更好 +3. Debug 可以知道最终配置是什么 + +--- + +# 9. 但是 Secret 不应该真的被普通内部结构展开 + +这里可以做得非常漂亮。 + +普通变量: + +```text +${env.OPENAI_MODEL} +``` + +解析成: + +```text +Value("gpt-5.6") +``` + +Secret: + +```text +${secret.OPENAI_API_KEY} +``` + +解析成: + +```text +SecretValue(...) +``` + +而不是普通 String。 + +于是 Runtime 可以知道: + +```text +Value +SecretValue +``` + +是两种不同类型。 + +例如: + +```text +print(api_key) +``` + +自动输出: + +```text + +``` + +而: + +```text +HTTP Authorization +``` + +可以正常得到真正值。 + +这实际上可以成为 Marqdo 类型系统的一部分: + +```text +String +Secret +Number +Boolean +Table +Document +... +``` + +--- + +# 10. Metadata 本身也应该支持类型 + +我建议不要把 front matter 看成: + +```text +YAML → Map +``` + +而是: + +```text +Metadata → Marqdo Value +``` + +所以: + +```yaml +--- +type: llm +temperature: 0.2 +stream: false +max_tokens: 2048 +tags: + - ai + - llm +--- +``` + +应该保留类型: + +```text +type String +temperature Number +stream Boolean +max_tokens Number +tags List +``` + +而: + +```yaml +model: ${env.OPENAI_MODEL} +``` + +解析之后也是: + +```text +String +``` + +--- + +# 11. 更进一步:Binding 应该允许转换 + +例如: + +```yaml +--- +max_tokens: ${env.MAX_TOKENS} +temperature: ${env.TEMPERATURE} +stream: ${env.STREAM} +--- +``` + +环境变量天然都是字符串。 + +但 Marqdo 应该允许: + +```text +${env.MAX_TOKENS -> int} +${env.TEMPERATURE -> float} +${env.STREAM -> bool} +``` + +或者更简洁: + +```text +${int(env.MAX_TOKENS)} +${float(env.TEMPERATURE)} +${bool(env.STREAM)} +``` + +我个人更倾向后者: + +```text +${int(...)} +``` + +因为它以后可以扩展。 + +--- + +# 12. 这实际上可以成为 Marqdo 的 Configuration Language + +例如: + +```yaml +--- +type: llm + +backend: openai-compatible + +model: ${env.OPENAI_MODEL ?? "gpt-5.6"} + +base_url: + ${env.OPENAI_BASE_URL ?? "https://api.openai.com/v1"} + +api_key: + ${secret.OPENAI_API_KEY!} + +temperature: + ${float(env.OPENAI_TEMPERATURE ?? "0.2")} + +max_tokens: + ${int(env.OPENAI_MAX_TOKENS ?? "4096")} +--- +``` + +这已经不是简单的 YAML 了。 + +它实际上是: + +> **Marqdo Artifact Declaration Language** + +--- + +# 13. 我建议增加 `config` 命名空间 + +还有一个很重要的场景: + +多个 `.mq.md` 共享配置。 + +例如: + +```text +config.mq.md +``` + +```yaml +--- +type: config + +llm: + model: ${env.OPENAI_MODEL ?? "gpt-5.6"} + base_url: ${env.OPENAI_BASE_URL ?? "..."} +--- +``` + +其他文件: + +```yaml +--- +type: llm +config: ./config.mq.md +--- +``` + +或者: + +```yaml +--- +type: llm +model: ${config.llm.model} +base_url: ${config.llm.base_url} +--- +``` + +于是: + +```text +Environment + ↓ +Config + ↓ +Artifact +``` + +形成配置层级。 + +--- + +# 14. 我甚至建议支持配置覆盖 + +这是 Agent Framework 非常需要的。 + +例如: + +```yaml +--- +type: llm +model: ${config.llm.model ?? "gpt-5.6"} +--- +``` + +运行: + +```text +marqdo run task.mq.md --model=gpt-5.6-mini +``` + +那么: + +```text +arg.model +``` + +可以覆盖: + +```text +config.llm.model +``` + +形成: + +```text +Priority + +explicit argument + ↓ +artifact metadata + ↓ +config + ↓ +environment + ↓ +default +``` + +但这里我要特别建议: + +> **不要让环境变量静默覆盖 Artifact 明确写出的配置。** + +否则: + +```yaml +model: gpt-5.6 +``` + +用户看到的是 gpt-5.6, + +机器却因为: + +```text +OPENAI_MODEL=gpt-5-mini +``` + +实际运行了另一个模型。 + +这是非常危险的可复现性问题。 + +--- + +# 15. 所以我建议明确一个 Resolution Policy + +例如: + +```text +explicit metadata + > +argument + > +config + > +environment + > +default +``` + +或者如果 CLI 参数需要最高优先级: + +```text +CLI argument + > +explicit metadata + > +config + > +environment + > +default +``` + +我更推荐: + +```text +CLI + ↓ +Artifact + ↓ +Config + ↓ +Environment + ↓ +Default +``` + +因为: + +```text +Artifact = 文档作者意图 +Environment = 运行环境 +``` + +这符合“代码即文档”的哲学。 + +--- + +# 16. 但环境变量应该允许显式 opt-in 覆盖 + +有时候部署环境确实需要: + +```text +production + ↓ +覆盖 model +``` + +所以可以提供: + +```yaml +--- +type: llm + +model: + value: gpt-5.6 + override: env.OPENAI_MODEL +--- +``` + +或者更简单: + +```yaml +--- +type: llm +model: gpt-5.6 +model_override: ${env.OPENAI_MODEL} +--- +``` + +这样**覆盖是显式的**,而不是隐式的。 + +--- + +# 17. 一个非常重要的设计:Binding 不应该只能出现在 Metadata + +既然这是核心语言能力,我建议: + +```text +${...} +``` + +可以在整个 Marqdo 中出现。 + +例如: + +```markdown +# Hello ${env.USER_NAME} + +Current model: ${config.llm.model} +``` + +代码中: + +```marqdo +**path = ${env.DATA_PATH}** +``` + +参数中: + +```marqdo +**answer = > llm.ask prompt=${file.prompt}** +``` + +表格中: + +```markdown +| model | ${env.OPENAI_MODEL} | +``` + +所以它真正是: + +> **Marqdo Interpolation / Binding** + +而不是: + +> YAML Environment Variable Feature。 + +--- + +# 18. 但要区分「文本插值」和「值绑定」 + +这是语言设计中很容易踩坑的地方。 + +例如: + +```text +"model=${env.OPENAI_MODEL}" +``` + +这是: + +```text +String interpolation +``` + +而: + +```yaml +model: ${env.OPENAI_MODEL} +``` + +应该是: + +```text +Value binding +``` + +两者不能完全一样。 + +比如: + +```yaml +temperature: ${env.TEMPERATURE} +``` + +如果环境变量是: + +```text +0.2 +``` + +应该得到: + +```text +Number(0.2) +``` + +而不是: + +```text +String("0.2") +``` + +因此我建议: + +```text +${...} +``` + +本身是一个 **Expression**。 + +当它独占一个 value 时: + +```text +Value Binding +``` + +当它出现在字符串内部: + +```text +Interpolation +``` + +例如: + +```text +${env.USER} +``` + +→ 原始值 + +而: + +```text +"Hello ${env.USER}" +``` + +→ String + +这会非常干净。 + +--- + +# 19. Binding 可以进一步支持文件 + +这对于 `.mq.md` 非常有意义。 + +例如: + +```yaml +--- +system_prompt: ${file.system_prompt} +knowledge: ${file.knowledge} +--- +``` + +甚至: + +```text +${file("./knowledge.md")} +``` + +这样: + +```text +Document + ↓ +Metadata + ↓ +External Artifact +``` + +天然形成知识组合。 + +这和你: + +> Executable Documents → Executable Knowledge Bases + +的研究方向又连接起来了。 + +--- + +# 20. 我建议最终定义 6 个核心 Binding Source + +第一版其实不用太多: + +```text +env +arg +sys +file +config +secret +``` + +分别: + +| Namespace | 意义 | +| --------- | ---------------- | +| `env` | 环境变量 | +| `arg` | CLI / Runtime 参数 | +| `sys` | Runtime 系统信息 | +| `file` | 外部文件 | +| `config` | 配置 Artifact | +| `secret` | Secret Provider | + +以后再扩展: + +```text +artifact +context +memory +agent +``` + +但第一版不要过度设计。 + +--- + +# 21. 我还建议把环境变量的“读取”从 `sys` 中移出来 + +你现在: + +```marqdo +sys.env_get +``` + +这是我认为非常值得改的地方。 + +`sys` 应该是: + +```text +sys.cwd +sys.platform +sys.time +sys.argv +``` + +而: + +```text +env +``` + +应该成为语言 Runtime 的 Binding Source。 + +于是用户不应该写: + +```marqdo +**api_key = > sys.env_get name="OPENAI_API_KEY"** +``` + +而应该: + +```yaml +api_key: ${secret.OPENAI_API_KEY!} +``` + +甚至程序里: + +```marqdo +**path = ${env.DATA_PATH}** +``` + +这会让代码明显干净很多。 + +--- + +# 22. 对你的 `llm.mq.md`,最终可以简化成非常漂亮的东西 + +例如: + +```yaml +--- +type: llm + +backend: openai-compatible + +model: ${env.OPENAI_MODEL ?? "gpt-5.6"} + +base_url: + ${env.OPENAI_BASE_URL ?? "https://api.openai.com/v1"} + +api_key: + ${secret.OPENAI_API_KEY!} +--- +``` + +然后下面: + +```marqdo +**## ask** + +Ask the configured language model. + ++ `prompt` + +**result = > self.complete prompt=`prompt`** + +*[text](result)* +``` + +这里 `complete` 只是 backend implementation。 + +用户层: + +```marqdo +**answer = > llm.ask** +``` + +就已经足够。 + +--- + +# 23. 最关键的是:Metadata 本身可以成为可执行配置 + +这一步我认为特别适合 Marqdo。 + +例如: + +```yaml +--- +type: llm + +backend: ${config.llm.backend ?? "openai-compatible"} + +model: ${arg.model ?? env.OPENAI_MODEL ?? "gpt-5.6"} + +temperature: ${float( + arg.temperature ?? + env.OPENAI_TEMPERATURE ?? + "0.2" +)} + +api_key: ${secret.OPENAI_API_KEY!} +--- +``` + +这已经具备: + +```text +Declaration ++ +Configuration ++ +Binding ++ +Default ++ +Type conversion +``` + +因此 `.mq.md` 不再只是: + +```text +Markdown + YAML +``` + +而是: + +> **Self-contained Executable Artifact** + +--- + +# 24. 我建议给这个机制一个正式名字 + +不要叫: + +> Environment Variables + +太窄。 + +我比较推荐: + +### **Marqdo Binding** + +语法: + +```text +${expression} +``` + +文档里可以称: + +> **Binding Expression** + +例如: + +```text +${env.OPENAI_API_KEY} +``` + +是: + +> Environment Binding + +```text +${config.llm.model} +``` + +是: + +> Configuration Binding + +```text +${secret.OPENAI_API_KEY} +``` + +是: + +> Secret Binding + +这套命名以后非常容易扩展。 + +--- + +# 25. 最终 Runtime 架构 + +我建议把它设计成: + +```text + .mq.md + │ + ▼ + ┌──────────────┐ + │ Metadata │ + │ + Body │ + └──────┬───────┘ + │ + ▼ + Binding Resolver + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ + env config secret + │ │ │ + └──────────────┼──────────────┘ + ▼ + Bound Artifact + │ + ┌───────┴────────┐ + ▼ ▼ + Runtime Document + │ + ▼ + Execution +``` + +然后最重要的一条: + +```text +Artifact Source + ↓ +Binding Resolution + ↓ +Execution +``` + +而不是: + +```text +Execution + ↓ +到处 getenv() +``` + +--- + +# 26. 这甚至可以成为 Marqdo 的一个语言卖点 + +如果未来论文里描述,我会把它归到: + +> **Executable Configuration** + +而不是简单的: + +> Environment Variable Support + +因为传统配置通常是: + +```text +Code + + +Config + + +Environment +``` + +三套东西。 + +Marqdo 可以变成: + +```text +Executable Artifact + │ + ├── Logic + ├── Knowledge + ├── Configuration + └── Environment Bindings +``` + +所以一个 `.mq.md` 可以真正做到: + +> **携带自己的执行语义,同时声明自己依赖哪些外部运行时资源。** + +这和你整个“Executable Document → Executable Knowledge Base”的方向是高度一致的。 + +--- + +## 我会把第一版核心规范收敛成这 7 条 + +```text +1. --- ... --- 是 Marqdo Artifact Metadata + +2. Metadata value 可以包含 ${expression} + +3. ${namespace.name} 是 Binding Expression + +4. 第一版 namespace: + env / arg / sys / config / file / secret + +5. Binding 在 Artifact Load 阶段统一解析 + +6. 独占 value 时保留原始类型; + 出现在字符串中时进行 interpolation + +7. Resolution: + CLI → Artifact → Config → Environment → Default +``` + +然后再加: + +```text +${expr ?? default} +${expr!} +${int(expr)} +${float(expr)} +${bool(expr)} +``` + +这就已经是一套相当完整的语言特性了。 + +而且我认为**最值得做的不是“让 LLM 配置更方便”,而是把它正式确立为 Marqdo 的 `Binding` 机制**。这样未来 `llm`、`agent`、`database`、`network`、`skill`、`knowledge` 都能统一使用同一套核心语法,而不会每个扩展自己发明一套环境变量读取方式。 diff --git a/doc/roadmap/ai-stack.md b/doc/roadmap/ai-stack.md index 525db6d..16d7ee5 100644 --- a/doc/roadmap/ai-stack.md +++ b/doc/roadmap/ai-stack.md @@ -35,6 +35,7 @@ | 拆 `ext/ai/llm/openai.mq.md` + `ollama` 后端 | **done** | | tool_calls 完整 Result | 暂缓 | | Transport → 可选 ABI 插件 | 暂缓 | +| Artifact Metadata Binding(`${env.*}` / `secret` · `sys.meta`) | **done**(Phase 1 Metadata-only;[binding.md](../design/binding.md)) | 作者面优先: diff --git a/ext/ai/llm.mq.md b/ext/ai/llm.mq.md index 023a390..abef4fc 100644 --- a/ext/ai/llm.mq.md +++ b/ext/ai/llm.mq.md @@ -174,12 +174,18 @@ Module convenience: create a default handle and return **answer text**. Pass `pa LLM handle — intelligence primitive, not an HTTP client. +Resolution: explicit args → entry Artifact Metadata (`sys.meta_get`) → `sys.env_get` → defaults. + 1. `api_key` **key = api_key** 2. * - **key = > sys.env_get name="OPENAI_API_KEY"** + **key = > sys.meta_get key="api_key"** 1. not `key` - **key = > sys.env_get name="MARQDO_LLM_API_KEY"** + **key = > sys.env_get name="OPENAI_API_KEY"** + 1. not `key` + **key = > sys.env_get name="MARQDO_LLM_API_KEY"** + 2. * + **_ = 1** 2. * **_ = 1** @@ -189,45 +195,53 @@ LLM handle — intelligence primitive, not an HTTP client. 2. * **_ = 1** 2. not `key` - > print text=ext/ai/llm: set OPENAI_API_KEY or MARQDO_LLM_API_KEY (or pass api_key=) + > print text=ext/ai/llm: set OPENAI_API_KEY or MARQDO_LLM_API_KEY (or pass api_key= / metadata api_key) > sys.exit code=1 3. * **_ = 1** 1. `base_url` **url = base_url** -2. `backend` == ollama - **url = > sys.env_get name="OLLAMA_HOST"** - 1. not `url` - **url = "http://127.0.0.1:11434/v1"** - 2. * - **_ = 1** -3. * - **url = > sys.env_get name="OPENAI_BASE_URL"** - 1. not `url` - **url = > sys.env_get name="MARQDO_LLM_BASE_URL"** - 2. * - **_ = 1** - 1. not `url` - **url = "https://api.openai.com/v1"** - 2. * +2. * + **url = > sys.meta_get key="base_url"** + 1. `url` **_ = 1** + 2. `backend` == ollama + **url = > sys.env_get name="OLLAMA_HOST"** + 1. not `url` + **url = "http://127.0.0.1:11434/v1"** + 2. * + **_ = 1** + 3. * + **url = > sys.env_get name="OPENAI_BASE_URL"** + 1. not `url` + **url = > sys.env_get name="MARQDO_LLM_BASE_URL"** + 2. * + **_ = 1** + 1. not `url` + **url = "https://api.openai.com/v1"** + 2. * + **_ = 1** 1. `model` **mdl = model** 2. * - **mdl = > sys.env_get name="OPENAI_MODEL"** - 1. not `mdl` - **mdl = > sys.env_get name="MARQDO_LLM_MODEL"** - 2. * + **mdl = > sys.meta_get key="model"** + 1. `mdl` **_ = 1** - 1. not `mdl` - 1. `backend` == ollama - **mdl = "llama3.2"** + 2. * + **mdl = > sys.env_get name="OPENAI_MODEL"** + 1. not `mdl` + **mdl = > sys.env_get name="MARQDO_LLM_MODEL"** 2. * - **mdl = "gpt-4o-mini"** - 2. * - **_ = 1** + **_ = 1** + 1. not `mdl` + 1. `backend` == ollama + **mdl = "llama3.2"** + 2. * + **mdl = "gpt-4o-mini"** + 2. * + **_ = 1** 1. `name` **nm = name** diff --git a/lib/sys.mq.md b/lib/sys.mq.md index f1b0e1c..5f8eaaf 100644 --- a/lib/sys.mq.md +++ b/lib/sys.mq.md @@ -39,6 +39,19 @@ Current working directory. *> host_cwd* +## meta + +Bound Artifact Metadata from the entry `.mq.md` (map). See `doc/design/binding.md`. + +*> host_meta* + +## meta_get + + `key` + +Lookup one metadata key; missing → None. + +*> host_meta_get key=`key`* + ## exit + `code` diff --git a/lib/系统.mq.md b/lib/系统.mq.md index 0583c91..f3af069 100644 --- a/lib/系统.mq.md +++ b/lib/系统.mq.md @@ -39,6 +39,19 @@ description: 中文系统包装(加载环境 可选 路径=) *> host_cwd* +## 元信息 + +入口 `.mq.md` 已绑定的 Artifact Metadata(映射)。见 `doc/design/binding.md`。 + +*> host_meta* + +## 取元信息 + + `键` + +按键取一条元信息;缺失 → None。 + +*> host_meta_get key=`键`* + ## 退出 + `码` diff --git a/src/aliases.rs b/src/aliases.rs index acc33ac..d34cee3 100644 --- a/src/aliases.rs +++ b/src/aliases.rs @@ -85,6 +85,7 @@ pub fn canonical_param(builtin: &str, param: &str) -> String { "path".into() } ("host_dotenv_load" | "load_dotenv", "path" | "路径") => "path".into(), + ("host_meta_get" | "meta_get", "key" | "键") => "key".into(), ("host_exec" | "exec", "cmd" | "命令") => "cmd".into(), ("host_exec" | "exec", "args" | "参数") => "args".into(), ("host_exec" | "exec", "capture" | "捕获") => "capture".into(), diff --git a/src/ast.rs b/src/ast.rs index 5b2b329..329ad4d 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -27,6 +27,10 @@ pub struct Module { pub functions: Vec, /// Populated by the loader: bind name → imported module (not flattened). pub import_modules: HashMap, + /// Unresolved Artifact Metadata (from frontmatter); filled by parse. + pub metadata_raw: Vec<(String, crate::binding::MetaValue)>, + /// Bound metadata after load-time Binding resolve (insertion order). + pub metadata: Vec<(String, crate::value::Value)>, } impl Default for Module { @@ -36,6 +40,8 @@ impl Default for Module { uses: Vec::new(), functions: Vec::new(), import_modules: HashMap::new(), + metadata_raw: Vec::new(), + metadata: Vec::new(), } } } diff --git a/src/binding/mod.rs b/src/binding/mod.rs new file mode 100644 index 0000000..0184a3a --- /dev/null +++ b/src/binding/mod.rs @@ -0,0 +1,531 @@ +//! Artifact Metadata Binding (`${ns.name}`) — Phase 1: frontmatter only. +//! +//! Spec: `doc/design/binding.md` · ADR 0006. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Result}; + +use crate::diagnostics::{bail_at, Span}; +use crate::value::Value; + +/// Unresolved frontmatter value (literal, sole binding, or interpolated text). +#[derive(Debug, Clone)] +pub enum MetaValue { + /// Already typed literal (no `${}`). + Literal(Value), + /// Whole value is one `${…}` expression. + Binding(BindingExpr), + /// Quoted/bare string containing `${…}` interpolations → Text after resolve. + Interpolated(String), +} + +/// Parsed binding expression (inside `${…}`). +#[derive(Debug, Clone)] +pub struct BindingExpr { + pub cast: Option, + pub primary: BindingRef, + pub default: Option, + pub required: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CastKind { + Int, + Float, + Bool, +} + +#[derive(Debug, Clone)] +pub struct BindingRef { + pub ns: String, + pub name: String, +} + +#[derive(Debug, Clone)] +pub enum DefaultVal { + Text(String), + /// Nested `${…}` not supported in Phase 1 defaults; only quoted/bare literals. + Binding(Box), +} + +/// Sources available while resolving metadata at load time. +#[derive(Debug, Clone, Default)] +pub struct BindingContext { + /// `--bind KEY=VALUE` → `arg.KEY` and post-resolve overlay. + pub args: HashMap, + pub cwd: PathBuf, +} + +impl BindingContext { + pub fn from_binds(binds: &[(String, String)], cwd: PathBuf) -> Self { + let mut args = HashMap::new(); + for (k, v) in binds { + args.insert(k.clone(), v.clone()); + } + Self { args, cwd } + } +} + +/// Parse a frontmatter RHS into a [`MetaValue`]. +pub fn parse_meta_value(raw: &str) -> Result { + let t = raw.trim(); + if t.is_empty() { + return Ok(MetaValue::Literal(Value::Text(String::new()))); + } + + // Sole binding: entire value is `${…}` + if let Some(inner) = sole_binding(t) { + let expr = parse_binding_expr(inner)?; + return Ok(MetaValue::Binding(expr)); + } + + // Quoted string (may interpolate) + if (t.starts_with('"') && t.ends_with('"') && t.len() >= 2) + || (t.starts_with('\'') && t.ends_with('\'') && t.len() >= 2) + { + let inner = &t[1..t.len() - 1]; + if inner.contains("${") { + return Ok(MetaValue::Interpolated(unquote_escapes(inner))); + } + return Ok(MetaValue::Literal(Value::Text(unquote_escapes(inner)))); + } + + // Bare with interpolation + if t.contains("${") { + return Ok(MetaValue::Interpolated(t.to_string())); + } + + Ok(MetaValue::Literal(parse_scalar_literal(t))) +} + +fn sole_binding(t: &str) -> Option<&str> { + let t = t.trim(); + if !t.starts_with("${") || !t.ends_with('}') { + return None; + } + // Ensure matching: first `${` closes at last `}` and nothing outside + let inner = &t[2..t.len() - 1]; + // Reject if another `${` appears that would mean concatenation (not sole) + // Allow nested parens inside cast: ${int(env.X ?? "1")} + Some(inner) +} + +fn unquote_escapes(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('\\') => out.push('\\'), + Some('"') => out.push('"'), + Some('\'') => out.push('\''), + Some(o) => { + out.push('\\'); + out.push(o); + } + None => out.push('\\'), + } + } else { + out.push(c); + } + } + out +} + +fn parse_scalar_literal(t: &str) -> Value { + match t { + "true" | "True" | "TRUE" => Value::Bool(true), + "false" | "False" | "FALSE" => Value::Bool(false), + "null" | "Null" | "None" | "~" => Value::None, + _ => { + if let Ok(n) = t.parse::() { + return Value::Int(n); + } + if let Ok(n) = t.parse::() { + if !n.is_nan() && !n.is_infinite() { + return Value::Num(n); + } + } + Value::Text(t.to_string()) + } + } +} + +/// Parse inside `${…}` (without braces). +pub fn parse_binding_expr(inner: &str) -> Result { + let s = inner.trim(); + // cast: int(…) / float(…) / bool(…) + if let Some((cast, rest)) = parse_cast_wrap(s)? { + let mut expr = parse_binding_expr(rest)?; + if expr.cast.is_some() { + bail!("nested casts are not allowed in binding expressions"); + } + expr.cast = Some(cast); + return Ok(expr); + } + + let (body, required) = if let Some(b) = s.strip_suffix('!') { + (b.trim(), true) + } else { + (s, false) + }; + + let (primary_s, default) = if let Some((left, right)) = split_default(body) { + (left.trim(), Some(parse_default_val(right.trim())?)) + } else { + (body, None) + }; + + if required && default.is_some() { + bail!("binding cannot combine `!` with `??` default"); + } + + let primary = parse_binding_ref(primary_s)?; + Ok(BindingExpr { + cast: None, + primary, + default, + required, + }) +} + +fn parse_cast_wrap(s: &str) -> Result> { + for (name, kind) in [ + ("int", CastKind::Int), + ("float", CastKind::Float), + ("bool", CastKind::Bool), + ] { + let prefix = format!("{name}("); + if let Some(rest) = s.strip_prefix(&prefix) { + if let Some(inner) = strip_trailing_paren(rest) { + return Ok(Some((kind, inner))); + } + bail!("unclosed `{name}(` in binding expression"); + } + } + Ok(None) +} + +fn strip_trailing_paren(s: &str) -> Option<&str> { + let s = s.trim_end(); + if !s.ends_with(')') { + return None; + } + Some(s[..s.len() - 1].trim()) +} + +fn split_default(s: &str) -> Option<(&str, &str)> { + // Split on top-level `??` + let bytes = s.as_bytes(); + let mut i = 0; + let mut depth = 0i32; + while i + 1 < bytes.len() { + match bytes[i] { + b'(' => depth += 1, + b')' => depth -= 1, + b'?' if depth == 0 && bytes[i + 1] == b'?' => { + return Some((&s[..i], &s[i + 2..])); + } + _ => {} + } + i += 1; + } + None +} + +fn parse_default_val(s: &str) -> Result { + let s = s.trim(); + if let Some(inner) = sole_binding(s) { + return Ok(DefaultVal::Binding(Box::new(parse_binding_expr(inner)?))); + } + if (s.starts_with('"') && s.ends_with('"') && s.len() >= 2) + || (s.starts_with('\'') && s.ends_with('\'') && s.len() >= 2) + { + return Ok(DefaultVal::Text(unquote_escapes(&s[1..s.len() - 1]))); + } + // bare default token + Ok(DefaultVal::Text(s.to_string())) +} + +fn parse_binding_ref(s: &str) -> Result { + let s = s.trim(); + let Some((ns, name)) = s.split_once('.') else { + bail!("binding path must be `namespace.name` (got `{s}`)"); + }; + let ns = ns.trim(); + let name = name.trim(); + if ns.is_empty() || name.is_empty() { + bail!("binding path must be `namespace.name` (got `{s}`)"); + } + match ns { + "env" | "arg" | "sys" | "secret" => {} + "config" | "file" => { + bail!("binding namespace `{ns}` is not available in Phase 1 (Metadata-only)") + } + other => bail!("unknown binding namespace `{other}`"), + } + if name.contains('.') { + // Phase 1: single segment name only + bail!("binding name must be a single segment (got `{s}`)"); + } + Ok(BindingRef { + ns: ns.to_string(), + name: name.to_string(), + }) +} + +/// Resolve raw metadata entries; then overlay `--bind` keys. +pub fn resolve_metadata( + raw: &[(String, MetaValue)], + ctx: &BindingContext, + span: Span, +) -> Result> { + let mut out: Vec<(String, Value)> = Vec::with_capacity(raw.len()); + for (k, mv) in raw { + let v = resolve_meta_value(mv, ctx, span) + .map_err(|e| bail_at(None, span, e.to_string()))?; + out.push((k.clone(), v)); + } + // CLI overlay + for (k, v) in &ctx.args { + if let Some(slot) = out.iter_mut().find(|(mk, _)| mk == k) { + slot.1 = Value::Text(v.clone()); + } else { + out.push((k.clone(), Value::Text(v.clone()))); + } + } + Ok(out) +} + +fn resolve_meta_value(mv: &MetaValue, ctx: &BindingContext, span: Span) -> Result { + match mv { + MetaValue::Literal(v) => Ok(v.clone()), + MetaValue::Binding(expr) => resolve_expr(expr, ctx, span), + MetaValue::Interpolated(s) => { + let text = interpolate(s, ctx, span)?; + Ok(Value::Text(text)) + } + } +} + +fn interpolate(s: &str, ctx: &BindingContext, span: Span) -> Result { + let mut out = String::new(); + let mut rest = s; + while let Some(start) = rest.find("${") { + out.push_str(&rest[..start]); + let after = &rest[start + 2..]; + let end = find_binding_close(after).ok_or_else(|| { + anyhow::anyhow!("{}:{}: unclosed `${{` in metadata interpolation", span.line, span.col) + })?; + let inner = &after[..end]; + let expr = parse_binding_expr(inner)?; + let v = resolve_expr(&expr, ctx, span)?; + out.push_str(&v.as_display()); + rest = &after[end + 1..]; + } + out.push_str(rest); + Ok(out) +} + +fn find_binding_close(s: &str) -> Option { + let mut depth = 0i32; + for (i, c) in s.char_indices() { + match c { + '(' => depth += 1, + ')' => depth -= 1, + '}' if depth == 0 => return Some(i), + _ => {} + } + } + None +} + +fn resolve_expr(expr: &BindingExpr, ctx: &BindingContext, span: Span) -> Result { + let got = lookup(&expr.primary, ctx)?; + let value = match got { + Some(v) => v, + None if expr.required => { + bail!( + "required binding `{}.{}` is missing", + expr.primary.ns, + expr.primary.name + ) + } + None => match &expr.default { + Some(DefaultVal::Text(t)) => Value::Text(t.clone()), + Some(DefaultVal::Binding(inner)) => resolve_expr(inner, ctx, span)?, + None => Value::None, + }, + }; + apply_cast(expr.cast, value) +} + +fn lookup(r: &BindingRef, ctx: &BindingContext) -> Result> { + match r.ns.as_str() { + "env" => Ok(std::env::var(&r.name).ok().map(Value::Text)), + "arg" => Ok(ctx.args.get(&r.name).cloned().map(Value::Text)), + "secret" => Ok(std::env::var(&r.name).ok().map(Value::Secret)), + "sys" => Ok(lookup_sys(&r.name, ctx)), + other => bail!("unknown binding namespace `{other}`"), + } +} + +fn lookup_sys(name: &str, ctx: &BindingContext) -> Option { + match name { + "cwd" => Some(Value::Text(ctx.cwd.display().to_string())), + "platform" => Some(Value::Text(std::env::consts::OS.to_string())), + _ => None, + } +} + +fn apply_cast(cast: Option, v: Value) -> Result { + let Some(c) = cast else { + return Ok(v); + }; + let text = match &v { + Value::None => { + bail!("cannot cast None to {:?}", c); + } + Value::Text(s) => s.clone(), + Value::Secret(s) => s.clone(), + Value::Int(n) => n.to_string(), + Value::Num(n) => n.to_string(), + Value::Bool(b) => { + if *b { + "true".into() + } else { + "false".into() + } + } + other => bail!("cannot cast {} to {:?}", other.as_display(), c), + }; + match c { + CastKind::Int => { + let n: i64 = text + .trim() + .parse() + .map_err(|_| anyhow::anyhow!("cannot cast `{text}` to int"))?; + Ok(Value::Int(n)) + } + CastKind::Float => { + let n: f64 = text + .trim() + .parse() + .map_err(|_| anyhow::anyhow!("cannot cast `{text}` to float"))?; + Ok(Value::Num(n)) + } + CastKind::Bool => { + let t = text.trim().to_ascii_lowercase(); + match t.as_str() { + "1" | "true" | "yes" | "on" => Ok(Value::Bool(true)), + "0" | "false" | "no" | "off" => Ok(Value::Bool(false)), + _ => bail!("cannot cast `{text}` to bool"), + } + } + } +} + +/// Split `KEY=VALUE` from CLI `--bind`. +pub fn parse_bind_kv(s: &str) -> Result<(String, String)> { + let Some((k, v)) = s.split_once('=') else { + bail!("--bind expects KEY=VALUE (got `{s}`)"); + }; + let k = k.trim(); + if k.is_empty() { + bail!("--bind key must be non-empty"); + } + Ok((k.to_string(), v.to_string())) +} + +pub fn cwd_for_path(path: Option<&Path>) -> PathBuf { + path.and_then(|p| p.parent()) + .map(|p| { + if p.as_os_str().is_empty() { + PathBuf::from(".") + } else { + p.to_path_buf() + } + }) + .unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sole_env_default() { + let mv = parse_meta_value(r#"${env.NO_SUCH_MARQDO_VAR_XYZ ?? "gpt"}"#).unwrap(); + let ctx = BindingContext::default(); + let v = resolve_meta_value(&mv, &ctx, Span::new(1, 1)).unwrap(); + assert_eq!(v, Value::Text("gpt".into())); + } + + #[test] + fn required_missing_errors() { + let mv = parse_meta_value("${env.NO_SUCH_MARQDO_REQ_XYZ!}").unwrap(); + let ctx = BindingContext::default(); + let err = resolve_meta_value(&mv, &ctx, Span::new(1, 1)).unwrap_err(); + assert!(err.to_string().contains("required binding")); + } + + #[test] + fn secret_masks_display() { + std::env::set_var("MARQDO_TEST_SECRET_BIND", "s3cr3t"); + let mv = parse_meta_value("${secret.MARQDO_TEST_SECRET_BIND}").unwrap(); + let ctx = BindingContext::default(); + let v = resolve_meta_value(&mv, &ctx, Span::new(1, 1)).unwrap(); + assert_eq!(v.as_display(), ""); + match v { + Value::Secret(s) => assert_eq!(s, "s3cr3t"), + other => panic!("expected Secret, got {other:?}"), + } + std::env::remove_var("MARQDO_TEST_SECRET_BIND"); + } + + #[test] + fn literal_not_overridden_by_env() { + std::env::set_var("OPENAI_MODEL", "from-env"); + let raw = vec![( + "model".into(), + MetaValue::Literal(Value::Text("gpt-literal".into())), + )]; + let ctx = BindingContext::default(); + let meta = resolve_metadata(&raw, &ctx, Span::new(1, 1)).unwrap(); + assert_eq!(meta[0].1, Value::Text("gpt-literal".into())); + std::env::remove_var("OPENAI_MODEL"); + } + + #[test] + fn bind_overlay_wins() { + let raw = vec![( + "model".into(), + MetaValue::Literal(Value::Text("gpt-literal".into())), + )]; + let ctx = BindingContext::from_binds(&[("model".into(), "from-cli".into())], PathBuf::from(".")); + let meta = resolve_metadata(&raw, &ctx, Span::new(1, 1)).unwrap(); + assert_eq!(meta[0].1, Value::Text("from-cli".into())); + } + + #[test] + fn int_cast() { + std::env::set_var("MARQDO_TEST_MAX_TOKENS", "2048"); + let mv = parse_meta_value("${int(env.MARQDO_TEST_MAX_TOKENS)}").unwrap(); + let v = resolve_meta_value(&mv, &BindingContext::default(), Span::new(1, 1)).unwrap(); + assert_eq!(v, Value::Int(2048)); + std::env::remove_var("MARQDO_TEST_MAX_TOKENS"); + } + + #[test] + fn interpolate_string() { + std::env::set_var("MARQDO_TEST_USER", "Ada"); + let mv = parse_meta_value(r#""Hello ${env.MARQDO_TEST_USER}""#).unwrap(); + let v = resolve_meta_value(&mv, &BindingContext::default(), Span::new(1, 1)).unwrap(); + assert_eq!(v, Value::Text("Hello Ada".into())); + std::env::remove_var("MARQDO_TEST_USER"); + } +} diff --git a/src/builtin.rs b/src/builtin.rs index c93b697..3859248 100644 --- a/src/builtin.rs +++ b/src/builtin.rs @@ -41,6 +41,7 @@ pub fn builtin_type(v: &Value) -> Value { Value::Int(_) => "int", Value::Num(_) => "num", Value::Text(_) => "text", + Value::Secret(_) => "secret", Value::List(_) => "list", Value::Map(_) => "map", Value::Formula(_) => "formula", diff --git a/src/bytecode/vm.rs b/src/bytecode/vm.rs index b1ac15b..cd9c996 100644 --- a/src/bytecode/vm.rs +++ b/src/bytecode/vm.rs @@ -698,6 +698,9 @@ fn add(a: Value, b: Value) -> Result { (Value::Int(x), Value::Num(y)) => Ok(Value::Num(x as f64 + y)), (Value::Num(x), Value::Int(y)) => Ok(Value::Num(x + y as f64)), (Value::Text(x), Value::Text(y)) => Ok(Value::Text(format!("{x}{y}"))), + (Value::Text(x), Value::Secret(y)) => Ok(Value::Text(format!("{x}{y}"))), + (Value::Secret(x), Value::Text(y)) => Ok(Value::Text(format!("{x}{y}"))), + (Value::Secret(x), Value::Secret(y)) => Ok(Value::Text(format!("{x}{y}"))), (Value::Text(x), y) => Ok(Value::Text(format!("{x}{}", y.as_display()))), (x, Value::Text(y)) => Ok(Value::Text(format!("{}{y}", x.as_display()))), (Value::Int(x), y) => Ok(Value::Text(format!("{}{}", x, y.as_display()))), diff --git a/src/contract.rs b/src/contract.rs index 3242d94..a083407 100644 --- a/src/contract.rs +++ b/src/contract.rs @@ -108,6 +108,7 @@ pub fn type_of(v: &Value) -> &'static str { Value::Int(_) => "int", Value::Num(_) => "num", Value::Text(_) => "text", + Value::Secret(_) => "secret", Value::List(_) => "list", Value::Map(_) => "map", Value::Formula(_) => "formula", diff --git a/src/host/dispatch.rs b/src/host/dispatch.rs index 8bd3e45..9d2aaf4 100644 --- a/src/host/dispatch.rs +++ b/src/host/dispatch.rs @@ -198,6 +198,8 @@ pub enum HostFn { StatsMean = 189, StatsMedian = 190, StatsStdev = 191, + Meta = 192, + MetaGet = 193, } @@ -386,6 +388,8 @@ impl HostFn { 189 => Self::StatsMean, 190 => Self::StatsMedian, 191 => Self::StatsStdev, + 192 => Self::Meta, + 193 => Self::MetaGet, _ => return None, }) } @@ -577,6 +581,8 @@ impl HostFn { "host_stats_mean" | "stats_mean" => Self::StatsMean, "host_stats_median" | "stats_median" => Self::StatsMedian, "host_stats_stdev" | "stats_stdev" => Self::StatsStdev, + "host_meta" | "meta" => Self::Meta, + "host_meta_get" | "meta_get" => Self::MetaGet, _ => return None, }) } @@ -755,6 +761,8 @@ impl HostFn { Self::StatsMean => "host_stats_mean", Self::StatsMedian => "host_stats_median", Self::StatsStdev => "host_stats_stdev", + Self::Meta => "host_meta", + Self::MetaGet => "host_meta_get", Self::WritebackRecord => "host_writeback_record", Self::WritebackGet => "host_writeback_get", Self::WritebackClear => "host_writeback_clear", @@ -778,11 +786,12 @@ impl HostFn { &["path"] } Self::WriteText | Self::AppendText => &["path", "text"], - Self::NowUnix | Self::NowMs | Self::Args | Self::Cwd => &[], + Self::NowUnix | Self::NowMs | Self::Args | Self::Cwd | Self::Meta => &[], Self::FormatTime => &["unix", "pattern"], Self::ParseTime => &["text", "pattern"], Self::SleepMs => &["ms"], Self::EnvGet => &["name"], + Self::MetaGet => &["key"], Self::EnvSet => &["name", "value"], Self::Exit => &["code"], Self::Exec => &["cmd"], @@ -980,6 +989,8 @@ pub fn call_host( HostFn::DotenvLoad => sys::dotenv_load(ctx, bound.get("path")), HostFn::Args => sys::args(ctx), HostFn::Cwd => sys::cwd(ctx), + HostFn::Meta => sys::meta(ctx), + HostFn::MetaGet => sys::meta_get(ctx, require(bound, "key")?), HostFn::Exit => sys::exit(ctx, require(bound, "code")?), HostFn::Exec => sys::exec( ctx, diff --git a/src/host/json.rs b/src/host/json.rs index bfe2060..f382f07 100644 --- a/src/host/json.rs +++ b/src/host/json.rs @@ -146,6 +146,7 @@ pub(crate) fn value_to_json(v: &Value) -> Result { } }, Value::Code(c) => serde_json::Value::String(c.source.clone()), + Value::Secret(_) => serde_json::Value::String("".into()), }) } diff --git a/src/host/mod.rs b/src/host/mod.rs index 71d72ee..7223d38 100644 --- a/src/host/mod.rs +++ b/src/host/mod.rs @@ -103,6 +103,8 @@ pub struct HostContext { pub current_line: u32, /// Entry-file line of each active user call (for writeback anchoring). pub call_site_lines: Vec, + /// Bound Artifact Metadata from the entry `.mq.md` (Phase 1 Binding). + pub entry_metadata: Vec<(String, crate::value::Value)>, /// Concurrent subtasks (`lib/subtask`): file / function / foreign. #[cfg_attr(not(feature = "exec-host"), allow(dead_code))] pub(crate) subtasks: HashMap, @@ -129,6 +131,7 @@ impl Default for HostContext { call_stack: Vec::new(), current_line: 1, call_site_lines: Vec::new(), + entry_metadata: Vec::new(), subtasks: HashMap::new(), subtask_seq: 0, log_min_level: 20, // Info diff --git a/src/host/subtask.rs b/src/host/subtask.rs index 06549db..5452060 100644 --- a/src/host/subtask.rs +++ b/src/host/subtask.rs @@ -408,6 +408,7 @@ fn spawn_fn( host.argv = argv; host.foreign_cmds = foreign_cmds; host.set_entry_source(Some(&module_path), &source); + host.entry_metadata = module.metadata.clone(); let mut interp = Interpreter::with_capture(Some(&module_path), false).with_host(host); interp .invoke_function(&module, &name, &call_args) diff --git a/src/host/sys.rs b/src/host/sys.rs index 85298e2..83d4aae 100644 --- a/src/host/sys.rs +++ b/src/host/sys.rs @@ -93,6 +93,22 @@ pub fn cwd(ctx: &HostContext) -> Result { Ok(Value::Text(ctx.cwd.display().to_string())) } +/// Bound entry Artifact Metadata (map). +pub fn meta(ctx: &HostContext) -> Result { + Ok(Value::Map(ctx.entry_metadata.clone())) +} + +/// Lookup one metadata key; missing → None. +pub fn meta_get(ctx: &HostContext, key: &Value) -> Result { + let k = as_text(key, "key")?; + Ok(ctx + .entry_metadata + .iter() + .find(|(name, _)| name == k) + .map(|(_, v)| v.clone()) + .unwrap_or(Value::None)) +} + pub fn exit(ctx: &HostContext, code: &Value) -> Result { let c = as_i64(code, "code")?; if ctx.soft_side_effects { diff --git a/src/interp.rs b/src/interp.rs index 000eea2..55a9e72 100644 --- a/src/interp.rs +++ b/src/interp.rs @@ -1285,6 +1285,9 @@ fn eval_binary(op: BinaryOp, l: &Value, r: &Value) -> Result { (Value::Int(a), Value::Num(b)) => Ok(Value::Num(*a as f64 + b)), (Value::Num(a), Value::Int(b)) => Ok(Value::Num(a + *b as f64)), (Value::Text(a), Value::Text(b)) => Ok(Value::Text(format!("{a}{b}"))), + (Value::Text(a), Value::Secret(b)) => Ok(Value::Text(format!("{a}{b}"))), + (Value::Secret(a), Value::Text(b)) => Ok(Value::Text(format!("{a}{b}"))), + (Value::Secret(a), Value::Secret(b)) => Ok(Value::Text(format!("{a}{b}"))), (Value::Text(a), b) => Ok(Value::Text(format!("{a}{}", b.as_display()))), (a, Value::Text(b)) => Ok(Value::Text(format!("{}{b}", a.as_display()))), _ => bail!("`+` needs ints, nums, or text"), diff --git a/src/lib.rs b/src/lib.rs index 509f690..ff434df 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ pub mod aliases; pub mod ast; +pub mod binding; pub mod browser_session; pub mod builtin; pub mod bytecode; @@ -51,7 +52,8 @@ use crate::bytecode::{compile_module, Vm}; use crate::capture::RunCapture; use crate::interp::Interpreter; use crate::lex::classify_source; -use crate::load::{load_module, load_module_from_source}; +use crate::load::{load_module_from_source, load_module_with_ctx}; +use crate::binding::{cwd_for_path, BindingContext}; pub use browser_session::{value_as_json, BrowserSession}; @@ -95,6 +97,8 @@ pub struct RunOptions { pub sleep_limit_ms: Option, /// When set, write `# main` return value as JSON to this path (file subtasks). pub emit_result: Option, + /// CLI `--bind KEY=VALUE` pairs for Artifact Metadata Binding (`arg` + overlay). + pub binds: Vec<(String, String)>, } impl Default for RunOptions { @@ -115,6 +119,7 @@ impl Default for RunOptions { fs_root: None, sleep_limit_ms: None, emit_result: None, + binds: Vec::new(), } } } @@ -173,7 +178,8 @@ pub fn run_file(path: &Path, opts: &RunOptions) -> Result { println!("=== marqdo: end tokens ==="); } - let module = load_module(path)?; + let bind_ctx = BindingContext::from_binds(&opts.binds, cwd_for_path(Some(path))); + let module = load_module_with_ctx(path, &bind_ctx)?; if opts.dump_ast { print!("{}", format_ast_dump(&path_label, &module)); @@ -189,6 +195,7 @@ pub fn run_file(path: &Path, opts: &RunOptions) -> Result { .collect::>() ); println!("imports: {:?}", module.imports); + println!("metadata: {:?}", module.metadata.iter().map(|(k, v)| (k, v.as_display())).collect::>()); println!("=== marqdo: end sema ==="); } @@ -196,6 +203,7 @@ pub fn run_file(path: &Path, opts: &RunOptions) -> Result { Backend::Tree => { let mut host = HostContext::for_run(Some(path), opts.host_caps(), opts.argv.clone()); host.set_entry_source(Some(path), &source); + host.entry_metadata = module.metadata.clone(); let mut interp = Interpreter::new(Some(path), opts.trace_eval) .with_stdin(stdin_lines.clone()) .with_host(host); @@ -217,6 +225,7 @@ pub fn run_file(path: &Path, opts: &RunOptions) -> Result { } let mut host = HostContext::for_run(Some(path), opts.host_caps(), opts.argv.clone()); host.set_entry_source(Some(path), &source); + host.entry_metadata = module.metadata.clone(); let mut vm = Vm::new(Some(path)) .with_stdin(stdin_lines) .with_trace(opts.trace_eval) @@ -261,12 +270,14 @@ pub fn run_file_capture(path: &Path, opts: &RunOptions) -> Result { let source = std::fs::read_to_string(path) .map_err(|e| anyhow::anyhow!("failed to read {}: {e}", path.display()))?; let stdin_lines = crate::input_feed::effective_stdin(&source, &opts.stdin_lines); - let module = load_module(path)?; + let bind_ctx = BindingContext::from_binds(&opts.binds, cwd_for_path(Some(path))); + let module = load_module_with_ctx(path, &bind_ctx)?; match opts.backend { Backend::Tree => { let mut host = HostContext::for_capture(Some(path), opts.host_caps()); host.set_entry_source(Some(path), &source); host.argv = opts.argv.clone(); + host.entry_metadata = module.metadata.clone(); if let Some(root) = &opts.fs_root { host.fs_root = Some(root.clone()); } @@ -290,6 +301,7 @@ pub fn run_file_capture(path: &Path, opts: &RunOptions) -> Result { let mut host = HostContext::for_capture(Some(path), opts.host_caps()); host.set_entry_source(Some(path), &source); host.argv = opts.argv.clone(); + host.entry_metadata = module.metadata.clone(); if let Some(root) = &opts.fs_root { host.fs_root = Some(root.clone()); } @@ -323,6 +335,7 @@ pub fn run_source(source: &str, opts: &RunOptions) -> Result { let mut host = HostContext::for_capture(None, opts.host_caps()); host.set_entry_source(None, source); host.argv = opts.argv.clone(); + host.entry_metadata = module.metadata.clone(); if let Some(root) = &opts.fs_root { host.fs_root = Some(root.clone()); } @@ -346,6 +359,7 @@ pub fn run_source(source: &str, opts: &RunOptions) -> Result { let mut host = HostContext::for_capture(None, opts.host_caps()); host.set_entry_source(None, source); host.argv = opts.argv.clone(); + host.entry_metadata = module.metadata.clone(); if let Some(root) = &opts.fs_root { host.fs_root = Some(root.clone()); } diff --git a/src/load.rs b/src/load.rs index 727e1b2..9f393ef 100644 --- a/src/load.rs +++ b/src/load.rs @@ -7,17 +7,27 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use crate::ast::Module; -use crate::diagnostics::Diagnostic; +use crate::binding::{cwd_for_path, resolve_metadata, BindingContext}; +use crate::diagnostics::{Diagnostic, Span}; use crate::embedded_lib; use crate::parse::parse_source; /// Load `path` and recursively bind imported modules (no flat merge). pub fn load_module(path: &Path) -> Result { - let mut visited = HashSet::new(); - load_module_inner(path, &mut visited) + load_module_with_ctx(path, &BindingContext::from_binds(&[], cwd_for_path(Some(path)))) } -fn load_module_inner(path: &Path, visited: &mut HashSet) -> Result { +/// Load with BindingContext (CLI `--bind`, cwd for `sys.cwd`). +pub fn load_module_with_ctx(path: &Path, ctx: &BindingContext) -> Result { + let mut visited = HashSet::new(); + load_module_inner(path, &mut visited, ctx) +} + +fn load_module_inner( + path: &Path, + visited: &mut HashSet, + ctx: &BindingContext, +) -> Result { let canon = path .canonicalize() .unwrap_or_else(|_| path.to_path_buf()); @@ -30,12 +40,20 @@ fn load_module_inner(path: &Path, visited: &mut HashSet) -> Result anyhow::Error { /// `lib/…` stdlib (no filesystem / `ext/` for this MVP path). pub fn load_module_from_source(source: &str) -> Result { let mut visited = HashSet::new(); - load_module_from_source_inner(source, "", &mut visited) + let ctx = BindingContext::default(); + load_module_from_source_inner(source, "", &mut visited, &ctx) } /// 给错误打上文件标签,**保留**结构化诊断(错误出口唯一;AI/MLSP 面不许拍平成字符串)。 @@ -270,6 +289,7 @@ fn load_module_from_source_inner( source: &str, label: &str, visited: &mut HashSet, + ctx: &BindingContext, ) -> Result { let key = PathBuf::from(label); if !visited.insert(key.clone()) { @@ -278,10 +298,12 @@ fn load_module_from_source_inner( let result = (|| { let mut module = parse_source(source).map_err(|e| label_error(label, e))?; + module.metadata = resolve_metadata(&module.metadata_raw, ctx, Span::new(1, 1)) + .map_err(|e| label_error(label, e))?; let imports = module.imports.clone(); let mut import_modules = HashMap::new(); for imp in imports { - let dep = load_embedded_import(&imp.path, visited)?; + let dep = load_embedded_import(&imp.path, visited, ctx)?; if import_modules.contains_key(&imp.bind) { bail!("duplicate import bind `{}` while loading {label}", imp.bind); } @@ -296,7 +318,11 @@ fn load_module_from_source_inner( result } -fn load_embedded_import(rel: &str, visited: &mut HashSet) -> Result { +fn load_embedded_import( + rel: &str, + visited: &mut HashSet, + ctx: &BindingContext, +) -> Result { let rel = rel.replace('\\', "/"); let rest = if let Some(r) = rel.strip_prefix("lib/") { r.to_string() @@ -314,7 +340,7 @@ fn load_embedded_import(rel: &str, visited: &mut HashSet) -> Result, + /// Artifact Metadata Binding: `KEY=VALUE` (repeatable). Feeds `arg.KEY` and overlays metadata. + #[arg(long = "bind", value_name = "KEY=VALUE")] + binds: Vec, + /// Emit diagnostics as JSON on stderr (machine-readable; AI/MLSP 面). #[arg(long)] json: bool, @@ -228,6 +232,7 @@ fn try_main(cli: Cli) -> Result { dump_all, stdin_file, emit_result, + binds, json: _, } => { let path = file.unwrap_or_else(|| PathBuf::from("index.mq.md")); @@ -235,10 +240,15 @@ fn try_main(cli: Cli) -> Result { Some(p) => load_stdin_file(&p)?, None => Vec::new(), }; + let mut bind_pairs = Vec::new(); + for b in &binds { + bind_pairs.push(marqdo::binding::parse_bind_kv(b)?); + } let mut opts = if dump_all { RunOptions { stdin_lines: stdin_lines.clone(), emit_result: emit_result.clone(), + binds: bind_pairs.clone(), ..RunOptions::dump_all() } } else { @@ -252,6 +262,7 @@ fn try_main(cli: Cli) -> Result { backend: backend.into(), stdin_lines, emit_result, + binds: bind_pairs, ..RunOptions::default() } }; diff --git a/src/parse/mod.rs b/src/parse/mod.rs index b11cffc..b313005 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -28,7 +28,7 @@ pub fn parse_source(source: &str) -> Result { pub fn parse_classified(lines: &[ClassifiedLine]) -> Result { let mut cur = Cursor::new(lines); - let (imports, uses) = cur.parse_frontmatter()?; + let (imports, uses, metadata_raw) = cur.parse_frontmatter()?; let mut functions = Vec::new(); while cur.skip_noise() { let Some(line) = cur.peek() else { break }; @@ -65,6 +65,8 @@ pub fn parse_classified(lines: &[ClassifiedLine]) -> Result { uses, functions, import_modules: std::collections::HashMap::new(), + metadata_raw, + metadata: Vec::new(), }) } @@ -134,17 +136,24 @@ impl<'a> Cursor<'a> { }) } - fn parse_frontmatter(&mut self) -> Result<(Vec, Vec)> { + fn parse_frontmatter( + &mut self, + ) -> Result<( + Vec, + Vec, + Vec<(String, crate::binding::MetaValue)>, + )> { let mut imports = Vec::new(); let mut uses = Vec::new(); + let mut metadata_raw = Vec::new(); if !self.skip_noise() { - return Ok((imports, uses)); + return Ok((imports, uses, metadata_raw)); } let Some(first) = self.peek() else { - return Ok((imports, uses)); + return Ok((imports, uses, metadata_raw)); }; if first.text.trim() != "---" { - return Ok((imports, uses)); + return Ok((imports, uses, metadata_raw)); } self.bump(); while let Some(line) = self.peek() { @@ -169,6 +178,8 @@ impl<'a> Cursor<'a> { )); } imports.push(imp); + self.bump(); + continue; } Ok(Some(ImportLine::Member(u))) => { if imports.iter().any(|i| i.bind == u.bind) @@ -181,13 +192,43 @@ impl<'a> Cursor<'a> { )); } uses.push(u); + self.bump(); + continue; } Ok(None) => {} Err(e) => return Err(bail_at(None, span, e.to_string())), } + // Artifact metadata key: value (flat Phase 1) + if let Some((key, rest)) = split_frontmatter_key(t) { + self.bump(); + let value_raw = if rest.is_empty() + || rest == "|" + || rest == "|-" + || rest == "|+" + || rest == ">" + || rest == ">-" + || rest == ">+" + { + collect_block_scalar(self, rest)? + } else { + rest.to_string() + }; + let mv = crate::binding::parse_meta_value(&value_raw) + .map_err(|e| bail_at(None, span, e.to_string()))?; + if metadata_raw.iter().any(|(k, _)| k == key) { + return Err(bail_at( + None, + span, + format!("duplicate metadata key `{key}`"), + )); + } + metadata_raw.push((key.to_string(), mv)); + continue; + } + // Unknown / non-key line: skip (compat with freeform frontmatter notes) self.bump(); } - Ok((imports, uses)) + Ok((imports, uses, metadata_raw)) } fn parse_function(&mut self, min_level: u8) -> Result { @@ -1169,6 +1210,80 @@ pub enum ImportLine { Member(Use), } +fn split_frontmatter_key(trimmed: &str) -> Option<(&str, &str)> { + let t = trimmed.trim(); + if t.is_empty() || t.starts_with('#') { + return None; + } + // import lines are handled earlier + let Some((key, rest)) = t.split_once(':') else { + return None; + }; + let key = key.trim(); + if key.is_empty() || key.contains(' ') { + return None; + } + // Avoid treating `http://…` style as key (no spaces in key already; also reject if key has `/`) + if key.contains('/') { + return None; + } + Some((key, rest.trim())) +} + +fn collect_block_scalar(cur: &mut Cursor<'_>, indicator: &str) -> Result { + let mut lines = Vec::new(); + while let Some(line) = cur.peek() { + let raw = line.text.trim_end_matches(['\r', '\n']); + let bt = raw.trim(); + if bt == "---" { + break; + } + // Next top-level key or import ends the block + if !raw.starts_with(' ') && !raw.starts_with('\t') && !bt.is_empty() { + if split_frontmatter_key(bt).is_some() || parse_import_line(bt).ok().flatten().is_some() + { + break; + } + if indicator.is_empty() { + // empty rest on key line without block indicator: no body + break; + } + } + if indicator.is_empty() { + break; + } + cur.bump(); + if raw.starts_with(' ') || raw.starts_with('\t') { + lines.push(strip_yaml_indent(raw)); + } else if bt.is_empty() { + lines.push(String::new()); + } else { + // folded content without indent — take as-is once + lines.push(bt.to_string()); + break; + } + } + if indicator.starts_with('>') { + Ok(lines + .into_iter() + .filter(|l| !l.is_empty()) + .collect::>() + .join(" ")) + } else { + Ok(lines.join("\n")) + } +} + +fn strip_yaml_indent(line: &str) -> String { + if line.starts_with(" ") { + line[2..].to_string() + } else if line.starts_with('\t') { + line[1..].to_string() + } else { + line.trim_start_matches([' ', '\t']).to_string() + } +} + fn legacy_frontmatter_import_hint(trimmed: &str) -> Option { let t = trimmed.trim(); if !t.starts_with('>') { diff --git a/src/value.rs b/src/value.rs index 7c2f03a..9b1cdf6 100644 --- a/src/value.rs +++ b/src/value.rs @@ -27,6 +27,8 @@ pub enum Value { Formula(FormulaExpr), /// External language source from ```lang fence (`type` → `code`). Code(CodeBlock), + /// Sensitive text from `${secret.*}` — Display shows ``; reveal on Text concat. + Secret(String), } impl Value { @@ -37,6 +39,7 @@ impl Value { Value::Int(n) => *n != 0, Value::Num(n) => *n != 0.0 && !n.is_nan(), Value::Text(s) => !s.is_empty(), + Value::Secret(s) => !s.is_empty(), Value::List(xs) => !xs.is_empty(), Value::Map(xs) => !xs.is_empty(), Value::Formula(_) => true, @@ -44,6 +47,14 @@ impl Value { } } + /// Plaintext for Text / Secret (Authorization etc.). Prefer [`as_display`] for logs. + pub fn reveal_text(&self) -> Option<&str> { + match self { + Value::Text(s) | Value::Secret(s) => Some(s.as_str()), + _ => None, + } + } + pub fn as_display(&self) -> String { match self { Value::None => "None".into(), @@ -52,6 +63,7 @@ impl Value { Value::Int(n) => n.to_string(), Value::Num(n) => crate::formula::format_num(*n), Value::Text(s) => s.clone(), + Value::Secret(_) => "".into(), Value::List(xs) => { let parts: Vec = xs.iter().map(|v| v.as_display()).collect(); format!("[{}]", parts.join(", ")) diff --git a/src/view/debug_api.rs b/src/view/debug_api.rs index 16b393c..a3e393f 100644 --- a/src/view/debug_api.rs +++ b/src/view/debug_api.rs @@ -98,6 +98,7 @@ pub fn api_debug_start(root: &Path, body: &str) -> serde_json::Value { }; let mut host = HostContext::for_capture(Some(&path_for_thread), opts.host_caps()); host.fs_root = Some(root_for_thread); + host.entry_metadata = module.metadata.clone(); if let Some(lim) = opts.sleep_limit_ms { host.sleep_limit_ms = Some(lim); } diff --git a/src/view/render.rs b/src/view/render.rs index 00a8e86..46f98d4 100644 --- a/src/view/render.rs +++ b/src/view/render.rs @@ -415,6 +415,7 @@ fn value_kind_label(v: &crate::value::Value) -> String { Value::Int(_) => "int".into(), Value::Num(_) => "num".into(), Value::Text(_) => "text".into(), + Value::Secret(_) => "secret".into(), Value::Formula(crate::formula::Expr::Matrix { rows, .. }) => { format!("formula · {}×{}", rows.len(), rows.first().map(|r| r.len()).unwrap_or(0)) } diff --git a/tests/gold.rs b/tests/gold.rs index d078f05..2fa94ac 100644 --- a/tests/gold.rs +++ b/tests/gold.rs @@ -31,6 +31,19 @@ fn run_with_stdin(args: &[&str], stdin: &str) -> (i32, String, String) { (code, stdout, stderr) } +fn run_with_env(args: &[&str], env: &[(&str, &str)]) -> (i32, String, String) { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_marqdo")); + cmd.args(args); + for (k, v) in env { + cmd.env(k, v); + } + let output = cmd.output().expect("failed to run marqdo"); + let code = output.status.code().unwrap_or(1); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + (code, stdout, stderr) +} + fn assert_out(path: &str, expect: &str) { let (code, stdout, stderr) = run(&["run", path]); assert_eq!(code, 0, "{path} stderr={stderr}"); @@ -508,6 +521,45 @@ fn structure_optional_param() { ); } +#[test] +fn structure_meta_binding() { + let (code, stdout, stderr) = run(&[ + "run", + "tests/structure/meta-binding.mq.md", + "--bind", + "extra=from-cli", + ]); + assert_eq!(code, 0, "meta-binding stderr={stderr}"); + assert_eq!( + stdout.trim_end(), + "default-model\nHello World\n16\nsk-test\nfrom-cli" + ); +} + +#[test] +fn structure_meta_binding_env_and_secret() { + let (code, stdout, stderr) = run_with_env( + &[ + "run", + "tests/structure/meta-binding.mq.md", + "--bind", + "model=from-bind", + ], + &[ + ("MARQDO_META_MODEL", "from-env"), + ("MARQDO_META_USER", "Ada"), + ("MARQDO_META_TOKENS", "32"), + ("MARQDO_META_SECRET", "s3cr3t"), + ], + ); + assert_eq!(code, 0, "meta-binding-env stderr={stderr}"); + // --bind overlays model; secret displays as + assert_eq!( + stdout.trim_end(), + "from-bind\nHello Ada\n32\n\nno-extra" + ); +} + #[test] fn keywords_print() { assert_out( diff --git a/tests/structure/meta-binding.mq.md b/tests/structure/meta-binding.mq.md new file mode 100644 index 0000000..26005a0 --- /dev/null +++ b/tests/structure/meta-binding.mq.md @@ -0,0 +1,28 @@ +--- +title: meta-binding +model: ${env.MARQDO_META_MODEL ?? "default-model"} +greeting: "Hello ${env.MARQDO_META_USER ?? "World"}" +tokens: ${int(env.MARQDO_META_TOKENS ?? "16")} +api_key: ${secret.MARQDO_META_SECRET ?? "sk-test"} +import sys:lib/sys.mq.md +--- + +# main + +**m = > sys.meta_get key="model"** +> print text=`m` + +**g = > sys.meta_get key="greeting"** +> print text=`g` + +**t = > sys.meta_get key="tokens"** +> print text=`t` + +**k = > sys.meta_get key="api_key"** +> print text=`k` + +**ov = > sys.meta_get key="extra"** +1. `ov` + > print text=`ov` +2. * + > print text=no-extra