From 506de65dae69d75f8d0c8209aa779ac9e8341a80 Mon Sep 17 00:00:00 2001 From: cflmy Date: Mon, 7 Sep 2026 20:27:19 +0800 Subject: [PATCH] feat(stdlib): Mid M4 csv and thickened text helpers Add lib/csv parse/stringify and expand lib/text with contains, replace, case, pad, and related golds plus public demos. Co-authored-by: Cursor --- CHANGELOG.md | 3 +- doc/design/stdlib-csv.md | 29 +++++ doc/design/stdlib-mid.md | 4 +- doc/design/stdlib-modules.md | 7 +- doc/design/stdlib-text-mid.md | 30 +++++ doc/roadmap/stdlib-mid.md | 13 ++- lib/csv.mq.md | 14 +++ lib/text.mq.md | 56 ++++++++- lib/文本.mq.md | 56 ++++++++- lib/逗号表.mq.md | 14 +++ public/README.md | 2 +- public/features/11-csv-text.mq.md | 24 ++++ public/features/11-逗号表文本.mq.md | 24 ++++ src/host/csv.rs | 172 ++++++++++++++++++++++++++++ src/host/dispatch.rs | 80 ++++++++++++- src/host/mod.rs | 2 + src/host/text_ops.rs | 153 +++++++++++++++++++++++++ tests/gold.rs | 26 +++++ tests/lib/csv-smoke.mq.md | 21 ++++ tests/lib/text-mid-smoke.mq.md | 21 ++++ tests/lib/文本-加厚-烟测.mq.md | 15 +++ tests/lib/逗号表-烟测.mq.md | 15 +++ 22 files changed, 766 insertions(+), 15 deletions(-) create mode 100644 doc/design/stdlib-csv.md create mode 100644 doc/design/stdlib-text-mid.md create mode 100644 lib/csv.mq.md create mode 100644 lib/逗号表.mq.md create mode 100644 public/features/11-csv-text.mq.md create mode 100644 public/features/11-逗号表文本.mq.md create mode 100644 src/host/csv.rs create mode 100644 src/host/text_ops.rs create mode 100644 tests/lib/csv-smoke.mq.md create mode 100644 tests/lib/text-mid-smoke.mq.md create mode 100644 tests/lib/文本-加厚-烟测.mq.md create mode 100644 tests/lib/逗号表-烟测.mq.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 92981bf..e0ab992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,11 @@ ## Unreleased ### Added +- **中层标准库 M4**:`lib/csv` / `逗号表`(parse/stringify)与 `lib/text` / `文本` 加厚(contains/replace/case/pad/…);金样与 public `11-csv-text`。设计 [stdlib-csv.md](doc/design/stdlib-csv.md) · [stdlib-text-mid.md](doc/design/stdlib-text-mid.md)。 - **中层标准库 M3**:`lib/hash` / `哈希`(sha256/sha1/md5/hmac_sha256)与 `lib/secrets` / `机密`(token_hex/token_urlsafe);金样与 public `10-hash-secrets`。设计 [stdlib-hash.md](doc/design/stdlib-hash.md) · [stdlib-secrets.md](doc/design/stdlib-secrets.md)。 - **中层标准库 M2**:`lib/path` / `lib/路径`(join/split/normalize/…)与 `fs`/`文件` 加厚(`copy_file`/`move`/`make_temp`);金样与 public `09-path-fs`。设计 [stdlib-path.md](doc/design/stdlib-path.md) · [stdlib-fs-mid.md](doc/design/stdlib-fs-mid.md)。 - **中层标准库 M1**:`lib/re` / `lib/正则`(`is_match`/`find`/`find_all`/`replace`/`split`)与 `lib/encoding` / `lib/编码`(base64/hex);host `host_re_*` + `host_encoding_*`;金样与 public `08-re-encoding`。设计 [stdlib-mid.md](doc/design/stdlib-mid.md) · [stdlib-re.md](doc/design/stdlib-re.md) · [stdlib-encoding.md](doc/design/stdlib-encoding.md)。 -- **中层标准库(规划)**:通用原语沉 `lib/`(csv / text++ …),减少「事事 ABI 插件」。路线 [stdlib-mid.md](doc/roadmap/stdlib-mid.md)。 +- **中层标准库(规划)**:通用原语沉 `lib/`(listx/cli/log …),减少「事事 ABI 插件」。路线 [stdlib-mid.md](doc/roadmap/stdlib-mid.md)。 - **`ext/linalg`**:L0–L6 + 公式文档面 **F1–F3**(中缀、`declare`、展示/`explicit` 默认化简、`T_ascii`、public「如何写公式文档」);示例 `linalg-transpose` · `linalg-svd` · `linalg-least-squares`。设计:[ext-linalg-formula-doc.md](doc/design/ext-linalg-formula-doc.md)。 ### Fixed diff --git a/doc/design/stdlib-csv.md b/doc/design/stdlib-csv.md new file mode 100644 index 0000000..b3bfff8 --- /dev/null +++ b/doc/design/stdlib-csv.md @@ -0,0 +1,29 @@ +# `lib/csv` — 逗号表(Mid M4) + +| | | +|---|---| +| 状态 | **已落地(M4)** | +| 日期 | 2026-09-07 | +| 相关 | [stdlib-text-mid.md](stdlib-text-mid.md) · [stdlib-mid.md](stdlib-mid.md) | +| 导入 | `lib/csv.mq.md` · `lib/逗号表.mq.md` | +| Host | `host_csv_*`(RFC4180 最小集,无第三方 crate) | + +--- + +## 1. API + +| 英文 | 中文 | 形参 | 结果 | +|------|------|------|------| +| `parse` | `解析` | `text` | **list of maps**:首行为表头 | +| `stringify` | `序列化` | `rows` | CSV 文本(表头 = 首行 map 的键顺序) | + +## 2. 方言(v1) + +- 分隔符 `,`;引号 `"`;`""` 转义。 +- 换行 `\n` / `\r\n`;忽略末尾空行。 +- 字段内可含逗号与换行(须加引号)。 +- **不**支持自定义分隔符 / 无表头模式(可后续加)。 + +## 3. 刻意不做 + +流式大文件、类型推断、Excel 方言。 diff --git a/doc/design/stdlib-mid.md b/doc/design/stdlib-mid.md index f45181d..5a5a675 100644 --- a/doc/design/stdlib-mid.md +++ b/doc/design/stdlib-mid.md @@ -2,7 +2,7 @@ | | | |---|---| -| 状态 | **规划中 · M1–M3 已实现** | +| 状态 | **规划中 · M1–M4 已实现** | | 日期 | 2026-09-07 | | 相关 | [stdlib.md](stdlib.md) · [stdlib-modules.md](stdlib-modules.md) · [ext-abi.md](ext-abi.md) · [roadmap/stdlib-mid.md](../roadmap/stdlib-mid.md) | | 目标 | 在接下来数个发版周期内,把「通用编程」能力沉到 **L0.5 宿主 + L1 `.mq.md`**,减少「凡是稍有能力就必须 ABI 插件」的路径依赖 | @@ -121,7 +121,7 @@ OS / 算法库(精选 crate,默认 features 最小化) | **M1** | `re` / `正则` · `encoding` / `编码` | match / find / replace;base64 / hex | **done** | | **M2** | `path` / `路径` · `fs` 加厚 | join / normalize;copy / move / temp | **done** | | **M3** | `hash` / `哈希` · `secrets` / `机密` | sha256 / hmac;token | **done** | -| **M4** | `csv` / `逗号表` · `text` 加厚 | parse / stringify;contains / replace | host 或纯 mq | +| **M4** | `csv` / `逗号表` · `text` 加厚 | parse / stringify;contains / replace | **done** | | **M5** | `listx` 或 `table` 加厚 · `cli` | sort / zip;flags | 优先纯 mq | | **M6** | `log` / `日志` · `uuid` / `标识` | level 日志;v4 | 薄 host 或纯 mq | | **可选** | `yaml` / `Toml` | 仅当 public/catalog 强需求 | 慎入 | diff --git a/doc/design/stdlib-modules.md b/doc/design/stdlib-modules.md index 988f390..510bdd9 100644 --- a/doc/design/stdlib-modules.md +++ b/doc/design/stdlib-modules.md @@ -2,7 +2,7 @@ | | | |---|---| -| 状态 | **核心已落地**;**中层 Mid:M1–M3 已落地** · [stdlib-mid.md](stdlib-mid.md) / [roadmap/stdlib-mid.md](../roadmap/stdlib-mid.md) | +| 状态 | **核心已落地**;**中层 Mid:M1–M4 已落地** · [stdlib-mid.md](stdlib-mid.md) / [roadmap/stdlib-mid.md](../roadmap/stdlib-mid.md) | | 日期 | 2026-09-07 | | 原则 | 全部经 frontmatter **导入**;除 JSON 外中英分文件;内核保持少而精;**通用能力优先 Mid,插件留给领域核** | | 相关 | [stdlib.md](stdlib.md) · [stdlib-i18n.md](stdlib-i18n.md) · [stdlib-mid.md](stdlib-mid.md) · [ext-abi.md](ext-abi.md) · [ext-llm.md](ext-llm.md) · [ext-agent.md](ext-agent.md) · [**module-namespace.md**](module-namespace.md)(导入命名空间;**M1/M2 已落地**:`库.成员` / `库.对象.成员` / `use`) | @@ -15,12 +15,13 @@ | 模块族 | 英文库 | 中文库 | 状态 | |--------|--------|--------|------| -| 文本 | `lib/text.mq.md` | `lib/文本.mq.md` | 已有 | +| 文本 | `lib/text.mq.md` | `lib/文本.mq.md` | 已有 + **Mid M4** 加厚 | | **正则** | `lib/re.mq.md` | `lib/正则.mq.md` | **Mid M1** | | **编解码** | `lib/encoding.mq.md` | `lib/编码.mq.md` | **Mid M1** | | **路径** | `lib/path.mq.md` | `lib/路径.mq.md` | **Mid M2** | | **哈希** | `lib/hash.mq.md` | `lib/哈希.mq.md` | **Mid M3** | | **机密** | `lib/secrets.mq.md` | `lib/机密.mq.md` | **Mid M3** | +| **逗号表** | `lib/csv.mq.md` | `lib/逗号表.mq.md` | **Mid M4** | | 表 | `lib/table.mq.md` | `lib/表.mq.md` | 已有(`put`/`改` + list/map;见 [stdlib-table.md](stdlib-table.md)) | | 浏览器效应 | `lib/browser.mq.md` | `lib/浏览器.mq.md` | WASM 客户端效应助手(GFM + helpers;勿用 `json.set` 链) | | 文件 | `lib/fs.mq.md` | `lib/文件.mq.md` | 已有 + Mid M2:`copy_file` / `move` / `make_temp` | @@ -36,7 +37,7 @@ 官方可选扩展(**非** stdlib):`ext/llm` · `ext/agent` · `ext/web` · `ext/quantum` · `ext/linalg` — 见各 `ext-*.md`。 -**中层下一波候选(未落地):** `csv` / text 加厚 / … — 见 [stdlib-mid.md](stdlib-mid.md)。**已落地:** M1–M3(`re`/`encoding`/`path`/`fs+`/`hash`/`secrets`)。 +**中层下一波候选(未落地):** table/listx + cli / log / uuid — 见 [stdlib-mid.md](stdlib-mid.md)。**已落地:** M1–M4。 设计细节:[stdlib-writeback.md](stdlib-writeback.md) · [stdlib-subtask.md](stdlib-subtask.md)。 diff --git a/doc/design/stdlib-text-mid.md b/doc/design/stdlib-text-mid.md new file mode 100644 index 0000000..1cc8715 --- /dev/null +++ b/doc/design/stdlib-text-mid.md @@ -0,0 +1,30 @@ +# `lib/text` 加厚(Mid M4) + +| | | +|---|---| +| 状态 | **已落地(M4)** | +| 日期 | 2026-09-07 | +| 相关 | [stdlib-csv.md](stdlib-csv.md) · [stdlib-mid.md](stdlib-mid.md) | +| 导入 | `lib/text.mq.md` · `lib/文本.mq.md` | +| Host | `host_text_*`(Unicode 大小写 / 子串;非正则替换) | + +--- + +## 1. 新增 API + +| 英文 | 中文 | 形参 | 结果 | +|------|------|------|------| +| `contains` | `包含` | `text`, `sub` | 布尔 | +| `starts_with` | `开头是` | `text`, `prefix` | 布尔 | +| `ends_with` | `结尾是` | `text`, `suffix` | 布尔 | +| `replace` | `替换` | `text`, `old`, `new`, 可选 `count` | 文本(字面替换,非正则) | +| `to_upper` | `大写` | `text` | 文本(Unicode) | +| `to_lower` | `小写` | `text` | 文本 | +| `repeat` | `重复` | `text`, `n` | 文本 | +| `pad` | `填充` | `text`, `width`, 可选 `fill`=`" "`,可选 `align`=`left`/`right`/`center` | 文本 | + +保留原有 `str_trim` / `str_split` / `str_join`(及中文去空白/拆分/拼接)。 + +## 2. 刻意不做 + +正则替换(用 `lib/re`)、完整 `format` 模板引擎。 diff --git a/doc/roadmap/stdlib-mid.md b/doc/roadmap/stdlib-mid.md index 8b2605e..112b78c 100644 --- a/doc/roadmap/stdlib-mid.md +++ b/doc/roadmap/stdlib-mid.md @@ -2,7 +2,7 @@ | | | |---|---| -| 状态 | **M1–M3 已落地 · M4–M6 规划中** | +| 状态 | **M1–M4 已落地 · M5–M6 规划中** | | 日期 | 2026-09-07 | | 设计 | [stdlib-mid.md](../design/stdlib-mid.md) | | 相关 | [stdlib-modules.md](../design/stdlib-modules.md) · [next-phase.md](next-phase.md) · [ext-cli.md](../design/ext-cli.md) | @@ -95,13 +95,16 @@ M6 可观测与标识 log + uuid(+ 可选 yaml) --- -### M4 — CSV 与文本加厚 +### M4 — CSV 与文本加厚 — **done** | 交付 | 说明 | |------|------| -| `lib/csv.mq.md` / `lib/逗号表.mq.md` | `parse` → list of maps;`stringify`;方言先 RFC4180 最小集 | -| `lib/text` / `文本` 加厚 | `contains` / `starts_with` / `ends_with` / `replace` / `to_upper`/`to_lower` / `repeat` / `pad`(Unicode 规则写清) | -| 验收 | public 数据清洗小例;与 `table.put` 管线可读(禁 json 袋胶水) | +| `lib/csv.mq.md` / `lib/逗号表.mq.md` | `parse` / `stringify`(首行表头 → list of maps) | +| `lib/text` / `文本` 加厚 | `contains` / `starts_with` / `ends_with` / `replace` / `to_upper`/`to_lower` / `repeat` / `pad` | +| Host | `host_csv_*`、`host_text_*` | +| 金样 | `csv-smoke` · `逗号表-烟测` · `text-mid-smoke` · `文本-加厚-烟测` | +| 文档 | [stdlib-csv.md](../design/stdlib-csv.md) · [stdlib-text-mid.md](../design/stdlib-text-mid.md) · public `11-csv-text` | +| 验收 | CSV 往返 + 文本清洗小例 | **可选同波:** 极简 `format`(`{name}` 替换),若与引用语法冲突则延后。 diff --git a/lib/csv.mq.md b/lib/csv.mq.md new file mode 100644 index 0000000..357e9e0 --- /dev/null +++ b/lib/csv.mq.md @@ -0,0 +1,14 @@ +--- +title: lib/csv +description: RFC4180-ish CSV parse/stringify (Mid M4). First row = headers → list of maps. +--- + +## parse + + `text` + +**> host_csv_parse text=`text`** + +## stringify + + `rows` + +**> host_csv_stringify rows=`rows`** diff --git a/lib/text.mq.md b/lib/text.mq.md index 760194a..17b4b9c 100644 --- a/lib/text.mq.md +++ b/lib/text.mq.md @@ -1,6 +1,6 @@ --- title: lib/text -description: English text helpers over trim / split / join +description: Text helpers (Mid M4 thicken). Literal replace — use lib/re for regex. --- ## str_trim @@ -19,3 +19,57 @@ description: English text helpers over trim / split / join + `sep` **> join value=`xs` sep=`sep`** + +## contains + + `text` + + `sub` + +**> host_text_contains text=`text` sub=`sub`** + +## starts_with + + `text` + + `prefix` + +**> host_text_starts_with text=`text` prefix=`prefix`** + +## ends_with + + `text` + + `suffix` + +**> host_text_ends_with text=`text` suffix=`suffix`** + +## replace + + `text` + + `old` + + `new` + + `count`=None + +Literal substring replace (`count` optional). + +**> host_text_replace text=`text` old=`old` new=`new` count=`count`** + +## to_upper + + `text` + +**> host_text_to_upper text=`text`** + +## to_lower + + `text` + +**> host_text_to_lower text=`text`** + +## repeat + + `text` + + `n` + +**> host_text_repeat text=`text` n=`n`** + +## pad + + `text` + + `width` + + `fill`=None + + `align`=None + +align: left (default), right, or center. + +**> host_text_pad text=`text` width=`width` fill=`fill` align=`align`** diff --git a/lib/文本.mq.md b/lib/文本.mq.md index 8e3e482..299b2e7 100644 --- a/lib/文本.mq.md +++ b/lib/文本.mq.md @@ -1,6 +1,6 @@ --- title: lib/文本 -description: 中文文本库:去空白 / 拆分 / 拼接 +description: 文本库加厚(中层 M4)。字面替换 — 正则请用 lib/正则。 --- ## 去空白 @@ -19,3 +19,57 @@ description: 中文文本库:去空白 / 拆分 / 拼接 + `sep` **> join value=`xs` sep=`sep`** + +## 包含 + + `内容` + + `子串` + +**> host_text_contains text=`内容` sub=`子串`** + +## 开头是 + + `内容` + + `前缀` + +**> host_text_starts_with text=`内容` prefix=`前缀`** + +## 结尾是 + + `内容` + + `后缀` + +**> host_text_ends_with text=`内容` suffix=`后缀`** + +## 替换 + + `内容` + + `旧` + + `新` + + `次数`=None + +字面子串替换(可选 `次数`)。 + +**> host_text_replace text=`内容` old=`旧` new=`新` count=`次数`** + +## 大写 + + `内容` + +**> host_text_to_upper text=`内容`** + +## 小写 + + `内容` + +**> host_text_to_lower text=`内容`** + +## 重复 + + `内容` + + `次数` + +**> host_text_repeat text=`内容` n=`次数`** + +## 填充 + + `内容` + + `宽度` + + `填充符`=None + + `对齐`=None + +对齐:left(默认)/ right / center。 + +**> host_text_pad text=`内容` width=`宽度` fill=`填充符` align=`对齐`** diff --git a/lib/逗号表.mq.md b/lib/逗号表.mq.md new file mode 100644 index 0000000..f8f74e4 --- /dev/null +++ b/lib/逗号表.mq.md @@ -0,0 +1,14 @@ +--- +title: lib/逗号表 +description: CSV 解析/序列化(中层 M4)。首行为表头 → 映射列表。 +--- + +## 解析 + + `内容` + +**> host_csv_parse text=`内容`** + +## 序列化 + + `行集` + +**> host_csv_stringify rows=`行集`** diff --git a/public/README.md b/public/README.md index 35b7148..937e544 100644 --- a/public/README.md +++ b/public/README.md @@ -14,7 +14,7 @@ 官方扩展(非 stdlib):`ext/llm`、`ext/agent`、`ext/web`、`ext/quantum`、`ext/linalg` — 见 `features/05-*.mq.md`、`06-*.mq.md`、`07-linalg.mq.md` / `07-线性代数.mq.md` 与 `doc/design/ext-*.md`。 -中层标准库(Mid):`08-re-encoding` · `09-path-fs` · `10-hash-secrets`(及对应中文页)。 +中层标准库(Mid):`08-re-encoding` · `09-path-fs` · `10-hash-secrets` · `11-csv-text`(及对应中文页)。 **不放** `errors/`:失败样例只在 `tests/errors/`。 diff --git a/public/features/11-csv-text.mq.md b/public/features/11-csv-text.mq.md new file mode 100644 index 0000000..c0455c6 --- /dev/null +++ b/public/features/11-csv-text.mq.md @@ -0,0 +1,24 @@ +--- +title: Mid stdlib — csv and text +description: lib/csv + thickened lib/text (Mid M4) +import csv:lib/csv.mq.md +import text:lib/text.mq.md +--- + +# CSV cleaning with text helpers + +Parse a tiny CSV, normalize a field, print — no plugins. + +# main + +*raw = "name,city\nAda,london\nBob,PARIS\n"* +*rows = > csv.parse text=`raw`* +*r0 = > at value=`rows` index=0* +*city = > text.to_upper text=`r0`[^city]* +> print text=`r0`[^name] +> print text=`city` +*n = > len `rows`* +1. `n` == 2 + > print text=mid-m4-ok +2. * + > print text=mid-m4-fail diff --git a/public/features/11-逗号表文本.mq.md b/public/features/11-逗号表文本.mq.md new file mode 100644 index 0000000..49aa4f4 --- /dev/null +++ b/public/features/11-逗号表文本.mq.md @@ -0,0 +1,24 @@ +--- +title: 中层标准库 — 逗号表与文本 +description: lib/逗号表 + 加厚 lib/文本(中层 M4) +导入 逗号表:lib/逗号表.mq.md +导入 文本:lib/文本.mq.md +--- + +# 用文本助手清洗 CSV + +解析小表、规范化字段、打印 — 无需插件。 + +# main + +*raw = "name,city\nAda,london\nBob,PARIS\n"* +*rows = > 逗号表.解析 内容=`raw`* +*r0 = > at value=`rows` index=0* +*city = > 文本.大写 内容=`r0`[^city]* +> 打印 内容=`r0`[^name] +> 打印 内容=`city` +*n = > 长度 `rows`* +1. `n` == 2 + > 打印 内容=mid-m4-ok +2. * + > 打印 内容=mid-m4-fail diff --git a/src/host/csv.rs b/src/host/csv.rs new file mode 100644 index 0000000..749fd23 --- /dev/null +++ b/src/host/csv.rs @@ -0,0 +1,172 @@ +//! Minimal RFC4180 CSV for Mid M4 (`lib/csv`). + +use crate::value::Value; + +fn as_text<'a>(v: &'a Value, label: &str) -> Result<&'a str, String> { + match v { + Value::Text(s) => Ok(s.as_str()), + _ => Err(format!("{label} needs text")), + } +} + +/// Parse CSV text into a list of maps (first row = headers). +pub fn parse(text: &Value) -> Result { + let s = as_text(text, "text")?; + let rows = parse_rows(s)?; + if rows.is_empty() { + return Ok(Value::List(vec![])); + } + let headers = rows[0].clone(); + if headers.is_empty() { + return Err("csv.parse: header row is empty".into()); + } + let mut out = Vec::new(); + for row in rows.into_iter().skip(1) { + let mut map = Vec::new(); + for (i, key) in headers.iter().enumerate() { + let val = row.get(i).cloned().unwrap_or_default(); + map.push((key.clone(), Value::Text(val))); + } + out.push(Value::Map(map)); + } + Ok(Value::List(out)) +} + +/// Serialize list of maps to CSV (header order = first row's keys). +pub fn stringify(rows: &Value) -> Result { + let list = match rows { + Value::List(xs) => xs, + _ => return Err("csv.stringify: rows needs list".into()), + }; + if list.is_empty() { + return Ok(Value::Text(String::new())); + } + let first = match &list[0] { + Value::Map(m) => m, + _ => return Err("csv.stringify: each row needs map".into()), + }; + let headers: Vec = first.iter().map(|(k, _)| k.clone()).collect(); + let mut lines = Vec::new(); + lines.push(headers.iter().map(|h| escape_field(h)).collect::>().join(",")); + for row in list { + let map = match row { + Value::Map(m) => m, + _ => return Err("csv.stringify: each row needs map".into()), + }; + let mut fields = Vec::new(); + for h in &headers { + let cell = map + .iter() + .find(|(k, _)| k == h) + .map(|(_, v)| value_to_cell(v)) + .unwrap_or_default(); + fields.push(escape_field(&cell)); + } + lines.push(fields.join(",")); + } + Ok(Value::Text(lines.join("\n"))) +} + +fn value_to_cell(v: &Value) -> String { + match v { + Value::Text(s) => s.clone(), + Value::Int(n) => n.to_string(), + Value::Num(n) => n.to_string(), + Value::Bool(b) => { + if *b { + "True".into() + } else { + "False".into() + } + } + Value::None => String::new(), + other => other.as_display(), + } +} + +fn escape_field(s: &str) -> String { + if s.contains(',') || s.contains('"') || s.contains('\n') || s.contains('\r') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.to_string() + } +} + +fn parse_rows(input: &str) -> Result>, String> { + let mut rows = Vec::new(); + let mut row = Vec::new(); + let mut field = String::new(); + let mut chars = input.chars().peekable(); + let mut in_quotes = false; + while let Some(c) = chars.next() { + if in_quotes { + if c == '"' { + if chars.peek() == Some(&'"') { + chars.next(); + field.push('"'); + } else { + in_quotes = false; + } + } else { + field.push(c); + } + } else { + match c { + '"' => in_quotes = true, + ',' => { + row.push(std::mem::take(&mut field)); + } + '\r' => { + // swallow optional \n + if chars.peek() == Some(&'\n') { + chars.next(); + } + row.push(std::mem::take(&mut field)); + rows.push(std::mem::take(&mut row)); + } + '\n' => { + row.push(std::mem::take(&mut field)); + rows.push(std::mem::take(&mut row)); + } + _ => field.push(c), + } + } + } + if in_quotes { + return Err("csv.parse: unterminated quote".into()); + } + if !field.is_empty() || !row.is_empty() { + row.push(field); + rows.push(row); + } else if !input.is_empty() && input.ends_with('\n') { + // trailing newline already flushed last row + } + // Drop a single trailing empty row produced by final newline only when last row empty? + while rows.last().is_some_and(|r| r.len() == 1 && r[0].is_empty()) { + // only drop if it's a spurious empty line at end with no fields + if rows.len() > 1 { + rows.pop(); + } else { + break; + } + } + Ok(rows) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrip_simple() { + let text = Value::Text("a,b\n1,2\n3,\"x,y\"\n".into()); + let rows = parse(&text).unwrap(); + let Value::List(ref xs) = rows else { + panic!("list"); + }; + assert_eq!(xs.len(), 2); + let out = stringify(&rows).unwrap(); + let again = parse(&out).unwrap(); + assert_eq!(again, rows); + } +} diff --git a/src/host/dispatch.rs b/src/host/dispatch.rs index 0cd3b3b..ecd06e8 100644 --- a/src/host/dispatch.rs +++ b/src/host/dispatch.rs @@ -3,7 +3,8 @@ use std::collections::HashMap; use crate::host::{ - encoding, fs, hash, json, math, net, path, re, secrets, sys, time, writeback, HostContext, + csv, encoding, fs, hash, json, math, net, path, re, secrets, sys, text_ops, time, writeback, + HostContext, }; #[cfg(feature = "exec-host")] use crate::host::{foreign, subtask}; @@ -155,6 +156,16 @@ pub enum HostFn { HashHmacSha256 = 147, SecretsTokenHex = 148, SecretsTokenUrlsafe = 149, + TextContains = 150, + TextStartsWith = 151, + TextEndsWith = 152, + TextReplace = 153, + TextToUpper = 154, + TextToLower = 155, + TextRepeat = 156, + TextPad = 157, + CsvParse = 158, + CsvStringify = 159, } @@ -301,6 +312,16 @@ impl HostFn { 147 => Self::HashHmacSha256, 148 => Self::SecretsTokenHex, 149 => Self::SecretsTokenUrlsafe, + 150 => Self::TextContains, + 151 => Self::TextStartsWith, + 152 => Self::TextEndsWith, + 153 => Self::TextReplace, + 154 => Self::TextToUpper, + 155 => Self::TextToLower, + 156 => Self::TextRepeat, + 157 => Self::TextPad, + 158 => Self::CsvParse, + 159 => Self::CsvStringify, _ => return None, }) } @@ -450,6 +471,16 @@ impl HostFn { "host_hash_hmac_sha256" | "hash_hmac_sha256" => Self::HashHmacSha256, "host_secrets_token_hex" | "secrets_token_hex" => Self::SecretsTokenHex, "host_secrets_token_urlsafe" | "secrets_token_urlsafe" => Self::SecretsTokenUrlsafe, + "host_text_contains" | "text_contains" => Self::TextContains, + "host_text_starts_with" | "text_starts_with" => Self::TextStartsWith, + "host_text_ends_with" | "text_ends_with" => Self::TextEndsWith, + "host_text_replace" | "text_replace" => Self::TextReplace, + "host_text_to_upper" | "text_to_upper" => Self::TextToUpper, + "host_text_to_lower" | "text_to_lower" => Self::TextToLower, + "host_text_repeat" | "text_repeat" => Self::TextRepeat, + "host_text_pad" | "text_pad" => Self::TextPad, + "host_csv_parse" | "csv_parse" => Self::CsvParse, + "host_csv_stringify" | "csv_stringify" => Self::CsvStringify, _ => return None, }) } @@ -586,6 +617,16 @@ impl HostFn { Self::HashHmacSha256 => "host_hash_hmac_sha256", Self::SecretsTokenHex => "host_secrets_token_hex", Self::SecretsTokenUrlsafe => "host_secrets_token_urlsafe", + Self::TextContains => "host_text_contains", + Self::TextStartsWith => "host_text_starts_with", + Self::TextEndsWith => "host_text_ends_with", + Self::TextReplace => "host_text_replace", + Self::TextToUpper => "host_text_to_upper", + Self::TextToLower => "host_text_to_lower", + Self::TextRepeat => "host_text_repeat", + Self::TextPad => "host_text_pad", + Self::CsvParse => "host_csv_parse", + Self::CsvStringify => "host_csv_stringify", Self::WritebackRecord => "host_writeback_record", Self::WritebackGet => "host_writeback_get", Self::WritebackClear => "host_writeback_clear", @@ -691,6 +732,15 @@ impl HostFn { Self::HashSha256 | Self::HashSha1 | Self::HashMd5 => &["text"], Self::HashHmacSha256 => &["key", "text"], Self::SecretsTokenHex | Self::SecretsTokenUrlsafe => &[], + Self::TextContains => &["text", "sub"], + Self::TextStartsWith => &["text", "prefix"], + Self::TextEndsWith => &["text", "suffix"], + Self::TextReplace => &["text", "old", "new"], + Self::TextToUpper | Self::TextToLower => &["text"], + Self::TextRepeat => &["text", "n"], + Self::TextPad => &["text", "width"], + Self::CsvParse => &["text"], + Self::CsvStringify => &["rows"], Self::WritebackRecord => &["value"], Self::WritebackGet | Self::WritebackClear => &[], Self::WritebackList => &[], @@ -726,6 +776,8 @@ impl HostFn { Self::ReReplace => &["count"], Self::FsMakeTemp => &["prefix"], Self::SecretsTokenHex | Self::SecretsTokenUrlsafe => &["n"], + Self::TextReplace => &["count"], + Self::TextPad => &["fill", "align"], _ => &[], } } @@ -1129,6 +1181,32 @@ pub fn call_host( } HostFn::SecretsTokenHex => secrets::token_hex(bound.get("n")), HostFn::SecretsTokenUrlsafe => secrets::token_urlsafe(bound.get("n")), + HostFn::TextContains => { + text_ops::contains(require(bound, "text")?, require(bound, "sub")?) + } + HostFn::TextStartsWith => { + text_ops::starts_with(require(bound, "text")?, require(bound, "prefix")?) + } + HostFn::TextEndsWith => { + text_ops::ends_with(require(bound, "text")?, require(bound, "suffix")?) + } + HostFn::TextReplace => text_ops::replace( + require(bound, "text")?, + require(bound, "old")?, + require(bound, "new")?, + bound.get("count"), + ), + HostFn::TextToUpper => text_ops::to_upper(require(bound, "text")?), + HostFn::TextToLower => text_ops::to_lower(require(bound, "text")?), + HostFn::TextRepeat => text_ops::repeat(require(bound, "text")?, require(bound, "n")?), + HostFn::TextPad => text_ops::pad( + require(bound, "text")?, + require(bound, "width")?, + bound.get("fill"), + bound.get("align"), + ), + HostFn::CsvParse => csv::parse(require(bound, "text")?), + HostFn::CsvStringify => csv::stringify(require(bound, "rows")?), HostFn::OuterCallLine => Ok(Value::Int( ctx.call_site_lines .first() diff --git a/src/host/mod.rs b/src/host/mod.rs index 30e0f39..23e4bbf 100644 --- a/src/host/mod.rs +++ b/src/host/mod.rs @@ -7,6 +7,7 @@ pub mod agent_rt; pub mod collection; mod dispatch; +mod csv; mod encoding; pub mod event_bus; #[cfg(feature = "exec-host")] @@ -19,6 +20,7 @@ mod net; mod path; mod re; mod secrets; +mod text_ops; #[cfg(feature = "plugin-host")] pub mod plugin; #[cfg(not(feature = "plugin-host"))] diff --git a/src/host/text_ops.rs b/src/host/text_ops.rs new file mode 100644 index 0000000..12264ea --- /dev/null +++ b/src/host/text_ops.rs @@ -0,0 +1,153 @@ +//! Text helpers for Mid M4 (`lib/text` thicken). + +use crate::value::Value; + +fn as_text<'a>(v: &'a Value, label: &str) -> Result<&'a str, String> { + match v { + Value::Text(s) => Ok(s.as_str()), + _ => Err(format!("{label} needs text")), + } +} + +fn as_usize(v: &Value, label: &str) -> Result { + match v { + Value::Int(n) if *n >= 0 => Ok(*n as usize), + Value::Int(_) => Err(format!("{label} must be non-negative")), + _ => Err(format!("{label} needs int")), + } +} + +pub fn contains(text: &Value, sub: &Value) -> Result { + let t = as_text(text, "text")?; + let s = as_text(sub, "sub")?; + Ok(Value::Bool(t.contains(s))) +} + +pub fn starts_with(text: &Value, prefix: &Value) -> Result { + let t = as_text(text, "text")?; + let p = as_text(prefix, "prefix")?; + Ok(Value::Bool(t.starts_with(p))) +} + +pub fn ends_with(text: &Value, suffix: &Value) -> Result { + let t = as_text(text, "text")?; + let s = as_text(suffix, "suffix")?; + Ok(Value::Bool(t.ends_with(s))) +} + +pub fn replace( + text: &Value, + old: &Value, + new: &Value, + count: Option<&Value>, +) -> Result { + let t = as_text(text, "text")?; + let old = as_text(old, "old")?; + let new = as_text(new, "new")?; + if old.is_empty() { + return Err("replace: old must be non-empty".into()); + } + let limit = match count { + None | Some(Value::None) => None, + Some(v) => Some(as_usize(v, "count")?), + }; + let out = match limit { + None => t.replace(old, new), + Some(0) => t.to_string(), + Some(n) => { + let mut out = String::new(); + let mut rest = t; + let mut done = 0usize; + while done < n { + if let Some(i) = rest.find(old) { + out.push_str(&rest[..i]); + out.push_str(new); + rest = &rest[i + old.len()..]; + done += 1; + } else { + break; + } + } + out.push_str(rest); + out + } + }; + Ok(Value::Text(out)) +} + +pub fn to_upper(text: &Value) -> Result { + Ok(Value::Text(as_text(text, "text")?.to_uppercase())) +} + +pub fn to_lower(text: &Value) -> Result { + Ok(Value::Text(as_text(text, "text")?.to_lowercase())) +} + +pub fn repeat(text: &Value, n: &Value) -> Result { + let t = as_text(text, "text")?; + let n = as_usize(n, "n")?; + if n > 10_000 { + return Err("repeat: n too large".into()); + } + Ok(Value::Text(t.repeat(n))) +} + +pub fn pad( + text: &Value, + width: &Value, + fill: Option<&Value>, + align: Option<&Value>, +) -> Result { + let t = as_text(text, "text")?; + let width = as_usize(width, "width")?; + let fill = match fill { + None | Some(Value::None) => " ".to_string(), + Some(v) => as_text(v, "fill")?.to_string(), + }; + if fill.is_empty() { + return Err("pad: fill must be non-empty".into()); + } + let align = match align { + None | Some(Value::None) => "left", + Some(v) => as_text(v, "align")?, + }; + let chars: Vec = t.chars().collect(); + if chars.len() >= width { + return Ok(Value::Text(t.to_string())); + } + let pad_n = width - chars.len(); + let fill_chars: Vec = fill.chars().collect(); + let make_pad = |n: usize| { + let mut s = String::new(); + for i in 0..n { + s.push(fill_chars[i % fill_chars.len()]); + } + s + }; + let out = match align { + "right" => format!("{}{}", make_pad(pad_n), t), + "center" => { + let left = pad_n / 2; + format!("{}{}{}", make_pad(left), t, make_pad(pad_n - left)) + } + _ => format!("{}{}", t, make_pad(pad_n)), + }; + Ok(Value::Text(out)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn case_and_contains() { + assert_eq!( + contains(&Value::Text("Hello".into()), &Value::Text("ell".into())).unwrap(), + Value::Bool(true) + ); + assert_eq!( + to_upper(&Value::Text("ß".into())).unwrap(), + Value::Text("SS".into()) + ); + } +} diff --git a/tests/gold.rs b/tests/gold.rs index b01fe2e..453c2b0 100644 --- a/tests/gold.rs +++ b/tests/gold.rs @@ -1121,6 +1121,32 @@ fn lib_secrets_smoke_zh() { assert_out("tests/lib/机密-烟测.mq.md", "16"); } +#[test] +fn lib_text_mid_smoke() { + assert_out( + "tests/lib/text-mid-smoke.mq.md", + "True\nAB\na_a_a\n...x\nababab", + ); +} + +#[test] +fn lib_text_mid_smoke_zh() { + assert_out("tests/lib/文本-加厚-烟测.mq.md", "True\nAB\na_a-a"); +} + +#[test] +fn lib_csv_smoke() { + assert_out( + "tests/lib/csv-smoke.mq.md", + "2\nalice\nhi, there\n2", + ); +} + +#[test] +fn lib_csv_smoke_zh() { + assert_out("tests/lib/逗号表-烟测.mq.md", "1\n1"); +} + #[test] fn lib_re_smoke() { assert_out( diff --git a/tests/lib/csv-smoke.mq.md b/tests/lib/csv-smoke.mq.md new file mode 100644 index 0000000..5844eb6 --- /dev/null +++ b/tests/lib/csv-smoke.mq.md @@ -0,0 +1,21 @@ +--- +title: lib/csv smoke +import csv:lib/csv.mq.md +--- + +# main + +*raw = "name,note\nalice,\"hi, there\"\nbob,ok\n"* + +*rows = > csv.parse text=`raw`* +*n = > len `rows`* +> print text=`n` + +*r0 = > at value=`rows` index=0* +> print text=`r0`[^name] +> print text=`r0`[^note] + +*out = > csv.stringify rows=`rows`* +*rows2 = > csv.parse text=`out`* +*n2 = > len `rows2`* +> print text=`n2` diff --git a/tests/lib/text-mid-smoke.mq.md b/tests/lib/text-mid-smoke.mq.md new file mode 100644 index 0000000..d6f8712 --- /dev/null +++ b/tests/lib/text-mid-smoke.mq.md @@ -0,0 +1,21 @@ +--- +title: lib/text mid smoke +import text:lib/text.mq.md +--- + +# main + +*c = > text.contains text="Hello" sub="ell"* +> print text=`c` + +*u = > text.to_upper text="ab"* +> print text=`u` + +*r = > text.replace text="a-a-a" old="-" new="_" count=2* +> print text=`r` + +*p = > text.pad text="x" width=4 fill="." align="right"* +> print text=`p` + +*rep = > text.repeat text="ab" n=3* +> print text=`rep` diff --git a/tests/lib/文本-加厚-烟测.mq.md b/tests/lib/文本-加厚-烟测.mq.md new file mode 100644 index 0000000..eccbe16 --- /dev/null +++ b/tests/lib/文本-加厚-烟测.mq.md @@ -0,0 +1,15 @@ +--- +title: lib/文本 加厚烟测 +导入 文本:lib/文本.mq.md +--- + +# main + +*c = > 文本.包含 内容="Hello" 子串="ell"* +> 打印 内容=`c` + +*u = > 文本.大写 内容="ab"* +> 打印 内容=`u` + +*r = > 文本.替换 内容="a-a-a" 旧="-" 新="_" 次数=1* +> 打印 内容=`r` diff --git a/tests/lib/逗号表-烟测.mq.md b/tests/lib/逗号表-烟测.mq.md new file mode 100644 index 0000000..b2aa1bc --- /dev/null +++ b/tests/lib/逗号表-烟测.mq.md @@ -0,0 +1,15 @@ +--- +title: lib/逗号表 烟测 +导入 逗号表:lib/逗号表.mq.md +--- + +# main + +*raw = "a,b\n1,2\n"* + +*rows = > 逗号表.解析 内容=`raw`* +*n = > 长度 `rows`* +> 打印 内容=`n` + +*r0 = > at value=`rows` index=0* +> 打印 内容=`r0`[^a]