From b0209041995ec238a290484d672cf31c0bbdd5d5 Mon Sep 17 00:00:00 2001 From: cflmy Date: Fri, 20 Nov 2026 23:14:38 +0800 Subject: [PATCH] test(p4): prove row-store/columnar table equivalence - Add semantic fixture 72: a .mq.md columnar List/Map table store, converted, sorted and filtered; compare headers/cells with the T0 row Table via a cell-wise helper (Table == is not structural in this language version). - Teach the direct-runtime semantic harness to resolve entry std: imports. - Update ADR-0032, P4 status, docs/changelog/evidence; semantic goldens 72. Gate: fmt, clippy -D warnings, 430 tests passed. --- CHANGELOG.md | 2 + README.md | 2 +- crates/cli/tests/conformance_semantics.rs | 37 +++ docs/STATUS.md | 10 +- ...ble-logical-value-and-stdlib-algorithms.md | 4 + .../2026-11-20-table-cross-storage.md | 47 ++++ tests/README.md | 2 +- .../semantics/72-table-columnar.expected.json | 22 ++ .../semantics/72-table-columnar.mq.md | 238 ++++++++++++++++++ 9 files changed, 359 insertions(+), 5 deletions(-) create mode 100644 docs/evidence/2026-11-20-table-cross-storage.md create mode 100644 tests/conformance/semantics/72-table-columnar.expected.json create mode 100644 tests/conformance/semantics/72-table-columnar.mq.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f811398..994d760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -127,6 +127,8 @@ currently `0.1.0`; no release tag has been cut. Dates use ISO 8601. `std:` resolution, tamper rejection and frozen-binary hash equality. - P2 closure gate: add a pure no-Foreign/no-Plugin lane for C01–C06/C11/C12 (20 fixtures + stdlib table), source shortcut scan and golden comparison. +- P4 cross-storage table equivalence: fixture 72 compares row-store Table + sort/filter output with a `.mq.md` columnar store cell-by-cell. - P5 HTTP framing/client: `stdlib/net/http.mq.md` parses status/headers/chunked responses and builds requests; `http_over_tcp.rs` runs a loopback TCP HTTP exchange through `resource.open("tcp")`. No host HTTP/TLS implementation. diff --git a/README.md b/README.md index 8954564..06cc2cd 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ executes anything. ## What works (implemented and tested) * **P0 contracts** — frozen Markdown profile, AST → program mapping, strict - expression grammar, evaluation semantics; 160 AST goldens, 71 semantic + expression grammar, evaluation semantics; 160 AST goldens, 72 semantic goldens, 52 compile-error fixtures and 21 runtime-error fixtures (`tests/`). * **P1 interpreter** — native Rust tree-walk interpreter: values, scopes, diff --git a/crates/cli/tests/conformance_semantics.rs b/crates/cli/tests/conformance_semantics.rs index 4acc531..f32b385 100644 --- a/crates/cli/tests/conformance_semantics.rs +++ b/crates/cli/tests/conformance_semantics.rs @@ -169,6 +169,7 @@ fn semantic_goldens_match_end_to_end_execution() { program.insert(analysis.module.clone()); program.main_module = Some(module_id.clone()); install_stdlib_print(&mut program); + install_entry_stdlib_imports(&mut program, &module_id, &analysis); let mut capabilities = Capabilities::pure(); if grant_foreign { @@ -334,3 +335,39 @@ fn load_stdlib_module( program.insert(analysis.module); Ok(module_id) } + +/// Resolve `std:` imports declared by a direct-runtime fixture entry. +fn install_entry_stdlib_imports( + program: &mut Program, + module_id: &ModuleId, + analysis: &marqdo_frontend::AnalysisResult, +) { + let root = repo_root().join("stdlib"); + let Some(imports) = analysis + .document + .frontmatter() + .and_then(|frontmatter| frontmatter.imports().ok()) + else { + return; + }; + let mut loaded = std::collections::BTreeSet::new(); + for (alias, target) in imports { + let Some(relative) = target.strip_prefix("std:") else { + continue; + }; + let relative = if relative.ends_with(".mq.md") { + relative.to_string() + } else { + format!("{relative}.mq.md") + }; + let to = match load_stdlib_module(&root, &relative, program, &mut loaded) { + Ok(id) => id, + Err(error) => panic!("cannot load stdlib import `{target}`: {error}"), + }; + program.resolved_imports.push(ResolvedImport { + from: module_id.clone(), + alias, + to, + }); + } +} diff --git a/docs/STATUS.md b/docs/STATUS.md index 5af9d96..ad2cb97 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -72,7 +72,7 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL | 2 | Cross-platform strict `cfg` | **Fixed locally; native CI UNVERIFIED** | `#[cfg]` dispatch + non-Linux regression test | | 3 | Strict probe→launch race | **Fixed locally; GitHub Linux CI UNVERIFIED** | fail-closed CLI contract + probe-OK/launch-EAGAIN seam test | | 4 | Parallel security matrix isolation | **Fixed** | unique paths + helper `OnceLock`; 7/7 in parallel workspace run | -| 5 | Markdown/power/control-flow corpus | **Done locally** | 160 AST, 71 semantic, 52 compile-negative, 21 runtime-negative fixtures | +| 5 | Markdown/power/control-flow corpus | **Done locally** | 160 AST, 72 semantic, 52 compile-negative, 21 runtime-negative fixtures | | 6 | Condition header + source spans | **Fixed** | executable tails rejected; literal-inside-backtick mapping tests | | 7 | Independent Tool consumers | **Done locally** | `sample-view`/`sample-knowledge`/`sample-proof` E2E test | | 8 | lock v3 reproducibility | **Implemented; tamper test added** | `package_env` PASS | @@ -87,7 +87,7 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL - Frozen Markdown profile `marqdo.md.v1` with 160 AST goldens and explicit CRLF byte semantics. -- Native Rust interpreter with 71 semantic goldens, 52 compile-error fixtures, +- Native Rust interpreter with 72 semantic goldens, 52 compile-error fixtures, and runtime error source attribution. - Native `**` power is pinned end-to-end: right associativity, unary-minus precedence, parentheses, float negative exponents, branch/loop conditions, @@ -147,6 +147,8 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL providers register additional resource kinds without parser/runtime/HostOp changes; project-mode + name grant required; pure/strict refuse provider kinds; C/Worker transport remains open. +- Cross-storage table equivalence (P4, ADR-0032): fixture 72 compares T0 row + Table results with a `.mq.md` columnar List/Map store for sort/filter. - Table stdlib algorithms (CM-A08, ADR-0032): `stdlib/data/table.mq.md` provides select/filter/sort/append/set/join/group/`转GFM` over the logical `Value::Table`; inputs are never mutated; GFM AST laziness and physical T0 @@ -203,7 +205,7 @@ behavior. This table records only what is in this tree now. | P1 Memory + place assignment | **Implemented locally** | `Value::Memory`, run-scoped `MemoryStore` (64 KiB pages, 64 MiB total / 16 MiB region), `memory.new/size/grow`, `m[i]` read, `m[i] = byte`; semantic fixtures 45–47, runtime negatives 13–18, `examples/memory-buffer.mq.md`; evidence [`evidence/2026-11-20-memory-p1.md`](evidence/2026-11-20-memory-p1.md) | | P2 Marqdo-written allocator/data structures + minimal structure observation | **PASS (local pure lane); external stdlib repo publication pending** | `.mq.md`-only evidence: bump allocator with u32 codec, linked list, failed-allocation atomicity (48–50); CM-A04 structure primitives and a table filter (51–54, ADR-0023); a first-fit free-list allocator with block reuse, splitting, address-adjacent coalescing and `Result` maps (55, ADR-0024), a `Memory`-backed binary tree with recursive insert/in-order traversal (56), a `.mq.md` CSV row splitter (57), an open-addressing hash table with linear probing, update and `hash.full` (58), a `.mq.md` UTF-8 encoder over `Memory` (59), a validating UTF-8 decoder rejecting truncated, bad-continuation and surrogate/overlong input (60), a `Memory`-backed graph adjacency list (61), a recursive JSON parser for nested arrays/objects, strings with escapes, integers, booleans, null and error Results (62), a `Memory`-backed stack bytecode VM with PUSH/ADD/MUL/JMP/JZ/HALT (63), big-endian 16/32/64-bit plus LEB128 integer codecs (64), a `.mq.md` table sort rebuilding a new table from `table.row/columns/cell` (65), a table join (67), a quoted/escaped CSV row parser (68), table group (69), and an Int matrix multiply (70); the first-fit allocator also detects double free (`alloc.double_free`). Evidence: [`memory-p2-closure`](evidence/2026-11-20-memory-p2-closure.md), [`structure-api-p2`](evidence/2026-11-20-structure-api-p2.md), [`result-and-free-list-p2`](evidence/2026-11-20-result-and-free-list-p2.md). Local gate PASS: `crates/cli/tests/p2_closure.rs` runs 20 C01–C06/C11/C12 fixtures plus the stdlib table self-test in `--mode pure` (no foreign/plugin/import grants), scans sources for foreign/plugin/import shortcuts and compares value/stdout to goldens; evidence [`p2-closure-gate`](evidence/2026-11-20-p2-closure-gate.md). The bundled `stdlib/` tree, `std:` imports, digest verification, Analysis identity and frozen-CLI closure test are implemented (ADR-0028). Remaining outside C01–C12: publication as a separate external stdlib repository, and no remote CI URL. | | P3 Resource/Stream ABI | **PARTIAL** | ADR-0025/0026 CM-A06 slices: opaque `Resource`, `resource.open/read/write/close`, stdout write in pure/project, granted project-file read and buffered project-file write in project, EOF/closed/handle errors, wire/host rejection; strict permits only stdio/stdout and refuses file/network resources. Fixture 66 + CLI/host tests. CM-A07 **Compat-C** is implemented: when a program calls `打印/print`, the loader injects the bundled `stdlib/io.mq.md` and the runtime routes to `io.写值` (all 15 value types, UTF-8 via `Memory`, `resource.write(stdout)`); `HostOp::Print` and the legacy `Value::display()` print path are removed, and a missing bundle fails with `kernel.unavailable`. Strict permits only the existing stdio console channel. Still missing: directories, network, generic provider registry, Table storage, strict file/network resources. | -| P4 Table logical/physical migration | **PARTIAL** | ADR-0032/CM-A08: `Value::Table` logical semantics unchanged (T0 `Vec>`); `stdlib/data/table.mq.md` provides select/filter/sort/append/set/join/group/`转GFM` as value-returning `.mq.md` functions with input immutability. GFM rendering is covered; no T1/T2/T3 physical storage, column store, cross-storage implementation or Memory-backed table. | +| P4 Table logical/physical migration | **PARTIAL** | ADR-0032/CM-A08: `Value::Table` logical semantics unchanged (T0 `Vec>`); `stdlib/data/table.mq.md` provides select/filter/sort/append/set/join/group/`转GFM` as value-returning `.mq.md` functions with input immutability. GFM rendering and cross-storage equivalence (fixture 72: row Table vs `.mq.md` columnar store, same headers/rows/cells) are covered. No T1/T2/T3 physical storage, native column store, zero-copy views or Memory-backed table yet. | | P5 third-party Provider / TCP / HTTP | **PARTIAL** | `stdlib/net/http.mq.md` + semantic fixture 71 implement HTTP/1.1 status/header/chunked parsing and request construction in `.mq.md`; `http_over_tcp.rs` runs a local HTTP exchange over `resource.open("tcp",...)`; no host HTTP/TLS and no external network test. ADR-0034 blocking TCP resource endpoint (`resource.open("tcp", ":", ["read","write"])`, exact IP grant, project-only, 5s timeouts; loopback host+CLI tests). ADR-0033 in-process `ResourceProvider` v1: explicitly trusted providers register by kind, project-mode only, gated by `Capabilities.allow_resource_provider`, using the existing `resource.*`/`Result` ABI; a mock read/write provider is tested. No C/Worker provider transport, host DNS/TLS/HTTP implementation, UDP, clock/random/process/device providers, or provider package installation. | | P6 performance / native backends | **NOT IMPLEMENTED** | no bytecode/JIT/GPU/QPU work | @@ -248,6 +250,8 @@ user-written allocator is explicitly **not** claimed. - [`evidence/2026-11-20-tcp-endpoint.md`](evidence/2026-11-20-tcp-endpoint.md) — ADR-0034 blocking TCP endpoint, IP grants, DNS refusal, loopback host/CLI echo tests. +- [`evidence/2026-11-20-table-cross-storage.md`](evidence/2026-11-20-table-cross-storage.md) + — P4 row-store Table vs `.mq.md` columnar store equivalence for sort/filter. - [`evidence/2026-11-20-http-over-tcp.md`](evidence/2026-11-20-http-over-tcp.md) — P5/C10 `.mq.md` HTTP framing over a loopback TCP stream, stdlib module and end-to-end client test. diff --git a/docs/adr/0032-table-logical-value-and-stdlib-algorithms.md b/docs/adr/0032-table-logical-value-and-stdlib-algorithms.md index e9b8fac..d0cb961 100644 --- a/docs/adr/0032-table-logical-value-and-stdlib-algorithms.md +++ b/docs/adr/0032-table-logical-value-and-stdlib-algorithms.md @@ -65,6 +65,10 @@ through the verified `std:` bundle and its digest. their existing behavior; `表` still validates row width. * Table source spans and document addresses are unchanged; a rebuilt Table is a logical value, not a rewrite of the source document. +* Cross-storage compatibility: a `.mq.md` columnar representation (List/Map of + columns) must produce the same headers/rows/cells as the T0 row-store Table + for the supported operations (fixture 72). This does not migrate the Rust + storage; it proves the logical contract is storage-independent. ### 4. What is deliberately not done diff --git a/docs/evidence/2026-11-20-table-cross-storage.md b/docs/evidence/2026-11-20-table-cross-storage.md new file mode 100644 index 0000000..60de65c --- /dev/null +++ b/docs/evidence/2026-11-20-table-cross-storage.md @@ -0,0 +1,47 @@ +# RFC-CM P4 evidence: cross-storage table equivalence + +Date: 2026-11-20 (Asia/Shanghai) +Repository: `marqdo-cli` +Baseline: `e981168` plus this step's changes. +Decision record: [ADR-0032](../adr/0032-table-logical-value-and-stdlib-algorithms.md) +Related: [CM-A08 table](2026-11-20-table-cm-a08.md), [STATUS §5](../STATUS.md). +Scope: logical Table (row store T0) vs a `.mq.md` columnar store. + +## Fixture + +`tests/conformance/semantics/72-table-columnar.mq.md`: + +* `列存 = {列: [...], 数据: [[column0...], [column1...]]}` is a columnar + representation built from List/Map; +* `转表` converts columnar -> logical `Table`; +* `列存排序` sorts the columnar representation by index order without a Table + value; `列存过滤` filters columns by retained indices; +* the same operations run on the logical Table via `std:data/table`; +* `表相等` compares headers + every cell (Table `==` is not structural in this + language version, so the fixture compares cells explicitly). + +Result: + +```json +[true, true, true] +``` + +i.e. conversion is lossless, and sorted/filtered outputs are identical between +the row-store Table and the columnar `.mq.md` representation. + +## What this proves / does not prove + +Proves: C06 algorithms are defined by logical Table semantics, not by the +current `Vec>` layout; a columnar `.mq.md` implementation can produce +the same observable result without changing the GFM AST or Rust runtime. + +Does not prove or implement: T1 controlled storage interface, T2 native column +store, T3 Memory-backed numeric columns, zero-copy views, or production +performance. Those remain P6/future work; the current Rust T0 storage is +unchanged. + +## Gate + +Local container: fixture passes via `conformance_semantics` and the full +workspace gate; GFM rendering (`转GFM`) is covered by +`crates/cli/tests/stdlib_table.rs`. diff --git a/tests/README.md b/tests/README.md index 786ece3..aa37d0f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -15,7 +15,7 @@ no test commits, copies or rewrites files outside a throw-away temp directory. tests/ ├── conformance/ │ ├── ast/ 160 × *.mq.md + sibling *.ast.json goldens -│ └── semantics/ 71 × *.mq.md + sibling *.expected.json goldens +│ └── semantics/ 72 × *.mq.md + sibling *.expected.json goldens ├── negative/ │ ├── *.mq.md 52 × compile-time hard-error fixtures + *.expected.json │ ├── warnings/ 6 × inert-executable warning fixtures + *.expected.json diff --git a/tests/conformance/semantics/72-table-columnar.expected.json b/tests/conformance/semantics/72-table-columnar.expected.json new file mode 100644 index 0000000..1c030c2 --- /dev/null +++ b/tests/conformance/semantics/72-table-columnar.expected.json @@ -0,0 +1,22 @@ +{ + "args": {}, + "function": "main", + "stdout": "", + "value": { + "items": [ + { + "type": "bool", + "value": true + }, + { + "type": "bool", + "value": true + }, + { + "type": "bool", + "value": true + } + ], + "type": "list" + } +} diff --git a/tests/conformance/semantics/72-table-columnar.mq.md b/tests/conformance/semantics/72-table-columnar.mq.md new file mode 100644 index 0000000..0b81de6 --- /dev/null +++ b/tests/conformance/semantics/72-table-columnar.mq.md @@ -0,0 +1,238 @@ +--- +import: + 数据表: "std:data/table" +--- + +## 转表 + +`(列存)` + +**列 = 列存["列"]** + +**数据 = 列存["数据"]** + +**行数 = 0** + +1. `长度(数据) > 0` + + **行数 = 长度(数据[0])** + +2. `true` + + **忽略 = 0** + +**行列表 = []** + +**行号 = 0** + +- `行号 < 行数` + + **行 = []** + + **列号 = 0** + + - `列号 < 长度(列)` + + **行 = list.append(行, 数据[列号][行号])** + + **列号 = 列号 + 1** + + **行列表 = list.append(行列表, 行)** + + **行号 = 行号 + 1** + +*表(列, 行列表)* + +## 列存排序 + +`(列存, 目标列)` + +**列 = 列存["列"]** + +**数据 = 列存["数据"]** + +**行数 = 长度(数据[目标列])** + +**顺序 = []** + +**行号 = 0** + +- `行号 < 行数` + + **顺序 = list.append(顺序, 行号)** + + **行号 = 行号 + 1** + +**下标 = 1** + +- `下标 < 行数` + + **当前 = 下标** + + **新顺序 = []** + + **已插入 = false** + + **位 = 0** + + - `位 < 长度(顺序)` + + **已有 = 顺序[位]** + + 1. `not 已插入 and 数据[目标列][已有] > 数据[目标列][当前]` + + **新顺序 = list.append(新顺序, 当前)** + + **新顺序 = list.append(新顺序, 已有)** + + **已插入 = true** + + 2. `true` + + **新顺序 = list.append(新顺序, 已有)** + + **位 = 位 + 1** + + 1. `not 已插入` + + **新顺序 = list.append(新顺序, 当前)** + + 2. `true` + + **忽略 = 0** + + **顺序 = 新顺序** + + **下标 = 下标 + 1** + +**新数据 = []** + +**列号 = 0** + +- `列号 < 长度(列)` + + **新列 = []** + + **位 = 0** + + - `位 < 行数` + + **新列 = list.append(新列, 数据[列号][顺序[位]])** + + **位 = 位 + 1** + + **新数据 = list.append(新数据, 新列)** + + **列号 = 列号 + 1** + +*{列: 列, 数据: 新数据}* + +## 列存过滤 + +`(列存, 目标列, 阈值)` + +**列 = 列存["列"]** + +**数据 = 列存["数据"]** + +**行数 = 长度(数据[目标列])** + +**保留 = []** + +**行号 = 0** + +- `行号 < 行数` + + 1. `数据[目标列][行号] > 阈值` + + **保留 = list.append(保留, 行号)** + + 2. `true` + + **忽略 = 0** + + **行号 = 行号 + 1** + +**新数据 = []** + +**列号 = 0** + +- `列号 < 长度(列)` + + **新列 = []** + + **位 = 0** + + - `位 < 长度(保留)` + + **新列 = list.append(新列, 数据[列号][保留[位]])** + + **位 = 位 + 1** + + **新数据 = list.append(新数据, 新列)** + + **列号 = 列号 + 1** + +*{列: 列, 数据: 新数据}* + +## 表相等 + +`(左, 右)` + +**左列 = table.columns(左)** + +**右列 = table.columns(右)** + +**相等 = 长度(左列) == 长度(右列) and 长度(左) == 长度(右)** + +**列号 = 0** + +- `列号 < 长度(左列) and 相等` + + 1. `左列[列号] != 右列[列号]` + + **相等 = false** + + 2. `true` + + **忽略 = 0** + + **列号 = 列号 + 1** + +**行号 = 0** + +- `行号 < 长度(左) and 相等` + + **列号 = 0** + + - `列号 < 长度(左列) and 相等` + + 1. `table.cell(左, 行号, 列号) != table.cell(右, 行号, 列号)` + + **相等 = false** + + 2. `true` + + **忽略 = 0** + + **列号 = 列号 + 1** + + **行号 = 行号 + 1** + +*相等* + +## main + +**列存 = {列: ["名称", "分数"], 数据: [["甲", "乙", "丙"], [87, 92, 71]]}** + +**行表 = 转表(列存)** + +**排序行 = 数据表.排序(行表, 1)** + +**排序列 = 转表(列存排序(列存, 1))** + +**过滤行 = 数据表.过滤数值大于(行表, 1, 80)** + +**过滤列 = 转表(列存过滤(列存, 1, 80))** + +*[表相等(转表(列存), 行表), 表相等(排序行, 排序列), 表相等(过滤行, 过滤列)]*