From b241d1d9789568ee6dbe64bdaf0fa68fae67d564 Mon Sep 17 00:00:00 2001 From: cflmy Date: Fri, 20 Nov 2026 23:17:09 +0800 Subject: [PATCH] feat(stdlib): add deterministic offline stdlib packaging - tools/pack-stdlib.py builds a byte-deterministic marqdo-stdlib-0.1.0.tar.gz (fixed metadata, gzip mtime 0) plus a schema-tagged descriptor with tree digest, archive SHA-256 and file list. - Verified repeated builds are identical and the extracted tree digest matches stdlib.json; archive is for the future external repo, not the CLI loader. - Update ADR-0028, status/changelog/tests README and evidence. Gate: fmt, clippy -D warnings, 430 tests passed. --- CHANGELOG.md | 3 + docs/STATUS.md | 5 +- .../0028-bundled-stdlib-and-std-imports.md | 8 ++ docs/evidence/2026-11-20-stdlib-package.json | 22 +++++ docs/evidence/2026-11-20-stdlib-package.md | 50 +++++++++++ tests/README.md | 3 + tools/pack-stdlib.py | 86 +++++++++++++++++++ 7 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 docs/evidence/2026-11-20-stdlib-package.json create mode 100644 docs/evidence/2026-11-20-stdlib-package.md create mode 100755 tools/pack-stdlib.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c9f677..cd33f64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -127,6 +127,9 @@ currently `0.1.0`; no release tag has been cut. Dates use ISO 8601. `std:` resolution, tamper rejection and frozen-binary hash equality. - P2 closure gate: add a pure no-Foreign/no-Plugin lane for C01–C06/C11/C12 (20 fixtures + stdlib table), source shortcut scan and golden comparison. +- CM-A10 offline packaging: `tools/pack-stdlib.py` emits a deterministic + `marqdo-stdlib-0.1.0.tar.gz` plus tree/archive digests and a file manifest; + the archive is for the future external repository, not the CLI loader. - P6 baseline profile: add `tools/perf/bench.py` and a checked-in 7-workload JSON report; no bytecode/JIT/native backend is introduced. - P4 cross-storage table equivalence: fixture 72 compares row-store Table diff --git a/docs/STATUS.md b/docs/STATUS.md index 41a1a9b..1523e5a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -162,7 +162,10 @@ This is still a **local VM lane**, not a self-hosted GitHub Actions security URL `project.stdlib`; a frozen-binary test asserts the CLI hash is unchanged after running a `std:io` project and that tampering is rejected. `打印/print` is routed to the bundled stdlib (Compat-C, ADR-0030/0031); there is no legacy - host formatter. + host formatter. `tools/pack-stdlib.py` additionally emits a byte-deterministic + offline archive + descriptor for the future external repository + ([evidence](evidence/2026-11-20-stdlib-package.md)); publication/signing is + still external. - Resource streams (RFC-CM CM-A06, ADR-0025): opaque host-issued `Resource` handles with `resource.open/read/write/close`; bytes move only through `Memory`; stdout write is capability/output-budget checked and project-file diff --git a/docs/adr/0028-bundled-stdlib-and-std-imports.md b/docs/adr/0028-bundled-stdlib-and-std-imports.md index b2b0eca..5e4b5b7 100644 --- a/docs/adr/0028-bundled-stdlib-and-std-imports.md +++ b/docs/adr/0028-bundled-stdlib-and-std-imports.md @@ -125,6 +125,14 @@ version and the CLI's compatibility table before release. * Windows/macOS packaging layout; only the source-tree/executable-adjacent layout is exercised locally. +## Offline packaging + +`tools/pack-stdlib.py` produces a deterministic `marqdo-stdlib-.tar.gz` +plus a schema-tagged descriptor (`tree_digest`, `archive_sha256`, file list) for +the future separate repository/release workflow. The archive is not loaded by +the CLI; the in-repo `stdlib/` tree remains the verified development/bundled +source. Publishing/signing the external repository is still an external step. + ## Consequences * Any project can `import: {io: "std:io"}` and use the print prototype diff --git a/docs/evidence/2026-11-20-stdlib-package.json b/docs/evidence/2026-11-20-stdlib-package.json new file mode 100644 index 0000000..610b8da --- /dev/null +++ b/docs/evidence/2026-11-20-stdlib-package.json @@ -0,0 +1,22 @@ +{ + "schema": "marqdo.stdlib-package", + "schema_version": 1, + "name": "marqdo-stdlib", + "version": "0.1.0", + "tree_digest": "sha256:832940f35fe5651eabe4ac672e12e2d8f2bf4e649cc430e455425b4f2a983121", + "archive": "marqdo-stdlib-0.1.0.tar.gz", + "archive_sha256": "sha256:94127ba3353bdef7430b2ef3c1914d99f9d7f48e4d72fa973eac7ca76c881394", + "files": [ + "README.md", + "data/table.mq.md", + "demo.mq.md", + "format.mq.md", + "io.mq.md", + "net/http.mq.md", + "stdlib.json", + "tests/http.mq.md", + "tests/io.mq.md", + "tests/table.mq.md", + "text.mq.md" + ] +} diff --git a/docs/evidence/2026-11-20-stdlib-package.md b/docs/evidence/2026-11-20-stdlib-package.md new file mode 100644 index 0000000..1f2f05f --- /dev/null +++ b/docs/evidence/2026-11-20-stdlib-package.md @@ -0,0 +1,50 @@ +# RFC-CM CM-A10 evidence: deterministic offline stdlib package + +Date: 2026-11-20 (Asia/Shanghai) +Repository: `marqdo-cli` +Baseline: `e985da2` plus this step's changes. +Related: [ADR-0028](../adr/0028-bundled-stdlib-and-std-imports.md), +[STATUS §5](../STATUS.md). +Scope: packaging artifact for the future external repository; no CLI command. + +## Artifact + +`tools/pack-stdlib.py` builds `dist/marqdo-stdlib-0.1.0.tar.gz` from the +verified `stdlib/` tree with: + +* sorted relative names, fixed mode/uid/gid/uname/gname and `mtime=0`; +* gzip `mtime=0` and an empty filename, so repeated builds are byte-identical; +* a JSON descriptor (`marqdo.stdlib-package`) with the tree digest, archive + SHA-256 and file list; a `.sha256` sidecar is also written. + +Descriptor: [`2026-11-20-stdlib-package.json`](2026-11-20-stdlib-package.json). + +```text +version 0.1.0 +tree digest sha256:832940f35fe5651eabe4ac672e12e2d8f2bf4e649cc430e455425b4f2a983121 +archive sha256:94127ba3353bdef7430b2ef3c1914d99f9d7f48e4d72fa973eac7ca76c881394 +files 11 (README, stdlib.json, *.mq.md) +``` + +Two consecutive builds compared byte-identically (`cmp`), and extracting the +archive and recomputing the custom tree digest reproduces the manifest digest. +`dist/` is gitignored, so the script and descriptor are the checked-in +reproducible inputs. + +## Boundary + +This does **not** publish the separate `marqdo-stdlib` repository or sign a +release; no GitHub credentials/remote is available from this host. It gives the +future external repository a deterministic archive layout and digest contract. + +The in-repo CLI still uses the bundled `stdlib/` tree with digest verification +and the frozen-binary closure test (ADR-0028); this archive is not loaded by +the CLI. + +## Reproduce + +```bash +export PATH=/root/.cargo/bin:$PATH +python3 tools/pack-stdlib.py > /tmp/stdlib-package.json +sha256sum dist/marqdo-stdlib-0.1.0.tar.gz +``` diff --git a/tests/README.md b/tests/README.md index c273f13..840fd84 100644 --- a/tests/README.md +++ b/tests/README.md @@ -185,6 +185,9 @@ grant, pure/strict, bad protocol version, reserved kinds and duplicate kinds are rejected. RFC-CM P2 closure (C01–C06, C11, C12) is pinned by +`tools/pack-stdlib.py` is the CM-A10 deterministic package builder (not a test; +its descriptor is checked in under `docs/evidence/`). + `tools/perf/bench.py` is the P6 baseline harness (not a test; it is not part of the cargo gate). diff --git a/tools/pack-stdlib.py b/tools/pack-stdlib.py new file mode 100755 index 0000000..7ed078c --- /dev/null +++ b/tools/pack-stdlib.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Build a deterministic offline marqdo-stdlib package. + +The archive is intended for the separate external stdlib repository/release +workflow, not for the CLI's in-repo development fallback. It contains the +Markdown stdlib tree, stdlib.json and README.md, with stable mtimes/ownership. +""" + +import hashlib +import io +import json +import os +import tarfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STDLIB = ROOT / "stdlib" +DIST = ROOT / "dist" + + +def tree_digest() -> str: + digest = hashlib.sha256() + for path in sorted(STDLIB.rglob("*.mq.md")): + relative = path.relative_to(STDLIB).as_posix() + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def main() -> int: + manifest = json.loads((STDLIB / "stdlib.json").read_text()) + version = manifest["version"] + digest = tree_digest() + if manifest["digest"] != f"sha256:{digest}": + raise SystemExit("stdlib.json digest does not match the tree") + + files = sorted( + [p for p in STDLIB.rglob("*") if p.is_file() and p.name != "stdlib.json"] + + [STDLIB / "stdlib.json"], + key=lambda p: p.relative_to(STDLIB).as_posix(), + ) + DIST.mkdir(exist_ok=True) + archive = DIST / f"marqdo-stdlib-{version}.tar.gz" + with archive.open("wb") as raw: + with gzip_open(raw) as gz: + with tarfile.open(fileobj=gz, mode="w") as tar: + for path in files: + relative = path.relative_to(STDLIB).as_posix() + info = tarfile.TarInfo(name=relative) + data = path.read_bytes() + info.size = len(data) + info.mode = 0o644 + info.uid = info.gid = 0 + info.uname = info.gname = "root" + info.mtime = 0 + tar.addfile(info, io.BytesIO(data)) + archive_sha = hashlib.sha256(archive.read_bytes()).hexdigest() + descriptor = { + "schema": "marqdo.stdlib-package", + "schema_version": 1, + "name": manifest["name"], + "version": version, + "tree_digest": f"sha256:{digest}", + "archive": archive.name, + "archive_sha256": f"sha256:{archive_sha}", + "files": [p.relative_to(STDLIB).as_posix() for p in files], + } + descriptor_path = DIST / f"marqdo-stdlib-{version}.json" + descriptor_path.write_text(json.dumps(descriptor, indent=2) + "\n") + (DIST / f"{archive.name}.sha256").write_text( + f"{archive_sha} {archive.name}\n" + ) + print(json.dumps(descriptor, indent=2)) + return 0 + + +def gzip_open(raw): + import gzip + + return gzip.GzipFile(fileobj=raw, mode="wb", mtime=0, filename="") + + +if __name__ == "__main__": + raise SystemExit(main())