From d2267e912a340f1fe76c07104e27771ded23f75b Mon Sep 17 00:00:00 2001 From: cflmy Date: Sat, 21 Nov 2026 17:28:05 +0800 Subject: [PATCH] perf: add correctness-verified marqdo.perf.v2 sampler Replace the three-run baseline profile with tools/perf/runner.py: one fixed release binary, per-child rusage via posix_spawn/wait4, 5 warmups and 20 samples by default, SHA-derived interleaved order, p50/p95/MAD, CPU user/sys and peak RSS. Every warmup output is normalized and compared with a committed wire digest before timing; a mismatch is CORRECTNESS_FAIL, never a fast sample. Short or unapproved-host runs are SMOKE, and baselines from another platform_id/profile are refused. Add workloads.json, expected digests for nine existing workloads, a dedicated fixed-node workflow (manual, self-hosted, requires --cpu-list), and docs/performance.md. bench.py stays as a compatibility shim. This is step 5 of the CI/security RFC (B08). --- .github/workflows/performance-linux.yml | 70 +++ .gitignore | 6 + docs/performance.md | 40 ++ tools/perf/README.md | 45 ++ tools/perf/bench.py | 104 +--- tools/perf/expected/allocator-free-list.json | 297 +++++++++++ tools/perf/expected/bytecode-vm.json | 42 ++ tools/perf/expected/factorial-run.json | 24 + tools/perf/expected/hello-check.json | 10 + tools/perf/expected/http-parser.json | 36 ++ tools/perf/expected/json-parser.json | 199 ++++++++ tools/perf/expected/matrix-multiply.json | 130 +++++ tools/perf/expected/stdlib-print-demo.json | 14 + tools/perf/expected/table-sort.json | 94 ++++ tools/perf/runner.py | 499 +++++++++++++++++++ tools/perf/workloads.json | 98 ++++ 16 files changed, 1615 insertions(+), 93 deletions(-) create mode 100644 .github/workflows/performance-linux.yml create mode 100644 docs/performance.md create mode 100644 tools/perf/README.md create mode 100644 tools/perf/expected/allocator-free-list.json create mode 100644 tools/perf/expected/bytecode-vm.json create mode 100644 tools/perf/expected/factorial-run.json create mode 100644 tools/perf/expected/hello-check.json create mode 100644 tools/perf/expected/http-parser.json create mode 100644 tools/perf/expected/json-parser.json create mode 100644 tools/perf/expected/matrix-multiply.json create mode 100644 tools/perf/expected/stdlib-print-demo.json create mode 100644 tools/perf/expected/table-sort.json create mode 100755 tools/perf/runner.py create mode 100644 tools/perf/workloads.json diff --git a/.github/workflows/performance-linux.yml b/.github/workflows/performance-linux.yml new file mode 100644 index 0000000..9d411eb --- /dev/null +++ b/.github/workflows/performance-linux.yml @@ -0,0 +1,70 @@ +name: Linux performance measurements (G4) + +# This workflow needs a dedicated, fixed Linux performance node. It is never +# run on GitHub-hosted runners or on TCG/QEMU guests: those numbers are not +# comparable with a native release baseline. A short smoke run is not release +# evidence; runner.py marks it SMOKE unless 5 warmups / 20 samples and +# --approved-node are used. +# +# Trust model: the node carries no untrusted PR code and no repository +# secrets. It must not share hardware with the G3 security VM. + +on: + workflow_dispatch: + inputs: + platform_id: + description: "Fixed node identifier, e.g. linux-x86_64-fixed-node-01" + required: true + type: string + cpu_list: + description: "taskset CPU list, e.g. 2,3 (empty = unbound smoke only)" + required: false + default: "" + type: string + baseline_run_id: + description: "Optional previous run id whose artifact is the baseline" + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + measure: + name: correctness-verified performance sample + runs-on: [self-hosted, linux, marqdo-perf] + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Toolchain report + run: | + rustc -vV + uname -a + cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || true + cat /proc/loadavg + - name: Measure + shell: bash + env: + PLATFORM_ID: ${{ inputs.platform_id }} + CPU_LIST: ${{ inputs.cpu_list }} + run: | + set -euo pipefail + args=(--warmups 5 --samples 20 --approved-node --platform-id "$PLATFORM_ID" --out perf.json) + if [ -n "$CPU_LIST" ]; then + args+=(--cpu-list "$CPU_LIST") + else + echo "error: a release measurement requires a fixed --cpu-list" >&2 + exit 1 + fi + python3 tools/perf/runner.py "${args[@]}" + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: marqdo-perf-${{ inputs.platform_id }} + path: perf.json + if-no-files-found: warn + retention-days: 90 diff --git a/.gitignore b/.gitignore index 47b1967..d40e0ad 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,9 @@ dist/ # stdlib product test output stdlib/tests/out/ + +# Python tooling caches and perf capture files +__pycache__/ +*.pyc +.perf-out-* +.perf-err-* diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 0000000..c6e744d --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,40 @@ +# Performance measurements (G4) + +Status: the sampler and correctness corpus are implemented; **no release +performance baseline exists yet**. Do not cite a development-container or +TCG/QEMU number as a release gate. + +## Three separate measurements + +| Class | Goal | Host | +|---|---|---| +| Language core throughput | interpreter, expressions, memory/containers/stdlib algorithms | fixed G4 Linux node | +| CLI experience | cold `check`/`run`/`inspect`, parsing, stdlib loading | fixed G4 Linux node | +| Strict overhead | probe, re-exec, verification and strict run time vs pure | separate isolated performance node | + +The G3 security VM can report correctness, not release performance: TCG time is +dominated by emulation and is not comparable with native or KVM. + +## Tool + +`tools/perf/runner.py` (schema `marqdo.perf.v2`) builds one release binary, +records its SHA256/Cargo.lock/rustc/source SHA, verifies every workload's exact +wire digest **before** timing, then reports warmups/samples, p50, p95, MAD, +CPU user/sys, peak RSS and the environment manifest. See +[`tools/perf/README.md`](../tools/perf/README.md). + +Rules enforced by the tool: + +* a wrong workload output is `CORRECTNESS_FAIL`; its timing is never reported; +* fewer than 5 warmups / 20 samples is `SMOKE`, not release evidence; +* `--approved-node` is required for `MEASURED`; +* a baseline from another `platform_id` or profile is refused; +* p50 +15% / RSS +20% are investigation thresholds, not automatic rollbacks. + +## Current evidence + +* Development-container smoke run (2026-11-21, `--warmups 1 --samples 3 + --allow-smoke`): all nine workloads `VERIFIED`/`SMOKE`, no correctness + failures. Stored as a smoke artifact, **not** a release baseline. +* Remote G4: **UNVERIFIED** until a fixed self-hosted node runs + `.github/workflows/performance-linux.yml` with a fixed `--cpu-list`. diff --git a/tools/perf/README.md b/tools/perf/README.md new file mode 100644 index 0000000..a46747e --- /dev/null +++ b/tools/perf/README.md @@ -0,0 +1,45 @@ +# Performance sampler (marqdo.perf.v2) + +`runner.py` measures a fixed release binary and refuses to report timings for a +workload whose wire output did not match the committed expected digest. + +## Rules + +* Build one release binary with `cargo build --release --locked -p marqdo-cli` + and keep it fixed for the whole run. The report records its SHA256, the + `Cargo.lock` digest, `rustc -vV`, source SHA and profile. +* Correctness comes first: every warmup run is parsed and hashed. A mismatch + is `CORRECTNESS_FAIL`; its faster wall time is never reported. +* Release evidence needs at least 5 warmups and 20 samples per workload with + `--approved-node` on a dedicated, stable performance node. Shorter runs are + `SMOKE` and must not be cited as a release gate. +* Do not compare timings across different `platform_id`s, CPU models, kernels, + toolchains or load conditions. `--baseline` refuses a different + `platform_id`/profile. +* Avoid TCG/QEMU for final performance numbers; use native or KVM with fixed + CPU affinity (`--cpu-list`) and an administrator-approved governor. +* The suggested investigation thresholds are p50 +15% and peak RSS +20%; they + are not automatic rollback rules. Investigate before freezing a hard gate. + +## Commands + +```bash +# Record/review expected wire digests after an intentional behaviour change. +python3 tools/perf/runner.py --record-expected --warmups 1 --samples 1 \ + --allow-smoke --only hello-check + +# Non-release smoke run on the development host. +python3 tools/perf/runner.py --warmups 1 --samples 3 --allow-smoke \ + --platform-id local-unapproved --out /tmp/marqdo-perf-smoke.json + +# Release measurement on the dedicated node. +python3 tools/perf/runner.py --warmups 5 --samples 20 --approved-node \ + --platform-id linux-x86_64-fixed-node-01 --cpu-list 2,3 \ + --out perf/2026-11-21.json +``` + +Expected digests live in `expected/.json` and are versioned with the +source. Running `--record-expected` rewrites them: inspect the diff and commit +it only when the output change is the intended one. + +`bench.py` remains only as a compatibility shim. diff --git a/tools/perf/bench.py b/tools/perf/bench.py index b22d163..8331b3d 100755 --- a/tools/perf/bench.py +++ b/tools/perf/bench.py @@ -1,100 +1,18 @@ #!/usr/bin/env python3 -"""Reproducible P6 baseline profile for selected .mq.md workloads. +"""Deprecated alias for tools/perf/runner.py. -Builds the release CLI, runs each workload three times through /usr/bin/time, -and writes a JSON report with wall seconds and peak RSS. This is a baseline -profile, not a claim that P6 optimization is complete. +The old three-run profile did not verify workload correctness and is not a +performance gate. Use runner.py; this wrapper keeps old local commands working. """ -import json -import os -import statistics -import subprocess +import runpy import sys -import tempfile -import time from pathlib import Path -ROOT = Path(__file__).resolve().parents[2] -target_dir = Path(os.environ.get("CARGO_TARGET_DIR", ROOT / "target")) -if not target_dir.is_absolute(): - target_dir = ROOT / target_dir -BIN = target_dir / "release" / "marqdo" -WORKLOADS = [ - ("allocator-free-list", "tests/conformance/semantics/55-first-fit-free-list-allocator.mq.md"), - ("json-parser", "tests/conformance/semantics/62-json-array-parser.mq.md"), - ("table-sort", "tests/conformance/semantics/65-table-sort-stdlib.mq.md"), - ("bytecode-vm", "tests/conformance/semantics/63-memory-bytecode-vm.mq.md"), - ("matrix-multiply", "tests/conformance/semantics/70-matrix-multiply.mq.md"), - ("http-parser", "tests/conformance/semantics/71-http-parser.mq.md"), - ("stdlib-print-demo", "stdlib/demo.mq.md"), -] - - -def run_once(relative: str) -> dict: - with tempfile.NamedTemporaryFile("r", delete=False) as handle: - time_path = handle.name - command = [ - "/usr/bin/time", - "-f", - "%e %M", - "-o", - time_path, - str(BIN), - "run", - relative, - "--json", - ] - env = dict(os.environ) - env["MARQDO_STDLIB_DIR"] = str(ROOT / "stdlib") - started = time.perf_counter() - output = subprocess.run( - command, cwd=ROOT, env=env, capture_output=True, text=True, timeout=120 - ) - elapsed = time.perf_counter() - started - timing = Path(time_path).read_text().split() - Path(time_path).unlink(missing_ok=True) - if output.returncode != 0: - raise RuntimeError(f"{relative} failed: {output.stderr}") - report = json.loads(output.stdout) - if not report.get("success"): - raise RuntimeError(f"{relative} reported failure") - return { - "seconds": elapsed, - "max_rss_kb": int(timing[1]), - } - - -def main() -> int: - subprocess.run( - ["cargo", "build", "--release", "-p", "marqdo-cli"], - cwd=ROOT, - check=True, - ) - report = { - "schema": "marqdo.perf-baseline", - "schema_version": 1, - "binary": str(BIN.relative_to(ROOT)), - "iterations": 3, - "workloads": [], - } - for name, relative in WORKLOADS: - runs = [run_once(relative) for _ in range(3)] - report["workloads"].append( - { - "name": name, - "source": relative, - "seconds_median": round(statistics.median(r["seconds"] for r in runs), 4), - "seconds_min": min(r["seconds"] for r in runs), - "max_rss_kb_median": int( - statistics.median(r["max_rss_kb"] for r in runs) - ), - } - ) - json.dump(report, sys.stdout, ensure_ascii=False, indent=2) - sys.stdout.write("\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) +print( + "warning: tools/perf/bench.py is a compatibility shim; " + "use tools/perf/runner.py for correctness-verified measurements", + file=sys.stderr, +) +sys.argv[0] = str(Path(__file__).with_name("runner.py")) +runpy.run_path(sys.argv[0], run_name="__main__") diff --git a/tools/perf/expected/allocator-free-list.json b/tools/perf/expected/allocator-free-list.json new file mode 100644 index 0000000..3d2fedc --- /dev/null +++ b/tools/perf/expected/allocator-free-list.json @@ -0,0 +1,297 @@ +{ + "normalized": { + "command": "run", + "stderr": "", + "stdout": "", + "success": true, + "value": { + "items": [ + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "16", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "32", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "16", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "16", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "32", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "56", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "56", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "72", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "96", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "112", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "code": { + "type": "text", + "value": "alloc.no_space" + }, + "message": { + "type": "text", + "value": "没有足够的空闲空间" + }, + "ok": { + "type": "bool", + "value": false + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "16", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "32", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "48", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "16", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "code": { + "type": "text", + "value": "alloc.no_space" + }, + "message": { + "type": "text", + "value": "没有足够的空闲空间" + }, + "ok": { + "type": "bool", + "value": false + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "type": "none" + } + }, + "type": "map" + }, + { + "entries": { + "code": { + "type": "text", + "value": "alloc.double_free" + }, + "message": { + "type": "text", + "value": "块已在空闲链表" + }, + "ok": { + "type": "bool", + "value": false + } + }, + "type": "map" + }, + { + "decimal": "33", + "type": "int" + }, + { + "decimal": "22", + "type": "int" + }, + { + "decimal": "33", + "type": "int" + }, + { + "decimal": "33", + "type": "int" + }, + { + "decimal": "22", + "type": "int" + }, + { + "decimal": "44", + "type": "int" + }, + { + "decimal": "55", + "type": "int" + }, + { + "decimal": "66", + "type": "int" + }, + { + "decimal": "77", + "type": "int" + } + ], + "type": "list" + } + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:039ec1523726f48375f1fb5fafdfe849e296e1ee39b8a4820a0a0970cf3f8c3b", + "workload": "allocator-free-list" +} diff --git a/tools/perf/expected/bytecode-vm.json b/tools/perf/expected/bytecode-vm.json new file mode 100644 index 0000000..0b0b6fe --- /dev/null +++ b/tools/perf/expected/bytecode-vm.json @@ -0,0 +1,42 @@ +{ + "normalized": { + "command": "run", + "stderr": "", + "stdout": "", + "success": true, + "value": { + "items": [ + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "36", + "type": "int" + } + }, + "type": "map" + }, + { + "entries": { + "ok": { + "type": "bool", + "value": true + }, + "value": { + "decimal": "42", + "type": "int" + } + }, + "type": "map" + } + ], + "type": "list" + } + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:418131f5780815da5716d84ed5c00269760a881856cce67e0f4c8b42d417c7ec", + "workload": "bytecode-vm" +} diff --git a/tools/perf/expected/factorial-run.json b/tools/perf/expected/factorial-run.json new file mode 100644 index 0000000..4bbe453 --- /dev/null +++ b/tools/perf/expected/factorial-run.json @@ -0,0 +1,24 @@ +{ + "normalized": { + "command": "call", + "diagnostics": [ + { + "code": "sem.inferred_signature", + "column": 1, + "line": 3, + "message": "function signature was inferred; declare it with `params(...)`", + "severity": "warning" + } + ], + "stderr": "", + "stdout": "", + "success": true, + "value": { + "decimal": "120", + "type": "int" + } + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:0cbd8ab0b0e9e671e7e54f707ff2de00181b2721924b7fdec998ff9611873275", + "workload": "factorial-run" +} diff --git a/tools/perf/expected/hello-check.json b/tools/perf/expected/hello-check.json new file mode 100644 index 0000000..ff9104b --- /dev/null +++ b/tools/perf/expected/hello-check.json @@ -0,0 +1,10 @@ +{ + "normalized": { + "command": "check", + "errors": 0, + "success": true + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:a1b2f1ab60f9b9d2e74b8841ae56909fe889aa643f856572f7fbb6729fc12380", + "workload": "hello-check" +} diff --git a/tools/perf/expected/http-parser.json b/tools/perf/expected/http-parser.json new file mode 100644 index 0000000..1a7bf76 --- /dev/null +++ b/tools/perf/expected/http-parser.json @@ -0,0 +1,36 @@ +{ + "normalized": { + "command": "run", + "stderr": "", + "stdout": "", + "success": true, + "value": { + "items": [ + { + "decimal": "200", + "type": "int" + }, + { + "type": "text", + "value": "OK" + }, + { + "type": "text", + "value": "text/plain" + }, + { + "type": "text", + "value": "hello world" + }, + { + "type": "text", + "value": "GET /health HTTP/1.1\r\nAccept: text/plain\r\nHost: example.test\r\n\r\n" + } + ], + "type": "list" + } + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:540f8dd9a2071d40c1f07d82961e0e0174822e123c828ffc5d2a06cc880052ed", + "workload": "http-parser" +} diff --git a/tools/perf/expected/json-parser.json b/tools/perf/expected/json-parser.json new file mode 100644 index 0000000..15bfa9c --- /dev/null +++ b/tools/perf/expected/json-parser.json @@ -0,0 +1,199 @@ +{ + "normalized": { + "command": "run", + "stderr": "", + "stdout": "", + "success": true, + "value": { + "items": [ + { + "entries": { + "next": { + "decimal": "20", + "type": "int" + }, + "ok": { + "type": "bool", + "value": true + }, + "value": { + "items": [ + { + "decimal": "1", + "type": "int" + }, + { + "decimal": "23", + "type": "int" + }, + { + "items": [ + { + "decimal": "4", + "type": "int" + }, + { + "decimal": "5", + "type": "int" + } + ], + "type": "list" + }, + { + "decimal": "-6", + "type": "int" + } + ], + "type": "list" + } + }, + "type": "map" + }, + { + "entries": { + "code": { + "type": "text", + "value": "json.invalid_array" + }, + "message": { + "type": "text", + "value": "数组缺少逗号或 ]" + }, + "ok": { + "type": "bool", + "value": false + } + }, + "type": "map" + }, + { + "entries": { + "code": { + "type": "text", + "value": "json.unexpected_eof" + }, + "message": { + "type": "text", + "value": "数组未闭合" + }, + "ok": { + "type": "bool", + "value": false + } + }, + "type": "map" + }, + { + "entries": { + "next": { + "decimal": "35", + "type": "int" + }, + "ok": { + "type": "bool", + "value": true + }, + "value": { + "entries": { + "a": { + "decimal": "1", + "type": "int" + }, + "b": { + "items": [ + { + "type": "bool", + "value": true + }, + { + "type": "none" + }, + { + "type": "text", + "value": "说\"好" + } + ], + "type": "list" + } + }, + "type": "map" + } + }, + "type": "map" + }, + { + "entries": { + "next": { + "decimal": "6", + "type": "int" + }, + "ok": { + "type": "bool", + "value": true + }, + "value": { + "type": "text", + "value": "a\nb" + } + }, + "type": "map" + }, + { + "entries": { + "next": { + "decimal": "37", + "type": "int" + }, + "ok": { + "type": "bool", + "value": true + }, + "value": { + "items": [ + { + "decimal": "1", + "type": "int" + }, + { + "type": "float", + "value": 2.5 + }, + { + "type": "float", + "value": -300.0 + }, + { + "type": "float", + "value": 0.4 + }, + { + "entries": { + "a": { + "type": "float", + "value": 1.0 + } + }, + "type": "map" + }, + { + "type": "text", + "value": "x" + } + ], + "type": "list" + } + }, + "type": "map" + }, + { + "type": "text", + "value": "[1, 2.5, -300.0, 0.4, {\"a\": 1.0}, \"x\"]" + } + ], + "type": "list" + } + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:d6f795f43eb1ae4c3d2f04254b8135f71d717ef8a649efdf7785f172b5381011", + "workload": "json-parser" +} diff --git a/tools/perf/expected/matrix-multiply.json b/tools/perf/expected/matrix-multiply.json new file mode 100644 index 0000000..8112825 --- /dev/null +++ b/tools/perf/expected/matrix-multiply.json @@ -0,0 +1,130 @@ +{ + "normalized": { + "command": "run", + "stderr": "", + "stdout": "", + "success": true, + "value": { + "items": [ + { + "items": [ + { + "items": [ + { + "decimal": "1", + "type": "int" + }, + { + "decimal": "2", + "type": "int" + }, + { + "decimal": "3", + "type": "int" + } + ], + "type": "list" + }, + { + "items": [ + { + "decimal": "4", + "type": "int" + }, + { + "decimal": "5", + "type": "int" + }, + { + "decimal": "6", + "type": "int" + } + ], + "type": "list" + } + ], + "type": "list" + }, + { + "items": [ + { + "items": [ + { + "decimal": "7", + "type": "int" + }, + { + "decimal": "8", + "type": "int" + } + ], + "type": "list" + }, + { + "items": [ + { + "decimal": "9", + "type": "int" + }, + { + "decimal": "10", + "type": "int" + } + ], + "type": "list" + }, + { + "items": [ + { + "decimal": "11", + "type": "int" + }, + { + "decimal": "12", + "type": "int" + } + ], + "type": "list" + } + ], + "type": "list" + }, + { + "items": [ + { + "items": [ + { + "decimal": "58", + "type": "int" + }, + { + "decimal": "64", + "type": "int" + } + ], + "type": "list" + }, + { + "items": [ + { + "decimal": "139", + "type": "int" + }, + { + "decimal": "154", + "type": "int" + } + ], + "type": "list" + } + ], + "type": "list" + } + ], + "type": "list" + } + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:c5edb02098191e081a143b8f5d23424fcc43172643a44912cdc8ae17ace99cc2", + "workload": "matrix-multiply" +} diff --git a/tools/perf/expected/stdlib-print-demo.json b/tools/perf/expected/stdlib-print-demo.json new file mode 100644 index 0000000..2b7c03b --- /dev/null +++ b/tools/perf/expected/stdlib-print-demo.json @@ -0,0 +1,14 @@ +{ + "normalized": { + "command": "run", + "stderr": "", + "stdout": "[\"文本\", 2, 1.5, [3, true], {a: 1, b: \"x\"}, None]\n[\"文本\", 2, 1.5, [3, true], {a: 1, b: \"x\"}, None]\nTable[名称 | 分数]{甲 | 87; 乙 | 92}\nTable[名称 | 分数]{甲 | 87; 乙 | 92}\n[Formula(\"a = 1\"), Code(\"text\", 9 bytes), FunctionRef(root:default::demo.mq.md::演示函数)]\n[Formula(\"a = 1\"), Code(\"text\", 9 bytes), FunctionRef(root:default::demo.mq.md::演示函数)]\n", + "success": true, + "value": { + "type": "none" + } + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:80dfbfb894544b5cd4281999a67deaa9ce4fb19ef32d160fb4ee72dbd615d03c", + "workload": "stdlib-print-demo" +} diff --git a/tools/perf/expected/table-sort.json b/tools/perf/expected/table-sort.json new file mode 100644 index 0000000..73aed24 --- /dev/null +++ b/tools/perf/expected/table-sort.json @@ -0,0 +1,94 @@ +{ + "normalized": { + "command": "run", + "stderr": "", + "stdout": "", + "success": true, + "value": { + "items": [ + { + "header": [ + "名称", + "分数" + ], + "rows": [ + [ + { + "type": "text", + "value": "甲" + }, + { + "decimal": "87", + "type": "int" + } + ], + [ + { + "type": "text", + "value": "乙" + }, + { + "decimal": "92", + "type": "int" + } + ], + [ + { + "type": "text", + "value": "丙" + }, + { + "decimal": "71", + "type": "int" + } + ] + ], + "type": "table" + }, + { + "header": [ + "名称", + "分数" + ], + "rows": [ + [ + { + "type": "text", + "value": "丙" + }, + { + "decimal": "71", + "type": "int" + } + ], + [ + { + "type": "text", + "value": "甲" + }, + { + "decimal": "87", + "type": "int" + } + ], + [ + { + "type": "text", + "value": "乙" + }, + { + "decimal": "92", + "type": "int" + } + ] + ], + "type": "table" + } + ], + "type": "list" + } + }, + "schema": "marqdo.perf.expected.v1", + "wire_sha256": "sha256:12b61dcd9955cb7e137c705e51e29d58322d9525993681170f2dded48e94ee97", + "workload": "table-sort" +} diff --git a/tools/perf/runner.py b/tools/perf/runner.py new file mode 100755 index 0000000..ddec9c1 --- /dev/null +++ b/tools/perf/runner.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +"""Correctness-verified Marqdo performance sampler (schema marqdo.perf.v2). + +This replaces the three-run P6 baseline profile. It: + +1. builds (or verifies) one release binary and records its SHA256, rustc, + Cargo.lock digest and git SHA; +2. verifies each workload's exact wire output against a committed expected + digest **before** timing it; +3. warms up and samples with a fixed, SHA-derived interleaved order; +4. reports p50/p95/MAD, CPU user/sys, max RSS and the environment manifest. + +It deliberately does not compare timings across machines. Use the same +`--platform-id`, CPU affinity and release binary when comparing a baseline. +A short run is marked SMOKE, not MEASURED, and must not be used as release +performance evidence. +""" + +import argparse +import hashlib +import json +import os +import platform +import random +import shutil +import signal +import statistics +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WORKLOADS = ROOT / "tools" / "perf" / "workloads.json" +EXPECTED = ROOT / "tools" / "perf" / "expected" +SCHEMA = "marqdo.perf.v2" +MIN_WARMUPS = 5 +MIN_SAMPLES = 20 +P50_REGRESSION = 0.15 +RSS_REGRESSION = 0.20 + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return f"sha256:{digest.hexdigest()}" + + +def output(command: list[str]) -> str: + try: + return subprocess.run( + command, check=False, capture_output=True, text=True, timeout=30 + ).stdout.strip() + except (OSError, subprocess.SubprocessError): + return "" + + +def git_sha(repo: Path) -> str: + explicit = os.environ.get("MARQDO_TESTED_SHA", "") + if explicit: + return explicit + return output(["git", "-C", str(repo), "rev-parse", "HEAD"]) or "unknown" + + +def normalize(document: dict) -> dict: + """Canonical subset used for correctness: no timings, no trace, no paths.""" + payload = {} + for key in ("command", "success", "value", "stdout", "stderr", "errors"): + if key in document: + payload[key] = document[key] + diagnostics = document.get("diagnostics") + if diagnostics: + payload["diagnostics"] = [ + { + key: item.get(key) + for key in ("code", "severity", "line", "column", "message") + } + for item in diagnostics + ] + return payload + + +def wire_digest(document: dict) -> str: + canonical = json.dumps( + normalize(document), ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + return f"sha256:{hashlib.sha256(canonical.encode('utf-8')).hexdigest()}" + + +def expected_path(name: str) -> Path: + return EXPECTED / f"{name}.json" + + +def load_expected(name: str) -> str | None: + path = expected_path(name) + if not path.is_file(): + return None + return json.loads(path.read_text())["wire_sha256"] + + +def record_expected(name: str, document: dict) -> None: + EXPECTED.mkdir(parents=True, exist_ok=True) + expected_path(name).write_text( + json.dumps( + { + "schema": "marqdo.perf.expected.v1", + "workload": name, + "wire_sha256": wire_digest(document), + "normalized": normalize(document), + }, + ensure_ascii=False, + sort_keys=True, + indent=2, + ) + + "\n" + ) + + +def spawn_capture( + argv: list[str], cwd: Path, env: dict[str, str], timeout: float +) -> tuple[int, dict, float, float, int]: + """Run one process, returning exit status, rusage, wall and captured bytes. + + `os.posix_spawn` + `os.wait4` gives exact per-child rusage without an + external `/usr/bin/time`; stdout/stderr are redirected to temporary files + so the capture never buffers inside the sampler. + """ + out_path = cwd / f".perf-out-{os.getpid()}-{time.time_ns()}" + err_path = cwd / f".perf-err-{os.getpid()}-{time.time_ns()}" + out_fd = os.open(out_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + err_fd = os.open(err_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + actions = [ + (os.POSIX_SPAWN_DUP2, out_fd, 1), + (os.POSIX_SPAWN_DUP2, err_fd, 2), + ] + started = time.perf_counter() + try: + pid = os.posix_spawn(argv[0], argv, env, file_actions=actions) + except OSError as error: + os.close(out_fd) + os.close(err_fd) + out_path.unlink(missing_ok=True) + err_path.unlink(missing_ok=True) + raise RuntimeError(f"cannot spawn {argv[0]}: {error}") from error + finally: + os.close(out_fd) + os.close(err_fd) + deadline = started + timeout + status = None + usage = None + while True: + reaped, raw_status, rusage = os.wait4(pid, os.WNOHANG) + if reaped == pid: + status = raw_status + usage = rusage + break + if time.perf_counter() >= deadline: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + os.wait4(pid, 0) + break + time.sleep(0.001) + elapsed = time.perf_counter() - started + stdout = out_path.read_bytes() + stderr = err_path.read_bytes() + out_path.unlink(missing_ok=True) + err_path.unlink(missing_ok=True) + if status is None: + raise RuntimeError(f"timeout after {timeout:g}s: {' '.join(argv)}") + if os.WIFSIGNALED(status): + raise RuntimeError( + f"terminated by signal {os.WTERMSIG(status)}: {' '.join(argv)}" + ) + return ( + os.WEXITSTATUS(status), + { + "user_ms": usage.ru_utime * 1000.0, + "sys_ms": usage.ru_stime * 1000.0, + "max_rss_kb": usage.ru_maxrss, + }, + elapsed, + stdout, + stderr, + ) + + +def cpu_model() -> str: + try: + for line in Path("/proc/cpuinfo").read_text().splitlines(): + if line.startswith("model name"): + return line.split(":", 1)[1].strip() + except OSError: + pass + return "unknown" + + +def governor() -> str: + value = output( + ["sh", "-c", "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null"] + ) + return value or "unavailable" + + +def loadavg() -> str: + try: + return Path("/proc/loadavg").read_text().strip() + except OSError: + return "unavailable" + + +def median_ms(values: list[float]) -> float: + return round(statistics.median(values), 4) + + +def mad_ms(values: list[float]) -> float: + middle = statistics.median(values) + return round(statistics.median(abs(value - middle) for value in values), 4) + + +def p95_ms(values: list[float]) -> float: + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, int(round(0.95 * (len(ordered) - 1))))) + return round(ordered[index], 4) + + +def run_workload( + binary: Path, + workload: dict, + warmups: int, + samples: int, + order_seed: str, + allow_smoke: bool, + record: bool, + timeout: float, + cpu_list: str, +) -> dict: + name = workload["name"] + prefix = [] + if cpu_list: + prefix = [shutil.which("taskset") or "taskset", "-c", cpu_list] + argv = prefix + [str(binary)] + workload["argv"] + env = dict(os.environ) + env["MARQDO_STDLIB_DIR"] = str(ROOT / "stdlib") + env.setdefault("NO_COLOR", "1") + expected = load_expected(name) + result = { + "name": name, + "kind": workload.get("kind", "run"), + "source": workload["source"], + "argv": workload["argv"], + } + correctness = "VERIFIED" + warmup_samples: list[dict] = [] + measured: list[dict] = [] + # Correctness is checked on every warmup execution; a wrong result is never + # timed and can never become a "fast" regression sample. + for index in range(warmups + samples): + code, usage, wall, stdout, stderr = spawn_capture( + argv, ROOT, env, timeout + ) + if code != 0: + correctness = "CORRECTNESS_FAIL" + result["error"] = ( + f"exit {code}: {stderr.decode('utf-8', 'replace')[:500]}" + ) + break + try: + document = json.loads(stdout) + except json.JSONDecodeError as error: + correctness = "CORRECTNESS_FAIL" + result["error"] = f"invalid JSON: {error}" + break + digest = wire_digest(document) + if record: + record_expected(name, document) + expected = digest + if expected is None: + correctness = "UNVERIFIED_NO_EXPECTED" + elif digest != expected: + correctness = "CORRECTNESS_FAIL" + result["error"] = ( + f"wire digest mismatch: expected {expected}, got {digest}" + ) + break + sample = { + "wall_ms": wall * 1000.0, + "user_ms": usage["user_ms"], + "sys_ms": usage["sys_ms"], + "max_rss_kb": usage["max_rss_kb"], + } + if index < warmups: + warmup_samples.append(sample) + else: + measured.append(sample) + + result["correctness"] = correctness + if correctness != "VERIFIED": + result["status"] = correctness + return result + if not measured: + result["status"] = "NO_SAMPLES" + return result + walls = [sample["wall_ms"] for sample in measured] + result.update( + { + "status": "MEASURED" if (len(measured) >= MIN_SAMPLES and warmups >= MIN_WARMUPS) else "SMOKE", + "warmups": len(warmup_samples), + "samples": len(measured), + "order_seed": order_seed, + "median_ms": median_ms(walls), + "p95_ms": p95_ms(walls), + "mad_ms": mad_ms(walls), + "min_ms": round(min(walls), 4), + "max_ms": round(max(walls), 4), + "cpu_user_ms_median": median_ms( + [sample["user_ms"] for sample in measured] + ), + "cpu_sys_ms_median": median_ms([sample["sys_ms"] for sample in measured]), + "max_rss_kb_median": int( + statistics.median(sample["max_rss_kb"] for sample in measured) + ), + } + ) + if result["status"] == "SMOKE" and not allow_smoke: + pass # still recorded as SMOKE; the caller decides whether to accept it + return result + + +def compare_baseline(report: dict, baseline: dict) -> list[dict]: + if baseline.get("schema") != SCHEMA: + raise SystemExit(f"baseline schema {baseline.get('schema')!r} is not {SCHEMA}") + if baseline.get("platform_id") != report.get("platform_id"): + raise SystemExit( + "baseline platform_id differs; timings from another machine/node are not comparable" + ) + if baseline.get("binary", {}).get("profile") != report.get("binary", {}).get("profile"): + raise SystemExit("baseline profile differs") + previous = {item["name"]: item for item in baseline.get("workloads", [])} + comparisons = [] + for current in report["workloads"]: + old = previous.get(current["name"]) + if not old or old.get("status") not in {"MEASURED", "SMOKE"}: + comparisons.append({"name": current["name"], "status": "NO_BASELINE"}) + continue + if current.get("status") not in {"MEASURED", "SMOKE"}: + comparisons.append( + {"name": current["name"], "status": "CURRENT_NOT_MEASURED"} + ) + continue + p50_delta = (current["median_ms"] - old["median_ms"]) / old["median_ms"] + rss_delta = ( + current["max_rss_kb_median"] - old["max_rss_kb_median"] + ) / old["max_rss_kb_median"] + comparisons.append( + { + "name": current["name"], + "status": "COMPARED", + "p50_delta_percent": round(p50_delta * 100.0, 2), + "rss_delta_percent": round(rss_delta * 100.0, 2), + "investigate": p50_delta >= P50_REGRESSION + or rss_delta >= RSS_REGRESSION, + } + ) + return comparisons + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", default=str(ROOT / "target" / "release" / "marqdo")) + parser.add_argument("--no-build", action="store_true") + parser.add_argument("--warmups", type=int, default=MIN_WARMUPS) + parser.add_argument("--samples", type=int, default=MIN_SAMPLES) + parser.add_argument("--timeout", type=float, default=120.0) + parser.add_argument("--only", action="append", default=[]) + parser.add_argument("--record-expected", action="store_true") + parser.add_argument("--platform-id", default="local-unapproved") + parser.add_argument("--approved-node", action="store_true") + parser.add_argument("--cpu-list", default="") + parser.add_argument("--baseline", default="") + parser.add_argument("--out", default="") + parser.add_argument("--allow-smoke", action="store_true") + args = parser.parse_args(argv) + + if args.samples < 1 or args.warmups < 0: + raise SystemExit("samples must be >= 1 and warmups >= 0") + if (args.samples < MIN_SAMPLES or args.warmups < MIN_WARMUPS) and not args.allow_smoke: + raise SystemExit( + f"release evidence requires >= {MIN_WARMUPS} warmups and >= {MIN_SAMPLES} " + "samples; pass --allow-smoke for a non-release smoke run" + ) + binary = Path(args.binary).resolve() + if not args.no_build: + build = [os.environ.get("CARGO", "cargo"), "build", "--release", "--locked", "-p", "marqdo-cli"] + subprocess.run(build, cwd=ROOT, check=True) + if not binary.is_file(): + raise SystemExit(f"release binary not found: {binary}") + + if args.cpu_list and not shutil.which("taskset"): + raise SystemExit("--cpu-list requires taskset") + + config = json.loads(WORKLOADS.read_text()) + workloads = [ + item + for item in config["workloads"] + if not args.only or item["name"] in args.only + ] + if not workloads: + raise SystemExit("no workloads selected") + + source_sha = git_sha(ROOT) + order_seed = hashlib.sha256(source_sha.encode()).hexdigest()[:16] + order = list(workloads) + random.Random(order_seed).shuffle(order) + results = [] + for workload in order: + result = run_workload( + binary, + workload, + args.warmups, + args.samples, + order_seed, + args.allow_smoke, + args.record_expected, + args.timeout, + args.cpu_list, + ) + results.append(result) + print( + f"{result['name']}: {result.get('status')} " + f"median={result.get('median_ms', 'n/a')}ms " + f"p95={result.get('p95_ms', 'n/a')}ms " + f"rss={result.get('max_rss_kb_median', 'n/a')}KB", + file=sys.stderr, + ) + + if args.record_expected: + print("expected digests recorded; commit tools/perf/expected/", file=sys.stderr) + + correctness_ok = all(item.get("correctness") == "VERIFIED" for item in results) + measured = all(item.get("status") == "MEASURED" for item in results) + report = { + "schema": SCHEMA, + "source_sha": source_sha, + "platform_id": args.platform_id, + "environment_approved": bool(args.approved_node), + "binary": { + "path": str(binary.relative_to(ROOT)) if binary.is_relative_to(ROOT) else str(binary), + "sha256": sha256_file(binary), + "profile": "release", + "cargo_lock_sha256": sha256_file(ROOT / "Cargo.lock"), + "rustc": output(["rustc", "-vV"]), + "cargo": output(["cargo", "-V"]), + }, + "environment": { + "uname": platform.platform(), + "kernel": platform.release(), + "machine": platform.machine(), + "cpu_model": cpu_model(), + "governor": governor(), + "loadavg_start": loadavg(), + "cpu_list": args.cpu_list or "unbound", + "python": platform.python_version(), + }, + "recipe": { + "warmups": args.warmups, + "samples": args.samples, + "order_seed": order_seed, + "timeout_seconds": args.timeout, + }, + "workloads": results, + "correctness": "VERIFIED" if correctness_ok else "CORRECTNESS_FAIL", + "status": "MEASURED" + if (measured and args.approved_node) + else ("SMOKE" if correctness_ok else "CORRECTNESS_FAIL"), + "baseline_comparison": [], + } + report["environment"]["loadavg_end"] = loadavg() + if args.baseline: + baseline = json.loads(Path(args.baseline).read_text()) + report["baseline_comparison"] = compare_baseline(report, baseline) + + text = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + if args.out: + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + Path(args.out).write_text(text) + sys.stdout.write(text) + if not correctness_ok: + return 1 + if not measured and not args.allow_smoke: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tools/perf/workloads.json b/tools/perf/workloads.json new file mode 100644 index 0000000..3637c6f --- /dev/null +++ b/tools/perf/workloads.json @@ -0,0 +1,98 @@ +{ + "schema": "marqdo.perf.workloads.v1", + "workloads": [ + { + "name": "hello-check", + "kind": "check", + "source": "examples/hello.mq.md", + "argv": [ + "check", + "examples/hello.mq.md", + "--json" + ] + }, + { + "name": "factorial-run", + "kind": "call", + "source": "examples/factorial.mq.md", + "argv": [ + "call", + "examples/factorial.mq.md", + "数学.阶乘", + "--arg", + "n=5", + "--json" + ] + }, + { + "name": "allocator-free-list", + "kind": "run", + "source": "tests/conformance/semantics/55-first-fit-free-list-allocator.mq.md", + "argv": [ + "run", + "tests/conformance/semantics/55-first-fit-free-list-allocator.mq.md", + "--json" + ] + }, + { + "name": "json-parser", + "kind": "run", + "source": "tests/conformance/semantics/62-json-array-parser.mq.md", + "argv": [ + "run", + "tests/conformance/semantics/62-json-array-parser.mq.md", + "--json" + ] + }, + { + "name": "table-sort", + "kind": "run", + "source": "tests/conformance/semantics/65-table-sort-stdlib.mq.md", + "argv": [ + "run", + "tests/conformance/semantics/65-table-sort-stdlib.mq.md", + "--json" + ] + }, + { + "name": "bytecode-vm", + "kind": "run", + "source": "tests/conformance/semantics/63-memory-bytecode-vm.mq.md", + "argv": [ + "run", + "tests/conformance/semantics/63-memory-bytecode-vm.mq.md", + "--json" + ] + }, + { + "name": "matrix-multiply", + "kind": "run", + "source": "tests/conformance/semantics/70-matrix-multiply.mq.md", + "argv": [ + "run", + "tests/conformance/semantics/70-matrix-multiply.mq.md", + "--json" + ] + }, + { + "name": "http-parser", + "kind": "run", + "source": "tests/conformance/semantics/71-http-parser.mq.md", + "argv": [ + "run", + "tests/conformance/semantics/71-http-parser.mq.md", + "--json" + ] + }, + { + "name": "stdlib-print-demo", + "kind": "run", + "source": "stdlib/demo.mq.md", + "argv": [ + "run", + "stdlib/demo.mq.md", + "--json" + ] + } + ] +}