Files
marqdo-cli/tools/perf/runner.py
T
cflmy d2267e912a
CI / fmt + clippy + test (Linux x86_64) (push) Waiting to run
CI / schema instance validation + consumers (push) Waiting to run
CI / runtime contract (macos-latest) (push) Waiting to run
CI / runtime contract (windows-latest) (push) Waiting to run
Linux strict security lane (G-C) / strict isolation escape matrix (push) Waiting to run
ci/woodpecker/push/woodpecker Pipeline was canceled
perf: add correctness-verified marqdo.perf.v2 sampler
Replace the three-run baseline profile with tools/perf/runner.py: one fixed
release binary, per-child rusage via posix_spawn/wait4, 5 warmups and 20
samples by default, SHA-derived interleaved order, p50/p95/MAD, CPU user/sys
and peak RSS. Every warmup output is normalized and compared with a committed
wire digest before timing; a mismatch is CORRECTNESS_FAIL, never a fast
sample. Short or unapproved-host runs are SMOKE, and baselines from another
platform_id/profile are refused.

Add workloads.json, expected digests for nine existing workloads, a dedicated
fixed-node workflow (manual, self-hosted, requires --cpu-list), and
docs/performance.md. bench.py stays as a compatibility shim. This is step 5 of
the CI/security RFC (B08).
2026-11-21 17:28:05 +08:00

500 lines
17 KiB
Python
Executable File

#!/usr/bin/env python3
"""Correctness-verified Marqdo performance sampler (schema marqdo.perf.v2).
This replaces the three-run P6 baseline profile. It:
1. builds (or verifies) one release binary and records its SHA256, rustc,
Cargo.lock digest and git SHA;
2. verifies each workload's exact wire output against a committed expected
digest **before** timing it;
3. warms up and samples with a fixed, SHA-derived interleaved order;
4. reports p50/p95/MAD, CPU user/sys, max RSS and the environment manifest.
It deliberately does not compare timings across machines. Use the same
`--platform-id`, CPU affinity and release binary when comparing a baseline.
A short run is marked SMOKE, not MEASURED, and must not be used as release
performance evidence.
"""
import argparse
import hashlib
import json
import os
import platform
import random
import shutil
import signal
import statistics
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
WORKLOADS = ROOT / "tools" / "perf" / "workloads.json"
EXPECTED = ROOT / "tools" / "perf" / "expected"
SCHEMA = "marqdo.perf.v2"
MIN_WARMUPS = 5
MIN_SAMPLES = 20
P50_REGRESSION = 0.15
RSS_REGRESSION = 0.20
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return f"sha256:{digest.hexdigest()}"
def output(command: list[str]) -> str:
try:
return subprocess.run(
command, check=False, capture_output=True, text=True, timeout=30
).stdout.strip()
except (OSError, subprocess.SubprocessError):
return ""
def git_sha(repo: Path) -> str:
explicit = os.environ.get("MARQDO_TESTED_SHA", "")
if explicit:
return explicit
return output(["git", "-C", str(repo), "rev-parse", "HEAD"]) or "unknown"
def normalize(document: dict) -> dict:
"""Canonical subset used for correctness: no timings, no trace, no paths."""
payload = {}
for key in ("command", "success", "value", "stdout", "stderr", "errors"):
if key in document:
payload[key] = document[key]
diagnostics = document.get("diagnostics")
if diagnostics:
payload["diagnostics"] = [
{
key: item.get(key)
for key in ("code", "severity", "line", "column", "message")
}
for item in diagnostics
]
return payload
def wire_digest(document: dict) -> str:
canonical = json.dumps(
normalize(document), ensure_ascii=False, sort_keys=True, separators=(",", ":")
)
return f"sha256:{hashlib.sha256(canonical.encode('utf-8')).hexdigest()}"
def expected_path(name: str) -> Path:
return EXPECTED / f"{name}.json"
def load_expected(name: str) -> str | None:
path = expected_path(name)
if not path.is_file():
return None
return json.loads(path.read_text())["wire_sha256"]
def record_expected(name: str, document: dict) -> None:
EXPECTED.mkdir(parents=True, exist_ok=True)
expected_path(name).write_text(
json.dumps(
{
"schema": "marqdo.perf.expected.v1",
"workload": name,
"wire_sha256": wire_digest(document),
"normalized": normalize(document),
},
ensure_ascii=False,
sort_keys=True,
indent=2,
)
+ "\n"
)
def spawn_capture(
argv: list[str], cwd: Path, env: dict[str, str], timeout: float
) -> tuple[int, dict, float, float, int]:
"""Run one process, returning exit status, rusage, wall and captured bytes.
`os.posix_spawn` + `os.wait4` gives exact per-child rusage without an
external `/usr/bin/time`; stdout/stderr are redirected to temporary files
so the capture never buffers inside the sampler.
"""
out_path = cwd / f".perf-out-{os.getpid()}-{time.time_ns()}"
err_path = cwd / f".perf-err-{os.getpid()}-{time.time_ns()}"
out_fd = os.open(out_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
err_fd = os.open(err_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
actions = [
(os.POSIX_SPAWN_DUP2, out_fd, 1),
(os.POSIX_SPAWN_DUP2, err_fd, 2),
]
started = time.perf_counter()
try:
pid = os.posix_spawn(argv[0], argv, env, file_actions=actions)
except OSError as error:
os.close(out_fd)
os.close(err_fd)
out_path.unlink(missing_ok=True)
err_path.unlink(missing_ok=True)
raise RuntimeError(f"cannot spawn {argv[0]}: {error}") from error
finally:
os.close(out_fd)
os.close(err_fd)
deadline = started + timeout
status = None
usage = None
while True:
reaped, raw_status, rusage = os.wait4(pid, os.WNOHANG)
if reaped == pid:
status = raw_status
usage = rusage
break
if time.perf_counter() >= deadline:
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
pass
os.wait4(pid, 0)
break
time.sleep(0.001)
elapsed = time.perf_counter() - started
stdout = out_path.read_bytes()
stderr = err_path.read_bytes()
out_path.unlink(missing_ok=True)
err_path.unlink(missing_ok=True)
if status is None:
raise RuntimeError(f"timeout after {timeout:g}s: {' '.join(argv)}")
if os.WIFSIGNALED(status):
raise RuntimeError(
f"terminated by signal {os.WTERMSIG(status)}: {' '.join(argv)}"
)
return (
os.WEXITSTATUS(status),
{
"user_ms": usage.ru_utime * 1000.0,
"sys_ms": usage.ru_stime * 1000.0,
"max_rss_kb": usage.ru_maxrss,
},
elapsed,
stdout,
stderr,
)
def cpu_model() -> str:
try:
for line in Path("/proc/cpuinfo").read_text().splitlines():
if line.startswith("model name"):
return line.split(":", 1)[1].strip()
except OSError:
pass
return "unknown"
def governor() -> str:
value = output(
["sh", "-c", "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null"]
)
return value or "unavailable"
def loadavg() -> str:
try:
return Path("/proc/loadavg").read_text().strip()
except OSError:
return "unavailable"
def median_ms(values: list[float]) -> float:
return round(statistics.median(values), 4)
def mad_ms(values: list[float]) -> float:
middle = statistics.median(values)
return round(statistics.median(abs(value - middle) for value in values), 4)
def p95_ms(values: list[float]) -> float:
ordered = sorted(values)
index = min(len(ordered) - 1, max(0, int(round(0.95 * (len(ordered) - 1)))))
return round(ordered[index], 4)
def run_workload(
binary: Path,
workload: dict,
warmups: int,
samples: int,
order_seed: str,
allow_smoke: bool,
record: bool,
timeout: float,
cpu_list: str,
) -> dict:
name = workload["name"]
prefix = []
if cpu_list:
prefix = [shutil.which("taskset") or "taskset", "-c", cpu_list]
argv = prefix + [str(binary)] + workload["argv"]
env = dict(os.environ)
env["MARQDO_STDLIB_DIR"] = str(ROOT / "stdlib")
env.setdefault("NO_COLOR", "1")
expected = load_expected(name)
result = {
"name": name,
"kind": workload.get("kind", "run"),
"source": workload["source"],
"argv": workload["argv"],
}
correctness = "VERIFIED"
warmup_samples: list[dict] = []
measured: list[dict] = []
# Correctness is checked on every warmup execution; a wrong result is never
# timed and can never become a "fast" regression sample.
for index in range(warmups + samples):
code, usage, wall, stdout, stderr = spawn_capture(
argv, ROOT, env, timeout
)
if code != 0:
correctness = "CORRECTNESS_FAIL"
result["error"] = (
f"exit {code}: {stderr.decode('utf-8', 'replace')[:500]}"
)
break
try:
document = json.loads(stdout)
except json.JSONDecodeError as error:
correctness = "CORRECTNESS_FAIL"
result["error"] = f"invalid JSON: {error}"
break
digest = wire_digest(document)
if record:
record_expected(name, document)
expected = digest
if expected is None:
correctness = "UNVERIFIED_NO_EXPECTED"
elif digest != expected:
correctness = "CORRECTNESS_FAIL"
result["error"] = (
f"wire digest mismatch: expected {expected}, got {digest}"
)
break
sample = {
"wall_ms": wall * 1000.0,
"user_ms": usage["user_ms"],
"sys_ms": usage["sys_ms"],
"max_rss_kb": usage["max_rss_kb"],
}
if index < warmups:
warmup_samples.append(sample)
else:
measured.append(sample)
result["correctness"] = correctness
if correctness != "VERIFIED":
result["status"] = correctness
return result
if not measured:
result["status"] = "NO_SAMPLES"
return result
walls = [sample["wall_ms"] for sample in measured]
result.update(
{
"status": "MEASURED" if (len(measured) >= MIN_SAMPLES and warmups >= MIN_WARMUPS) else "SMOKE",
"warmups": len(warmup_samples),
"samples": len(measured),
"order_seed": order_seed,
"median_ms": median_ms(walls),
"p95_ms": p95_ms(walls),
"mad_ms": mad_ms(walls),
"min_ms": round(min(walls), 4),
"max_ms": round(max(walls), 4),
"cpu_user_ms_median": median_ms(
[sample["user_ms"] for sample in measured]
),
"cpu_sys_ms_median": median_ms([sample["sys_ms"] for sample in measured]),
"max_rss_kb_median": int(
statistics.median(sample["max_rss_kb"] for sample in measured)
),
}
)
if result["status"] == "SMOKE" and not allow_smoke:
pass # still recorded as SMOKE; the caller decides whether to accept it
return result
def compare_baseline(report: dict, baseline: dict) -> list[dict]:
if baseline.get("schema") != SCHEMA:
raise SystemExit(f"baseline schema {baseline.get('schema')!r} is not {SCHEMA}")
if baseline.get("platform_id") != report.get("platform_id"):
raise SystemExit(
"baseline platform_id differs; timings from another machine/node are not comparable"
)
if baseline.get("binary", {}).get("profile") != report.get("binary", {}).get("profile"):
raise SystemExit("baseline profile differs")
previous = {item["name"]: item for item in baseline.get("workloads", [])}
comparisons = []
for current in report["workloads"]:
old = previous.get(current["name"])
if not old or old.get("status") not in {"MEASURED", "SMOKE"}:
comparisons.append({"name": current["name"], "status": "NO_BASELINE"})
continue
if current.get("status") not in {"MEASURED", "SMOKE"}:
comparisons.append(
{"name": current["name"], "status": "CURRENT_NOT_MEASURED"}
)
continue
p50_delta = (current["median_ms"] - old["median_ms"]) / old["median_ms"]
rss_delta = (
current["max_rss_kb_median"] - old["max_rss_kb_median"]
) / old["max_rss_kb_median"]
comparisons.append(
{
"name": current["name"],
"status": "COMPARED",
"p50_delta_percent": round(p50_delta * 100.0, 2),
"rss_delta_percent": round(rss_delta * 100.0, 2),
"investigate": p50_delta >= P50_REGRESSION
or rss_delta >= RSS_REGRESSION,
}
)
return comparisons
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--binary", default=str(ROOT / "target" / "release" / "marqdo"))
parser.add_argument("--no-build", action="store_true")
parser.add_argument("--warmups", type=int, default=MIN_WARMUPS)
parser.add_argument("--samples", type=int, default=MIN_SAMPLES)
parser.add_argument("--timeout", type=float, default=120.0)
parser.add_argument("--only", action="append", default=[])
parser.add_argument("--record-expected", action="store_true")
parser.add_argument("--platform-id", default="local-unapproved")
parser.add_argument("--approved-node", action="store_true")
parser.add_argument("--cpu-list", default="")
parser.add_argument("--baseline", default="")
parser.add_argument("--out", default="")
parser.add_argument("--allow-smoke", action="store_true")
args = parser.parse_args(argv)
if args.samples < 1 or args.warmups < 0:
raise SystemExit("samples must be >= 1 and warmups >= 0")
if (args.samples < MIN_SAMPLES or args.warmups < MIN_WARMUPS) and not args.allow_smoke:
raise SystemExit(
f"release evidence requires >= {MIN_WARMUPS} warmups and >= {MIN_SAMPLES} "
"samples; pass --allow-smoke for a non-release smoke run"
)
binary = Path(args.binary).resolve()
if not args.no_build:
build = [os.environ.get("CARGO", "cargo"), "build", "--release", "--locked", "-p", "marqdo-cli"]
subprocess.run(build, cwd=ROOT, check=True)
if not binary.is_file():
raise SystemExit(f"release binary not found: {binary}")
if args.cpu_list and not shutil.which("taskset"):
raise SystemExit("--cpu-list requires taskset")
config = json.loads(WORKLOADS.read_text())
workloads = [
item
for item in config["workloads"]
if not args.only or item["name"] in args.only
]
if not workloads:
raise SystemExit("no workloads selected")
source_sha = git_sha(ROOT)
order_seed = hashlib.sha256(source_sha.encode()).hexdigest()[:16]
order = list(workloads)
random.Random(order_seed).shuffle(order)
results = []
for workload in order:
result = run_workload(
binary,
workload,
args.warmups,
args.samples,
order_seed,
args.allow_smoke,
args.record_expected,
args.timeout,
args.cpu_list,
)
results.append(result)
print(
f"{result['name']}: {result.get('status')} "
f"median={result.get('median_ms', 'n/a')}ms "
f"p95={result.get('p95_ms', 'n/a')}ms "
f"rss={result.get('max_rss_kb_median', 'n/a')}KB",
file=sys.stderr,
)
if args.record_expected:
print("expected digests recorded; commit tools/perf/expected/", file=sys.stderr)
correctness_ok = all(item.get("correctness") == "VERIFIED" for item in results)
measured = all(item.get("status") == "MEASURED" for item in results)
report = {
"schema": SCHEMA,
"source_sha": source_sha,
"platform_id": args.platform_id,
"environment_approved": bool(args.approved_node),
"binary": {
"path": str(binary.relative_to(ROOT)) if binary.is_relative_to(ROOT) else str(binary),
"sha256": sha256_file(binary),
"profile": "release",
"cargo_lock_sha256": sha256_file(ROOT / "Cargo.lock"),
"rustc": output(["rustc", "-vV"]),
"cargo": output(["cargo", "-V"]),
},
"environment": {
"uname": platform.platform(),
"kernel": platform.release(),
"machine": platform.machine(),
"cpu_model": cpu_model(),
"governor": governor(),
"loadavg_start": loadavg(),
"cpu_list": args.cpu_list or "unbound",
"python": platform.python_version(),
},
"recipe": {
"warmups": args.warmups,
"samples": args.samples,
"order_seed": order_seed,
"timeout_seconds": args.timeout,
},
"workloads": results,
"correctness": "VERIFIED" if correctness_ok else "CORRECTNESS_FAIL",
"status": "MEASURED"
if (measured and args.approved_node)
else ("SMOKE" if correctness_ok else "CORRECTNESS_FAIL"),
"baseline_comparison": [],
}
report["environment"]["loadavg_end"] = loadavg()
if args.baseline:
baseline = json.loads(Path(args.baseline).read_text())
report["baseline_comparison"] = compare_baseline(report, baseline)
text = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if args.out:
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
Path(args.out).write_text(text)
sys.stdout.write(text)
if not correctness_ok:
return 1
if not measured and not args.allow_smoke:
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))