Files
marqdo-cli/tools/ci/verify_extension_contracts.py
T

276 lines
10 KiB
Python
Executable File

#!/usr/bin/env python3
"""Frozen-CLI external-extension evidence generator and validator.
RFC-EXT-0001 §10.2/§11.3: a release candidate must prove that independent
repositories work against an *unchanged* CLI binary. This script:
1. hashes the CLI binary before and after the external suite and refuses to
emit PASS if the hash moved;
2. records the exact marqdo-cli source SHA and the external fixture repo SHA;
3. runs the fixture repository's own black-box suite with ``MARQDO_BIN`` set to
the frozen binary (the suite itself must not run cargo or read the CLI
source tree);
4. validates/normalises the per-case results into the repository evidence
schema (``docs/evidence/extension/schema.json``);
5. writes ``docs/evidence/extension/<tested_sha>/manifest.json``.
Usage:
python3 tools/ci/verify_extension_contracts.py \
--cli target/release/marqdo \
--fixtures ../marqdo-extension-conformance \
--out docs/evidence/extension/<sha>/manifest.json
python3 tools/ci/verify_extension_contracts.py --validate manifest.json
Only Python's standard library is used. No network access is performed by this
script; the external suite documents its own offline cache behaviour.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import platform
import subprocess
import sys
import tempfile
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
SCHEMA = REPO / "docs" / "evidence" / "extension" / "schema.json"
REQUIRED_RESULTS = [
"callable_registry",
"plugin_worker",
"foreign_kernel",
"resource_provider_cli",
"package_pinned_git",
"tool_analysis_consumer",
"negative_security",
]
ALLOWED = {"PASS", "FAIL", "UNSUPPORTED", "SKIPPED"}
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def git_sha(directory: Path) -> str:
output = subprocess.run(
["git", "-C", str(directory), "rev-parse", "HEAD"],
check=True,
capture_output=True,
text=True,
)
return output.stdout.strip()
def os_name() -> str:
name = platform.system().lower()
if name.startswith("linux"):
return "linux"
if name == "darwin":
return "macos"
if name.startswith("windows"):
return "windows"
raise SystemExit(f"unsupported OS for extension evidence: {name}")
def validate_manifest(manifest: dict) -> list[str]:
problems: list[str] = []
if manifest.get("schema") != "marqdo.extension-evidence":
problems.append("schema must be marqdo.extension-evidence")
if manifest.get("schema_version") != 1:
problems.append("schema_version must be 1")
for field in ["tested_sha", "fixture_repo_sha"]:
value = manifest.get(field, "")
if not (isinstance(value, str) and len(value) == 40 and all(c in "0123456789abcdef" for c in value)):
problems.append(f"{field} must be a 40-character lowercase git SHA")
cli = manifest.get("cli_binary_sha256", "")
if not (isinstance(cli, str) and len(cli) == 64 and all(c in "0123456789abcdef" for c in cli)):
problems.append("cli_binary_sha256 must be a 64-character lowercase sha256")
if manifest.get("os") not in {"linux", "macos", "windows"}:
problems.append("os must be linux, macos or windows")
results = manifest.get("results")
if not isinstance(results, dict):
problems.append("results must be an object")
else:
for name in REQUIRED_RESULTS:
if results.get(name) not in ALLOWED:
problems.append(f"results.{name} must be one of {sorted(ALLOWED)}")
for key, value in results.items():
if key not in REQUIRED_RESULTS:
problems.append(f"unknown result key {key}")
if value not in ALLOWED:
problems.append(f"result {key}={value} is not allowed")
commands = manifest.get("commands")
if not isinstance(commands, list) or not commands or not all(isinstance(c, str) for c in commands):
problems.append("commands must be a non-empty array of strings")
for field in ["failures", "unverified"]:
if not isinstance(manifest.get(field), list):
problems.append(f"{field} must be an array")
if not problems:
for name, value in (manifest.get("results") or {}).items():
if value in {"FAIL"}:
problems.append(f"result {name} is FAIL")
return problems
def run_suite(fixtures: Path, cli: Path) -> tuple[bool, dict, list[str]]:
"""Run the external suite and return (ok, results, commands)."""
results: dict[str, str] = {}
commands: list[str] = []
env = dict(os.environ)
env["MARQDO_BIN"] = str(cli)
env["MARQDO_FROZEN_CLI"] = str(cli)
runner = fixtures / "tests" / "run_all.py"
if not runner.is_file():
return False, results, commands
commands.append(f"python3 {runner} (cwd={fixtures})")
completed = subprocess.run(
[sys.executable, str(runner)],
cwd=fixtures,
env=env,
capture_output=True,
text=True,
)
print(completed.stdout, end="")
print(completed.stderr, end="", file=sys.stderr)
results_path = fixtures / "evidence" / "results.json"
if results_path.is_file():
with results_path.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
for key, value in (payload.get("results") or {}).items():
results[key] = str(value).upper()
return completed.returncode == 0, results, commands
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--cli", type=Path, help="Absolute/relative frozen marqdo binary")
parser.add_argument("--fixtures", type=Path, help="External fixture repository root")
parser.add_argument("--source-repo", type=Path, default=REPO, help="marqdo-cli checkout")
parser.add_argument("--out", type=Path, help="Evidence manifest output path")
parser.add_argument("--validate", type=Path, help="Only validate an existing manifest")
parser.add_argument("--allow-unverified", action="store_true", help="Record UNSUPPORTED/SKIPPED without failing")
parser.add_argument(
"--tested-sha",
help="Require the source checkout HEAD to be exactly this 40-hex SHA before testing",
)
parser.add_argument(
"--allow-dirty-source",
action="store_true",
help="Development only: permit a dirty source checkout (never valid for release evidence)",
)
args = parser.parse_args()
if args.validate:
manifest = json.loads(args.validate.read_text(encoding="utf-8"))
problems = validate_manifest(manifest)
if problems:
for problem in problems:
print(f"INVALID: {problem}", file=sys.stderr)
return 1
print(f"valid extension evidence: {args.validate}")
return 0
if not args.cli or not args.fixtures:
parser.error("--cli and --fixtures are required unless --validate is used")
cli = args.cli.resolve()
fixtures = args.fixtures.resolve()
if not cli.is_file():
print(f"error: CLI binary not found: {cli}", file=sys.stderr)
return 2
if not (fixtures / ".git").exists():
print(f"error: fixture path is not a git repository: {fixtures}", file=sys.stderr)
return 2
if subprocess.run(
["git", "-C", str(fixtures), "status", "--porcelain"],
capture_output=True,
text=True,
check=True,
).stdout.strip():
print("error: fixture repository has uncommitted changes", file=sys.stderr)
return 2
source_repo = args.source_repo.resolve()
if not args.allow_dirty_source:
dirty = subprocess.run(
["git", "-C", str(source_repo), "status", "--porcelain"],
capture_output=True,
text=True,
check=True,
).stdout.strip()
if dirty:
print(
"error: source checkout has uncommitted changes; release evidence must "
"name a committed SHA (use --allow-dirty-source only for local development)",
file=sys.stderr,
)
return 2
cli_before = sha256_file(cli)
tested_sha = git_sha(source_repo)
if args.tested_sha and args.tested_sha != tested_sha:
print(
f"error: --tested-sha {args.tested_sha} does not match checkout HEAD {tested_sha}",
file=sys.stderr,
)
return 2
fixture_sha = git_sha(fixtures)
ok, results, commands = run_suite(fixtures, cli)
cli_after = sha256_file(cli)
if cli_before != cli_after:
print("error: frozen CLI hash changed during the external suite", file=sys.stderr)
return 1
failures = [name for name, value in results.items() if value == "FAIL"]
unverified = [
name for name, value in results.items() if value in {"UNSUPPORTED", "SKIPPED"}
]
missing = [name for name in REQUIRED_RESULTS if name not in results]
if missing:
failures.extend(f"missing:{name}" for name in missing)
manifest = {
"schema": "marqdo.extension-evidence",
"schema_version": 1,
"tested_sha": tested_sha,
"cli_binary_sha256": cli_before,
"fixture_repo_sha": fixture_sha,
"os": os_name(),
"arch": platform.machine(),
"results": {name: results.get(name, "SKIPPED") for name in REQUIRED_RESULTS},
"commands": commands,
"failures": failures,
"unverified": unverified,
}
problems = validate_manifest(manifest)
if unverified and not args.allow_unverified:
problems.append(
"UNSUPPORTED/SKIPPED results require --allow-unverified and an explicit report entry"
)
if not ok and not failures:
failures.append("external suite exited non-zero without structured results")
problems.append("external suite failed")
if problems:
for problem in problems:
print(f"FAIL: {problem}", file=sys.stderr)
print(json.dumps(manifest, indent=2))
return 1
out = args.out or (REPO / "docs" / "evidence" / "extension" / tested_sha / "manifest.json")
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
print(f"wrote extension evidence: {out}")
return 0
if __name__ == "__main__":
raise SystemExit(main())