Files

641 lines
22 KiB
Python

#!/usr/bin/env python3
"""Black-box conformance suite for a frozen Marqdo CLI.
Run from this repository after `scripts/build_all.sh`:
MARQDO_BIN=/abs/path/to/marqdo python3 tests/run_all.py
The suite never runs Cargo, never imports a Rust crate, and never writes into
the CLI workspace. It refuses to report success if the frozen binary hash
changes during the run (see tests/verify_frozen_cli.py).
Results are written to evidence/results.json in the shape consumed by
`marqdo-cli/tools/ci/verify_extension_contracts.py`.
"""
from __future__ import annotations
import hashlib
import json
import os
import platform
import shutil
import subprocess
import sys
import tempfile
from dataclasses import dataclass, field
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
RESULTS = EVIDENCE / "results.json"
CATEGORIES = [
"callable_registry",
"plugin_worker",
"foreign_kernel",
"resource_provider_cli",
"package_pinned_git",
"tool_analysis_consumer",
"negative_security",
]
@dataclass
class Case:
category: str
name: str
status: str = "SKIPPED"
detail: str = ""
commands: list[str] = field(default_factory=list)
error_codes: list[str] = field(default_factory=list)
class Runner:
def __init__(self, cli: Path) -> None:
self.cli = cli
self.cases: list[Case] = []
self.cli_hash = sha256_file(cli)
self.tool_sha = git_sha(ROOT)
self.env = dict(os.environ)
self.env["MARQDO_BIN"] = str(cli)
# -- helpers ---------------------------------------------------------
def case(self, category: str, name: str) -> Case:
item = Case(category=category, name=name)
self.cases.append(item)
return item
def run(
self,
args: list[str],
cwd: Path | None = None,
env: dict[str, str] | None = None,
timeout: int = 60,
) -> subprocess.CompletedProcess:
merged = dict(self.env)
if env:
merged.update(env)
completed = subprocess.run(
[str(self.cli), *args],
cwd=str(cwd or ROOT),
env=merged,
capture_output=True,
text=True,
timeout=timeout,
)
return completed
def finish(
self,
case: Case,
completed: subprocess.CompletedProcess,
expect_exit: int,
*,
stdout: str | None = None,
stderr_contains: str | None = None,
stderr_not_contains: str | None = None,
) -> bool:
case.commands.append(" ".join(str(part) for part in completed.args))
ok = completed.returncode == expect_exit
if ok and stdout is not None:
ok = completed.stdout == stdout
if ok and stderr_contains is not None:
ok = stderr_contains in completed.stderr
if ok and stderr_not_contains is not None:
ok = stderr_not_contains not in completed.stderr
if ok:
case.status = "PASS"
case.detail = f"exit {completed.returncode}"
else:
case.status = "FAIL"
case.detail = (
f"exit {completed.returncode} (wanted {expect_exit}); "
f"stdout={completed.stdout[:300]!r}; stderr={completed.stderr[:500]!r}"
)
for line in completed.stderr.splitlines():
if line.startswith("error[") and "]" in line:
case.error_codes.append(line.split("[", 1)[1].split("]", 1)[0])
return ok
def skip(self, case: Case, reason: str) -> None:
case.status = "SKIPPED"
case.detail = reason
# -- fixture builds --------------------------------------------------
def build(self) -> None:
completed = subprocess.run(
[str(ROOT / "scripts" / "build_all.sh")],
cwd=str(ROOT),
env=self.env,
capture_output=True,
text=True,
)
if completed.returncode != 0:
raise SystemExit(
"fixture build failed:\n" + completed.stdout + completed.stderr
)
# -- categories ------------------------------------------------------
def check_plugin(self) -> None:
plugin = ROOT / "plugin"
build = plugin / "build"
demo = plugin / "demo.mq.md"
case = self.case("plugin_worker", "plugin check is static and succeeds")
check = self.run(["check", str(demo), "--plugin", str(build)])
self.finish(case, check, 0)
case = self.case("plugin_worker", "worker plugin executes three exports")
run = self.run(
[
"run",
str(demo),
"--plugin",
str(build),
"--allow-plugin",
"demo.ext",
]
)
self.finish(case, run, 0, stdout="42\n-7\n15\n")
case = self.case("callable_registry", "missing plugin export is symbol.unknown")
missing = self.run(
[
"run",
str(plugin / "negative-missing-export.mq.md"),
"--plugin",
str(build),
"--allow-plugin",
"demo.ext",
]
)
self.finish(case, missing, 1, stderr_contains="symbol.unknown")
case = self.case("callable_registry", "bad plugin args fail statically")
bad = self.run(
[
"check",
str(plugin / "negative-bad-args.mq.md"),
"--plugin",
str(build),
]
)
self.finish(case, bad, 1, stderr_contains="call.arity")
case = self.case("negative_security", "plugin without grant is permission.denied")
no_grant = self.run(
[
"run",
str(demo),
"--plugin",
str(build),
]
)
self.finish(case, no_grant, 1, stderr_contains="permission.denied")
case = self.case(
"negative_security", "tampered plugin artifact fails before main"
)
tampered = Path(tempfile.mkdtemp(prefix="ext-plugin-tamper-"))
try:
for entry in build.iterdir():
if entry.is_file():
shutil.copy(entry, tampered / entry.name)
manifest = self.read_yaml_artifact(tampered / "plugin.yaml")
artifact = tampered / manifest
artifact.write_bytes(artifact.read_bytes() + b"\0")
tamper_run = self.run(
[
"run",
str(demo),
"--plugin",
str(tampered),
"--allow-plugin",
"demo.ext",
]
)
self.finish(
case,
tamper_run,
1,
stdout="",
stderr_contains="ext.artifact_integrity",
)
finally:
shutil.rmtree(tampered, ignore_errors=True)
case = self.case(
"negative_security", "check does not require the plugin artifact to exist"
)
missing_artifact = Path(tempfile.mkdtemp(prefix="ext-plugin-missing-"))
try:
shutil.copytree(build, missing_artifact, dirs_exist_ok=True)
artifact_name = self.read_yaml_artifact(missing_artifact / "plugin.yaml")
(missing_artifact / artifact_name).unlink()
static_check = self.run(
[
"check",
str(demo),
"--plugin",
str(missing_artifact),
]
)
self.finish(case, static_check, 0)
finally:
shutil.rmtree(missing_artifact, ignore_errors=True)
def check_kernel(self) -> None:
kernel = ROOT / "kernel" / "build" / "kernel.json"
demo = ROOT / "kernel" / "demo.mq.md"
case = self.case("foreign_kernel", "unknown language mocklang executes")
result = self.run(
[
"run",
str(demo),
"--kernel",
str(kernel),
"--allow-foreign",
]
)
self.finish(case, result, 0, stdout="42\n")
case = self.case("negative_security", "foreign kernel without grant is denied")
denied = self.run(["run", str(demo), "--kernel", str(kernel)])
self.finish(case, denied, 1, stderr_contains="permission.denied")
def check_provider(self) -> None:
provider_dir = ROOT / "provider"
build = provider_dir / "build"
demo = provider_dir / "demo.mq.md"
help_text = self.run(["run", "--help"])
supports = "--provider" in help_text.stdout
case = self.case(
"resource_provider_cli", "provider v2 reads five bytes through the CLI"
)
if not supports:
self.skip(
case,
"frozen CLI has no --provider/--allow-provider (RFC-EXT-0001 PR-03 not built yet)",
)
return
helper = self.cli.with_name("marqdo-provider-worker")
if not helper.is_file():
self.skip(
case,
f"provider helper is not shipped next to the frozen CLI: {helper}",
)
return
result = self.run(
[
"run",
str(demo),
"--provider",
str(build / "provider.json"),
"--allow-provider",
"demo.sensor",
"--json",
]
)
self.finish(case, result, 0)
if case.status == "PASS":
payload = json.loads(result.stdout)
if payload.get("value") != {"type": "int", "decimal": "5"}:
case.status = "FAIL"
case.detail = f"unexpected provider value: {result.stdout[:500]}"
case = self.case("negative_security", "provider without grant is denied")
denied = self.run(["run", str(demo), "--provider", str(build / "provider.json")])
self.finish(case, denied, 1, stderr_contains="permission.denied")
case = self.case(
"negative_security", "provider v1 schema is rejected without migration"
)
temp = Path(tempfile.mkdtemp(prefix="ext-provider-v1-"))
try:
shutil.copytree(build, temp, dirs_exist_ok=True)
manifest = json.loads((temp / "provider.json").read_text())
manifest["schema_version"] = 1
(temp / "provider.json").write_text(json.dumps(manifest))
v1 = self.run(
[
"run",
str(demo),
"--provider",
str(temp / "provider.json"),
"--allow-provider",
"demo.sensor",
]
)
self.finish(
case,
v1,
2,
stderr_contains="ext.unsupported_provider_schema",
)
finally:
shutil.rmtree(temp, ignore_errors=True)
case = self.case(
"negative_security", "provider artifact digest tamper is rejected"
)
temp = Path(tempfile.mkdtemp(prefix="ext-provider-tamper-"))
try:
shutil.copytree(build, temp, dirs_exist_ok=True)
manifest = json.loads((temp / "provider.json").read_text())
manifest["artifacts"][0]["sha256"] = "0" * 64
(temp / "provider.json").write_text(json.dumps(manifest))
tampered = self.run(
[
"run",
str(demo),
"--provider",
str(temp / "provider.json"),
"--allow-provider",
"demo.sensor",
]
)
self.finish(
case,
tampered,
1,
stderr_contains="ext.artifact_integrity",
)
finally:
shutil.rmtree(temp, ignore_errors=True)
case = self.case(
"negative_security", "pure and strict modes reject provider flags"
)
pure = self.run(
[
"run",
str(demo),
"--mode",
"pure",
"--provider",
str(build / "provider.json"),
"--allow-provider",
"demo.sensor",
]
)
strict = self.run(
[
"run",
str(demo),
"--mode",
"strict",
"--provider",
str(build / "provider.json"),
"--allow-provider",
"demo.sensor",
]
)
ok_pure = self.finish(case, pure, 2)
if ok_pure:
strict_case = self.case(
"negative_security", "strict mode rejects provider flags"
)
self.finish(strict_case, strict, 2)
def check_tool(self) -> None:
case = self.case(
"tool_analysis_consumer", "independent tool consumes Analysis v2"
)
home = Path(tempfile.mkdtemp(prefix="ext-tool-home-"))
try:
env = {
"HOME": str(home),
"XDG_CONFIG_HOME": str(home / ".config"),
"XDG_DATA_HOME": str(home / ".local" / "share"),
}
install = self.run(
["tool", "install", str(ROOT / "tool" / "tool.json")], env=env
)
if not self.finish(case, install, 0):
return
case = self.case(
"tool_analysis_consumer", "registered tool command dispatches"
)
invoke = self.run(
["inspect-ext", str(ROOT / "tool" / "demo.mq.md")], env=env
)
ok = self.finish(case, invoke, 0)
if ok:
payload = json.loads(invoke.stdout)
functions = payload.get("functions", [])
ok = any(item.get("local_name") == "main" for item in functions)
if not ok:
case.status = "FAIL"
case.detail = f"tool saw no main function: {invoke.stdout[:500]}"
case = self.case(
"negative_security", "tool install rejects unknown entrypoint traversal"
)
bad = Path(tempfile.mkdtemp(prefix="ext-tool-bad-"))
try:
shutil.copytree(ROOT / "tool", bad, dirs_exist_ok=True)
bad_manifest = json.loads((bad / "tool.json").read_text())
bad_manifest["entrypoint"] = "../../escape.py"
(bad / "tool.json").write_text(json.dumps(bad_manifest))
rejected = self.run(["tool", "install", str(bad)], env=env)
self.finish(case, rejected, 1)
finally:
shutil.rmtree(bad, ignore_errors=True)
remove = self.run(["tool", "remove", "inspect-ext"], env=env)
self.finish(
self.case("tool_analysis_consumer", "tool remove unregisters"),
remove,
0,
)
finally:
shutil.rmtree(home, ignore_errors=True)
def check_package(self) -> None:
case = self.case(
"package_pinned_git", "pinned Git .mq.md library runs from a separate repo"
)
workspace = Path(tempfile.mkdtemp(prefix="ext-package-"))
try:
source = workspace / "math-src"
shutil.copytree(ROOT / "pure-lib", source)
git_commit(source, "math library snapshot")
rev = git_sha(source)
consumer = workspace / "consumer"
consumer.mkdir()
shutil.copy(ROOT / "consumer" / "index.mq.md", consumer / "index.mq.md")
(consumer / "marqdo.yaml").write_text(
"project:\n"
" name: ext-consumer\n"
" version: 0.1.0\n"
"dependencies:\n"
" math:\n"
f" git: \"file://{source}\"\n"
f" rev: \"{rev}\"\n"
)
lock = self.run(["lock"], cwd=consumer)
if not self.finish(case, lock, 0):
return
sync = self.run(["sync"], cwd=consumer)
if not self.finish(case, sync, 0):
return
case = self.case(
"package_pinned_git", "consumer runs the pinned dependency"
)
check = self.run(["check", "index.mq.md"], cwd=consumer)
ok = self.finish(case, check, 0)
run = self.run(["run", "index.mq.md", "--json"], cwd=consumer)
ok = ok and self.finish(case, run, 0)
if ok:
payload = json.loads(run.stdout)
if payload.get("value") != {"type": "int", "decimal": "18"}:
case.status = "FAIL"
case.detail = f"unexpected consumer value: {run.stdout[:500]}"
ok = False
if not ok:
return
# Offline/reproducibility: move the project, delete the source Git
# repository, and run again from the verified cache only.
moved = workspace / "moved-consumer"
shutil.move(str(consumer), moved)
shutil.rmtree(source)
case = self.case(
"package_pinned_git",
"verified cache still runs after moving the checkout and deleting the source",
)
offline = self.run(["run", "index.mq.md"], cwd=moved)
self.finish(case, offline, 0)
finally:
shutil.rmtree(workspace, ignore_errors=True)
# -- utilities -------------------------------------------------------
@staticmethod
def read_yaml_artifact(manifest: Path) -> str:
for line in manifest.read_text().splitlines():
stripped = line.strip()
if stripped.startswith("file:"):
return stripped.split("file:", 1)[1].strip().strip('"')
raise SystemExit(f"manifest has no artifact file: {manifest}")
def report(self) -> int:
final_hash = sha256_file(self.cli)
if final_hash != self.cli_hash:
case = self.case("negative_security", "frozen CLI binary did not change")
case.status = "FAIL"
case.detail = f"sha256 changed {self.cli_hash} -> {final_hash}"
else:
case = self.case("negative_security", "frozen CLI binary did not change")
case.status = "PASS"
case.detail = f"sha256 {self.cli_hash}"
results: dict[str, str] = {}
for category in CATEGORIES:
cases = [item for item in self.cases if item.category == category]
if not cases:
results[category] = "UNSUPPORTED"
elif any(item.status == "FAIL" for item in cases):
results[category] = "FAIL"
elif all(item.status == "SKIPPED" for item in cases):
results[category] = "UNSUPPORTED"
elif any(item.status == "SKIPPED" for item in cases):
results[category] = "PASS" if all(
item.status in {"PASS", "SKIPPED"} for item in cases
) else "FAIL"
else:
results[category] = "PASS"
payload = {
"schema": "marqdo.ext-fixture-results",
"schema_version": 1,
"tool_sha": self.tool_sha,
"cli_binary_sha256": final_hash,
"os": os_name(),
"arch": platform.machine(),
"results": results,
"cases": [
{
"category": item.category,
"name": item.name,
"status": item.status,
"detail": item.detail,
"commands": item.commands,
"error_codes": item.error_codes,
}
for item in self.cases
],
}
EVIDENCE.mkdir(exist_ok=True)
RESULTS.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
failures = [item for item in self.cases if item.status == "FAIL"]
for item in self.cases:
marker = {"PASS": "PASS", "FAIL": "FAIL", "SKIPPED": "SKIP"}[item.status]
print(f"[{marker}] {item.category}: {item.name}")
if item.status != "PASS":
print(f" {item.detail}")
print(json.dumps(results, indent=2))
return 1 if failures else 0
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def git_sha(directory: Path) -> str:
output = subprocess.run(
["git", "-C", str(directory), "rev-parse", "HEAD"],
check=True,
capture_output=True,
text=True,
)
return output.stdout.strip()
def git_commit(directory: Path, message: str) -> None:
def git(*args: str) -> None:
subprocess.run(
["git", "-C", str(directory), *args], check=True, capture_output=True
)
if not (directory / ".git").is_dir():
git("init", "-q", "-b", "main")
git("config", "user.name", "Marqdo Extension Fixtures")
git("config", "user.email", "fixtures@example.invalid")
git("add", "-A")
git("commit", "-q", "-m", message)
def os_name() -> str:
name = platform.system().lower()
if name.startswith("linux"):
return "linux"
if name == "darwin":
return "macos"
if name.startswith("windows"):
return "windows"
return name
def main() -> int:
raw = os.environ.get("MARQDO_BIN")
if not raw:
print("error: set MARQDO_BIN to the absolute frozen CLI path", file=sys.stderr)
return 2
cli = Path(raw).resolve()
if not cli.is_file():
print(f"error: MARQDO_BIN is not a file: {cli}", file=sys.stderr)
return 2
runner = Runner(cli)
print(f"frozen CLI: {cli} sha256 {runner.cli_hash}")
runner.build()
runner.check_plugin()
runner.check_kernel()
runner.check_provider()
runner.check_tool()
runner.check_package()
return runner.report()
if __name__ == "__main__":
raise SystemExit(main())