641 lines
22 KiB
Python
641 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Black-box conformance suite for a frozen Marqdo CLI.
|
|
|
|
Run from this repository after `scripts/build_all.sh`:
|
|
|
|
MARQDO_BIN=/abs/path/to/marqdo python3 tests/run_all.py
|
|
|
|
The suite never runs Cargo, never imports a Rust crate, and never writes into
|
|
the CLI workspace. It refuses to report success if the frozen binary hash
|
|
changes during the run (see tests/verify_frozen_cli.py).
|
|
|
|
Results are written to evidence/results.json in the shape consumed by
|
|
`marqdo-cli/tools/ci/verify_extension_contracts.py`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVIDENCE = ROOT / "evidence"
|
|
RESULTS = EVIDENCE / "results.json"
|
|
|
|
CATEGORIES = [
|
|
"callable_registry",
|
|
"plugin_worker",
|
|
"foreign_kernel",
|
|
"resource_provider_cli",
|
|
"package_pinned_git",
|
|
"tool_analysis_consumer",
|
|
"negative_security",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class Case:
|
|
category: str
|
|
name: str
|
|
status: str = "SKIPPED"
|
|
detail: str = ""
|
|
commands: list[str] = field(default_factory=list)
|
|
error_codes: list[str] = field(default_factory=list)
|
|
|
|
|
|
class Runner:
|
|
def __init__(self, cli: Path) -> None:
|
|
self.cli = cli
|
|
self.cases: list[Case] = []
|
|
self.cli_hash = sha256_file(cli)
|
|
self.tool_sha = git_sha(ROOT)
|
|
self.env = dict(os.environ)
|
|
self.env["MARQDO_BIN"] = str(cli)
|
|
|
|
# -- helpers ---------------------------------------------------------
|
|
def case(self, category: str, name: str) -> Case:
|
|
item = Case(category=category, name=name)
|
|
self.cases.append(item)
|
|
return item
|
|
|
|
def run(
|
|
self,
|
|
args: list[str],
|
|
cwd: Path | None = None,
|
|
env: dict[str, str] | None = None,
|
|
timeout: int = 60,
|
|
) -> subprocess.CompletedProcess:
|
|
merged = dict(self.env)
|
|
if env:
|
|
merged.update(env)
|
|
completed = subprocess.run(
|
|
[str(self.cli), *args],
|
|
cwd=str(cwd or ROOT),
|
|
env=merged,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
return completed
|
|
|
|
def finish(
|
|
self,
|
|
case: Case,
|
|
completed: subprocess.CompletedProcess,
|
|
expect_exit: int,
|
|
*,
|
|
stdout: str | None = None,
|
|
stderr_contains: str | None = None,
|
|
stderr_not_contains: str | None = None,
|
|
) -> bool:
|
|
case.commands.append(" ".join(str(part) for part in completed.args))
|
|
ok = completed.returncode == expect_exit
|
|
if ok and stdout is not None:
|
|
ok = completed.stdout == stdout
|
|
if ok and stderr_contains is not None:
|
|
ok = stderr_contains in completed.stderr
|
|
if ok and stderr_not_contains is not None:
|
|
ok = stderr_not_contains not in completed.stderr
|
|
if ok:
|
|
case.status = "PASS"
|
|
case.detail = f"exit {completed.returncode}"
|
|
else:
|
|
case.status = "FAIL"
|
|
case.detail = (
|
|
f"exit {completed.returncode} (wanted {expect_exit}); "
|
|
f"stdout={completed.stdout[:300]!r}; stderr={completed.stderr[:500]!r}"
|
|
)
|
|
for line in completed.stderr.splitlines():
|
|
if line.startswith("error[") and "]" in line:
|
|
case.error_codes.append(line.split("[", 1)[1].split("]", 1)[0])
|
|
return ok
|
|
|
|
def skip(self, case: Case, reason: str) -> None:
|
|
case.status = "SKIPPED"
|
|
case.detail = reason
|
|
|
|
# -- fixture builds --------------------------------------------------
|
|
def build(self) -> None:
|
|
completed = subprocess.run(
|
|
[str(ROOT / "scripts" / "build_all.sh")],
|
|
cwd=str(ROOT),
|
|
env=self.env,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(
|
|
"fixture build failed:\n" + completed.stdout + completed.stderr
|
|
)
|
|
|
|
# -- categories ------------------------------------------------------
|
|
def check_plugin(self) -> None:
|
|
plugin = ROOT / "plugin"
|
|
build = plugin / "build"
|
|
demo = plugin / "demo.mq.md"
|
|
|
|
case = self.case("plugin_worker", "plugin check is static and succeeds")
|
|
check = self.run(["check", str(demo), "--plugin", str(build)])
|
|
self.finish(case, check, 0)
|
|
|
|
case = self.case("plugin_worker", "worker plugin executes three exports")
|
|
run = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--plugin",
|
|
str(build),
|
|
"--allow-plugin",
|
|
"demo.ext",
|
|
]
|
|
)
|
|
self.finish(case, run, 0, stdout="42\n-7\n15\n")
|
|
|
|
case = self.case("callable_registry", "missing plugin export is symbol.unknown")
|
|
missing = self.run(
|
|
[
|
|
"run",
|
|
str(plugin / "negative-missing-export.mq.md"),
|
|
"--plugin",
|
|
str(build),
|
|
"--allow-plugin",
|
|
"demo.ext",
|
|
]
|
|
)
|
|
self.finish(case, missing, 1, stderr_contains="symbol.unknown")
|
|
|
|
case = self.case("callable_registry", "bad plugin args fail statically")
|
|
bad = self.run(
|
|
[
|
|
"check",
|
|
str(plugin / "negative-bad-args.mq.md"),
|
|
"--plugin",
|
|
str(build),
|
|
]
|
|
)
|
|
self.finish(case, bad, 1, stderr_contains="call.arity")
|
|
|
|
case = self.case("negative_security", "plugin without grant is permission.denied")
|
|
no_grant = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--plugin",
|
|
str(build),
|
|
]
|
|
)
|
|
self.finish(case, no_grant, 1, stderr_contains="permission.denied")
|
|
|
|
case = self.case(
|
|
"negative_security", "tampered plugin artifact fails before main"
|
|
)
|
|
tampered = Path(tempfile.mkdtemp(prefix="ext-plugin-tamper-"))
|
|
try:
|
|
for entry in build.iterdir():
|
|
if entry.is_file():
|
|
shutil.copy(entry, tampered / entry.name)
|
|
manifest = self.read_yaml_artifact(tampered / "plugin.yaml")
|
|
artifact = tampered / manifest
|
|
artifact.write_bytes(artifact.read_bytes() + b"\0")
|
|
tamper_run = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--plugin",
|
|
str(tampered),
|
|
"--allow-plugin",
|
|
"demo.ext",
|
|
]
|
|
)
|
|
self.finish(
|
|
case,
|
|
tamper_run,
|
|
1,
|
|
stdout="",
|
|
stderr_contains="ext.artifact_integrity",
|
|
)
|
|
finally:
|
|
shutil.rmtree(tampered, ignore_errors=True)
|
|
|
|
case = self.case(
|
|
"negative_security", "check does not require the plugin artifact to exist"
|
|
)
|
|
missing_artifact = Path(tempfile.mkdtemp(prefix="ext-plugin-missing-"))
|
|
try:
|
|
shutil.copytree(build, missing_artifact, dirs_exist_ok=True)
|
|
artifact_name = self.read_yaml_artifact(missing_artifact / "plugin.yaml")
|
|
(missing_artifact / artifact_name).unlink()
|
|
static_check = self.run(
|
|
[
|
|
"check",
|
|
str(demo),
|
|
"--plugin",
|
|
str(missing_artifact),
|
|
]
|
|
)
|
|
self.finish(case, static_check, 0)
|
|
finally:
|
|
shutil.rmtree(missing_artifact, ignore_errors=True)
|
|
|
|
def check_kernel(self) -> None:
|
|
kernel = ROOT / "kernel" / "build" / "kernel.json"
|
|
demo = ROOT / "kernel" / "demo.mq.md"
|
|
case = self.case("foreign_kernel", "unknown language mocklang executes")
|
|
result = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--kernel",
|
|
str(kernel),
|
|
"--allow-foreign",
|
|
]
|
|
)
|
|
self.finish(case, result, 0, stdout="42\n")
|
|
|
|
case = self.case("negative_security", "foreign kernel without grant is denied")
|
|
denied = self.run(["run", str(demo), "--kernel", str(kernel)])
|
|
self.finish(case, denied, 1, stderr_contains="permission.denied")
|
|
|
|
def check_provider(self) -> None:
|
|
provider_dir = ROOT / "provider"
|
|
build = provider_dir / "build"
|
|
demo = provider_dir / "demo.mq.md"
|
|
help_text = self.run(["run", "--help"])
|
|
supports = "--provider" in help_text.stdout
|
|
case = self.case(
|
|
"resource_provider_cli", "provider v2 reads five bytes through the CLI"
|
|
)
|
|
if not supports:
|
|
self.skip(
|
|
case,
|
|
"frozen CLI has no --provider/--allow-provider (RFC-EXT-0001 PR-03 not built yet)",
|
|
)
|
|
return
|
|
helper = self.cli.with_name("marqdo-provider-worker")
|
|
if not helper.is_file():
|
|
self.skip(
|
|
case,
|
|
f"provider helper is not shipped next to the frozen CLI: {helper}",
|
|
)
|
|
return
|
|
result = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--provider",
|
|
str(build / "provider.json"),
|
|
"--allow-provider",
|
|
"demo.sensor",
|
|
"--json",
|
|
]
|
|
)
|
|
self.finish(case, result, 0)
|
|
if case.status == "PASS":
|
|
payload = json.loads(result.stdout)
|
|
if payload.get("value") != {"type": "int", "decimal": "5"}:
|
|
case.status = "FAIL"
|
|
case.detail = f"unexpected provider value: {result.stdout[:500]}"
|
|
|
|
case = self.case("negative_security", "provider without grant is denied")
|
|
denied = self.run(["run", str(demo), "--provider", str(build / "provider.json")])
|
|
self.finish(case, denied, 1, stderr_contains="permission.denied")
|
|
|
|
case = self.case(
|
|
"negative_security", "provider v1 schema is rejected without migration"
|
|
)
|
|
temp = Path(tempfile.mkdtemp(prefix="ext-provider-v1-"))
|
|
try:
|
|
shutil.copytree(build, temp, dirs_exist_ok=True)
|
|
manifest = json.loads((temp / "provider.json").read_text())
|
|
manifest["schema_version"] = 1
|
|
(temp / "provider.json").write_text(json.dumps(manifest))
|
|
v1 = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--provider",
|
|
str(temp / "provider.json"),
|
|
"--allow-provider",
|
|
"demo.sensor",
|
|
]
|
|
)
|
|
self.finish(
|
|
case,
|
|
v1,
|
|
2,
|
|
stderr_contains="ext.unsupported_provider_schema",
|
|
)
|
|
finally:
|
|
shutil.rmtree(temp, ignore_errors=True)
|
|
|
|
case = self.case(
|
|
"negative_security", "provider artifact digest tamper is rejected"
|
|
)
|
|
temp = Path(tempfile.mkdtemp(prefix="ext-provider-tamper-"))
|
|
try:
|
|
shutil.copytree(build, temp, dirs_exist_ok=True)
|
|
manifest = json.loads((temp / "provider.json").read_text())
|
|
manifest["artifacts"][0]["sha256"] = "0" * 64
|
|
(temp / "provider.json").write_text(json.dumps(manifest))
|
|
tampered = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--provider",
|
|
str(temp / "provider.json"),
|
|
"--allow-provider",
|
|
"demo.sensor",
|
|
]
|
|
)
|
|
self.finish(
|
|
case,
|
|
tampered,
|
|
1,
|
|
stderr_contains="ext.artifact_integrity",
|
|
)
|
|
finally:
|
|
shutil.rmtree(temp, ignore_errors=True)
|
|
|
|
case = self.case(
|
|
"negative_security", "pure and strict modes reject provider flags"
|
|
)
|
|
pure = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--mode",
|
|
"pure",
|
|
"--provider",
|
|
str(build / "provider.json"),
|
|
"--allow-provider",
|
|
"demo.sensor",
|
|
]
|
|
)
|
|
strict = self.run(
|
|
[
|
|
"run",
|
|
str(demo),
|
|
"--mode",
|
|
"strict",
|
|
"--provider",
|
|
str(build / "provider.json"),
|
|
"--allow-provider",
|
|
"demo.sensor",
|
|
]
|
|
)
|
|
ok_pure = self.finish(case, pure, 2)
|
|
if ok_pure:
|
|
strict_case = self.case(
|
|
"negative_security", "strict mode rejects provider flags"
|
|
)
|
|
self.finish(strict_case, strict, 2)
|
|
|
|
def check_tool(self) -> None:
|
|
case = self.case(
|
|
"tool_analysis_consumer", "independent tool consumes Analysis v2"
|
|
)
|
|
home = Path(tempfile.mkdtemp(prefix="ext-tool-home-"))
|
|
try:
|
|
env = {
|
|
"HOME": str(home),
|
|
"XDG_CONFIG_HOME": str(home / ".config"),
|
|
"XDG_DATA_HOME": str(home / ".local" / "share"),
|
|
}
|
|
install = self.run(
|
|
["tool", "install", str(ROOT / "tool" / "tool.json")], env=env
|
|
)
|
|
if not self.finish(case, install, 0):
|
|
return
|
|
case = self.case(
|
|
"tool_analysis_consumer", "registered tool command dispatches"
|
|
)
|
|
invoke = self.run(
|
|
["inspect-ext", str(ROOT / "tool" / "demo.mq.md")], env=env
|
|
)
|
|
ok = self.finish(case, invoke, 0)
|
|
if ok:
|
|
payload = json.loads(invoke.stdout)
|
|
functions = payload.get("functions", [])
|
|
ok = any(item.get("local_name") == "main" for item in functions)
|
|
if not ok:
|
|
case.status = "FAIL"
|
|
case.detail = f"tool saw no main function: {invoke.stdout[:500]}"
|
|
case = self.case(
|
|
"negative_security", "tool install rejects unknown entrypoint traversal"
|
|
)
|
|
bad = Path(tempfile.mkdtemp(prefix="ext-tool-bad-"))
|
|
try:
|
|
shutil.copytree(ROOT / "tool", bad, dirs_exist_ok=True)
|
|
bad_manifest = json.loads((bad / "tool.json").read_text())
|
|
bad_manifest["entrypoint"] = "../../escape.py"
|
|
(bad / "tool.json").write_text(json.dumps(bad_manifest))
|
|
rejected = self.run(["tool", "install", str(bad)], env=env)
|
|
self.finish(case, rejected, 1)
|
|
finally:
|
|
shutil.rmtree(bad, ignore_errors=True)
|
|
remove = self.run(["tool", "remove", "inspect-ext"], env=env)
|
|
self.finish(
|
|
self.case("tool_analysis_consumer", "tool remove unregisters"),
|
|
remove,
|
|
0,
|
|
)
|
|
finally:
|
|
shutil.rmtree(home, ignore_errors=True)
|
|
|
|
def check_package(self) -> None:
|
|
case = self.case(
|
|
"package_pinned_git", "pinned Git .mq.md library runs from a separate repo"
|
|
)
|
|
workspace = Path(tempfile.mkdtemp(prefix="ext-package-"))
|
|
try:
|
|
source = workspace / "math-src"
|
|
shutil.copytree(ROOT / "pure-lib", source)
|
|
git_commit(source, "math library snapshot")
|
|
rev = git_sha(source)
|
|
consumer = workspace / "consumer"
|
|
consumer.mkdir()
|
|
shutil.copy(ROOT / "consumer" / "index.mq.md", consumer / "index.mq.md")
|
|
(consumer / "marqdo.yaml").write_text(
|
|
"project:\n"
|
|
" name: ext-consumer\n"
|
|
" version: 0.1.0\n"
|
|
"dependencies:\n"
|
|
" math:\n"
|
|
f" git: \"file://{source}\"\n"
|
|
f" rev: \"{rev}\"\n"
|
|
)
|
|
lock = self.run(["lock"], cwd=consumer)
|
|
if not self.finish(case, lock, 0):
|
|
return
|
|
sync = self.run(["sync"], cwd=consumer)
|
|
if not self.finish(case, sync, 0):
|
|
return
|
|
case = self.case(
|
|
"package_pinned_git", "consumer runs the pinned dependency"
|
|
)
|
|
check = self.run(["check", "index.mq.md"], cwd=consumer)
|
|
ok = self.finish(case, check, 0)
|
|
run = self.run(["run", "index.mq.md", "--json"], cwd=consumer)
|
|
ok = ok and self.finish(case, run, 0)
|
|
if ok:
|
|
payload = json.loads(run.stdout)
|
|
if payload.get("value") != {"type": "int", "decimal": "18"}:
|
|
case.status = "FAIL"
|
|
case.detail = f"unexpected consumer value: {run.stdout[:500]}"
|
|
ok = False
|
|
if not ok:
|
|
return
|
|
# Offline/reproducibility: move the project, delete the source Git
|
|
# repository, and run again from the verified cache only.
|
|
moved = workspace / "moved-consumer"
|
|
shutil.move(str(consumer), moved)
|
|
shutil.rmtree(source)
|
|
case = self.case(
|
|
"package_pinned_git",
|
|
"verified cache still runs after moving the checkout and deleting the source",
|
|
)
|
|
offline = self.run(["run", "index.mq.md"], cwd=moved)
|
|
self.finish(case, offline, 0)
|
|
finally:
|
|
shutil.rmtree(workspace, ignore_errors=True)
|
|
|
|
# -- utilities -------------------------------------------------------
|
|
@staticmethod
|
|
def read_yaml_artifact(manifest: Path) -> str:
|
|
for line in manifest.read_text().splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith("file:"):
|
|
return stripped.split("file:", 1)[1].strip().strip('"')
|
|
raise SystemExit(f"manifest has no artifact file: {manifest}")
|
|
|
|
def report(self) -> int:
|
|
final_hash = sha256_file(self.cli)
|
|
if final_hash != self.cli_hash:
|
|
case = self.case("negative_security", "frozen CLI binary did not change")
|
|
case.status = "FAIL"
|
|
case.detail = f"sha256 changed {self.cli_hash} -> {final_hash}"
|
|
else:
|
|
case = self.case("negative_security", "frozen CLI binary did not change")
|
|
case.status = "PASS"
|
|
case.detail = f"sha256 {self.cli_hash}"
|
|
results: dict[str, str] = {}
|
|
for category in CATEGORIES:
|
|
cases = [item for item in self.cases if item.category == category]
|
|
if not cases:
|
|
results[category] = "UNSUPPORTED"
|
|
elif any(item.status == "FAIL" for item in cases):
|
|
results[category] = "FAIL"
|
|
elif all(item.status == "SKIPPED" for item in cases):
|
|
results[category] = "UNSUPPORTED"
|
|
elif any(item.status == "SKIPPED" for item in cases):
|
|
results[category] = "PASS" if all(
|
|
item.status in {"PASS", "SKIPPED"} for item in cases
|
|
) else "FAIL"
|
|
else:
|
|
results[category] = "PASS"
|
|
payload = {
|
|
"schema": "marqdo.ext-fixture-results",
|
|
"schema_version": 1,
|
|
"tool_sha": self.tool_sha,
|
|
"cli_binary_sha256": final_hash,
|
|
"os": os_name(),
|
|
"arch": platform.machine(),
|
|
"results": results,
|
|
"cases": [
|
|
{
|
|
"category": item.category,
|
|
"name": item.name,
|
|
"status": item.status,
|
|
"detail": item.detail,
|
|
"commands": item.commands,
|
|
"error_codes": item.error_codes,
|
|
}
|
|
for item in self.cases
|
|
],
|
|
}
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
RESULTS.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
|
|
failures = [item for item in self.cases if item.status == "FAIL"]
|
|
for item in self.cases:
|
|
marker = {"PASS": "PASS", "FAIL": "FAIL", "SKIPPED": "SKIP"}[item.status]
|
|
print(f"[{marker}] {item.category}: {item.name}")
|
|
if item.status != "PASS":
|
|
print(f" {item.detail}")
|
|
print(json.dumps(results, indent=2))
|
|
return 1 if failures else 0
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def git_sha(directory: Path) -> str:
|
|
output = subprocess.run(
|
|
["git", "-C", str(directory), "rev-parse", "HEAD"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return output.stdout.strip()
|
|
|
|
|
|
def git_commit(directory: Path, message: str) -> None:
|
|
def git(*args: str) -> None:
|
|
subprocess.run(
|
|
["git", "-C", str(directory), *args], check=True, capture_output=True
|
|
)
|
|
|
|
if not (directory / ".git").is_dir():
|
|
git("init", "-q", "-b", "main")
|
|
git("config", "user.name", "Marqdo Extension Fixtures")
|
|
git("config", "user.email", "fixtures@example.invalid")
|
|
git("add", "-A")
|
|
git("commit", "-q", "-m", message)
|
|
|
|
|
|
def os_name() -> str:
|
|
name = platform.system().lower()
|
|
if name.startswith("linux"):
|
|
return "linux"
|
|
if name == "darwin":
|
|
return "macos"
|
|
if name.startswith("windows"):
|
|
return "windows"
|
|
return name
|
|
|
|
|
|
def main() -> int:
|
|
raw = os.environ.get("MARQDO_BIN")
|
|
if not raw:
|
|
print("error: set MARQDO_BIN to the absolute frozen CLI path", file=sys.stderr)
|
|
return 2
|
|
cli = Path(raw).resolve()
|
|
if not cli.is_file():
|
|
print(f"error: MARQDO_BIN is not a file: {cli}", file=sys.stderr)
|
|
return 2
|
|
runner = Runner(cli)
|
|
print(f"frozen CLI: {cli} sha256 {runner.cli_hash}")
|
|
runner.build()
|
|
runner.check_plugin()
|
|
runner.check_kernel()
|
|
runner.check_provider()
|
|
runner.check_tool()
|
|
runner.check_package()
|
|
return runner.report()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|