Files
marqdo-cli/tools/ci/test_release_evidence.py
T
cflmy 0d7e11b6f1
CI / fmt + clippy + test (Linux x86_64) (push) Waiting to run
CI / schema instance validation + consumers (push) Waiting to run
CI / runtime contract (macos-latest) (push) Waiting to run
CI / runtime contract (windows-latest) (push) Waiting to run
Linux strict security lane (G-C) / strict isolation escape matrix (push) Waiting to run
ci/woodpecker/push/woodpecker Pipeline was successful
ci: aggregate SHA-bound release evidence and clear status drift
Add tools/ci/release_evidence.py (marqdo.release-gates.v1) plus unit tests:
missing/skipped/FAIL/UNVERIFIED and tested_sha mismatches stay
DO_NOT_RELEASE, performance may be PASS/WARN, unknown gate names are rejected.
The GitHub CI now has a release-evidence job that maps the Linux/schema and
native matrix results onto the manifest and uploads it, and both CI systems
run the evidence-tool tests.

Clear the contradictions the RFC calls out: remove the stale CURRENT.md rows
that claimed DNS/TLS/UDP and fs metadata were absent while the same table said
they were implemented; correct docs/ci.md to state that the GitHub-hosted
Windows/macOS matrix exists (Woodpecker has no such agent); keep the old
convergence plan historical; require the evidence manifest in releasing.md;
and add a docs-consistency guard for these claims.

This is step 6 of the CI/security RFC (B05/B06/B07).
2026-11-21 17:29:48 +08:00

95 lines
3.2 KiB
Python

import importlib.util
import unittest
from pathlib import Path
MODULE = Path(__file__).with_name("release_evidence.py")
spec = importlib.util.spec_from_file_location("release_evidence", MODULE)
release_evidence = importlib.util.module_from_spec(spec)
spec.loader.exec_module(release_evidence)
SHA = "a" * 40
OTHER = "b" * 40
def gate(name: str, state: str, sha: str = SHA, url: str = "https://example.test/run") -> str:
return f"{name}={state}:{sha}:{url}"
class ReleaseEvidenceTests(unittest.TestCase):
def test_all_required_pass_is_candidate(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "PASS"),
gate("windows", "PASS"),
gate("macos", "PASS"),
gate("strict_vm", "PASS"),
gate("performance", "WARN"),
],
)
self.assertEqual(manifest["release_decision"], "RELEASE_CANDIDATE")
self.assertEqual(manifest["reasons"], [])
def test_missing_strict_vm_is_not_release(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "PASS"),
gate("windows", "PASS"),
gate("macos", "PASS"),
gate("performance", "PASS"),
],
)
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
self.assertTrue(any("strict_vm" in reason for reason in manifest["reasons"]))
def test_sha_mismatch_is_not_release(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "PASS"),
gate("windows", "PASS", OTHER),
gate("macos", "PASS"),
gate("strict_vm", "PASS"),
gate("performance", "PASS"),
],
)
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
self.assertTrue(any("windows" in reason for reason in manifest["reasons"]))
def test_failure_is_not_release(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "FAIL"),
gate("windows", "PASS"),
gate("macos", "PASS"),
gate("strict_vm", "PASS"),
gate("performance", "PASS"),
],
)
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
def test_unverified_performance_is_not_release(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "PASS"),
gate("windows", "PASS"),
gate("macos", "PASS"),
gate("strict_vm", "PASS"),
gate("performance", "UNVERIFIED"),
],
)
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
def test_unknown_gate_and_bad_sha_are_rejected(self) -> None:
with self.assertRaises(ValueError):
release_evidence.build_manifest(SHA, [gate("solaris", "PASS")])
with self.assertRaises(ValueError):
release_evidence.build_manifest("not-a-sha", [])
if __name__ == "__main__":
unittest.main()