CI / fmt + clippy + test (Linux x86_64) (push) Waiting to run
CI / schema instance validation + consumers (push) Waiting to run
CI / runtime contract (macos-latest) (push) Waiting to run
CI / runtime contract (windows-latest) (push) Waiting to run
Linux strict security lane (G-C) / strict isolation escape matrix (push) Waiting to run
ci/woodpecker/push/woodpecker Pipeline was successful
Add tools/ci/release_evidence.py (marqdo.release-gates.v1) plus unit tests: missing/skipped/FAIL/UNVERIFIED and tested_sha mismatches stay DO_NOT_RELEASE, performance may be PASS/WARN, unknown gate names are rejected. The GitHub CI now has a release-evidence job that maps the Linux/schema and native matrix results onto the manifest and uploads it, and both CI systems run the evidence-tool tests. Clear the contradictions the RFC calls out: remove the stale CURRENT.md rows that claimed DNS/TLS/UDP and fs metadata were absent while the same table said they were implemented; correct docs/ci.md to state that the GitHub-hosted Windows/macOS matrix exists (Woodpecker has no such agent); keep the old convergence plan historical; require the evidence manifest in releasing.md; and add a docs-consistency guard for these claims. This is step 6 of the CI/security RFC (B05/B06/B07).
95 lines
3.2 KiB
Python
95 lines
3.2 KiB
Python
import importlib.util
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
MODULE = Path(__file__).with_name("release_evidence.py")
|
|
spec = importlib.util.spec_from_file_location("release_evidence", MODULE)
|
|
release_evidence = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(release_evidence)
|
|
|
|
SHA = "a" * 40
|
|
OTHER = "b" * 40
|
|
|
|
|
|
def gate(name: str, state: str, sha: str = SHA, url: str = "https://example.test/run") -> str:
|
|
return f"{name}={state}:{sha}:{url}"
|
|
|
|
|
|
class ReleaseEvidenceTests(unittest.TestCase):
|
|
def test_all_required_pass_is_candidate(self) -> None:
|
|
manifest = release_evidence.build_manifest(
|
|
SHA,
|
|
[
|
|
gate("linux", "PASS"),
|
|
gate("windows", "PASS"),
|
|
gate("macos", "PASS"),
|
|
gate("strict_vm", "PASS"),
|
|
gate("performance", "WARN"),
|
|
],
|
|
)
|
|
self.assertEqual(manifest["release_decision"], "RELEASE_CANDIDATE")
|
|
self.assertEqual(manifest["reasons"], [])
|
|
|
|
def test_missing_strict_vm_is_not_release(self) -> None:
|
|
manifest = release_evidence.build_manifest(
|
|
SHA,
|
|
[
|
|
gate("linux", "PASS"),
|
|
gate("windows", "PASS"),
|
|
gate("macos", "PASS"),
|
|
gate("performance", "PASS"),
|
|
],
|
|
)
|
|
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
|
|
self.assertTrue(any("strict_vm" in reason for reason in manifest["reasons"]))
|
|
|
|
def test_sha_mismatch_is_not_release(self) -> None:
|
|
manifest = release_evidence.build_manifest(
|
|
SHA,
|
|
[
|
|
gate("linux", "PASS"),
|
|
gate("windows", "PASS", OTHER),
|
|
gate("macos", "PASS"),
|
|
gate("strict_vm", "PASS"),
|
|
gate("performance", "PASS"),
|
|
],
|
|
)
|
|
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
|
|
self.assertTrue(any("windows" in reason for reason in manifest["reasons"]))
|
|
|
|
def test_failure_is_not_release(self) -> None:
|
|
manifest = release_evidence.build_manifest(
|
|
SHA,
|
|
[
|
|
gate("linux", "FAIL"),
|
|
gate("windows", "PASS"),
|
|
gate("macos", "PASS"),
|
|
gate("strict_vm", "PASS"),
|
|
gate("performance", "PASS"),
|
|
],
|
|
)
|
|
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
|
|
|
|
def test_unverified_performance_is_not_release(self) -> None:
|
|
manifest = release_evidence.build_manifest(
|
|
SHA,
|
|
[
|
|
gate("linux", "PASS"),
|
|
gate("windows", "PASS"),
|
|
gate("macos", "PASS"),
|
|
gate("strict_vm", "PASS"),
|
|
gate("performance", "UNVERIFIED"),
|
|
],
|
|
)
|
|
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
|
|
|
|
def test_unknown_gate_and_bad_sha_are_rejected(self) -> None:
|
|
with self.assertRaises(ValueError):
|
|
release_evidence.build_manifest(SHA, [gate("solaris", "PASS")])
|
|
with self.assertRaises(ValueError):
|
|
release_evidence.build_manifest("not-a-sha", [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|