diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f54639..50ef67a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,8 @@ jobs: run: cargo test -p marqdo-cli --test docs_consistency - name: Architecture boundaries run: cargo test -p marqdo-cli --test architecture_boundaries + - name: CI evidence tool tests + run: python3 -m unittest discover -s tools/ci -p "test_*.py" - name: Whole stdlib product tests run: cargo test -p marqdo-cli --test stdlib_product - name: Validate public JSON schemas @@ -144,6 +146,55 @@ jobs: test "$code" -eq 1 echo "$output" | grep -qi "strict" echo "$output" + - name: Strict JSON refusal is exactly one document + shell: bash + run: | + set -euo pipefail + json=$(cargo run -q -p marqdo-cli -- run examples/hello.mq.md --mode strict --json) + JSON_DOC="$json" python3 - <<'PY' + import json, os + document = json.loads(os.environ["JSON_DOC"]) + assert document["success"] is False, document + assert document["command"] == "run", document + assert document["code"] == "strict.probe_unavailable", document + PY + + release-evidence: + name: release evidence manifest + needs: [test, schema, cross-platform] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build SHA-bound manifest + shell: bash + env: + TARGET_SHA: ${{ github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + LINUX_RESULT: ${{ needs.test.result }} + SCHEMA_RESULT: ${{ needs.schema.result }} + PORTABLE_RESULT: ${{ needs.cross-platform.result }} + run: | + set -euo pipefail + state() { + case "$1" in + success) echo PASS ;; + failure) echo FAIL ;; + *) echo UNVERIFIED ;; + esac + } + linux="$(state "$LINUX_RESULT")" + portable="$(state "$PORTABLE_RESULT")" + if [ "$(state "$SCHEMA_RESULT")" != "PASS" ]; then + linux="FAIL" + fi + python3 tools/ci/release_evidence.py --target-sha "$TARGET_SHA" --out "docs/evidence/ci/$TARGET_SHA/manifest.json" --gate "linux=$linux:$TARGET_SHA:$RUN_URL" --gate "windows=$portable:$TARGET_SHA:$RUN_URL" --gate "macos=$portable:$TARGET_SHA:$RUN_URL" --gate "strict_vm=UNVERIFIED::${{ github.server_url }}/${{ github.repository }}/actions/workflows/security-linux.yml" --gate "performance=UNVERIFIED::" + - name: Upload release manifest + uses: actions/upload-artifact@v4 + with: + name: release-evidence-${{ github.sha }} + path: docs/evidence/ci/${{ github.sha }}/manifest.json + if-no-files-found: error release-candidate: name: release candidate artifact (${{ matrix.os }}) diff --git a/.woodpecker.yml b/.woodpecker.yml index 4f3ec00..2d5681e 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -10,7 +10,7 @@ steps: commands: - rustup component add clippy rustfmt - apt-get update - - apt-get install -y --no-install-recommends curl ca-certificates + - apt-get install -y --no-install-recommends curl ca-certificates python3 - curl -fsSL https://deb.nodesource.com/setup_22.x | bash - - apt-get install -y --no-install-recommends nodejs - cargo fmt --all -- --check @@ -18,6 +18,7 @@ steps: - cargo test --workspace --locked - cargo test -p marqdo-cli --test docs_consistency - cargo test -p marqdo-cli --test architecture_boundaries + - python3 -m unittest discover -s tools/ci -p "test_*.py" - cargo test -p marqdo-cli --test stdlib_product build: diff --git a/crates/cli/tests/docs_consistency.rs b/crates/cli/tests/docs_consistency.rs index 02644b3..2a44c30 100644 --- a/crates/cli/tests/docs_consistency.rs +++ b/crates/cli/tests/docs_consistency.rs @@ -122,6 +122,40 @@ fn current_relative_markdown_links_resolve() { } } +/// CI/security status drift is a defect: the current architecture table must +/// not simultaneously claim a network endpoint is implemented and absent, and +/// the CI doc must not claim the GitHub-hosted Windows/macOS matrix does not +/// exist (it does; Woodpecker has no such agent). +#[test] +fn ci_and_architecture_status_do_not_contradict_the_workflows() { + let root = repo_root(); + let current = read(&root.join("docs/architecture/CURRENT.md")); + assert!( + !current.contains("no DNS/TLS/server socket/UDP"), + "CURRENT.md repeats the stale TCP row that contradicts the network endpoint row" + ); + assert!( + !current.contains("file metadata / directory / rename / remove / mkdir"), + "CURRENT.md repeats the stale not-implemented filesystem row" + ); + + let ci = read(&root.join("docs/ci.md")); + assert!( + !ci.contains("no runners are configured"), + "docs/ci.md denies the existing GitHub-hosted Windows/macOS matrix" + ); + assert!( + ci.contains("windows-latest") && ci.contains("macos-latest"), + "docs/ci.md must name the native matrix it cannot replace with Woodpecker" + ); + + let historical = read(&root.join("docs/NEXT_RELEASE_CI_CONVERGENCE.md")); + assert!( + historical.contains("Historical / Superseded baseline"), + "the old convergence plan must keep its historical banner" + ); +} + #[test] fn evidence_files_are_sha_pinned() { let evidence = repo_root().join("docs/evidence"); diff --git a/docs/NEXT_RELEASE_CI_CONVERGENCE.md b/docs/NEXT_RELEASE_CI_CONVERGENCE.md index 2273faa..7999675 100644 --- a/docs/NEXT_RELEASE_CI_CONVERGENCE.md +++ b/docs/NEXT_RELEASE_CI_CONVERGENCE.md @@ -15,6 +15,13 @@ > 为目标语法并标注历史;现行规范见 `spec/mapping.md` / `spec/expression.ebnf`。 +> **CI status refresh (2026-11-21):** the GitHub-hosted Windows/macOS matrix +> exists in `.github/workflows/ci.yml`; it is not "unconfigured". The platform +> `E0425`/`E0433` defect and the strict `--json` empty-stdout defect are fixed in +> the RFC-CI-0001 series (ADR-0053). Those fixes do not turn this file's +> `c9f01aa8` rows into current evidence: re-run the matrix on the candidate SHA +> and record the result in `docs/evidence/ci//manifest.json`. +> > **文档性质:** 可直接入库的执行规范 / Release Convergence Plan。供未阅读任何历史聊天的维护者、贡献者与 DeepSeek Harness 使用。 > > **仓库:** `https://github.com/cflmy/marqdo-cli`;开发者的本地 Git 远端可能是 Gitee,不应混淆远端提交、GitHub 镜像与真实 CI 状态。 diff --git a/docs/architecture/CURRENT.md b/docs/architecture/CURRENT.md index 7511cf1..7af5b83 100644 --- a/docs/architecture/CURRENT.md +++ b/docs/architecture/CURRENT.md @@ -111,8 +111,6 @@ review RFC, not a new crate per provider. | `std:compress` | `compress.压缩/解压` -> gzip byte list <-> Text; 8 MiB caps | implemented (project mode; no grant) | `resource_streams`, `stdlib_product` | | network endpoints | `tcp`/`udp`/`tcp-listener`/`tls` open + `resource.accept`; exact host grant, DNS after grant, TLS certificate validation | implemented (project mode) | `resource_streams` | | `std:net/http` | byte-exact HTTP/1.1 parse/build; chunk sizes are byte counts; text wrapper compatible | implemented (pure `.mq.md`) | `stdlib_http` | -| file metadata / directory / rename / remove / mkdir | not part of the current Resource contract | **not implemented**; needs a separate Resource Operation ADR | — | -| TCP | `resource.open("tcp", ip:port, ["read","write"])`; exact IP grant, 5 s timeout | implemented; no DNS/TLS/server socket/UDP | `tcp_endpoint`, `http_over_tcp` | | HTTP/1.1 | `stdlib/net/http.mq.md`: status/header/chunked-byte parse, request build | implemented over an already-authorized TCP stream; not a DNS/TLS client | `stdlib_http`, `http_over_tcp`, `stdlib_product` | | print | bundled `io.mq.md`/`console.mq.md` through `resource.write`; legacy `HostOp::Print` removed | implemented (Compat-C) | `print_routing`, `stdlib_product` | | strict mode | only the pure/Math execution path and the built-in stdout console channel; project/foreign/plugin/tool/trace/file/network grants are rejected | implemented L1; L2 not implemented | `tests/security`, `resource_streams`, `stdlib_product` | diff --git a/docs/ci.md b/docs/ci.md index 255ae95..191c419 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -14,10 +14,11 @@ The default Docker pipeline runs on push and pull request: ```text test: rustup component add clippy rustfmt - apt install curl/ca-certificates + Node.js 22 + apt install curl/ca-certificates/python3 + Node.js 22 cargo fmt --all -- --check cargo clippy --workspace --all-targets -- -D warnings cargo test --workspace --locked + python3 -m unittest discover -s tools/ci -p "test_*.py" build (depends on test): cargo build --release --locked -p marqdo-cli @@ -38,10 +39,21 @@ syntax; Debian bookworm's Node 18 fails those tests. ## What this pipeline does not cover - **Strict security lane**: needs a disposable VM/self-hosted runner with - Bubblewrap/AppArmor namespace support. See [security-vm.md](security-vm.md). - The Docker pipeline is not an OS-isolation proof. -- **Windows/macOS native lanes**: no runners are configured. -- **Release publication/signing**: no release job is defined. + Bubblewrap/AppArmor namespace support. See [security-vm.md](security-vm.md) + and [security-gate.md](security-gate.md). The Docker pipeline is not an + OS-isolation proof. +- **Windows/macOS native lanes**: the GitHub-hosted + `.github/workflows/ci.yml` matrix **does exist and does run** on + `windows-latest` and `macos-latest`. The Woodpecker instance itself has no + Windows/macOS agent, so a Woodpecker-only green pipeline cannot substitute + for those native jobs. A remote Windows/macOS PASS requires the GitHub + matrix (or an equivalent real native runner) on the same SHA. +- **Performance baseline (G4)**: needs a fixed Linux node; see + [performance.md](performance.md). Development-container or TCG/QEMU timings + are `SMOKE`, never release evidence. +- **Release publication/signing**: no publication job is defined. The + `release-evidence` job produces a SHA-bound manifest artifact; it does not + tag or upload a release. ## Local reproduction diff --git a/docs/evidence/ci/README.md b/docs/evidence/ci/README.md new file mode 100644 index 0000000..39552ec --- /dev/null +++ b/docs/evidence/ci/README.md @@ -0,0 +1,21 @@ +# CI release-evidence manifests + +`docs/evidence/ci//manifest.json` is generated by +`tools/ci/release_evidence.py` (schema `marqdo.release-gates.v1`) and uploaded +by the `release-evidence` CI job. Do not hand-edit a manifest into `PASS`. + +Rules: + +* every gate records its own `tested_sha`; it must equal `target_sha`; +* skipped, cancelled, queued, timed-out, missing or older-SHA jobs are + `UNVERIFIED`/`FAIL`, never `PASS`; +* `strict_vm` needs a real disposable-VM `strict-evidence.json` (schema + `marqdo.strict-evidence.v1`) with `outcome: PASS`, non-root `uid` and the + same SHA; a URL alone is not enough; +* `performance` needs a `MEASURED` report from the fixed node; TCG/QEMU or + development-container numbers are not a baseline; +* `release_decision` is `RELEASE_CANDIDATE` only when every required gate is + PASS with the same SHA and performance is PASS/WARN. Anything else is + `DO_NOT_RELEASE`. + +The manifest is evidence, not permission to tag; owner approval is separate. diff --git a/docs/releasing.md b/docs/releasing.md index 4b54136..5266e5d 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -7,8 +7,12 @@ required CI matrix on the exact commit. ## 1. Preconditions - All applicable gates in - [`NEXT_RELEASE_CI_CONVERGENCE.md`](NEXT_RELEASE_CI_CONVERGENCE.md) §7 are - PASS with a CI URL for the same SHA. + [`STABLE_RELEASE_IMPLEMENTATION.md`](STABLE_RELEASE_IMPLEMENTATION.md) are + PASS, and `docs/evidence/ci//manifest.json` has + `release_decision: RELEASE_CANDIDATE` with every gate's `tested_sha` equal to + the release SHA. The manifest is produced by + `tools/ci/release_evidence.py`; a missing, skipped, timed-out or + SHA-mismatched gate is not PASS. - **Language migration is closed through the release SHA.** The source language is [RFC-LANG-0002](architecture/RFC-LANG-0002-unified-semantics.md); the release notes must state that this is a **breaking** change and that pre-RFC @@ -18,8 +22,14 @@ required CI matrix on the exact commit. release SHA. Platform and strict-lane evidence are still required before this is a releasable state; see [`STATUS.md`](STATUS.md) and [`migration/README.md`](migration/README.md). -- G-C is either PASS with a security-lane URL, or the release explicitly says - strict isolation is unavailable. Never claim G-C from local root/sudo runs. +- G-C is either PASS with a security-lane URL **and** a + `strict-evidence.json` whose `tested_sha` is the release SHA, or the release + explicitly says strict isolation is unavailable with owner approval. Never + claim G-C from a local root/sudo run, a container, or a TCG/QEMU timing run. + See [`security-gate.md`](security-gate.md) and [`security-vm.md`](security-vm.md). +- G4 performance is `MEASURED` on the fixed node with `--approved-node`, or + explicitly `UNVERIFIED`/`WARN` with the reason recorded. Never copy a + development-container or TCG number. See [`performance.md`](performance.md). - `git status --porcelain` is clean apart from intended release files. - `Cargo.lock` is committed and the build uses `--locked`. diff --git a/docs/security-vm.md b/docs/security-vm.md index a298e70..90de2da 100644 --- a/docs/security-vm.md +++ b/docs/security-vm.md @@ -5,12 +5,13 @@ development host is a container that cannot create namespaces. This document records how that VM was built so the result can be reproduced and so the VM can be registered as a dedicated self-hosted security runner. -> **Language-syntax migration note.** The lane fixtures are `.mq.md` sources and -> are migrated to RFC-LANG-0002 +> **Current evidence rule.** The lane fixtures are `.mq.md` sources under +> RFC-LANG-0002 > ([`architecture/RFC-LANG-0002-unified-semantics.md`](architecture/RFC-LANG-0002-unified-semantics.md)). -> The lane has not been re-run at `ffd1cc9`; run `tests/security/run-lane.sh` on -> the migrated tree before using this VM as evidence for the current syntax. The -> VM/OS-isolation procedure itself is unchanged. See [`STATUS.md`](STATUS.md). +> The lane is only evidence for the **exact SHA in its +> `strict-evidence.json`**; a PASS at an older SHA or on another host is +> historical. Every candidate release must rerun `tests/security/run-lane.sh` +> and archive the evidence document. See [`STATUS.md`](STATUS.md). **Do not run this on a daily host with valuable data.** The VM is disposable; the commands below do not disable AppArmor or any global host protection. @@ -28,14 +29,36 @@ gitignored and is not part of the repository. ## 2. Cloud image and seed +**Pin the base image.** Do not run a release gate from a mutable `current` +URL. Download one dated release, verify it against Ubuntu's published +`SHA256SUMS`, record the digest, and keep that file immutable: + ```bash mkdir -p .tools/vm && cd .tools/vm -curl -L -o resolute.img \ - https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img -qemu-img create -f qcow2 -b resolute.img -F qcow2 disk.qcow2 20G -ssh-keygen -t ed25519 -N '' -f marqdo_vm_key +# Replace with the reviewed dated release URL, never .../current/... +curl -L -o ubuntu-base.img \ + https://cloud-images.ubuntu.com/releases/26.04/release-20261120/ubuntu-26.04-server-cloudimg-amd64.img +curl -L -o SHA256SUMS \ + https://cloud-images.ubuntu.com/releases/26.04/release-20261120/SHA256SUMS +grep 'ubuntu-26.04-server-cloudimg-amd64.img' SHA256SUMS | sha256sum -c - +sha256sum ubuntu-base.img | tee ubuntu-base.img.sha256 +qemu-img create -f qcow2 -b ubuntu-base.img -F qcow2 \ + "strict-${SOURCE_SHA}.qcow2" 24G +ssh-keygen -t ed25519 -N '' -f "marqdo_vm_key-${SOURCE_SHA}" ``` +The local reference run used this base image; the digest belongs in the +evidence document as `runner_image_sha256`, not in a claim about a remote +runner: + +```text +ubuntu-base.img sha256: +7b90593f5532b49b7e3fe3d2743f0d1f0d822e5b07cebcd8e00c0df8069ecab9 +``` + +Create a fresh qcow2 overlay and a fresh cloud-init seed for every tested SHA. +Never reuse a writable overlay that may contain state from another run. + Create `user-data` with an ordinary `dev` user (passwordless sudo only for test setup), `meta-data` with a unique instance id, and wrap both into a NoCloud ISO with volume label `cidata`. Any ISO builder works; the reference @@ -115,37 +138,68 @@ globally. ## 5. Copy the source and run the lane -From the host repository root (or from `git archive `): +Build the archive from the reviewed commit, never from a dirty working tree: ```bash -tar --exclude=.git --exclude=.tools --exclude=.cache --exclude=.cargo-home \ - --exclude=target --exclude=dist -czf /tmp/marqdo-src.tgz . -scp -P 2222 -i .tools/vm/marqdo_vm_key /tmp/marqdo-src.tgz dev@127.0.0.1:/home/dev/ -ssh -p 2222 -i .tools/vm/marqdo_vm_key dev@127.0.0.1 +export SOURCE_SHA="$(git rev-parse HEAD)" # must be the reviewed SHA +git status --porcelain # must be empty +git archive --format=tar "$SOURCE_SHA" > "marqdo-${SOURCE_SHA}.tar" +sha256sum "marqdo-${SOURCE_SHA}.tar" | tee "marqdo-${SOURCE_SHA}.tar.sha256" +scp -P 2222 -i .tools/vm/marqdo_vm_key \ + "marqdo-${SOURCE_SHA}.tar" dev@127.0.0.1:/home/dev/ ``` -Inside the guest: +Inside the guest (ordinary user, not root): ```bash -mkdir -p ~/marqdo-src && tar -C ~/marqdo-src -xzf ~/marqdo-src.tgz -cd ~/marqdo-src +sed -n '1p' ~/marqdo-${SOURCE_SHA}.tar.sha256 +sha256sum ~/marqdo-${SOURCE_SHA}.tar +mkdir -p ~/marqdo-${SOURCE_SHA} +tar -C ~/marqdo-${SOURCE_SHA} -xf ~/marqdo-${SOURCE_SHA}.tar +cd ~/marqdo-${SOURCE_SHA} export PATH="$HOME/.cargo/bin:$PATH" +export MARQDO_TESTED_SHA="$SOURCE_SHA" +export MARQDO_SOURCE_ARCHIVE="$HOME/marqdo-${SOURCE_SHA}.tar" +export MARQDO_SOURCE_ARCHIVE_SHA256="sha256:" +export MARQDO_STRICT_EVIDENCE="$HOME/evidence-${SOURCE_SHA}/strict-evidence.json" +export MARQDO_STRICT_LANE_LOG="$HOME/evidence-${SOURCE_SHA}/gc-lane.log" +mkdir -p "$(dirname "$MARQDO_STRICT_EVIDENCE")" ./tests/security/run-lane.sh ``` -Expected final line: `G-C security lane: PASS`. The lane also checks that the read-only bundled stdlib works for `打印` and that a project-file `resource.open` is refused in strict mode. The script exports -`MARQDO_REQUIRE_STRICT_SECURITY=1`; a missing bubblewrap, compiler or helper is -a failure, never a skip. +Expected final line: `G-C security lane: PASS`, followed by the evidence path. +The evidence document records the tested SHA, archive digest, `uid`, `uname`, +bwrap/rustc/cargo versions, cgroup2 view, runner kind and outcome. A root-only +PASS is rejected. A missing bubblewrap, compiler, helper or skipped test is a +failure, never a skip. + +If the archive digest in the guest does not match the host digest, stop: the +transfer or the extraction is wrong and the run is not evidence. ## 6. Registering as a security runner Register this VM (not the development container) as a GitHub self-hosted runner with labels `linux` and `marqdo-security`. The workflow `.github/workflows/security-linux.yml` targets -`runs-on: [self-hosted, linux, marqdo-security]`. Keep the runner disposable -and recreate it from this document; do not give the runner access to secrets or -the primary workspace. A CI URL from that runner is still required before a -release may call G-C CI-verified. +`runs-on: [self-hosted, linux, marqdo-security]`. + +The runner must be **ephemeral**: + +* provision a fresh VM (or a self-hosted runner with a genuinely one-shot + lifetime) for each job, from a pinned base image; +* no persistent home, host mounts, Docker socket, SSH/Git/cloud credentials or + release token; only the short-lived runner registration credential; +* job `permissions: contents: read`, `actions/checkout` with + `persist-credentials: false` (already in the workflow); +* do not run arbitrary fork PRs on it: the workflow only runs on `master` + pushes or an explicit reviewed `source_sha`; +* destroy the whole VM after the job and keep the uploaded + `strict-evidence.json` and lane log. + +A PASS from a persistent runner with production credentials is not acceptable +evidence, even if the test output is green. If no ephemeral runner is +available, report "local disposable-VM PASS, remote G3 UNVERIFIED" instead of a +green CI claim. ## Reference environment diff --git a/tools/ci/README.md b/tools/ci/README.md new file mode 100644 index 0000000..87da2f3 --- /dev/null +++ b/tools/ci/README.md @@ -0,0 +1,37 @@ +# CI evidence tools + +## `strict-evidence.py` + +Writes `marqdo.strict-evidence.v1`, called by `tests/security/run-lane.sh` when +`MARQDO_STRICT_EVIDENCE` is set and by the self-hosted security workflow. It +records only measured or explicitly supplied facts: + +* `tested_sha` and whether it came from `git rev-parse` or an operator env; +* source archive SHA256 (and whether it matched the expected digest); +* kernel/uname/cgroup2/container, `uid`/root, bwrap/rustc/cargo/cc versions; +* runner kind, image digest, run URL, start/finish and outcome. + +A root-only `PASS` is rejected. `PASS` from a container or a persistent +self-hosted runner with production credentials is not valid G-C evidence; see +[`docs/security-vm.md`](../../docs/security-vm.md). + +## `release_evidence.py` + +Aggregates per-gate results into `marqdo.release-gates.v1`: + +```bash +python3 tools/ci/release_evidence.py \ + --target-sha "$SHA" \ + --gate "linux=PASS:$SHA:https://ci.example/run/1" \ + --gate "windows=PASS:$SHA:https://ci.example/run/2" \ + --gate "macos=PASS:$SHA:https://ci.example/run/3" \ + --gate "strict_vm=PASS:$SHA:https://ci.example/run/4" \ + --gate "performance=WARN:$SHA:https://ci.example/run/5" \ + --out "docs/evidence/ci/$SHA/manifest.json" +``` + +A missing gate becomes `UNVERIFIED`. Any non-PASS required gate, a +`tested_sha` different from `target_sha`, an unknown gate name or a malformed +SHA forces `DO_NOT_RELEASE`. `performance` may be `PASS` or `WARN`. + +Unit tests: `python3 -m unittest discover -s tools/ci -p 'test_*.py'`. diff --git a/tools/ci/release_evidence.py b/tools/ci/release_evidence.py new file mode 100755 index 0000000..9c40f16 --- /dev/null +++ b/tools/ci/release_evidence.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Aggregate per-gate CI evidence into a SHA-bound release manifest. + +Schema: marqdo.release-gates.v1. A missing, skipped, timed-out, failed or +SHA-mismatched gate is never PASS. The tool only records supplied facts; it +does not query CI and cannot turn an UNVERIFIED gate into a release. + +Gate spec: `name=STATE:tested_sha:run_url` (url may contain colons). +Required gates: linux, windows, macos, strict_vm; performance must be PASS or +WARN. Any other gate name is rejected so a typo cannot silently disappear. +""" + +import argparse +import datetime +import json +import re +import sys +from pathlib import Path + +SCHEMA = "marqdo.release-gates.v1" +REQUIRED = ("linux", "windows", "macos", "strict_vm") +OPTIONAL = ("performance",) +STATES = { + "linux": {"PASS", "FAIL", "UNVERIFIED"}, + "windows": {"PASS", "FAIL", "UNVERIFIED"}, + "macos": {"PASS", "FAIL", "UNVERIFIED"}, + "strict_vm": {"PASS", "FAIL", "UNVERIFIED"}, + "performance": {"PASS", "WARN", "FAIL", "UNVERIFIED"}, +} +SHA_RE = re.compile(r"^[0-9a-f]{40}$") + + +def utc_now() -> str: + return ( + datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + + +def parse_gate(text: str) -> tuple[str, dict]: + if "=" not in text: + raise ValueError(f"gate must be name=STATE:sha:url, got {text!r}") + name, rest = text.split("=", 1) + parts = rest.split(":", 2) + if len(parts) < 3: + raise ValueError(f"gate must be name=STATE:sha:url, got {text!r}") + state, tested_sha, run_url = parts + if name not in STATES: + raise ValueError(f"unknown gate {name!r}; expected {list(STATES)}") + if state not in STATES[name]: + raise ValueError(f"invalid state {state!r} for gate {name!r}") + if tested_sha and not SHA_RE.fullmatch(tested_sha): + raise ValueError(f"gate {name!r} tested_sha is not a full 40-hex SHA: {tested_sha!r}") + return name, {"state": state, "tested_sha": tested_sha, "run_url": run_url} + + +def build_manifest(target_sha: str, gates: list[str]) -> dict: + if not SHA_RE.fullmatch(target_sha): + raise ValueError(f"target SHA must be 40 lowercase hex characters: {target_sha!r}") + parsed = {} + for spec in gates: + name, gate = parse_gate(spec) + if name in parsed: + raise ValueError(f"duplicate gate {name!r}") + parsed[name] = gate + + reasons = [] + for name in REQUIRED + OPTIONAL: + gate = parsed.get(name) + if gate is None: + parsed[name] = {"state": "UNVERIFIED", "tested_sha": "", "run_url": ""} + reasons.append(f"{name}: missing gate") + continue + if gate["tested_sha"] != target_sha: + reasons.append( + f"{name}: tested_sha {gate['tested_sha'] or '(missing)'} != target_sha {target_sha}" + ) + elif gate["state"] == "FAIL": + reasons.append(f"{name}: FAIL") + elif gate["state"] == "UNVERIFIED": + reasons.append(f"{name}: UNVERIFIED") + elif name in REQUIRED and gate["state"] != "PASS": + reasons.append(f"{name}: {gate['state']} is not PASS") + elif name == "performance" and gate["state"] not in {"PASS", "WARN"}: + reasons.append(f"performance: {gate['state']} is not PASS/WARN") + + decision = "RELEASE_CANDIDATE" if not reasons else "DO_NOT_RELEASE" + recorded = utc_now() + return { + "schema": SCHEMA, + "target_sha": target_sha, + "recorded_utc": recorded, + "gates": { + name: { + **gate, + "recorded_utc": recorded, + } + for name, gate in parsed.items() + }, + "release_decision": decision, + "reasons": reasons, + } + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target-sha", required=True) + parser.add_argument("--gate", action="append", default=[]) + parser.add_argument("--out", default="") + args = parser.parse_args(argv) + try: + manifest = build_manifest(args.target_sha, args.gate) + except ValueError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + text = json.dumps(manifest, ensure_ascii=False, indent=2) + "\n" + if args.out: + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(text) + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tools/ci/test_release_evidence.py b/tools/ci/test_release_evidence.py new file mode 100644 index 0000000..8907779 --- /dev/null +++ b/tools/ci/test_release_evidence.py @@ -0,0 +1,94 @@ +import importlib.util +import unittest +from pathlib import Path + +MODULE = Path(__file__).with_name("release_evidence.py") +spec = importlib.util.spec_from_file_location("release_evidence", MODULE) +release_evidence = importlib.util.module_from_spec(spec) +spec.loader.exec_module(release_evidence) + +SHA = "a" * 40 +OTHER = "b" * 40 + + +def gate(name: str, state: str, sha: str = SHA, url: str = "https://example.test/run") -> str: + return f"{name}={state}:{sha}:{url}" + + +class ReleaseEvidenceTests(unittest.TestCase): + def test_all_required_pass_is_candidate(self) -> None: + manifest = release_evidence.build_manifest( + SHA, + [ + gate("linux", "PASS"), + gate("windows", "PASS"), + gate("macos", "PASS"), + gate("strict_vm", "PASS"), + gate("performance", "WARN"), + ], + ) + self.assertEqual(manifest["release_decision"], "RELEASE_CANDIDATE") + self.assertEqual(manifest["reasons"], []) + + def test_missing_strict_vm_is_not_release(self) -> None: + manifest = release_evidence.build_manifest( + SHA, + [ + gate("linux", "PASS"), + gate("windows", "PASS"), + gate("macos", "PASS"), + gate("performance", "PASS"), + ], + ) + self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE") + self.assertTrue(any("strict_vm" in reason for reason in manifest["reasons"])) + + def test_sha_mismatch_is_not_release(self) -> None: + manifest = release_evidence.build_manifest( + SHA, + [ + gate("linux", "PASS"), + gate("windows", "PASS", OTHER), + gate("macos", "PASS"), + gate("strict_vm", "PASS"), + gate("performance", "PASS"), + ], + ) + self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE") + self.assertTrue(any("windows" in reason for reason in manifest["reasons"])) + + def test_failure_is_not_release(self) -> None: + manifest = release_evidence.build_manifest( + SHA, + [ + gate("linux", "FAIL"), + gate("windows", "PASS"), + gate("macos", "PASS"), + gate("strict_vm", "PASS"), + gate("performance", "PASS"), + ], + ) + self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE") + + def test_unverified_performance_is_not_release(self) -> None: + manifest = release_evidence.build_manifest( + SHA, + [ + gate("linux", "PASS"), + gate("windows", "PASS"), + gate("macos", "PASS"), + gate("strict_vm", "PASS"), + gate("performance", "UNVERIFIED"), + ], + ) + self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE") + + def test_unknown_gate_and_bad_sha_are_rejected(self) -> None: + with self.assertRaises(ValueError): + release_evidence.build_manifest(SHA, [gate("solaris", "PASS")]) + with self.assertRaises(ValueError): + release_evidence.build_manifest("not-a-sha", []) + + +if __name__ == "__main__": + unittest.main()