ci: aggregate SHA-bound release evidence and clear status drift
CI / fmt + clippy + test (Linux x86_64) (push) Waiting to run
CI / schema instance validation + consumers (push) Waiting to run
CI / runtime contract (macos-latest) (push) Waiting to run
CI / runtime contract (windows-latest) (push) Waiting to run
Linux strict security lane (G-C) / strict isolation escape matrix (push) Waiting to run
ci/woodpecker/push/woodpecker Pipeline was successful

Add tools/ci/release_evidence.py (marqdo.release-gates.v1) plus unit tests:
missing/skipped/FAIL/UNVERIFIED and tested_sha mismatches stay
DO_NOT_RELEASE, performance may be PASS/WARN, unknown gate names are rejected.
The GitHub CI now has a release-evidence job that maps the Linux/schema and
native matrix results onto the manifest and uploads it, and both CI systems
run the evidence-tool tests.

Clear the contradictions the RFC calls out: remove the stale CURRENT.md rows
that claimed DNS/TLS/UDP and fs metadata were absent while the same table said
they were implemented; correct docs/ci.md to state that the GitHub-hosted
Windows/macOS matrix exists (Woodpecker has no such agent); keep the old
convergence plan historical; require the evidence manifest in releasing.md;
and add a docs-consistency guard for these claims.

This is step 6 of the CI/security RFC (B05/B06/B07).
This commit is contained in:
cflmy committed 2026-11-21 17:29:48 +08:00
1 parent d2267e912a
commit 0d7e11b6f1
12 files changed
+483 -36

No files matched your search

+51
View File
@@ -29,6 +29,8 @@ jobs:
run: cargo test -p marqdo-cli --test docs_consistency run: cargo test -p marqdo-cli --test docs_consistency
- name: Architecture boundaries - name: Architecture boundaries
run: cargo test -p marqdo-cli --test architecture_boundaries run: cargo test -p marqdo-cli --test architecture_boundaries
- name: CI evidence tool tests
run: python3 -m unittest discover -s tools/ci -p "test_*.py"
- name: Whole stdlib product tests - name: Whole stdlib product tests
run: cargo test -p marqdo-cli --test stdlib_product run: cargo test -p marqdo-cli --test stdlib_product
- name: Validate public JSON schemas - name: Validate public JSON schemas
@@ -144,6 +146,55 @@ jobs:
test "$code" -eq 1 test "$code" -eq 1
echo "$output" | grep -qi "strict" echo "$output" | grep -qi "strict"
echo "$output" echo "$output"
- name: Strict JSON refusal is exactly one document
shell: bash
run: |
set -euo pipefail
json=$(cargo run -q -p marqdo-cli -- run examples/hello.mq.md --mode strict --json)
JSON_DOC="$json" python3 - <<'PY'
import json, os
document = json.loads(os.environ["JSON_DOC"])
assert document["success"] is False, document
assert document["command"] == "run", document
assert document["code"] == "strict.probe_unavailable", document
PY
release-evidence:
name: release evidence manifest
needs: [test, schema, cross-platform]
if: always()
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build SHA-bound manifest
shell: bash
env:
TARGET_SHA: ${{ github.sha }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
LINUX_RESULT: ${{ needs.test.result }}
SCHEMA_RESULT: ${{ needs.schema.result }}
PORTABLE_RESULT: ${{ needs.cross-platform.result }}
run: |
set -euo pipefail
state() {
case "$1" in
success) echo PASS ;;
failure) echo FAIL ;;
*) echo UNVERIFIED ;;
esac
}
linux="$(state "$LINUX_RESULT")"
portable="$(state "$PORTABLE_RESULT")"
if [ "$(state "$SCHEMA_RESULT")" != "PASS" ]; then
linux="FAIL"
fi
python3 tools/ci/release_evidence.py --target-sha "$TARGET_SHA" --out "docs/evidence/ci/$TARGET_SHA/manifest.json" --gate "linux=$linux:$TARGET_SHA:$RUN_URL" --gate "windows=$portable:$TARGET_SHA:$RUN_URL" --gate "macos=$portable:$TARGET_SHA:$RUN_URL" --gate "strict_vm=UNVERIFIED::${{ github.server_url }}/${{ github.repository }}/actions/workflows/security-linux.yml" --gate "performance=UNVERIFIED::"
- name: Upload release manifest
uses: actions/upload-artifact@v4
with:
name: release-evidence-${{ github.sha }}
path: docs/evidence/ci/${{ github.sha }}/manifest.json
if-no-files-found: error
release-candidate: release-candidate:
name: release candidate artifact (${{ matrix.os }}) name: release candidate artifact (${{ matrix.os }})
+2 -1
View File
@@ -10,7 +10,7 @@ steps:
commands: commands:
- rustup component add clippy rustfmt - rustup component add clippy rustfmt
- apt-get update - apt-get update
- apt-get install -y --no-install-recommends curl ca-certificates - apt-get install -y --no-install-recommends curl ca-certificates python3
- curl -fsSL https://deb.nodesource.com/setup_22.x | bash - - curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
- apt-get install -y --no-install-recommends nodejs - apt-get install -y --no-install-recommends nodejs
- cargo fmt --all -- --check - cargo fmt --all -- --check
@@ -18,6 +18,7 @@ steps:
- cargo test --workspace --locked - cargo test --workspace --locked
- cargo test -p marqdo-cli --test docs_consistency - cargo test -p marqdo-cli --test docs_consistency
- cargo test -p marqdo-cli --test architecture_boundaries - cargo test -p marqdo-cli --test architecture_boundaries
- python3 -m unittest discover -s tools/ci -p "test_*.py"
- cargo test -p marqdo-cli --test stdlib_product - cargo test -p marqdo-cli --test stdlib_product
build: build:
+34
View File
@@ -122,6 +122,40 @@ fn current_relative_markdown_links_resolve() {
} }
} }
/// CI/security status drift is a defect: the current architecture table must
/// not simultaneously claim a network endpoint is implemented and absent, and
/// the CI doc must not claim the GitHub-hosted Windows/macOS matrix does not
/// exist (it does; Woodpecker has no such agent).
#[test]
fn ci_and_architecture_status_do_not_contradict_the_workflows() {
let root = repo_root();
let current = read(&root.join("docs/architecture/CURRENT.md"));
assert!(
!current.contains("no DNS/TLS/server socket/UDP"),
"CURRENT.md repeats the stale TCP row that contradicts the network endpoint row"
);
assert!(
!current.contains("file metadata / directory / rename / remove / mkdir"),
"CURRENT.md repeats the stale not-implemented filesystem row"
);
let ci = read(&root.join("docs/ci.md"));
assert!(
!ci.contains("no runners are configured"),
"docs/ci.md denies the existing GitHub-hosted Windows/macOS matrix"
);
assert!(
ci.contains("windows-latest") && ci.contains("macos-latest"),
"docs/ci.md must name the native matrix it cannot replace with Woodpecker"
);
let historical = read(&root.join("docs/NEXT_RELEASE_CI_CONVERGENCE.md"));
assert!(
historical.contains("Historical / Superseded baseline"),
"the old convergence plan must keep its historical banner"
);
}
#[test] #[test]
fn evidence_files_are_sha_pinned() { fn evidence_files_are_sha_pinned() {
let evidence = repo_root().join("docs/evidence"); let evidence = repo_root().join("docs/evidence");
+7
View File
@@ -15,6 +15,13 @@
> 为目标语法并标注历史;现行规范见 `spec/mapping.md` / `spec/expression.ebnf`。 > 为目标语法并标注历史;现行规范见 `spec/mapping.md` / `spec/expression.ebnf`。
> **CI status refresh (2026-11-21):** the GitHub-hosted Windows/macOS matrix
> exists in `.github/workflows/ci.yml`; it is not "unconfigured". The platform
> `E0425`/`E0433` defect and the strict `--json` empty-stdout defect are fixed in
> the RFC-CI-0001 series (ADR-0053). Those fixes do not turn this file's
> `c9f01aa8` rows into current evidence: re-run the matrix on the candidate SHA
> and record the result in `docs/evidence/ci/<SHA>/manifest.json`.
>
> **文档性质:** 可直接入库的执行规范 / Release Convergence Plan。供未阅读任何历史聊天的维护者、贡献者与 DeepSeek Harness 使用。 > **文档性质:** 可直接入库的执行规范 / Release Convergence Plan。供未阅读任何历史聊天的维护者、贡献者与 DeepSeek Harness 使用。
> >
> **仓库:** `https://github.com/cflmy/marqdo-cli`;开发者的本地 Git 远端可能是 Gitee,不应混淆远端提交、GitHub 镜像与真实 CI 状态。 > **仓库:** `https://github.com/cflmy/marqdo-cli`;开发者的本地 Git 远端可能是 Gitee,不应混淆远端提交、GitHub 镜像与真实 CI 状态。
-2
View File
@@ -111,8 +111,6 @@ review RFC, not a new crate per provider.
| `std:compress` | `compress.压缩/解压` -> gzip byte list <-> Text; 8 MiB caps | implemented (project mode; no grant) | `resource_streams`, `stdlib_product` | | `std:compress` | `compress.压缩/解压` -> gzip byte list <-> Text; 8 MiB caps | implemented (project mode; no grant) | `resource_streams`, `stdlib_product` |
| network endpoints | `tcp`/`udp`/`tcp-listener`/`tls` open + `resource.accept`; exact host grant, DNS after grant, TLS certificate validation | implemented (project mode) | `resource_streams` | | network endpoints | `tcp`/`udp`/`tcp-listener`/`tls` open + `resource.accept`; exact host grant, DNS after grant, TLS certificate validation | implemented (project mode) | `resource_streams` |
| `std:net/http` | byte-exact HTTP/1.1 parse/build; chunk sizes are byte counts; text wrapper compatible | implemented (pure `.mq.md`) | `stdlib_http` | | `std:net/http` | byte-exact HTTP/1.1 parse/build; chunk sizes are byte counts; text wrapper compatible | implemented (pure `.mq.md`) | `stdlib_http` |
| file metadata / directory / rename / remove / mkdir | not part of the current Resource contract | **not implemented**; needs a separate Resource Operation ADR | — |
| TCP | `resource.open("tcp", ip:port, ["read","write"])`; exact IP grant, 5 s timeout | implemented; no DNS/TLS/server socket/UDP | `tcp_endpoint`, `http_over_tcp` |
| HTTP/1.1 | `stdlib/net/http.mq.md`: status/header/chunked-byte parse, request build | implemented over an already-authorized TCP stream; not a DNS/TLS client | `stdlib_http`, `http_over_tcp`, `stdlib_product` | | HTTP/1.1 | `stdlib/net/http.mq.md`: status/header/chunked-byte parse, request build | implemented over an already-authorized TCP stream; not a DNS/TLS client | `stdlib_http`, `http_over_tcp`, `stdlib_product` |
| print | bundled `io.mq.md`/`console.mq.md` through `resource.write`; legacy `HostOp::Print` removed | implemented (Compat-C) | `print_routing`, `stdlib_product` | | print | bundled `io.mq.md`/`console.mq.md` through `resource.write`; legacy `HostOp::Print` removed | implemented (Compat-C) | `print_routing`, `stdlib_product` |
| strict mode | only the pure/Math execution path and the built-in stdout console channel; project/foreign/plugin/tool/trace/file/network grants are rejected | implemented L1; L2 not implemented | `tests/security`, `resource_streams`, `stdlib_product` | | strict mode | only the pure/Math execution path and the built-in stdout console channel; project/foreign/plugin/tool/trace/file/network grants are rejected | implemented L1; L2 not implemented | `tests/security`, `resource_streams`, `stdlib_product` |
+17 -5
View File
@@ -14,10 +14,11 @@ The default Docker pipeline runs on push and pull request:
```text ```text
test: test:
rustup component add clippy rustfmt rustup component add clippy rustfmt
apt install curl/ca-certificates + Node.js 22 apt install curl/ca-certificates/python3 + Node.js 22
cargo fmt --all -- --check cargo fmt --all -- --check
cargo clippy --workspace --all-targets -- -D warnings cargo clippy --workspace --all-targets -- -D warnings
cargo test --workspace --locked cargo test --workspace --locked
python3 -m unittest discover -s tools/ci -p "test_*.py"
build (depends on test): build (depends on test):
cargo build --release --locked -p marqdo-cli cargo build --release --locked -p marqdo-cli
@@ -38,10 +39,21 @@ syntax; Debian bookworm's Node 18 fails those tests.
## What this pipeline does not cover ## What this pipeline does not cover
- **Strict security lane**: needs a disposable VM/self-hosted runner with - **Strict security lane**: needs a disposable VM/self-hosted runner with
Bubblewrap/AppArmor namespace support. See [security-vm.md](security-vm.md). Bubblewrap/AppArmor namespace support. See [security-vm.md](security-vm.md)
The Docker pipeline is not an OS-isolation proof. and [security-gate.md](security-gate.md). The Docker pipeline is not an
- **Windows/macOS native lanes**: no runners are configured. OS-isolation proof.
- **Release publication/signing**: no release job is defined. - **Windows/macOS native lanes**: the GitHub-hosted
`.github/workflows/ci.yml` matrix **does exist and does run** on
`windows-latest` and `macos-latest`. The Woodpecker instance itself has no
Windows/macOS agent, so a Woodpecker-only green pipeline cannot substitute
for those native jobs. A remote Windows/macOS PASS requires the GitHub
matrix (or an equivalent real native runner) on the same SHA.
- **Performance baseline (G4)**: needs a fixed Linux node; see
[performance.md](performance.md). Development-container or TCG/QEMU timings
are `SMOKE`, never release evidence.
- **Release publication/signing**: no publication job is defined. The
`release-evidence` job produces a SHA-bound manifest artifact; it does not
tag or upload a release.
## Local reproduction ## Local reproduction
+21
View File
@@ -0,0 +1,21 @@
# CI release-evidence manifests
`docs/evidence/ci/<full-40-hex-sha>/manifest.json` is generated by
`tools/ci/release_evidence.py` (schema `marqdo.release-gates.v1`) and uploaded
by the `release-evidence` CI job. Do not hand-edit a manifest into `PASS`.
Rules:
* every gate records its own `tested_sha`; it must equal `target_sha`;
* skipped, cancelled, queued, timed-out, missing or older-SHA jobs are
`UNVERIFIED`/`FAIL`, never `PASS`;
* `strict_vm` needs a real disposable-VM `strict-evidence.json` (schema
`marqdo.strict-evidence.v1`) with `outcome: PASS`, non-root `uid` and the
same SHA; a URL alone is not enough;
* `performance` needs a `MEASURED` report from the fixed node; TCG/QEMU or
development-container numbers are not a baseline;
* `release_decision` is `RELEASE_CANDIDATE` only when every required gate is
PASS with the same SHA and performance is PASS/WARN. Anything else is
`DO_NOT_RELEASE`.
The manifest is evidence, not permission to tag; owner approval is separate.
+14 -4
View File
@@ -7,8 +7,12 @@ required CI matrix on the exact commit.
## 1. Preconditions ## 1. Preconditions
- All applicable gates in - All applicable gates in
[`NEXT_RELEASE_CI_CONVERGENCE.md`](NEXT_RELEASE_CI_CONVERGENCE.md) §7 are [`STABLE_RELEASE_IMPLEMENTATION.md`](STABLE_RELEASE_IMPLEMENTATION.md) are
PASS with a CI URL for the same SHA. PASS, and `docs/evidence/ci/<SHA>/manifest.json` has
`release_decision: RELEASE_CANDIDATE` with every gate's `tested_sha` equal to
the release SHA. The manifest is produced by
`tools/ci/release_evidence.py`; a missing, skipped, timed-out or
SHA-mismatched gate is not PASS.
- **Language migration is closed through the release SHA.** The source language - **Language migration is closed through the release SHA.** The source language
is [RFC-LANG-0002](architecture/RFC-LANG-0002-unified-semantics.md); the is [RFC-LANG-0002](architecture/RFC-LANG-0002-unified-semantics.md); the
release notes must state that this is a **breaking** change and that pre-RFC release notes must state that this is a **breaking** change and that pre-RFC
@@ -18,8 +22,14 @@ required CI matrix on the exact commit.
release SHA. Platform and strict-lane evidence are still required before this release SHA. Platform and strict-lane evidence are still required before this
is a releasable state; see [`STATUS.md`](STATUS.md) and is a releasable state; see [`STATUS.md`](STATUS.md) and
[`migration/README.md`](migration/README.md). [`migration/README.md`](migration/README.md).
- G-C is either PASS with a security-lane URL, or the release explicitly says - G-C is either PASS with a security-lane URL **and** a
strict isolation is unavailable. Never claim G-C from local root/sudo runs. `strict-evidence.json` whose `tested_sha` is the release SHA, or the release
explicitly says strict isolation is unavailable with owner approval. Never
claim G-C from a local root/sudo run, a container, or a TCG/QEMU timing run.
See [`security-gate.md`](security-gate.md) and [`security-vm.md`](security-vm.md).
- G4 performance is `MEASURED` on the fixed node with `--approved-node`, or
explicitly `UNVERIFIED`/`WARN` with the reason recorded. Never copy a
development-container or TCG number. See [`performance.md`](performance.md).
- `git status --porcelain` is clean apart from intended release files. - `git status --porcelain` is clean apart from intended release files.
- `Cargo.lock` is committed and the build uses `--locked`. - `Cargo.lock` is committed and the build uses `--locked`.
+78 -24
View File
@@ -5,12 +5,13 @@ development host is a container that cannot create namespaces. This document
records how that VM was built so the result can be reproduced and so the VM can records how that VM was built so the result can be reproduced and so the VM can
be registered as a dedicated self-hosted security runner. be registered as a dedicated self-hosted security runner.
> **Language-syntax migration note.** The lane fixtures are `.mq.md` sources and > **Current evidence rule.** The lane fixtures are `.mq.md` sources under
> are migrated to RFC-LANG-0002 > RFC-LANG-0002
> ([`architecture/RFC-LANG-0002-unified-semantics.md`](architecture/RFC-LANG-0002-unified-semantics.md)). > ([`architecture/RFC-LANG-0002-unified-semantics.md`](architecture/RFC-LANG-0002-unified-semantics.md)).
> The lane has not been re-run at `ffd1cc9`; run `tests/security/run-lane.sh` on > The lane is only evidence for the **exact SHA in its
> the migrated tree before using this VM as evidence for the current syntax. The > `strict-evidence.json`**; a PASS at an older SHA or on another host is
> VM/OS-isolation procedure itself is unchanged. See [`STATUS.md`](STATUS.md). > historical. Every candidate release must rerun `tests/security/run-lane.sh`
> and archive the evidence document. See [`STATUS.md`](STATUS.md).
**Do not run this on a daily host with valuable data.** The VM is disposable; **Do not run this on a daily host with valuable data.** The VM is disposable;
the commands below do not disable AppArmor or any global host protection. the commands below do not disable AppArmor or any global host protection.
@@ -28,14 +29,36 @@ gitignored and is not part of the repository.
## 2. Cloud image and seed ## 2. Cloud image and seed
**Pin the base image.** Do not run a release gate from a mutable `current`
URL. Download one dated release, verify it against Ubuntu's published
`SHA256SUMS`, record the digest, and keep that file immutable:
```bash ```bash
mkdir -p .tools/vm && cd .tools/vm mkdir -p .tools/vm && cd .tools/vm
curl -L -o resolute.img \ # Replace with the reviewed dated release URL, never .../current/...
https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img curl -L -o ubuntu-base.img \
qemu-img create -f qcow2 -b resolute.img -F qcow2 disk.qcow2 20G https://cloud-images.ubuntu.com/releases/26.04/release-20261120/ubuntu-26.04-server-cloudimg-amd64.img
ssh-keygen -t ed25519 -N '' -f marqdo_vm_key curl -L -o SHA256SUMS \
https://cloud-images.ubuntu.com/releases/26.04/release-20261120/SHA256SUMS
grep 'ubuntu-26.04-server-cloudimg-amd64.img' SHA256SUMS | sha256sum -c -
sha256sum ubuntu-base.img | tee ubuntu-base.img.sha256
qemu-img create -f qcow2 -b ubuntu-base.img -F qcow2 \
"strict-${SOURCE_SHA}.qcow2" 24G
ssh-keygen -t ed25519 -N '' -f "marqdo_vm_key-${SOURCE_SHA}"
``` ```
The local reference run used this base image; the digest belongs in the
evidence document as `runner_image_sha256`, not in a claim about a remote
runner:
```text
ubuntu-base.img sha256:
7b90593f5532b49b7e3fe3d2743f0d1f0d822e5b07cebcd8e00c0df8069ecab9
```
Create a fresh qcow2 overlay and a fresh cloud-init seed for every tested SHA.
Never reuse a writable overlay that may contain state from another run.
Create `user-data` with an ordinary `dev` user (passwordless sudo only for Create `user-data` with an ordinary `dev` user (passwordless sudo only for
test setup), `meta-data` with a unique instance id, and wrap both into a test setup), `meta-data` with a unique instance id, and wrap both into a
NoCloud ISO with volume label `cidata`. Any ISO builder works; the reference NoCloud ISO with volume label `cidata`. Any ISO builder works; the reference
@@ -115,37 +138,68 @@ globally.
## 5. Copy the source and run the lane ## 5. Copy the source and run the lane
From the host repository root (or from `git archive <sha>`): Build the archive from the reviewed commit, never from a dirty working tree:
```bash ```bash
tar --exclude=.git --exclude=.tools --exclude=.cache --exclude=.cargo-home \ export SOURCE_SHA="$(git rev-parse HEAD)" # must be the reviewed SHA
--exclude=target --exclude=dist -czf /tmp/marqdo-src.tgz . git status --porcelain # must be empty
scp -P 2222 -i .tools/vm/marqdo_vm_key /tmp/marqdo-src.tgz dev@127.0.0.1:/home/dev/ git archive --format=tar "$SOURCE_SHA" > "marqdo-${SOURCE_SHA}.tar"
ssh -p 2222 -i .tools/vm/marqdo_vm_key dev@127.0.0.1 sha256sum "marqdo-${SOURCE_SHA}.tar" | tee "marqdo-${SOURCE_SHA}.tar.sha256"
scp -P 2222 -i .tools/vm/marqdo_vm_key \
"marqdo-${SOURCE_SHA}.tar" dev@127.0.0.1:/home/dev/
``` ```
Inside the guest: Inside the guest (ordinary user, not root):
```bash ```bash
mkdir -p ~/marqdo-src && tar -C ~/marqdo-src -xzf ~/marqdo-src.tgz sed -n '1p' ~/marqdo-${SOURCE_SHA}.tar.sha256
cd ~/marqdo-src sha256sum ~/marqdo-${SOURCE_SHA}.tar
mkdir -p ~/marqdo-${SOURCE_SHA}
tar -C ~/marqdo-${SOURCE_SHA} -xf ~/marqdo-${SOURCE_SHA}.tar
cd ~/marqdo-${SOURCE_SHA}
export PATH="$HOME/.cargo/bin:$PATH" export PATH="$HOME/.cargo/bin:$PATH"
export MARQDO_TESTED_SHA="$SOURCE_SHA"
export MARQDO_SOURCE_ARCHIVE="$HOME/marqdo-${SOURCE_SHA}.tar"
export MARQDO_SOURCE_ARCHIVE_SHA256="sha256:<digest from the host>"
export MARQDO_STRICT_EVIDENCE="$HOME/evidence-${SOURCE_SHA}/strict-evidence.json"
export MARQDO_STRICT_LANE_LOG="$HOME/evidence-${SOURCE_SHA}/gc-lane.log"
mkdir -p "$(dirname "$MARQDO_STRICT_EVIDENCE")"
./tests/security/run-lane.sh ./tests/security/run-lane.sh
``` ```
Expected final line: `G-C security lane: PASS`. The lane also checks that the read-only bundled stdlib works for `打印` and that a project-file `resource.open` is refused in strict mode. The script exports Expected final line: `G-C security lane: PASS`, followed by the evidence path.
`MARQDO_REQUIRE_STRICT_SECURITY=1`; a missing bubblewrap, compiler or helper is The evidence document records the tested SHA, archive digest, `uid`, `uname`,
a failure, never a skip. bwrap/rustc/cargo versions, cgroup2 view, runner kind and outcome. A root-only
PASS is rejected. A missing bubblewrap, compiler, helper or skipped test is a
failure, never a skip.
If the archive digest in the guest does not match the host digest, stop: the
transfer or the extraction is wrong and the run is not evidence.
## 6. Registering as a security runner ## 6. Registering as a security runner
Register this VM (not the development container) as a GitHub self-hosted Register this VM (not the development container) as a GitHub self-hosted
runner with labels `linux` and `marqdo-security`. The workflow runner with labels `linux` and `marqdo-security`. The workflow
`.github/workflows/security-linux.yml` targets `.github/workflows/security-linux.yml` targets
`runs-on: [self-hosted, linux, marqdo-security]`. Keep the runner disposable `runs-on: [self-hosted, linux, marqdo-security]`.
and recreate it from this document; do not give the runner access to secrets or
the primary workspace. A CI URL from that runner is still required before a The runner must be **ephemeral**:
release may call G-C CI-verified.
* provision a fresh VM (or a self-hosted runner with a genuinely one-shot
lifetime) for each job, from a pinned base image;
* no persistent home, host mounts, Docker socket, SSH/Git/cloud credentials or
release token; only the short-lived runner registration credential;
* job `permissions: contents: read`, `actions/checkout` with
`persist-credentials: false` (already in the workflow);
* do not run arbitrary fork PRs on it: the workflow only runs on `master`
pushes or an explicit reviewed `source_sha`;
* destroy the whole VM after the job and keep the uploaded
`strict-evidence.json` and lane log.
A PASS from a persistent runner with production credentials is not acceptable
evidence, even if the test output is green. If no ephemeral runner is
available, report "local disposable-VM PASS, remote G3 UNVERIFIED" instead of a
green CI claim.
## Reference environment ## Reference environment
+37
View File
@@ -0,0 +1,37 @@
# CI evidence tools
## `strict-evidence.py`
Writes `marqdo.strict-evidence.v1`, called by `tests/security/run-lane.sh` when
`MARQDO_STRICT_EVIDENCE` is set and by the self-hosted security workflow. It
records only measured or explicitly supplied facts:
* `tested_sha` and whether it came from `git rev-parse` or an operator env;
* source archive SHA256 (and whether it matched the expected digest);
* kernel/uname/cgroup2/container, `uid`/root, bwrap/rustc/cargo/cc versions;
* runner kind, image digest, run URL, start/finish and outcome.
A root-only `PASS` is rejected. `PASS` from a container or a persistent
self-hosted runner with production credentials is not valid G-C evidence; see
[`docs/security-vm.md`](../../docs/security-vm.md).
## `release_evidence.py`
Aggregates per-gate results into `marqdo.release-gates.v1`:
```bash
python3 tools/ci/release_evidence.py \
--target-sha "$SHA" \
--gate "linux=PASS:$SHA:https://ci.example/run/1" \
--gate "windows=PASS:$SHA:https://ci.example/run/2" \
--gate "macos=PASS:$SHA:https://ci.example/run/3" \
--gate "strict_vm=PASS:$SHA:https://ci.example/run/4" \
--gate "performance=WARN:$SHA:https://ci.example/run/5" \
--out "docs/evidence/ci/$SHA/manifest.json"
```
A missing gate becomes `UNVERIFIED`. Any non-PASS required gate, a
`tested_sha` different from `target_sha`, an unknown gate name or a malformed
SHA forces `DO_NOT_RELEASE`. `performance` may be `PASS` or `WARN`.
Unit tests: `python3 -m unittest discover -s tools/ci -p 'test_*.py'`.
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
"""Aggregate per-gate CI evidence into a SHA-bound release manifest.
Schema: marqdo.release-gates.v1. A missing, skipped, timed-out, failed or
SHA-mismatched gate is never PASS. The tool only records supplied facts; it
does not query CI and cannot turn an UNVERIFIED gate into a release.
Gate spec: `name=STATE:tested_sha:run_url` (url may contain colons).
Required gates: linux, windows, macos, strict_vm; performance must be PASS or
WARN. Any other gate name is rejected so a typo cannot silently disappear.
"""
import argparse
import datetime
import json
import re
import sys
from pathlib import Path
SCHEMA = "marqdo.release-gates.v1"
REQUIRED = ("linux", "windows", "macos", "strict_vm")
OPTIONAL = ("performance",)
STATES = {
"linux": {"PASS", "FAIL", "UNVERIFIED"},
"windows": {"PASS", "FAIL", "UNVERIFIED"},
"macos": {"PASS", "FAIL", "UNVERIFIED"},
"strict_vm": {"PASS", "FAIL", "UNVERIFIED"},
"performance": {"PASS", "WARN", "FAIL", "UNVERIFIED"},
}
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
def utc_now() -> str:
return (
datetime.datetime.now(datetime.timezone.utc)
.replace(microsecond=0)
.isoformat()
.replace("+00:00", "Z")
)
def parse_gate(text: str) -> tuple[str, dict]:
if "=" not in text:
raise ValueError(f"gate must be name=STATE:sha:url, got {text!r}")
name, rest = text.split("=", 1)
parts = rest.split(":", 2)
if len(parts) < 3:
raise ValueError(f"gate must be name=STATE:sha:url, got {text!r}")
state, tested_sha, run_url = parts
if name not in STATES:
raise ValueError(f"unknown gate {name!r}; expected {list(STATES)}")
if state not in STATES[name]:
raise ValueError(f"invalid state {state!r} for gate {name!r}")
if tested_sha and not SHA_RE.fullmatch(tested_sha):
raise ValueError(f"gate {name!r} tested_sha is not a full 40-hex SHA: {tested_sha!r}")
return name, {"state": state, "tested_sha": tested_sha, "run_url": run_url}
def build_manifest(target_sha: str, gates: list[str]) -> dict:
if not SHA_RE.fullmatch(target_sha):
raise ValueError(f"target SHA must be 40 lowercase hex characters: {target_sha!r}")
parsed = {}
for spec in gates:
name, gate = parse_gate(spec)
if name in parsed:
raise ValueError(f"duplicate gate {name!r}")
parsed[name] = gate
reasons = []
for name in REQUIRED + OPTIONAL:
gate = parsed.get(name)
if gate is None:
parsed[name] = {"state": "UNVERIFIED", "tested_sha": "", "run_url": ""}
reasons.append(f"{name}: missing gate")
continue
if gate["tested_sha"] != target_sha:
reasons.append(
f"{name}: tested_sha {gate['tested_sha'] or '(missing)'} != target_sha {target_sha}"
)
elif gate["state"] == "FAIL":
reasons.append(f"{name}: FAIL")
elif gate["state"] == "UNVERIFIED":
reasons.append(f"{name}: UNVERIFIED")
elif name in REQUIRED and gate["state"] != "PASS":
reasons.append(f"{name}: {gate['state']} is not PASS")
elif name == "performance" and gate["state"] not in {"PASS", "WARN"}:
reasons.append(f"performance: {gate['state']} is not PASS/WARN")
decision = "RELEASE_CANDIDATE" if not reasons else "DO_NOT_RELEASE"
recorded = utc_now()
return {
"schema": SCHEMA,
"target_sha": target_sha,
"recorded_utc": recorded,
"gates": {
name: {
**gate,
"recorded_utc": recorded,
}
for name, gate in parsed.items()
},
"release_decision": decision,
"reasons": reasons,
}
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--target-sha", required=True)
parser.add_argument("--gate", action="append", default=[])
parser.add_argument("--out", default="")
args = parser.parse_args(argv)
try:
manifest = build_manifest(args.target_sha, args.gate)
except ValueError as error:
print(f"error: {error}", file=sys.stderr)
return 2
text = json.dumps(manifest, ensure_ascii=False, indent=2) + "\n"
if args.out:
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(text)
sys.stdout.write(text)
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))
+94
View File
@@ -0,0 +1,94 @@
import importlib.util
import unittest
from pathlib import Path
MODULE = Path(__file__).with_name("release_evidence.py")
spec = importlib.util.spec_from_file_location("release_evidence", MODULE)
release_evidence = importlib.util.module_from_spec(spec)
spec.loader.exec_module(release_evidence)
SHA = "a" * 40
OTHER = "b" * 40
def gate(name: str, state: str, sha: str = SHA, url: str = "https://example.test/run") -> str:
return f"{name}={state}:{sha}:{url}"
class ReleaseEvidenceTests(unittest.TestCase):
def test_all_required_pass_is_candidate(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "PASS"),
gate("windows", "PASS"),
gate("macos", "PASS"),
gate("strict_vm", "PASS"),
gate("performance", "WARN"),
],
)
self.assertEqual(manifest["release_decision"], "RELEASE_CANDIDATE")
self.assertEqual(manifest["reasons"], [])
def test_missing_strict_vm_is_not_release(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "PASS"),
gate("windows", "PASS"),
gate("macos", "PASS"),
gate("performance", "PASS"),
],
)
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
self.assertTrue(any("strict_vm" in reason for reason in manifest["reasons"]))
def test_sha_mismatch_is_not_release(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "PASS"),
gate("windows", "PASS", OTHER),
gate("macos", "PASS"),
gate("strict_vm", "PASS"),
gate("performance", "PASS"),
],
)
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
self.assertTrue(any("windows" in reason for reason in manifest["reasons"]))
def test_failure_is_not_release(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "FAIL"),
gate("windows", "PASS"),
gate("macos", "PASS"),
gate("strict_vm", "PASS"),
gate("performance", "PASS"),
],
)
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
def test_unverified_performance_is_not_release(self) -> None:
manifest = release_evidence.build_manifest(
SHA,
[
gate("linux", "PASS"),
gate("windows", "PASS"),
gate("macos", "PASS"),
gate("strict_vm", "PASS"),
gate("performance", "UNVERIFIED"),
],
)
self.assertEqual(manifest["release_decision"], "DO_NOT_RELEASE")
def test_unknown_gate_and_bad_sha_are_rejected(self) -> None:
with self.assertRaises(ValueError):
release_evidence.build_manifest(SHA, [gate("solaris", "PASS")])
with self.assertRaises(ValueError):
release_evidence.build_manifest("not-a-sha", [])
if __name__ == "__main__":
unittest.main()