From 20d15fd5a2d093bca9078e34384c78baa48989cf Mon Sep 17 00:00:00 2001 From: cflmy Date: Fri, 20 Nov 2026 13:03:38 +0800 Subject: [PATCH] ci: validate real protocol artifacts and run core tests cross-platform (B-13 B-14) - add an independent schema job with a pinned Draft 2020-12 validator that validates real Analysis v2/Trace v2/Tool artifacts and negative mutations - tighten the tool schema (SemVer, package-relative entrypoint, non-empty protocol arrays) - run cargo test --workspace plus portable CLI/analysis/trace smoke and a strict-refusal assertion on Windows and macOS - record honestly that Tool/C-ABI/worker runtime coverage remains Unix-only --- .github/workflows/ci.yml | 49 ++++++++++++++- docs/STATUS.md | 21 ++++++- spec/schemas/tool-v1.schema.json | 60 +++++++++++++++--- tests/consumers/validate_protocols.py | 91 +++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 14 deletions(-) create mode 100644 tests/consumers/validate_protocols.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ef8322..08777ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,8 +74,30 @@ jobs: --trust-native-plugin \ --allow-plugin demo.solver + schema: + name: schema instance validation + consumers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Install JSON Schema validator + run: python3 -m pip install --disable-pip-version-check "jsonschema==4.23.0" + - name: Validate real artifacts + run: | + cargo build -p marqdo-cli + ./target/debug/marqdo check examples/multi-module/index.mq.md --refs \ + --emit /tmp/analysis-v2.json + ./target/debug/marqdo run examples/multi-module/index.mq.md \ + --trace-file /tmp/trace-v2.json + python3 tests/consumers/validate_protocols.py \ + /tmp/analysis-v2.json /tmp/trace-v2.json examples/tool-unknown-name/tool.json + python3 tests/consumers/analysis_consumer.py /tmp/analysis-v2.json + python3 tests/consumers/trace_consumer.py \ + /tmp/analysis-v2.json /tmp/trace-v2.json + cross-platform: - name: cargo check (${{ matrix.os }}) + name: runtime contract (${{ matrix.os }}) strategy: fail-fast: false matrix: @@ -85,5 +107,26 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - name: Check all targets - run: cargo check --workspace --all-targets + - name: Workspace tests + run: cargo test --workspace + - name: Portable CLI, analysis and trace smoke + shell: bash + run: | + set -e + cargo run -q -p marqdo-cli -- check examples/multi-module/index.mq.md \ + --refs --emit /tmp/marqdo-portable-analysis.json + python3 tests/consumers/analysis_consumer.py /tmp/marqdo-portable-analysis.json + cargo run -q -p marqdo-cli -- run examples/multi-module/index.mq.md \ + --trace-file /tmp/marqdo-portable-trace.json + python3 tests/consumers/trace_consumer.py \ + /tmp/marqdo-portable-analysis.json /tmp/marqdo-portable-trace.json + - name: Strict isolation is unavailable and must fail closed + shell: bash + run: | + set +e + output=$(cargo run -q -p marqdo-cli -- run examples/hello.mq.md --mode strict 2>&1) + code=$? + set -e + test "$code" -eq 1 + echo "$output" | grep -qi "strict" + echo "$output" diff --git a/docs/STATUS.md b/docs/STATUS.md index 9d2713a..96c9822 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -170,6 +170,24 @@ Tool Protocol hardening (`spec/tool-v1.md`): * Tests cover accepted/rejected protocol declarations, auxiliary-file/extra-file tampering, entrypoint tampering and reserved `record.json`. +### S4-A/B-13 B-14 record (current) + +* CI has an independent `schema` job that installs a pinned Draft 2020-12 + validator (`jsonschema==4.23.0`), emits real Analysis/Trace artifacts, + validates each against its matching schema, and runs the independent Python + consumers. `tests/consumers/validate_protocols.py` also asserts negative + mutations (wrong type/version, missing required fields, unsupported + truncation reason, traversal entrypoint, invalid SemVer). +* Tool schema tightened: SemVer version pattern, package-relative entrypoint + pattern and non-empty protocol arrays. +* The Windows/macOS CI lane now runs `cargo test --workspace` plus portable CLI, + Analysis and Trace smoke tests and a strict-mode refusal assertion, instead + of `cargo check` only. +* Still Unix-only in CI: compiled C ABI plugins, worker transport tests, + package/Git/process tests and the shell-fixture Tool black-box suite. They are + not counted as cross-platform evidence; `docs/STATUS.md` records this rather + than treating `cargo test` skips as passes. + ### M0.6 implementation progress | Step | State | Evidence | @@ -180,13 +198,14 @@ Tool Protocol hardening (`spec/tool-v1.md`): | S2-B/H2-A Analysis v2 | **Done** | logical project identity, resources array, machine-path-free diagnostics, emitted-bytes digest, v2 schema + independent consumer (see §S2-B/H2-A) | | S2-C/H2-B Trace v2 | **Done** | logical root/digest, module-owned events, bounded/truncated trace, independent consumer, hardened worker frames (see §S2-C/H2-B) | | S3-A/H2-C Tool protocol | **Done** | SemVer negotiation at install+dispatch, whole-bundle manifest/digest verification, atomic install/remove, reserved record file (see §S3-A/H2-C) | +| S4-A/B-13 B-14 quality gates | **Partial** | real schema instance validation + negative mutations in CI; Windows/macOS runtime tests for core/Analysis/Trace and strict refusal. Tool/C-ABI/worker runtime evidence remains Unix-only (see §S4-A) | | R0 baseline + ADRs | **Done** | this section; `docs/adr/0008`–`0012`; full workspace gate | | R1 package-aware identity + callable registry | **Done for statics; superseded by S1 lock v3** | `ModuleId { instance, relative_path }`; `PackageResolver` path sources; lock v3 transitive graph (ADR-0016) replaces the flat v2 lock; `CallableRegistry` for builtins/marqdo/declared ABI exports; external manifest wiring is R2 | | R2 ABI/Kernel/Broker | **Done (worker = crash isolation)** | `KernelRegistry` external providers; complete-name export routing; `CapabilityBroker`; static `check --plugin`; multi-export C test; isolated worker with timeout/crash tests; host handle table. Native/worker are not an OS sandbox | | R3 Tool Protocol | **Done (S3 hardened)** | `tool install/list/remove`; unknown-command fallback; SemVer + Analysis/Trace negotiation; whole-bundle digest verification; staged install and rename-based removal; black-box tests for arbitrary names, Unicode args, conflicts, traversal, PATH hijacking and bundle tampering | | R4 Analysis/Trace v2 | **Done** | `check --emit` Analysis v2 and `run/call --trace-file` Trace v2 with logical identity, resources, module-owned events, capacity bounds, schemas and independent Python consumers; stale-artifact removal; value redaction | | R5 environment/isolation | **Partial** | Pinned Git fetch with verified cache markers, transitive `@alias` resolution, full-closure atomic `.marqdo/env` materialization, foreign process-group cleanup. Strict OS isolation remains explicitly unavailable (fail-closed) | -| R6 release/consumers | **Done for Linux** | examples for unknown-name Tool, multi-export plugin, external kernel and multi-version packages; Analysis/Trace/Tool JSON Schemas; CI Linux gate + Windows/macOS `cargo check`; R6 example tests | +| R6 release/consumers | **Partial** | examples for unknown-name Tool, multi-export plugin, external kernel and multi-version packages; Analysis/Trace/Tool JSON Schemas with instance validation; Linux full gate + Windows/macOS workspace tests and portable analysis/trace smoke; Unix-only Tool/ABI/worker runtime evidence | Reproduce the evidence: diff --git a/spec/schemas/tool-v1.schema.json b/spec/schemas/tool-v1.schema.json index ea3845f..e8d1be4 100644 --- a/spec/schemas/tool-v1.schema.json +++ b/spec/schemas/tool-v1.schema.json @@ -3,20 +3,60 @@ "$id": "https://marqdo.dev/schemas/tool-v1.schema.json", "title": "Marqdo CLI Tool Manifest v1", "type": "object", - "required": ["schema", "schema_version", "name", "version", "command", "entrypoint"], + "required": [ + "schema", + "schema_version", + "name", + "version", + "command", + "entrypoint" + ], "properties": { - "schema": {"const": "marqdo.cli-tool"}, - "schema_version": {"const": 1}, - "name": {"type": "string", "minLength": 1}, - "version": {"type": "string", "minLength": 1}, - "command": {"type": "string", "pattern": "^[a-z][a-z0-9-]*$"}, - "entrypoint": {"type": "string", "minLength": 1}, + "schema": { + "const": "marqdo.cli-tool" + }, + "schema_version": { + "const": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?$" + }, + "command": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$" + }, + "entrypoint": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" + }, "compatibility": { "type": "object", "properties": { - "marqdo": {"type": "string"}, - "analysis": {"type": "array", "items": {"type": "integer", "minimum": 1}}, - "trace": {"type": "array", "items": {"type": "integer", "minimum": 1}} + "marqdo": { + "type": "string" + }, + "analysis": { + "type": "array", + "minItems": 1, + "items": { + "type": "integer", + "minimum": 1 + } + }, + "trace": { + "type": "array", + "minItems": 1, + "items": { + "type": "integer", + "minimum": 1 + } + } }, "additionalProperties": true } diff --git a/tests/consumers/validate_protocols.py b/tests/consumers/validate_protocols.py new file mode 100644 index 0000000..0e13d4f --- /dev/null +++ b/tests/consumers/validate_protocols.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Validate real Marqdo protocol artifacts against their JSON Schemas. + +This is an independent consumer: it imports only Python's standard library and +the `jsonschema` package, never the Rust workspace. +""" +import copy +import json +import pathlib +import sys + +from jsonschema import Draft202012Validator + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCHEMA_DIR = ROOT / "spec" / "schemas" + + +def load_schema(name): + path = SCHEMA_DIR / name + schema = json.loads(path.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(schema) + return schema + + +def validator(schema): + return Draft202012Validator(schema) + + +def require_valid(schema_name, document, label): + errors = sorted(validator(load_schema(schema_name)).iter_errors(document), key=str) + if errors: + raise SystemExit(f"{label}: schema validation failed: {errors[0].message}") + + +def require_invalid(schema_name, document, label): + errors = list(validator(load_schema(schema_name)).iter_errors(document)) + if not errors: + raise SystemExit(f"{label}: expected schema validation to fail") + + +def main(): + if len(sys.argv) != 4: + raise SystemExit( + "usage: validate_protocols.py ANALYSIS.json TRACE.json TOOL.json" + ) + analysis_path, trace_path, tool_path = map(pathlib.Path, sys.argv[1:]) + analysis = json.loads(analysis_path.read_text(encoding="utf-8")) + trace = json.loads(trace_path.read_text(encoding="utf-8")) + tool = json.loads(tool_path.read_text(encoding="utf-8")) + + require_valid("analysis-v2.schema.json", analysis, "analysis") + require_valid("trace-v2.schema.json", trace, "trace") + require_valid("tool-v1.schema.json", tool, "tool") + + # Negative mutations: structure, version and traversal must fail closed. + bad = copy.deepcopy(analysis) + bad["schema_version"] = 999 + require_invalid("analysis-v2.schema.json", bad, "analysis unsupported version") + + bad = copy.deepcopy(analysis) + bad.pop("modules", None) + require_invalid("analysis-v2.schema.json", bad, "analysis missing modules") + + bad = copy.deepcopy(analysis) + bad["modules"][0]["resources"] = 3 + require_invalid("analysis-v2.schema.json", bad, "analysis resources integer") + + bad = copy.deepcopy(trace) + bad["events"][0]["seq"] = "zero" + require_invalid("trace-v2.schema.json", bad, "trace event sequence type") + + bad = copy.deepcopy(trace) + bad["truncated"] = {"reason": "made_up", "limit": 1, "written": 1} + require_invalid("trace-v2.schema.json", bad, "trace truncation reason") + + bad = copy.deepcopy(tool) + bad["entrypoint"] = "../outside" + require_invalid("tool-v1.schema.json", bad, "tool traversal entrypoint") + + bad = copy.deepcopy(tool) + bad["version"] = "not semver" + require_invalid("tool-v1.schema.json", bad, "tool invalid version") + + print( + "validated analysis/trace/tool artifacts " + f"({len(analysis['modules'])} modules, {len(trace['events'])} events)" + ) + + +if __name__ == "__main__": + main()