docs(spec): adopt RFC-LANG-0002 unified language semantics
Replace the Strong/Emph/List/Quote mapping in place with the single @exec/@knowledge heading-mode model, unified payload carriers and keyword control flow. - add the authoritative RFC, ADR-0052 and a historical ADR index - rewrite spec/mapping.md and spec/expression.ebnf as the normative mapping/statement grammar; update evaluation, references, math, foreign, package-format, plugin-abi, security and schema notes - rewrite README, AGENTS, COMPATIBILITY and CHANGELOG for the breaking language change; update the architecture index/overview and active docs - mark superseded planning docs, compute RFCs, evidence and syntax-role ADRs as historical/superseded without rewriting their decisions - add migration guide, per-file inventory and dev-only audit scripts The frontend/stdlib migration is completed in the following commits; this commit contains documentation only.
This commit is contained in:
1 parent
0d8ed44010
commit
49835424ee
47 files changed
+4941
-603
No files matched your search
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Review aid for the RFC-LANG-0002 migration (NOT a parser).
|
||||
|
||||
Checks migrated `.mq.md` files for common mistakes:
|
||||
* unclosed/unmatched if/else/while/end inside an executable segment;
|
||||
* `if`/`while`/`return` without an expression; `else`/`end` with trailing text;
|
||||
* semicolon-separated statements in one payload;
|
||||
* unknown `sem.*` codes in negative expected JSON.
|
||||
|
||||
It intentionally ignores Knowledge sections and does not try to be a
|
||||
normative validator; the Rust frontend migration must provide the real
|
||||
validation. Usage: python3 tools/dev-only/check-mq-syntax.py <dir>...
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
KNOWN_SEM = {
|
||||
"sem.invalid_mode_marker", "sem.empty_unit_name", "sem.duplicate_unit_name", "sem.exec_without_function",
|
||||
"sem.payload_nested", "sem.empty_payload", "sem.syntax", "sem.unmatched_else",
|
||||
"sem.duplicate_else", "sem.unmatched_end", "sem.unclosed_control",
|
||||
"sem.signature_position", "sem.unsupported_version_directive",
|
||||
"sem.no_entry_function", "sem.inferred_signature",
|
||||
}
|
||||
HEADING = re.compile(r"^(#{1,6})\s+(.*)$")
|
||||
MODE = re.compile(r"^@(exec|knowledge)\b\s*(.*)$")
|
||||
|
||||
|
||||
def payload_of(line: str):
|
||||
s = line.strip()
|
||||
while True:
|
||||
n = re.sub(r"^(?:>|[-+*]\s+|\d+[.)]\s+)", "", s).strip()
|
||||
if n == s:
|
||||
break
|
||||
s = n
|
||||
if len(s) >= 4 and s.startswith("**") and s.endswith("**") and "**" not in s[2:-2]:
|
||||
return s[2:-2]
|
||||
if len(s) >= 2 and s.startswith("*") and s.endswith("*") and not s.startswith("**") and "*" not in s[1:-1]:
|
||||
return s[1:-1]
|
||||
m = re.match(r"^(`+)(.*)\1$", s, re.S)
|
||||
if m:
|
||||
return m.group(2)
|
||||
return None
|
||||
|
||||
|
||||
def check(path: Path) -> list[str]:
|
||||
errors: list[str] = []
|
||||
stack: list[tuple[str, int]] = []
|
||||
headings: list[tuple[int, str | None, int]] = [] # level, explicit mode
|
||||
segment = "root"
|
||||
|
||||
def effective(level: int) -> str:
|
||||
for hl, mode, _ln in reversed(headings):
|
||||
if hl < level:
|
||||
if mode:
|
||||
return mode
|
||||
# continue to a higher explicit ancestor
|
||||
return "knowledge"
|
||||
|
||||
def close_segment(line: int) -> None:
|
||||
nonlocal stack
|
||||
if stack:
|
||||
kind, op = stack[-1]
|
||||
errors.append(f"{path}:{line}: unclosed {kind} from line {op} in {segment}")
|
||||
stack = []
|
||||
|
||||
for i, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
m = HEADING.match(raw)
|
||||
if m:
|
||||
close_segment(i)
|
||||
level, body = len(m.group(1)), m.group(2)
|
||||
mm = MODE.match(body)
|
||||
mode = mm.group(1) if mm else None
|
||||
name = mm.group(2).strip() if mm else body.strip()
|
||||
if mm and not name:
|
||||
errors.append(f"{path}:{i}: mode without heading name")
|
||||
if not mm and re.match(r"^@[A-Za-z]", name):
|
||||
errors.append(f"{path}:{i}: unknown mode marker {name.split()[0]}")
|
||||
while headings and headings[-1][0] >= level:
|
||||
headings.pop()
|
||||
headings.append((level, mode, i))
|
||||
segment = f"H{level} {name}"
|
||||
continue
|
||||
text = payload_of(raw)
|
||||
if text is None:
|
||||
continue
|
||||
text = text.strip()
|
||||
if not text:
|
||||
errors.append(f"{path}:{i}: empty executable payload")
|
||||
continue
|
||||
owning = False
|
||||
for hl, _mode, _ln in reversed(headings):
|
||||
if hl == 2:
|
||||
owning = effective(2) == "exec"
|
||||
break
|
||||
if not owning:
|
||||
continue
|
||||
if ";" in text:
|
||||
errors.append(f"{path}:{i}: semicolon in one payload: {text!r}")
|
||||
first = text.split(None, 1)[0]
|
||||
if first in ("if", "while"):
|
||||
if len(text.split()) == 1:
|
||||
errors.append(f"{path}:{i}: {first} without expression")
|
||||
stack.append((first, i))
|
||||
elif first == "else":
|
||||
if text != "else":
|
||||
errors.append(f"{path}:{i}: else with trailing tokens")
|
||||
if not stack or not stack[-1][0].startswith("if"):
|
||||
errors.append(f"{path}:{i}: unmatched else")
|
||||
elif stack[-1][0] == "if_else":
|
||||
errors.append(f"{path}:{i}: duplicate else")
|
||||
else:
|
||||
stack[-1] = ("if_else", stack[-1][1])
|
||||
elif first == "end":
|
||||
if text != "end":
|
||||
errors.append(f"{path}:{i}: end with trailing tokens")
|
||||
if not stack:
|
||||
errors.append(f"{path}:{i}: unmatched end")
|
||||
else:
|
||||
stack.pop()
|
||||
elif first == "return" and len(text.split()) == 1:
|
||||
errors.append(f"{path}:{i}: return without expression")
|
||||
elif first == "params" and not re.match(r"^params\s*\(.*\)$", text, re.S):
|
||||
errors.append(f"{path}:{i}: malformed params")
|
||||
close_segment(len(headings) + 1)
|
||||
return errors
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
roots = [Path(a) for a in argv] or [Path("tests"), Path("examples")]
|
||||
failures: list[str] = []
|
||||
count = 0
|
||||
for root in roots:
|
||||
paths = [root] if root.is_file() else sorted(root.rglob("*"))
|
||||
for path in paths:
|
||||
if path.name.endswith(".mq.md"):
|
||||
count += 1
|
||||
failures.extend(check(path))
|
||||
elif path.suffix == ".json" and "negative" in path.as_posix():
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception as error: # noqa: BLE001 - review aid
|
||||
failures.append(f"{path}: invalid JSON: {error}")
|
||||
continue
|
||||
for code in data.get("codes", []):
|
||||
if code.startswith("sem.") and code not in KNOWN_SEM:
|
||||
failures.append(f"{path}: unknown sem code {code}")
|
||||
for failure in failures:
|
||||
print(failure)
|
||||
print(f"checked {count} .mq.md files, {len(failures)} issue(s)")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate docs/migration/semantic-rewrite-inventory.tsv.
|
||||
|
||||
The inventory classifies every git-tracked file for the RFC-LANG-0002
|
||||
single-syntax migration. It is a repository audit aid, not a compiler and not
|
||||
a substitute for `cargo test` or the real frontend.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
OLD_MARKERS = re.compile(
|
||||
r"^##\s+[^@\s].*$|^\*[^*\n].*\*$|^\*\*[^*\n].*\*\*$|"
|
||||
r"markdown\.inert_executable|markdown\.invalid_construct|"
|
||||
r"marqdo_semantics|--legacy|map_v1",
|
||||
re.M,
|
||||
)
|
||||
NEW_MARKERS = re.compile(r"@exec|@knowledge")
|
||||
FRONTEND = {
|
||||
"crates/marqdo-frontend/src/units.rs",
|
||||
"crates/marqdo-frontend/src/map.rs",
|
||||
"crates/marqdo-frontend/src/payload.rs",
|
||||
"crates/marqdo-frontend/src/signature.rs",
|
||||
"crates/marqdo-frontend/src/analyze.rs",
|
||||
"crates/marqdo-expression/src/lexer.rs",
|
||||
"crates/marqdo-expression/src/parser.rs",
|
||||
"crates/marqdo-expression/src/ast.rs",
|
||||
"crates/marqdo-ir/src/lib.rs",
|
||||
}
|
||||
|
||||
|
||||
def tracked() -> list[str]:
|
||||
"""Tracked files plus untracked-but-not-ignored files.
|
||||
|
||||
The migration adds new fixtures/docs before they are committed; an inventory
|
||||
that only reads `git ls-files` would silently omit them.
|
||||
"""
|
||||
listed = subprocess.run(
|
||||
["git", "ls-files"], cwd=ROOT, check=True, text=True, capture_output=True
|
||||
).stdout.splitlines()
|
||||
untracked = subprocess.run(
|
||||
["git", "ls-files", "--others", "--exclude-standard"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
).stdout.splitlines()
|
||||
return sorted({line for line in listed + untracked if line})
|
||||
|
||||
|
||||
def deleted() -> set[str]:
|
||||
out = subprocess.run(
|
||||
["git", "ls-files", "--deleted"], cwd=ROOT, check=True, text=True, capture_output=True
|
||||
).stdout
|
||||
return {line for line in out.splitlines() if line}
|
||||
|
||||
|
||||
def changed() -> set[str]:
|
||||
out = subprocess.run(
|
||||
["git", "status", "--porcelain"], cwd=ROOT, check=True, text=True, capture_output=True
|
||||
).stdout
|
||||
result = set()
|
||||
for line in out.splitlines():
|
||||
if len(line) > 3:
|
||||
result.add(line[3:].strip())
|
||||
return result
|
||||
|
||||
|
||||
def classify(path: str, changed_set: set[str]) -> tuple[str, str, str, str, str, str, str]:
|
||||
"""Return category, active, old_semantics, action, owner, verification, status."""
|
||||
p = Path(path)
|
||||
text = ""
|
||||
full = ROOT / path
|
||||
if full.is_file() and full.stat().st_size < 2_000_000:
|
||||
try:
|
||||
text = full.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = ""
|
||||
is_changed = path in changed_set
|
||||
|
||||
if path == "docs/adr/0052-unified-markdown-semantics.md":
|
||||
return ("decision", "yes", "old mapping", "adopt current language decision", "docs", "review", "migrated")
|
||||
if path == "docs/adr/README.md":
|
||||
return ("decision", "yes", "n/a", "index current decisions/history", "docs", "review", "migrated")
|
||||
if path == "docs/evidence/2026-11-21-semantic-docs-migration.md":
|
||||
return ("evidence", "yes", "old syntax", "record migration evidence", "docs", "docs_consistency", "migrated")
|
||||
if path.startswith("docs/adr/"):
|
||||
return ("historical", "no", "decision at its date", "banner if syntax-role", "docs", "review", "historical")
|
||||
if path.startswith("docs/history/") or path.startswith("docs/evidence/"):
|
||||
return ("historical", "no", "archived", "banner only", "docs", "review", "historical")
|
||||
if path.startswith("docs/migration/"):
|
||||
return ("migration", "yes", "n/a", "maintain inventory/guide", "docs", "review", "migrated")
|
||||
if path.startswith("docs/architecture/RFC-LANG-0002"):
|
||||
return ("architecture", "yes", "old mapping", "adopt as sole authority", "docs", "review", "migrated")
|
||||
if path.startswith("docs/architecture/Marqdo_CLI_Latest"):
|
||||
return ("architecture", "no", "review input", "historical banner", "docs", "review", "historical")
|
||||
if path.startswith("docs/architecture/"):
|
||||
return ("architecture", "yes", "status/index", "update to RFC-LANG-0002", "docs", "docs_consistency", "migrated")
|
||||
if path.startswith("docs/MARQDO_COMPUTE"):
|
||||
return ("architecture", "yes", "compute RFC", "migrate examples / mark historical spelling", "docs", "review", "migrated")
|
||||
if path.startswith("docs/"):
|
||||
if is_changed:
|
||||
return ("tutorial", "yes", "old mapping", "rewrite + historical banner where needed", "docs", "docs_consistency", "migrated")
|
||||
return ("tutorial", "yes", "none", "reviewed", "docs", "review", "reviewed")
|
||||
if path.startswith("spec/"):
|
||||
return ("spec", "yes", "old mapping", "in-place rewrite", "language", "cargo test / review", "migrated")
|
||||
if path.endswith(".mq.md"):
|
||||
if path.startswith("stdlib/"):
|
||||
return ("stdlib", "yes", "old syntax", "migrate with frontend + digest", "stdlib", "stdlib_product", "pending-implementation")
|
||||
if path.startswith("tests/conformance/ast/"):
|
||||
return ("test", "yes", "Markdown AST only", "keep AST baseline", "tests", "conformance_ast", "reviewed")
|
||||
if path.startswith("tests/negative/"):
|
||||
return ("test", "yes", "old negative shape", "rebuild for sem.* diagnostics", "tests", "negative", "migrated")
|
||||
if path.startswith("examples/"):
|
||||
return ("example", "yes", "old syntax", "migrate source", "examples", "examples_r6 / run", "migrated")
|
||||
if NEW_MARKERS.search(text):
|
||||
return ("test", "yes", "old syntax", "migrate source", "tests", "conformance/negative", "migrated")
|
||||
return ("test", "yes", "old syntax", "needs migration", "tests", "review", "pending")
|
||||
if path.startswith("tests/"):
|
||||
if is_changed:
|
||||
return ("test", "yes", "old syntax", "migrate source/expectations", "tests", "negative/properties", "migrated")
|
||||
return ("test", "yes", "harness", "review", "tests", "review", "reviewed")
|
||||
if path.startswith("crates/"):
|
||||
if path in FRONTEND:
|
||||
return ("rust-source", "yes", "old mapping", "implement RFC-LANG-0002", "language", "cargo test", "pending-implementation")
|
||||
if "/tests/" in path or path.endswith("/tests.rs"):
|
||||
return ("rust-test", "yes", "embedded old syntax", "migrate embedded .mq.md strings", "language", "cargo test", "migrated" if is_changed else "reviewed")
|
||||
if is_changed:
|
||||
return ("rust-source", "yes", "embedded text/help", "migrate embedded .mq.md text only", "language", "cargo test", "migrated")
|
||||
return ("rust-source", "yes", "not syntax-bearing", "review", "language", "cargo test", "reviewed")
|
||||
if path.startswith("stdlib/"):
|
||||
return ("stdlib", "yes", "old syntax", "migrate with frontend + digest", "stdlib", "stdlib_product", "pending-implementation")
|
||||
if path.startswith("tools/"):
|
||||
return ("tooling", "yes", "migration aid", "review", "docs", "review", "reviewed")
|
||||
if path.startswith("examples/"):
|
||||
return ("example", "yes", "old syntax", "review", "examples", "run", "reviewed")
|
||||
if path in {"README.md", "COMPATIBILITY.md", "CHANGELOG.md", "AGENTS.md"}:
|
||||
return ("tutorial", "yes", "old mapping", "rewrite", "docs", "review", "migrated")
|
||||
return ("non-semantic", "no", "none", "none", "-", "n/a", "reviewed")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
changed_set = changed()
|
||||
deleted_set = deleted()
|
||||
rows = []
|
||||
for path in tracked():
|
||||
if path in deleted_set:
|
||||
continue
|
||||
rows.append((path, *classify(path, changed_set)))
|
||||
rows.sort(key=lambda row: row[0])
|
||||
out = ROOT / "docs/migration/semantic-rewrite-inventory.tsv"
|
||||
with out.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(
|
||||
"path\tcategory\tactive\told_semantics\taction\towner\tverification\tstatus\n"
|
||||
)
|
||||
for row in rows:
|
||||
handle.write("\t".join(row) + "\n")
|
||||
counts: dict[str, int] = {}
|
||||
for row in rows:
|
||||
counts[row[-1]] = counts.get(row[-1], 0) + 1
|
||||
print(f"wrote {out} ({len(rows)} rows)")
|
||||
for status, count in sorted(counts.items()):
|
||||
print(f" {status}: {count}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user