docs(spec): adopt RFC-LANG-0002 unified language semantics

Replace the Strong/Emph/List/Quote mapping in place with the single
@exec/@knowledge heading-mode model, unified payload carriers and keyword
control flow.

- add the authoritative RFC, ADR-0052 and a historical ADR index
- rewrite spec/mapping.md and spec/expression.ebnf as the normative
  mapping/statement grammar; update evaluation, references, math,
  foreign, package-format, plugin-abi, security and schema notes
- rewrite README, AGENTS, COMPATIBILITY and CHANGELOG for the breaking
  language change; update the architecture index/overview and active docs
- mark superseded planning docs, compute RFCs, evidence and syntax-role
  ADRs as historical/superseded without rewriting their decisions
- add migration guide, per-file inventory and dev-only audit scripts

The frontend/stdlib migration is completed in the following commits; this
commit contains documentation only.
This commit is contained in:
cflmy committed 2026-11-21 12:24:19 +08:00
1 parent 0d8ed44010
commit 49835424ee
47 files changed
+4941 -603

No files matched your search

+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
"""Review aid for the RFC-LANG-0002 migration (NOT a parser).
Checks migrated `.mq.md` files for common mistakes:
* unclosed/unmatched if/else/while/end inside an executable segment;
* `if`/`while`/`return` without an expression; `else`/`end` with trailing text;
* semicolon-separated statements in one payload;
* unknown `sem.*` codes in negative expected JSON.
It intentionally ignores Knowledge sections and does not try to be a
normative validator; the Rust frontend migration must provide the real
validation. Usage: python3 tools/dev-only/check-mq-syntax.py <dir>...
"""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
KNOWN_SEM = {
"sem.invalid_mode_marker", "sem.empty_unit_name", "sem.duplicate_unit_name", "sem.exec_without_function",
"sem.payload_nested", "sem.empty_payload", "sem.syntax", "sem.unmatched_else",
"sem.duplicate_else", "sem.unmatched_end", "sem.unclosed_control",
"sem.signature_position", "sem.unsupported_version_directive",
"sem.no_entry_function", "sem.inferred_signature",
}
HEADING = re.compile(r"^(#{1,6})\s+(.*)$")
MODE = re.compile(r"^@(exec|knowledge)\b\s*(.*)$")
def payload_of(line: str):
s = line.strip()
while True:
n = re.sub(r"^(?:>|[-+*]\s+|\d+[.)]\s+)", "", s).strip()
if n == s:
break
s = n
if len(s) >= 4 and s.startswith("**") and s.endswith("**") and "**" not in s[2:-2]:
return s[2:-2]
if len(s) >= 2 and s.startswith("*") and s.endswith("*") and not s.startswith("**") and "*" not in s[1:-1]:
return s[1:-1]
m = re.match(r"^(`+)(.*)\1$", s, re.S)
if m:
return m.group(2)
return None
def check(path: Path) -> list[str]:
errors: list[str] = []
stack: list[tuple[str, int]] = []
headings: list[tuple[int, str | None, int]] = [] # level, explicit mode
segment = "root"
def effective(level: int) -> str:
for hl, mode, _ln in reversed(headings):
if hl < level:
if mode:
return mode
# continue to a higher explicit ancestor
return "knowledge"
def close_segment(line: int) -> None:
nonlocal stack
if stack:
kind, op = stack[-1]
errors.append(f"{path}:{line}: unclosed {kind} from line {op} in {segment}")
stack = []
for i, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
m = HEADING.match(raw)
if m:
close_segment(i)
level, body = len(m.group(1)), m.group(2)
mm = MODE.match(body)
mode = mm.group(1) if mm else None
name = mm.group(2).strip() if mm else body.strip()
if mm and not name:
errors.append(f"{path}:{i}: mode without heading name")
if not mm and re.match(r"^@[A-Za-z]", name):
errors.append(f"{path}:{i}: unknown mode marker {name.split()[0]}")
while headings and headings[-1][0] >= level:
headings.pop()
headings.append((level, mode, i))
segment = f"H{level} {name}"
continue
text = payload_of(raw)
if text is None:
continue
text = text.strip()
if not text:
errors.append(f"{path}:{i}: empty executable payload")
continue
owning = False
for hl, _mode, _ln in reversed(headings):
if hl == 2:
owning = effective(2) == "exec"
break
if not owning:
continue
if ";" in text:
errors.append(f"{path}:{i}: semicolon in one payload: {text!r}")
first = text.split(None, 1)[0]
if first in ("if", "while"):
if len(text.split()) == 1:
errors.append(f"{path}:{i}: {first} without expression")
stack.append((first, i))
elif first == "else":
if text != "else":
errors.append(f"{path}:{i}: else with trailing tokens")
if not stack or not stack[-1][0].startswith("if"):
errors.append(f"{path}:{i}: unmatched else")
elif stack[-1][0] == "if_else":
errors.append(f"{path}:{i}: duplicate else")
else:
stack[-1] = ("if_else", stack[-1][1])
elif first == "end":
if text != "end":
errors.append(f"{path}:{i}: end with trailing tokens")
if not stack:
errors.append(f"{path}:{i}: unmatched end")
else:
stack.pop()
elif first == "return" and len(text.split()) == 1:
errors.append(f"{path}:{i}: return without expression")
elif first == "params" and not re.match(r"^params\s*\(.*\)$", text, re.S):
errors.append(f"{path}:{i}: malformed params")
close_segment(len(headings) + 1)
return errors
def main(argv: list[str]) -> int:
roots = [Path(a) for a in argv] or [Path("tests"), Path("examples")]
failures: list[str] = []
count = 0
for root in roots:
paths = [root] if root.is_file() else sorted(root.rglob("*"))
for path in paths:
if path.name.endswith(".mq.md"):
count += 1
failures.extend(check(path))
elif path.suffix == ".json" and "negative" in path.as_posix():
try:
data = json.loads(path.read_text(encoding="utf-8"))
except Exception as error: # noqa: BLE001 - review aid
failures.append(f"{path}: invalid JSON: {error}")
continue
for code in data.get("codes", []):
if code.startswith("sem.") and code not in KNOWN_SEM:
failures.append(f"{path}: unknown sem code {code}")
for failure in failures:
print(failure)
print(f"checked {count} .mq.md files, {len(failures)} issue(s)")
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Generate docs/migration/semantic-rewrite-inventory.tsv.
The inventory classifies every git-tracked file for the RFC-LANG-0002
single-syntax migration. It is a repository audit aid, not a compiler and not
a substitute for `cargo test` or the real frontend.
"""
from __future__ import annotations
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
OLD_MARKERS = re.compile(
r"^##\s+[^@\s].*$|^\*[^*\n].*\*$|^\*\*[^*\n].*\*\*$|"
r"markdown\.inert_executable|markdown\.invalid_construct|"
r"marqdo_semantics|--legacy|map_v1",
re.M,
)
NEW_MARKERS = re.compile(r"@exec|@knowledge")
FRONTEND = {
"crates/marqdo-frontend/src/units.rs",
"crates/marqdo-frontend/src/map.rs",
"crates/marqdo-frontend/src/payload.rs",
"crates/marqdo-frontend/src/signature.rs",
"crates/marqdo-frontend/src/analyze.rs",
"crates/marqdo-expression/src/lexer.rs",
"crates/marqdo-expression/src/parser.rs",
"crates/marqdo-expression/src/ast.rs",
"crates/marqdo-ir/src/lib.rs",
}
def tracked() -> list[str]:
"""Tracked files plus untracked-but-not-ignored files.
The migration adds new fixtures/docs before they are committed; an inventory
that only reads `git ls-files` would silently omit them.
"""
listed = subprocess.run(
["git", "ls-files"], cwd=ROOT, check=True, text=True, capture_output=True
).stdout.splitlines()
untracked = subprocess.run(
["git", "ls-files", "--others", "--exclude-standard"],
cwd=ROOT,
check=True,
text=True,
capture_output=True,
).stdout.splitlines()
return sorted({line for line in listed + untracked if line})
def deleted() -> set[str]:
out = subprocess.run(
["git", "ls-files", "--deleted"], cwd=ROOT, check=True, text=True, capture_output=True
).stdout
return {line for line in out.splitlines() if line}
def changed() -> set[str]:
out = subprocess.run(
["git", "status", "--porcelain"], cwd=ROOT, check=True, text=True, capture_output=True
).stdout
result = set()
for line in out.splitlines():
if len(line) > 3:
result.add(line[3:].strip())
return result
def classify(path: str, changed_set: set[str]) -> tuple[str, str, str, str, str, str, str]:
"""Return category, active, old_semantics, action, owner, verification, status."""
p = Path(path)
text = ""
full = ROOT / path
if full.is_file() and full.stat().st_size < 2_000_000:
try:
text = full.read_text(encoding="utf-8")
except UnicodeDecodeError:
text = ""
is_changed = path in changed_set
if path == "docs/adr/0052-unified-markdown-semantics.md":
return ("decision", "yes", "old mapping", "adopt current language decision", "docs", "review", "migrated")
if path == "docs/adr/README.md":
return ("decision", "yes", "n/a", "index current decisions/history", "docs", "review", "migrated")
if path == "docs/evidence/2026-11-21-semantic-docs-migration.md":
return ("evidence", "yes", "old syntax", "record migration evidence", "docs", "docs_consistency", "migrated")
if path.startswith("docs/adr/"):
return ("historical", "no", "decision at its date", "banner if syntax-role", "docs", "review", "historical")
if path.startswith("docs/history/") or path.startswith("docs/evidence/"):
return ("historical", "no", "archived", "banner only", "docs", "review", "historical")
if path.startswith("docs/migration/"):
return ("migration", "yes", "n/a", "maintain inventory/guide", "docs", "review", "migrated")
if path.startswith("docs/architecture/RFC-LANG-0002"):
return ("architecture", "yes", "old mapping", "adopt as sole authority", "docs", "review", "migrated")
if path.startswith("docs/architecture/Marqdo_CLI_Latest"):
return ("architecture", "no", "review input", "historical banner", "docs", "review", "historical")
if path.startswith("docs/architecture/"):
return ("architecture", "yes", "status/index", "update to RFC-LANG-0002", "docs", "docs_consistency", "migrated")
if path.startswith("docs/MARQDO_COMPUTE"):
return ("architecture", "yes", "compute RFC", "migrate examples / mark historical spelling", "docs", "review", "migrated")
if path.startswith("docs/"):
if is_changed:
return ("tutorial", "yes", "old mapping", "rewrite + historical banner where needed", "docs", "docs_consistency", "migrated")
return ("tutorial", "yes", "none", "reviewed", "docs", "review", "reviewed")
if path.startswith("spec/"):
return ("spec", "yes", "old mapping", "in-place rewrite", "language", "cargo test / review", "migrated")
if path.endswith(".mq.md"):
if path.startswith("stdlib/"):
return ("stdlib", "yes", "old syntax", "migrate with frontend + digest", "stdlib", "stdlib_product", "pending-implementation")
if path.startswith("tests/conformance/ast/"):
return ("test", "yes", "Markdown AST only", "keep AST baseline", "tests", "conformance_ast", "reviewed")
if path.startswith("tests/negative/"):
return ("test", "yes", "old negative shape", "rebuild for sem.* diagnostics", "tests", "negative", "migrated")
if path.startswith("examples/"):
return ("example", "yes", "old syntax", "migrate source", "examples", "examples_r6 / run", "migrated")
if NEW_MARKERS.search(text):
return ("test", "yes", "old syntax", "migrate source", "tests", "conformance/negative", "migrated")
return ("test", "yes", "old syntax", "needs migration", "tests", "review", "pending")
if path.startswith("tests/"):
if is_changed:
return ("test", "yes", "old syntax", "migrate source/expectations", "tests", "negative/properties", "migrated")
return ("test", "yes", "harness", "review", "tests", "review", "reviewed")
if path.startswith("crates/"):
if path in FRONTEND:
return ("rust-source", "yes", "old mapping", "implement RFC-LANG-0002", "language", "cargo test", "pending-implementation")
if "/tests/" in path or path.endswith("/tests.rs"):
return ("rust-test", "yes", "embedded old syntax", "migrate embedded .mq.md strings", "language", "cargo test", "migrated" if is_changed else "reviewed")
if is_changed:
return ("rust-source", "yes", "embedded text/help", "migrate embedded .mq.md text only", "language", "cargo test", "migrated")
return ("rust-source", "yes", "not syntax-bearing", "review", "language", "cargo test", "reviewed")
if path.startswith("stdlib/"):
return ("stdlib", "yes", "old syntax", "migrate with frontend + digest", "stdlib", "stdlib_product", "pending-implementation")
if path.startswith("tools/"):
return ("tooling", "yes", "migration aid", "review", "docs", "review", "reviewed")
if path.startswith("examples/"):
return ("example", "yes", "old syntax", "review", "examples", "run", "reviewed")
if path in {"README.md", "COMPATIBILITY.md", "CHANGELOG.md", "AGENTS.md"}:
return ("tutorial", "yes", "old mapping", "rewrite", "docs", "review", "migrated")
return ("non-semantic", "no", "none", "none", "-", "n/a", "reviewed")
def main() -> int:
changed_set = changed()
deleted_set = deleted()
rows = []
for path in tracked():
if path in deleted_set:
continue
rows.append((path, *classify(path, changed_set)))
rows.sort(key=lambda row: row[0])
out = ROOT / "docs/migration/semantic-rewrite-inventory.tsv"
with out.open("w", encoding="utf-8", newline="\n") as handle:
handle.write(
"path\tcategory\tactive\told_semantics\taction\towner\tverification\tstatus\n"
)
for row in rows:
handle.write("\t".join(row) + "\n")
counts: dict[str, int] = {}
for row in rows:
counts[row[-1]] = counts.get(row[-1], 0) + 1
print(f"wrote {out} ({len(rows)} rows)")
for status, count in sorted(counts.items()):
print(f" {status}: {count}")
return 0
if __name__ == "__main__":
sys.exit(main())