Files
marqdo-cli/tools/dev-only/gen-migration-inventory.py
T
cflmy bbeb1de768 docs(status): record the landed RFC-LANG-0002 implementation
Update status, compatibility, changelog, README, architecture overview,
ADR status, migration guide, contributor/CI/security/release notes and
the per-file inventory from 'migration pending' to the verified state:

- language SHA ffd1cc9: fmt/clippy clean, workspace 501 passed / 0 failed
  / 2 ignored, stdlib digest regenerated and pack-verified
- green evidence report added; the prior intermediate evidence is marked
  superseded
- inventory regenerated: 1015 classified files, 0 pending
- explicit non-claims: Windows/macOS native lanes and the disposable-VM
  strict lane were not re-run at this SHA, so the release decision remains
  DO NOT RELEASE

Verified: cargo test -p marqdo-cli --test docs_consistency (4/4) and
git diff --check.
2026-11-21 12:31:38 +08:00

173 lines
8.1 KiB
Python

#!/usr/bin/env python3
"""Generate docs/migration/semantic-rewrite-inventory.tsv.
The inventory classifies every git-tracked file for the RFC-LANG-0002
single-syntax migration. It is a repository audit aid, not a compiler and not
a substitute for `cargo test` or the real frontend.
"""
from __future__ import annotations
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
OLD_MARKERS = re.compile(
r"^##\s+[^@\s].*$|^\*[^*\n].*\*$|^\*\*[^*\n].*\*\*$|"
r"markdown\.inert_executable|markdown\.invalid_construct|"
r"marqdo_semantics|--legacy|map_v1",
re.M,
)
NEW_MARKERS = re.compile(r"@exec|@knowledge")
FRONTEND = {
"crates/marqdo-frontend/src/units.rs",
"crates/marqdo-frontend/src/map.rs",
"crates/marqdo-frontend/src/payload.rs",
"crates/marqdo-frontend/src/signature.rs",
"crates/marqdo-frontend/src/analyze.rs",
"crates/marqdo-expression/src/lexer.rs",
"crates/marqdo-expression/src/parser.rs",
"crates/marqdo-expression/src/ast.rs",
"crates/marqdo-ir/src/lib.rs",
}
def tracked() -> list[str]:
"""Tracked files plus untracked-but-not-ignored files.
The migration adds new fixtures/docs before they are committed; an inventory
that only reads `git ls-files` would silently omit them.
"""
listed = subprocess.run(
["git", "ls-files"], cwd=ROOT, check=True, text=True, capture_output=True
).stdout.splitlines()
untracked = subprocess.run(
["git", "ls-files", "--others", "--exclude-standard"],
cwd=ROOT,
check=True,
text=True,
capture_output=True,
).stdout.splitlines()
return sorted({line for line in listed + untracked if line})
def deleted() -> set[str]:
out = subprocess.run(
["git", "ls-files", "--deleted"], cwd=ROOT, check=True, text=True, capture_output=True
).stdout
return {line for line in out.splitlines() if line}
def changed() -> set[str]:
out = subprocess.run(
["git", "status", "--porcelain"], cwd=ROOT, check=True, text=True, capture_output=True
).stdout
result = set()
for line in out.splitlines():
if len(line) > 3:
result.add(line[3:].strip())
return result
def classify(path: str, changed_set: set[str]) -> tuple[str, str, str, str, str, str, str]:
"""Return category, active, old_semantics, action, owner, verification, status."""
p = Path(path)
text = ""
full = ROOT / path
if full.is_file() and full.stat().st_size < 2_000_000:
try:
text = full.read_text(encoding="utf-8")
except UnicodeDecodeError:
text = ""
is_changed = path in changed_set
if path == "docs/adr/0052-unified-markdown-semantics.md":
return ("decision", "yes", "old mapping", "adopt current language decision", "docs", "review", "migrated")
if path == "docs/adr/README.md":
return ("decision", "yes", "n/a", "index current decisions/history", "docs", "review", "migrated")
if path == "docs/evidence/2026-11-21-semantic-docs-migration.md":
return ("evidence", "yes", "old syntax", "record migration evidence", "docs", "docs_consistency", "migrated")
if path.startswith("docs/adr/"):
return ("historical", "no", "decision at its date", "banner if syntax-role", "docs", "review", "historical")
if path.startswith("docs/history/") or path.startswith("docs/evidence/"):
return ("historical", "no", "archived", "banner only", "docs", "review", "historical")
if path.startswith("docs/migration/"):
return ("migration", "yes", "n/a", "maintain inventory/guide", "docs", "review", "migrated")
if path.startswith("docs/architecture/RFC-LANG-0002"):
return ("architecture", "yes", "old mapping", "adopt as sole authority", "docs", "review", "migrated")
if path.startswith("docs/architecture/Marqdo_CLI_Latest"):
return ("architecture", "no", "review input", "historical banner", "docs", "review", "historical")
if path.startswith("docs/architecture/"):
return ("architecture", "yes", "status/index", "update to RFC-LANG-0002", "docs", "docs_consistency", "migrated")
if path.startswith("docs/MARQDO_COMPUTE"):
return ("architecture", "yes", "compute RFC", "migrate examples / mark historical spelling", "docs", "review", "migrated")
if path.startswith("docs/"):
if is_changed:
return ("tutorial", "yes", "old mapping", "rewrite + historical banner where needed", "docs", "docs_consistency", "migrated")
return ("tutorial", "yes", "none", "reviewed", "docs", "review", "reviewed")
if path.startswith("spec/"):
return ("spec", "yes", "old mapping", "in-place rewrite", "language", "cargo test / review", "migrated")
if path.endswith(".mq.md"):
if path.startswith("stdlib/"):
status = "migrated" if NEW_MARKERS.search(text) else "pending"
return ("stdlib", "yes", "old syntax", "migrate with frontend + digest", "stdlib", "stdlib_product", status)
if path.startswith("tests/conformance/ast/"):
return ("test", "yes", "Markdown AST only", "keep AST baseline", "tests", "conformance_ast", "reviewed")
if path.startswith("tests/negative/"):
return ("test", "yes", "old negative shape", "rebuild for sem.* diagnostics", "tests", "negative", "migrated")
if path.startswith("examples/"):
return ("example", "yes", "old syntax", "migrate source", "examples", "examples_r6 / run", "migrated")
if NEW_MARKERS.search(text):
return ("test", "yes", "old syntax", "migrate source", "tests", "conformance/negative", "migrated")
return ("test", "yes", "old syntax", "needs migration", "tests", "review", "pending")
if path.startswith("tests/"):
if is_changed:
return ("test", "yes", "old syntax", "migrate source/expectations", "tests", "negative/properties", "migrated")
return ("test", "yes", "harness", "review", "tests", "review", "reviewed")
if path.startswith("crates/"):
if path in FRONTEND:
return ("rust-source", "yes", "old mapping", "implemented RFC-LANG-0002", "language", "cargo test", "migrated")
if "/tests/" in path or path.endswith("/tests.rs"):
return ("rust-test", "yes", "embedded old syntax", "migrate embedded .mq.md strings", "language", "cargo test", "migrated" if is_changed else "reviewed")
if is_changed:
return ("rust-source", "yes", "embedded text/help", "migrate embedded .mq.md text only", "language", "cargo test", "migrated")
return ("rust-source", "yes", "not syntax-bearing", "review", "language", "cargo test", "reviewed")
if path.startswith("stdlib/"):
return ("stdlib", "yes", "old syntax", "migrate with frontend + digest", "stdlib", "stdlib_product", "migrated" if is_changed else "reviewed")
if path.startswith("tools/"):
return ("tooling", "yes", "migration aid", "review", "docs", "review", "reviewed")
if path.startswith("examples/"):
return ("example", "yes", "old syntax", "review", "examples", "run", "reviewed")
if path in {"README.md", "COMPATIBILITY.md", "CHANGELOG.md", "AGENTS.md"}:
return ("tutorial", "yes", "old mapping", "rewrite", "docs", "review", "migrated")
return ("non-semantic", "no", "none", "none", "-", "n/a", "reviewed")
def main() -> int:
changed_set = changed()
deleted_set = deleted()
rows = []
for path in tracked():
if path in deleted_set:
continue
rows.append((path, *classify(path, changed_set)))
rows.sort(key=lambda row: row[0])
out = ROOT / "docs/migration/semantic-rewrite-inventory.tsv"
with out.open("w", encoding="utf-8", newline="\n") as handle:
handle.write(
"path\tcategory\tactive\told_semantics\taction\towner\tverification\tstatus\n"
)
for row in rows:
handle.write("\t".join(row) + "\n")
counts: dict[str, int] = {}
for row in rows:
counts[row[-1]] = counts.get(row[-1], 0) + 1
print(f"wrote {out} ({len(rows)} rows)")
for status, count in sorted(counts.items()):
print(f" {status}: {count}")
return 0
if __name__ == "__main__":
sys.exit(main())