Files
marqdo/src/knowledge/compile.rs
cflmyandCursor 16f26d7d60 fix(knowledge): warm-load IR for preflight/find/reuse queries
Stop full recompile on every knowledge preflight by writing ekc/graph/ir.json
and loading it unless --refresh or missing projections, cutting chat预检 wall time.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-27 09:18:27 +08:00

676 lines
19 KiB
Rust

//! Compile KnowledgeGraph into `.marqdo/` projections.
use std::fs;
use std::path::Path;
use anyhow::{Context, Result};
use serde_json::json;
use crate::knowledge::extract::{extract_graph, ExtractOptions};
use crate::knowledge::ir::KnowledgeGraph;
use crate::knowledge::util::{module_stem, slugify, yaml_escape};
#[derive(Debug, Clone)]
pub struct KnowledgeOptions {
pub path: std::path::PathBuf,
pub out_dir: std::path::PathBuf,
}
pub fn compile(opts: KnowledgeOptions) -> Result<KnowledgeGraph> {
// Authoritative execution/binding graph: SemanticProject under `graph/`.
// Legacy EKC retrieval catalog writes only under `ekc/` and must never
// overwrite `graph/edges.json` (G01).
let semantic_root = if opts.path.is_file() {
opts.path.parent().unwrap_or_else(|| Path::new(".")).to_path_buf()
} else {
opts.path.clone()
};
let t0 = std::time::Instant::now();
let semantic = crate::sema::compile_project(&crate::sema::CompileOptions {
project_root: semantic_root,
entry: opts.path.is_file().then(|| opts.path.clone()),
lock_mode: crate::sema::LockMode::Off,
})?;
let sema_ms = t0.elapsed().as_millis();
let t1 = std::time::Instant::now();
crate::sema::write_semantic_projections(&opts.out_dir, &semantic)?;
let sema_write_ms = t1.elapsed().as_millis();
let extract_opts = ExtractOptions {
path: opts.path.clone(),
out_dir: opts.out_dir.clone(),
};
let t2 = std::time::Instant::now();
let graph = extract_graph(&extract_opts)?;
let extract_ms = t2.elapsed().as_millis();
let t3 = std::time::Instant::now();
write_projections(&opts.out_dir, &graph)?;
let retrieve_write_ms = t3.elapsed().as_millis();
let t4 = std::time::Instant::now();
write_publish_manifest(&opts.out_dir, &semantic, &graph)?;
let manifest_ms = t4.elapsed().as_millis();
eprintln!(
"marqdo knowledge: {} module(s), {} symbol(s), {} capability(ies), {} knowledge → {} (semantic graph/; retrieval ekc/)",
graph.modules.len(),
graph.symbols.len(),
graph.capabilities.len(),
graph.knowledge.len(),
opts.out_dir.display()
);
eprintln!(
"marqdo knowledge timing_ms: sema={sema_ms} sema_write={sema_write_ms} extract={extract_ms} retrieve_write={retrieve_write_ms} manifest={manifest_ms} total={}",
t0.elapsed().as_millis()
);
Ok(graph)
}
/// M06: associate semantic generation, retrieval generation, and source digests.
fn write_publish_manifest(
out_dir: &Path,
semantic: &crate::sema::Compilation,
graph: &KnowledgeGraph,
) -> Result<()> {
let current = fs::read_to_string(out_dir.join("CURRENT")).unwrap_or_default();
let generation = current.trim().to_string();
let source_digest = {
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
for a in &semantic.semantics.artifacts {
h.update(a.hashes.artifact.0.as_bytes());
}
format!("sha256:{:x}", h.finalize())
};
let evidence_digest = {
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
for a in &semantic.semantics.artifacts {
h.update(a.hashes.evidence.0.as_bytes());
}
format!("sha256:{:x}", h.finalize())
};
let manifest = json!({
"schema_version": 1,
"kind": "publish_manifest",
"semantic_generation": generation,
"retrieval": {
"ekc_graph": "ekc/graph",
"modules": graph.modules.len(),
"symbols": graph.symbols.len(),
"capabilities": graph.capabilities.len(),
"knowledge": graph.knowledge.len(),
},
"source_digest": source_digest,
"evidence_digest": evidence_digest,
"excluded_dirs": crate::project::EXCLUDED_DIRS,
"project_id": semantic.semantics.project_id,
});
let path = out_dir.join("publish_manifest.json");
fs::write(&path, serde_json::to_string_pretty(&manifest)?)
.with_context(|| format!("write {}", path.display()))?;
Ok(())
}
/// Directory for the legacy EKC retrieval projection (not execution authority).
pub fn ekc_graph_dir(out_dir: &Path) -> std::path::PathBuf {
out_dir.join("ekc").join("graph")
}
/// Path of the full IR snapshot used by warm query paths (preflight/find/reuse).
pub fn ir_snapshot_path(out_dir: &Path) -> std::path::PathBuf {
ekc_graph_dir(out_dir).join("ir.json")
}
/// Load a previously compiled KnowledgeGraph without re-extracting sources.
/// Prefers `ekc/graph/ir.json`. Does **not** walk the source tree.
pub fn load_graph(out_dir: &Path) -> Result<KnowledgeGraph> {
let ir_path = ir_snapshot_path(out_dir);
if ir_path.exists() {
let raw = fs::read_to_string(&ir_path)
.with_context(|| format!("read {}", ir_path.display()))?;
let graph: KnowledgeGraph = serde_json::from_str(&raw)
.with_context(|| format!("parse {}", ir_path.display()))?;
return Ok(graph);
}
let graph_path = ekc_graph_dir(out_dir).join("graph.json");
let edges_path = ekc_graph_dir(out_dir).join("edges.json");
if !graph_path.exists() {
anyhow::bail!(
"no EKC retrieval graph at {} — run `marqdo knowledge` first",
graph_path.display()
);
}
// Legacy projections without ir.json: edges-only fallback (no source walk).
let edges_raw = fs::read_to_string(&edges_path).unwrap_or_else(|_| "{\"edges\":[]}".into());
let edges_v: serde_json::Value = serde_json::from_str(&edges_raw)?;
let graph_raw = fs::read_to_string(&graph_path)?;
let graph_v: serde_json::Value = serde_json::from_str(&graph_raw)?;
let mut edges = Vec::new();
if let Some(arr) = edges_v.get("edges").and_then(|e| e.as_array()) {
for e in arr {
let kind = crate::knowledge::ir::EdgeKind::parse(
e.get("kind").and_then(|k| k.as_str()).unwrap_or(""),
);
edges.push(crate::knowledge::ir::Edge {
from: e.get("from").and_then(|v| v.as_str()).unwrap_or("").into(),
to: e.get("to").and_then(|v| v.as_str()).unwrap_or("").into(),
kind,
note: e.get("note").and_then(|v| v.as_str()).map(|s| s.to_string()),
});
}
}
Ok(KnowledgeGraph {
repository: crate::knowledge::ir::RepositoryInfo {
title: graph_v
.pointer("/repository/title")
.and_then(|v| v.as_str())
.unwrap_or("marqdo-project")
.into(),
root: graph_v
.pointer("/repository/root")
.and_then(|v| v.as_str())
.unwrap_or(".")
.into(),
languages: vec!["marqdo".into()],
module_count: 0,
symbol_count: 0,
capability_count: 0,
knowledge_count: 0,
},
modules: Vec::new(),
symbols: Vec::new(),
capabilities: Vec::new(),
knowledge: Vec::new(),
edges,
generated_by: graph_v
.pointer("/generated/by")
.and_then(|v| v.as_str())
.unwrap_or("marqdo")
.into(),
})
}
/// Force a full knowledge compile (write path / `--refresh`).
pub fn ensure_compiled(path: &Path, out_dir: &Path) -> Result<KnowledgeGraph> {
compile(KnowledgeOptions {
path: path.to_path_buf(),
out_dir: out_dir.to_path_buf(),
})
}
/// Query path loader: compile if missing (or `refresh`), else hot-load `ir.json`.
pub fn graph_for_query(path: &Path, out_dir: &Path, refresh: bool) -> Result<KnowledgeGraph> {
if refresh {
return ensure_compiled(path, out_dir);
}
let has_ir = ir_snapshot_path(out_dir).exists();
let has_graph = ekc_graph_dir(out_dir).join("graph.json").exists();
if has_ir || has_graph {
return load_graph(out_dir);
}
ensure_compiled(path, out_dir)
}
fn write_projections(out_dir: &Path, graph: &KnowledgeGraph) -> Result<()> {
let dirs = [
"catalog/modules",
"catalog/symbols",
"catalog/capabilities",
"knowledge/facts",
"knowledge/decisions",
"knowledge/constraints",
"knowledge/patterns",
"knowledge/failures",
"knowledge/migrations",
"ekc/graph",
"agent/policies",
"agent/skills",
"agent/contexts",
"agent/episodes",
"tests/reuse",
"tests/knowledge",
"tests/retrieval",
];
for d in dirs {
fs::create_dir_all(out_dir.join(d))
.with_context(|| format!("create {}/{}", out_dir.display(), d))?;
}
// L0 repository
fs::write(
out_dir.join("catalog/repository.mq.md"),
render_repository(graph),
)?;
// L1 modules
for m in &graph.modules {
let stem = module_stem(&m.resource);
fs::write(
out_dir.join("catalog/modules").join(format!("{stem}.md")),
render_module(m, graph),
)?;
}
// L2 symbols
for s in &graph.symbols {
let stem = slugify(&s.id.replace("::", "__").replace('/', "__"));
fs::write(
out_dir.join("catalog/symbols").join(format!("{stem}.md")),
render_symbol(s),
)?;
}
// L3 capabilities
for c in &graph.capabilities {
let stem = slugify(&c.id);
fs::write(
out_dir
.join("catalog/capabilities")
.join(format!("{stem}.md")),
render_capability(c),
)?;
}
// L4 knowledge
for k in &graph.knowledge {
let dir = out_dir.join("knowledge").join(k.kind.subdir());
let stem = slugify(&k.id.rsplit('/').next().unwrap_or(&k.id));
let path = dir.join(format!("{stem}.md"));
let body = if let Some(ref src) = k.source_text {
// Ensure generated marker without destroying authored content
if src.contains("GENERATED by marqdo") {
src.clone()
} else {
format!(
"<!-- projected by marqdo knowledge from {} -->\n{src}",
k.resource
)
}
} else {
render_knowledge(k)
};
fs::write(&path, body)?;
}
// Retrieval-only EKC graph (must not touch authoritative `graph/`).
let ekc = ekc_graph_dir(out_dir);
let graph_json = serde_json::to_string_pretty(&graph.to_graph_json())?;
fs::write(ekc.join("graph.json"), graph_json)?;
let edges_json = serde_json::to_string_pretty(&graph.to_edges_json())?;
fs::write(ekc.join("edges.json"), edges_json)?;
// Full IR snapshot for warm query paths (preflight/find/reuse) — no re-extract.
let ir_json = serde_json::to_string(&graph)?;
fs::write(ekc.join("ir.json"), ir_json)?;
// Search index
let index = json!({
"type": "Marqdo Knowledge Index",
"role": "retrieval_only",
"note": "Not an execution authority; binding uses graph/edges.json (SemanticProject).",
"generated": { "by": graph.generated_by },
"entries": graph.index_map(),
});
fs::write(
ekc.join("index.json"),
serde_json::to_string_pretty(&index)?,
)?;
// Root index.mq.md
fs::write(out_dir.join("index.mq.md"), render_index_mq(graph))?;
// Augment root catalog.yaml if present (append engineering summary)
write_engineering_summary(out_dir, graph)?;
Ok(())
}
fn write_engineering_summary(out_dir: &Path, graph: &KnowledgeGraph) -> Result<()> {
let path = out_dir.join("engineering.yaml");
let mut out = String::new();
out.push_str("# GENERATED by marqdo knowledge — do not edit by hand\n");
out.push_str("type: Marqdo Engineering Knowledge\n");
out.push_str(&format!("title: {}\n", yaml_escape(&graph.repository.title)));
out.push_str("generated:\n");
out.push_str(&format!(" by: {}\n", yaml_escape(&graph.generated_by)));
out.push_str("layers:\n");
out.push_str(&format!(" modules: {}\n", graph.modules.len()));
out.push_str(&format!(" symbols: {}\n", graph.symbols.len()));
out.push_str(&format!(" capabilities: {}\n", graph.capabilities.len()));
out.push_str(&format!(" knowledge: {}\n", graph.knowledge.len()));
out.push_str(&format!(" edges: {}\n", graph.edges.len()));
out.push_str("capabilities:\n");
if graph.capabilities.is_empty() {
out.push_str(" []\n");
} else {
for c in &graph.capabilities {
out.push_str(&format!(" - id: {}\n", yaml_escape(&c.id)));
out.push_str(&format!(" name: {}\n", yaml_escape(&c.name)));
out.push_str(&format!(" status: {}\n", c.status.as_str()));
if !c.implemented_by.is_empty() {
out.push_str(" implemented_by:\n");
for s in &c.implemented_by {
out.push_str(&format!(" - {}\n", yaml_escape(s)));
}
}
if !c.possible_duplicates.is_empty() {
out.push_str(" possible_duplicates:\n");
for s in &c.possible_duplicates {
out.push_str(&format!(" - {}\n", yaml_escape(s)));
}
}
}
}
out.push_str("knowledge:\n");
if graph.knowledge.is_empty() {
out.push_str(" []\n");
} else {
for k in &graph.knowledge {
out.push_str(&format!(" - id: {}\n", yaml_escape(&k.id)));
out.push_str(&format!(" type: {}\n", k.kind.as_str()));
out.push_str(&format!(" title: {}\n", yaml_escape(&k.title)));
out.push_str(&format!(" status: {}\n", k.status.as_str()));
}
}
fs::write(path, out)?;
Ok(())
}
fn render_repository(graph: &KnowledgeGraph) -> String {
let r = &graph.repository;
format!(
r#"---
type: Marqdo Repository
title: {title}
generated:
by: {by}
status: stable
---
# {title}
> GENERATED by Marqdo Engineering Knowledge Compiler. Do not edit by hand.
## Repository
| Field | Value |
|-------|-------|
| Languages | {langs} |
| Modules | {modules} |
| Symbols | {symbols} |
| Capabilities | {caps} |
| Knowledge | {know} |
| Edges | {edges} |
## Layers
- L0 Repository — this page
- L1 Modules — `catalog/modules/`
- L2 Symbols — `catalog/symbols/`
- L3 Capabilities — `catalog/capabilities/`
- L4 Engineering Knowledge — `knowledge/`
"#,
title = r.title,
by = graph.generated_by,
langs = r.languages.join(", "),
modules = r.module_count,
symbols = r.symbol_count,
caps = r.capability_count,
know = r.knowledge_count,
edges = graph.edges.len(),
)
}
fn render_module(m: &crate::knowledge::ir::ModuleInfo, graph: &KnowledgeGraph) -> String {
let mut exports = String::new();
if m.exports.is_empty() {
exports.push_str("_None_\n");
} else {
exports.push_str("| Symbol | Kind |\n|--------|------|\n");
for e in &m.exports {
let kind = graph
.symbols
.iter()
.find(|s| s.module == m.id && s.name == *e)
.map(|s| s.kind.as_str())
.unwrap_or("fn");
exports.push_str(&format!("| `{e}` | {kind} |\n"));
}
}
let resp = m.responsibility.as_deref().unwrap_or("_unspecified_");
format!(
r#"---
type: Marqdo Module
title: {title}
resource: {resource}
status: {status}
generated:
by: {by}
---
# {title}
> Auto-generated from `{resource}`. Do not edit by hand.
## Responsibility
{resp}
## Exports
{exports}
"#,
title = m.title,
resource = m.resource,
status = m.status.as_str(),
by = graph.generated_by,
resp = resp,
exports = exports,
)
}
fn render_symbol(s: &crate::knowledge::ir::SymbolInfo) -> String {
let params = if s.params.is_empty() {
"[]".into()
} else {
s.params
.iter()
.map(|p| format!(" - {p}"))
.collect::<Vec<_>>()
.join("\n")
};
let use_when = s.use_when.as_deref().unwrap_or("_unspecified_");
let do_not = s.do_not_use_when.as_deref().unwrap_or("_unspecified_");
format!(
r#"---
type: Function
name: {name}
module: {module}
symbol: {name}
resource: {resource}
status: {status}
visibility: {vis}
fingerprint: {fp}
capability: {cap}
generated:
by: marqdo/knowledge
---
# {name}
## Use when
{use_when}
## Do not use when
{do_not}
## Signature
params:
{params}
## Calls
{calls}
## Fingerprint
```
{fpt}
```
"#,
name = s.name,
module = s.module,
resource = s.resource,
status = s.status.as_str(),
vis = s.visibility,
fp = s.fingerprint,
cap = s.capability.as_deref().unwrap_or(""),
use_when = use_when,
do_not = do_not,
params = params,
calls = if s.calls.is_empty() {
"_None_".into()
} else {
s.calls
.iter()
.map(|c| format!("- `{c}`"))
.collect::<Vec<_>>()
.join("\n")
},
fpt = s.fingerprint_text,
)
}
fn render_capability(c: &crate::knowledge::ir::CapabilityInfo) -> String {
let use_when = c.use_when.as_deref().unwrap_or(&c.description);
let do_not = c.do_not_use_when.as_deref().unwrap_or("_unspecified_");
format!(
r#"---
type: Capability
id: {id}
name: {name}
status: {status}
generated:
by: marqdo/knowledge
---
# {name}
{desc}
## Use when
{use_when}
## Do not use when
{do_not}
## Implemented by
{impls}
## Possible duplicates
{dups}
"#,
id = c.id,
name = c.name,
status = c.status.as_str(),
desc = c.description,
use_when = use_when,
do_not = do_not,
impls = if c.implemented_by.is_empty() {
"_None_".into()
} else {
c.implemented_by
.iter()
.map(|s| format!("- `{s}`"))
.collect::<Vec<_>>()
.join("\n")
},
dups = if c.possible_duplicates.is_empty() {
"_None_".into()
} else {
c.possible_duplicates
.iter()
.map(|s| format!("- `{s}`"))
.collect::<Vec<_>>()
.join("\n")
},
)
}
fn render_knowledge(k: &crate::knowledge::ir::KnowledgeItem) -> String {
format!(
r#"---
type: {ty}
id: {id}
title: {title}
status: {status}
generated:
by: marqdo/knowledge
---
# {title}
{summary}
"#,
ty = k.kind.as_str(),
id = k.id,
title = k.title,
status = k.status.as_str(),
summary = k.body_summary,
)
}
fn render_index_mq(graph: &KnowledgeGraph) -> String {
format!(
r#"---
type: Marqdo Engineering Knowledge
title: Engineering Knowledge Index
generated:
by: {by}
---
# Engineering Knowledge Index
> GENERATED. Source of truth remains `*.mq.md`. This tree is a Knowledge Projection.
## Counts
| Layer | Count |
|-------|------:|
| Modules | {modules} |
| Symbols | {symbols} |
| Capabilities | {caps} |
| Knowledge | {know} |
| Edges | {edges} |
## Commands
```
marqdo knowledge
marqdo find "query"
marqdo reuse "task"
marqdo duplicate
marqdo impact PATH
marqdo conflicts
marqdo stale
marqdo verify
```
"#,
by = graph.generated_by,
modules = graph.modules.len(),
symbols = graph.symbols.len(),
caps = graph.capabilities.len(),
know = graph.knowledge.len(),
edges = graph.edges.len(),
)
}