Add local n-gram near match for OKF agent-kb reuse.

Sparse bigram cosine closes near-duplicate goal fragmentation; soft_match parents now judge ranked scored candidates instead of a flat task list.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
cflmyandCursor committed 2026-08-11 10:07:03 +08:00
1 parent f240bdf156
commit ab8ea8438f
9 files changed
+427 -88

No files matched your search

+248 -4
View File
@@ -1,5 +1,6 @@
//! OKF agent-kb helpers (plugin-side). No dependency on marqdo `src/host`.
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
@@ -606,6 +607,158 @@ pub fn kb_canonicalize(args: &Value) -> Result<Value, String> {
Ok(Value::String(canonicalize_goal_str(goal)))
}
fn arg_bool(args: &Value, key: &str, default: bool) -> bool {
match args.get(key) {
None | Some(Value::Null) => default,
Some(Value::Bool(b)) => *b,
Some(Value::Number(n)) => n.as_i64().unwrap_or(0) != 0,
Some(Value::String(s)) => matches!(
s.trim(),
"1" | "true" | "True" | "TRUE" | "yes" | "Yes" | "on" | "ON"
),
_ => default,
}
}
fn arg_f64(args: &Value, key: &str, default: f64) -> f64 {
match args.get(key) {
None | Some(Value::Null) => default,
Some(Value::Number(n)) => n.as_f64().unwrap_or(default),
Some(Value::String(s)) => s.parse().unwrap_or(default),
_ => default,
}
}
/// Character bigram bag (whitespace stripped). Sparse lexical vectorization — not an embedding model.
pub fn char_bigram_tf(text: &str) -> HashMap<String, f64> {
let chars: Vec<char> = text.chars().filter(|c| !c.is_whitespace()).collect();
let mut m = HashMap::new();
if chars.is_empty() {
return m;
}
if chars.len() == 1 {
m.insert(chars[0].to_string(), 1.0);
return m;
}
for w in chars.windows(2) {
let key = format!("{}{}", w[0], w[1]);
*m.entry(key).or_insert(0.0) += 1.0;
}
m
}
pub fn cosine_tf(a: &HashMap<String, f64>, b: &HashMap<String, f64>) -> f64 {
let mut na = 0.0;
let mut nb = 0.0;
for v in a.values() {
na += v * v;
}
for v in b.values() {
nb += v * v;
}
if na == 0.0 || nb == 0.0 {
return 0.0;
}
let mut dot = 0.0;
if a.len() <= b.len() {
for (k, va) in a {
if let Some(vb) = b.get(k) {
dot += va * vb;
}
}
} else {
for (k, vb) in b {
if let Some(va) = a.get(k) {
dot += va * vb;
}
}
}
dot / (na.sqrt() * nb.sqrt())
}
#[cfg(test)]
fn near_score(query: &str, candidate: &str) -> f64 {
let q = canonicalize_goal_str(query);
let c = canonicalize_goal_str(candidate);
if q.is_empty() || c.is_empty() {
return 0.0;
}
cosine_tf(&char_bigram_tf(&q), &char_bigram_tf(&c))
}
fn task_near_material(src: &str, slug: &str) -> String {
let title = extract_fm_field(src, "title").unwrap_or_else(|| slug.to_string());
let desc = extract_fm_field(src, "description").unwrap_or_default();
let aliases = extract_fm_aliases(src);
let mut parts = vec![title, desc];
parts.extend(aliases);
parts.join(" ")
}
#[derive(Clone)]
struct NearCand {
slug: String,
title: String,
score: f64,
}
fn rank_near_tasks(root: &Path, goal: &str, limit: usize) -> Vec<NearCand> {
let tasks_dir = root.join("concepts/tasks");
let skills_dir = root.join("concepts/skills");
let res_dir = root.join("resources");
if !tasks_dir.is_dir() {
return Vec::new();
}
let q = canonicalize_goal_str(goal);
let qv = char_bigram_tf(&q);
if qv.is_empty() {
return Vec::new();
}
let mut out = Vec::new();
let Ok(rd) = fs::read_dir(&tasks_dir) else {
return Vec::new();
};
for ent in rd.flatten() {
let p = ent.path();
if p.extension().and_then(|e| e.to_str()) != Some("md") {
continue;
}
let Ok(src) = fs::read_to_string(&p) else {
continue;
};
let slug = p
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_default();
if slug.is_empty() {
continue;
}
if !skills_dir.join(format!("{slug}.md")).is_file()
|| !res_dir.join(format!("{slug}.mq.md")).is_file()
{
continue;
}
let title = extract_fm_field(&src, "title").unwrap_or_else(|| slug.clone());
let material = task_near_material(&src, &slug);
let score = cosine_tf(&qv, &char_bigram_tf(&canonicalize_goal_str(&material)));
if score > 0.0 {
out.push(NearCand {
slug,
title,
score,
});
}
}
out.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| a.slug.cmp(&b.slug))
});
out.truncate(limit.max(1));
out
}
fn lookup_hit_payload(
kb: &str,
root: &Path,
@@ -613,6 +766,7 @@ fn lookup_hit_payload(
slug: &str,
match_kind: &str,
query_sig: &str,
score: Option<f64>,
) -> Result<Value, String> {
let skill_src =
fs::read_to_string(skill_path).map_err(|e| format!("kb_lookup read skill: {e}"))?;
@@ -645,7 +799,7 @@ fn lookup_hit_payload(
}
}
Ok(json!({
let mut out = json!({
"hit": true,
"match": match_kind,
"sig": skill_sig,
@@ -658,7 +812,13 @@ fn lookup_hit_payload(
"resource": kb_rel(kb, &format!("resources/{slug}.mq.md")),
"task": kb_rel(kb, &format!("concepts/tasks/{slug}.md")),
"skill": kb_rel(kb, &format!("concepts/skills/{slug}.md")),
}))
});
if let Some(s) = score {
if let Some(obj) = out.as_object_mut() {
obj.insert("score".into(), json!((s * 1000.0).round() / 1000.0));
}
}
Ok(out)
}
fn resolve_skill_paths_by_slug(
@@ -699,7 +859,7 @@ pub fn kb_lookup(args: &Value) -> Result<Value, String> {
.and_then(|v| v.as_str())
.map(|g| goal_sig_hex(&material_for(g, tools)))
.unwrap_or_default();
return lookup_hit_payload(kb, &root, &skill_path, &slug, "soft", &query_sig);
return lookup_hit_payload(kb, &root, &skill_path, &slug, "soft", &query_sig, None);
}
let goal = arg_text(args, "goal")?;
@@ -707,6 +867,7 @@ pub fn kb_lookup(args: &Value) -> Result<Value, String> {
let query_sig = goal_sig_hex(&material_for(goal, tools));
let slug_hint = goal_slug_str(goal);
let mut match_kind = "exact";
let mut near_score_hit: Option<f64> = None;
let resolved = resolve_skill_paths(&root, &query_sig, &slug_hint)
.or_else(|| {
match_kind = "alias";
@@ -722,12 +883,72 @@ pub fn kb_lookup(args: &Value) -> Result<Value, String> {
let slug2 = goal_slug_str(&canon);
resolve_skill_paths(&root, &sig2, &slug2)
.or_else(|| resolve_skill_paths_by_alias(&root, &canon))
})
.or_else(|| {
if !arg_bool(args, "near_match", false) {
return None;
}
let threshold = arg_f64(args, "near_threshold", 0.78);
let ranked = rank_near_tasks(&root, goal, 8);
let best = ranked.first()?;
if best.score < threshold {
return None;
}
match_kind = "near";
near_score_hit = Some(best.score);
resolve_skill_paths_by_slug(&root, &best.slug)
});
let Some((_task, skill_path, _res, slug)) = resolved else {
return Ok(Value::Null);
};
lookup_hit_payload(kb, &root, &skill_path, &slug, match_kind, &query_sig)
lookup_hit_payload(
kb,
&root,
&skill_path,
&slug,
match_kind,
&query_sig,
near_score_hit,
)
}
pub fn kb_near_match(args: &Value) -> Result<Value, String> {
let kb = arg_text(args, "kb_dir")?;
let goal = arg_text(args, "goal")?;
let limit = match args.get("limit") {
None | Some(Value::Null) => 8usize,
Some(Value::Number(n)) => n.as_u64().unwrap_or(8) as usize,
Some(Value::String(s)) => s.parse().unwrap_or(8),
_ => 8,
}
.clamp(1, 40);
let root = resolve_path(kb);
let ranked = rank_near_tasks(&root, goal, limit);
let candidates: Vec<Value> = ranked
.iter()
.map(|c| {
json!({
"slug": c.slug,
"title": c.title,
"score": (c.score * 1000.0).round() / 1000.0,
})
})
.collect();
let best = ranked.first().map(|c| {
json!({
"slug": c.slug,
"title": c.title,
"score": (c.score * 1000.0).round() / 1000.0,
})
});
let score = ranked.first().map(|c| (c.score * 1000.0).round() / 1000.0);
Ok(json!({
"candidates": candidates,
"count": ranked.len() as i64,
"best": best,
"score": score,
}))
}
pub fn kb_list_tasks(args: &Value) -> Result<Value, String> {
@@ -1102,4 +1323,27 @@ mod tests {
let c2 = canonicalize_goal_str("你是一个智能体,帮我规划明天的行程?");
assert_eq!(c2, "帮我规划明天的行程");
}
#[test]
fn near_score_trip_near_dupes_high() {
let a = "帮我规划明天行程";
let b = "帮我规划明天的行程";
let s = near_score(a, b);
assert!(s >= 0.78, "expected high near score, got {s}");
let c = "帮我完成明天的行程规划";
let s2 = near_score(b, c);
let unrelated = near_score(b, "明天做什么好");
assert!(
s2 > unrelated,
"paraphrase should beat unrelated: paraphrase={s2} unrelated={unrelated}"
);
}
#[test]
fn near_score_unrelated_low() {
let a = "帮我规划明天的行程";
let b = "明天做什么好";
let s = near_score(a, b);
assert!(s < 0.78, "expected below default threshold, got {s}");
}
}
+4
View File
@@ -879,6 +879,7 @@ kb_ffi!(agent_goal_sig, kb::goal_sig, "agent_goal_sig");
kb_ffi!(agent_goal_slug, kb::goal_slug, "agent_goal_slug");
kb_ffi!(agent_kb_canonicalize, kb::kb_canonicalize, "agent_kb_canonicalize");
kb_ffi!(agent_kb_lookup, kb::kb_lookup, "agent_kb_lookup");
kb_ffi!(agent_kb_near_match, kb::kb_near_match, "agent_kb_near_match");
kb_ffi!(agent_kb_list_tasks, kb::kb_list_tasks, "agent_kb_list_tasks");
kb_ffi!(agent_kb_add_alias, kb::kb_add_alias, "agent_kb_add_alias");
kb_ffi!(agent_kb_promote, kb::kb_promote, "agent_kb_promote");
@@ -971,6 +972,9 @@ pub unsafe extern "C" fn marqdo_plugin_init(host: *const MarqdoHostApi) -> c_int
if register(host, "agent_kb_lookup", "kb_dir,goal", agent_kb_lookup) != 0 {
return 1;
}
if register(host, "agent_kb_near_match", "kb_dir,goal", agent_kb_near_match) != 0 {
return 1;
}
if register(host, "agent_kb_list_tasks", "kb_dir", agent_kb_list_tasks) != 0 {
return 1;
}