feat(agent): CheckpointV2 atomic resume and AgentErrorV1
Replace bare resume writes with temp/fsync/rename, mark legacy loads as manual_review, and expose bounded read-only retry decisions while keeping write auto-resume closed without a ledger. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
303c06f773
commit
5912164039
6 files changed
+618
-30
No files matched your search
+46
-2
@@ -677,8 +677,12 @@ Close a stdio MCP client session.
|
||||
+ `route`=None
|
||||
+ `result`=None
|
||||
+ `resume_dir`=.marqdo/agent-resume
|
||||
+ `checkpoint_v2`=False
|
||||
+ `attempt_id`=None
|
||||
+ `effect_ledger_watermark`=None
|
||||
|
||||
Persist a plan/run checkpoint as `.marqdo/agent-resume/<id>.json`.
|
||||
Persist a plan/run checkpoint as `.marqdo/agent-resume/<id>.json` (atomic write).
|
||||
Pass `checkpoint_v2=True` for CheckpointV2 schema. `resume_dir` must be host-relative.
|
||||
|
||||
**p = > plugin.native_path name="agent"**
|
||||
1. `p`
|
||||
@@ -687,16 +691,56 @@ Persist a plan/run checkpoint as `.marqdo/agent-resume/<id>.json`.
|
||||
> print text=ext/ai/agent: native agent plugin not found
|
||||
> sys.exit code=1
|
||||
|
||||
*> agent_resume_save id=`id` goal=`goal` task=`task` status=`status` round=`round` workbook=`workbook` events=`events` route=`route` result=`result` resume_dir=`resume_dir`*
|
||||
*> agent_resume_save id=`id` goal=`goal` task=`task` status=`status` round=`round` workbook=`workbook` events=`events` route=`route` result=`result` resume_dir=`resume_dir` checkpoint_v2=`checkpoint_v2` attempt_id=`attempt_id` effect_ledger_watermark=`effect_ledger_watermark`*
|
||||
|
||||
## resume_load
|
||||
+ `id`
|
||||
+ `resume_dir`=.marqdo/agent-resume
|
||||
|
||||
Load a resume checkpoint (or `{ok:false,found:false}`).
|
||||
Legacy / missing effect ledger → `recovery_mode=manual_review`, `write_auto_resume=false`.
|
||||
|
||||
*> agent_resume_load id=`id` resume_dir=`resume_dir`*
|
||||
|
||||
## error_make
|
||||
+ `code`
|
||||
+ `message`=None
|
||||
+ `category`=None
|
||||
+ `stage`=agent
|
||||
+ `retryable`=None
|
||||
+ `effect_state`=none
|
||||
+ `recovery_action`=None
|
||||
+ `call_id`=None
|
||||
|
||||
Build an `AgentErrorV1` document (M-A1).
|
||||
|
||||
**p = > plugin.native_path name="agent"**
|
||||
1. `p`
|
||||
> plugin.load path=`p`
|
||||
2. *
|
||||
> print text=ext/ai/agent: native agent plugin not found
|
||||
> sys.exit code=1
|
||||
|
||||
*> agent_error_make code=`code` message=`message` category=`category` stage=`stage` retryable=`retryable` effect_state=`effect_state` recovery_action=`recovery_action` call_id=`call_id`*
|
||||
|
||||
## error_decide_retry
|
||||
+ `error`
|
||||
+ `attempts`=0
|
||||
+ `effect_class`=read
|
||||
+ `cancelled`=False
|
||||
+ `fingerprint_repeats`=0
|
||||
|
||||
Bounded recovery decision. Write / unknown effect → `manual_review` (no auto redo).
|
||||
|
||||
**p = > plugin.native_path name="agent"**
|
||||
1. `p`
|
||||
> plugin.load path=`p`
|
||||
2. *
|
||||
> print text=ext/ai/agent: native agent plugin not found
|
||||
> sys.exit code=1
|
||||
|
||||
*> agent_error_decide_retry error=`error` attempts=`attempts` effect_class=`effect_class` cancelled=`cancelled` fingerprint_repeats=`fingerprint_repeats`*
|
||||
|
||||
## resume_clear
|
||||
+ `id`
|
||||
+ `resume_dir`=.marqdo/agent-resume
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
//! AgentErrorV1 — classified errors and bounded recovery decisions (M-A1).
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// Build an AgentErrorV1 document from caller fields (compat: missing → defaults).
|
||||
pub fn make_error(args: &Value) -> Result<Value, String> {
|
||||
let code = args
|
||||
.get("code")
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
.ok_or_else(|| "missing text `code`".to_string())?;
|
||||
let category = args
|
||||
.get("category")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_else(|| infer_category(code));
|
||||
let stage = args
|
||||
.get("stage")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("agent");
|
||||
let message = args
|
||||
.get("message")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or(code);
|
||||
let retryable = args
|
||||
.get("retryable")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or_else(|| default_retryable(category));
|
||||
let effect_state = args
|
||||
.get("effect_state")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("none");
|
||||
let recovery_action = args
|
||||
.get("recovery_action")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_else(|| default_recovery(category, effect_state));
|
||||
|
||||
let mut out = json!({
|
||||
"schema": "AgentErrorV1",
|
||||
"schema_version": 1,
|
||||
"code": code,
|
||||
"category": category,
|
||||
"stage": stage,
|
||||
"message": message,
|
||||
"retryable": retryable,
|
||||
"effect_state": effect_state,
|
||||
"recovery_action": recovery_action,
|
||||
});
|
||||
if let Some(obj) = out.as_object_mut() {
|
||||
if let Some(v) = args.get("retry_after_ms") {
|
||||
obj.insert("retry_after_ms".into(), v.clone());
|
||||
} else {
|
||||
obj.insert("retry_after_ms".into(), Value::Null);
|
||||
}
|
||||
for key in ["call_id", "details_ref", "fingerprint"] {
|
||||
if let Some(v) = args.get(key) {
|
||||
if !v.is_null() {
|
||||
obj.insert(key.into(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Decide whether a step may auto-retry (read-only / no-effect only in M-A1).
|
||||
///
|
||||
/// Returns `{ok, action, reason, attempts_used, max_attempts}`.
|
||||
pub fn decide_retry(args: &Value) -> Result<Value, String> {
|
||||
let error = args
|
||||
.get("error")
|
||||
.cloned()
|
||||
.ok_or_else(|| "missing map `error`".to_string())?;
|
||||
let attempts = args
|
||||
.get("attempts")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0) as u32;
|
||||
let max_transport = args
|
||||
.get("max_transport_attempts")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(3) as u32; // 1 initial + 2 retries
|
||||
let max_arg_repair = args
|
||||
.get("max_arg_repairs")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(1) as u32;
|
||||
let effect_class = args
|
||||
.get("effect_class")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("read");
|
||||
let cancelled = args
|
||||
.get("cancelled")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let fingerprint_repeats = args
|
||||
.get("fingerprint_repeats")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0) as u32;
|
||||
|
||||
if cancelled {
|
||||
return Ok(decision("stop", "cancelled", attempts, max_transport));
|
||||
}
|
||||
|
||||
let category = error
|
||||
.get("category")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
let retryable = error
|
||||
.get("retryable")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let effect_state = error
|
||||
.get("effect_state")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("none");
|
||||
let recovery = error
|
||||
.get("recovery_action")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("none");
|
||||
|
||||
// Write / unknown effect: never auto-retry in M-A1 (ledger required).
|
||||
if effect_class != "read" || matches!(effect_state, "unknown" | "started" | "committed") {
|
||||
return Ok(decision(
|
||||
"manual_review",
|
||||
"write_or_unknown_effect_requires_ledger",
|
||||
attempts,
|
||||
max_transport,
|
||||
));
|
||||
}
|
||||
|
||||
if fingerprint_repeats >= 2 {
|
||||
return Ok(decision(
|
||||
"stop",
|
||||
"error_fingerprint_loop",
|
||||
attempts,
|
||||
max_transport,
|
||||
));
|
||||
}
|
||||
|
||||
match category {
|
||||
"transient" if retryable && attempts + 1 < max_transport => Ok(decision(
|
||||
"retry_step",
|
||||
"transient_read_safe",
|
||||
attempts,
|
||||
max_transport,
|
||||
)),
|
||||
"invalid_arguments" if recovery == "repair_args" && attempts < max_arg_repair => Ok(
|
||||
decision("repair_args", "one_arg_repair", attempts, max_arg_repair),
|
||||
),
|
||||
"auth" | "forbidden" | "policy_denied" => Ok(decision(
|
||||
"needs_input",
|
||||
category,
|
||||
attempts,
|
||||
max_transport,
|
||||
)),
|
||||
"budget_exhausted" => Ok(decision("partial", category, attempts, max_transport)),
|
||||
"cancelled" => Ok(decision("stop", "cancelled", attempts, max_transport)),
|
||||
"unknown_effect" | "worker_crash" => Ok(decision(
|
||||
"manual_review",
|
||||
category,
|
||||
attempts,
|
||||
max_transport,
|
||||
)),
|
||||
_ => Ok(decision("stop", "not_auto_retryable", attempts, max_transport)),
|
||||
}
|
||||
}
|
||||
|
||||
fn decision(action: &str, reason: &str, attempts: u32, max_attempts: u32) -> Value {
|
||||
json!({
|
||||
"ok": true,
|
||||
"schema": "AgentRetryDecisionV1",
|
||||
"schema_version": 1,
|
||||
"action": action,
|
||||
"reason": reason,
|
||||
"attempts_used": attempts,
|
||||
"max_attempts": max_attempts,
|
||||
"auto": matches!(action, "retry_step" | "repair_args"),
|
||||
})
|
||||
}
|
||||
|
||||
fn infer_category(code: &str) -> &'static str {
|
||||
let c = code.to_ascii_lowercase();
|
||||
if c.contains("timeout") || c.contains("429") || c.contains("5xx") || c.contains("connect") {
|
||||
"transient"
|
||||
} else if c.contains("schema") || c.contains("argument") {
|
||||
"invalid_arguments"
|
||||
} else if c.contains("auth") {
|
||||
"auth"
|
||||
} else if c.contains("forbid") {
|
||||
"forbidden"
|
||||
} else if c.contains("policy") {
|
||||
"policy_denied"
|
||||
} else if c.contains("budget") {
|
||||
"budget_exhausted"
|
||||
} else if c.contains("cancel") {
|
||||
"cancelled"
|
||||
} else if c.contains("conflict") || c.contains("stale") {
|
||||
"conflict"
|
||||
} else if c.contains("effect") || c.contains("crash") {
|
||||
"unknown_effect"
|
||||
} else {
|
||||
"unknown"
|
||||
}
|
||||
}
|
||||
|
||||
fn default_retryable(category: &str) -> bool {
|
||||
matches!(category, "transient")
|
||||
}
|
||||
|
||||
fn default_recovery(category: &str, effect_state: &str) -> &'static str {
|
||||
if effect_state == "unknown" {
|
||||
return "reconcile";
|
||||
}
|
||||
match category {
|
||||
"transient" => "retry_step",
|
||||
"invalid_arguments" => "repair_args",
|
||||
"auth" | "forbidden" | "policy_denied" | "needs_input" => "needs_input",
|
||||
"budget_exhausted" => "return_partial",
|
||||
"cancelled" => "stop",
|
||||
"unknown_effect" | "worker_crash" => "manual_review",
|
||||
_ => "stop",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn make_error_defaults() {
|
||||
let err = make_error(&json!({"code": "tool.timeout", "message": "t"})).unwrap();
|
||||
assert_eq!(err["schema"], "AgentErrorV1");
|
||||
assert_eq!(err["category"], "transient");
|
||||
assert_eq!(err["retryable"], true);
|
||||
assert_eq!(err["recovery_action"], "retry_step");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_transient_retries_once() {
|
||||
let err = make_error(&json!({"code": "tool.timeout"})).unwrap();
|
||||
let d = decide_retry(&json!({
|
||||
"error": err,
|
||||
"attempts": 0,
|
||||
"effect_class": "read"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(d["action"], "retry_step");
|
||||
assert_eq!(d["auto"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_never_auto_retries() {
|
||||
let err = make_error(&json!({"code": "tool.timeout"})).unwrap();
|
||||
let d = decide_retry(&json!({
|
||||
"error": err,
|
||||
"attempts": 0,
|
||||
"effect_class": "write"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(d["action"], "manual_review");
|
||||
assert_eq!(d["auto"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_loop_stops() {
|
||||
let err = make_error(&json!({"code": "tool.timeout"})).unwrap();
|
||||
let d = decide_retry(&json!({
|
||||
"error": err,
|
||||
"attempts": 0,
|
||||
"effect_class": "read",
|
||||
"fingerprint_repeats": 2
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(d["action"], "stop");
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
mod corpus;
|
||||
mod eng;
|
||||
mod error;
|
||||
mod kb;
|
||||
mod mcp_client;
|
||||
mod mcp_server;
|
||||
@@ -829,6 +830,8 @@ kb_ffi!(agent_resume_save, resume::resume_save, "agent_resume_save");
|
||||
kb_ffi!(agent_resume_load, resume::resume_load, "agent_resume_load");
|
||||
kb_ffi!(agent_resume_clear, resume::resume_clear, "agent_resume_clear");
|
||||
kb_ffi!(agent_resume_list, resume::resume_list, "agent_resume_list");
|
||||
kb_ffi!(agent_error_make, error::make_error, "agent_error_make");
|
||||
kb_ffi!(agent_error_decide_retry, error::decide_retry, "agent_error_decide_retry");
|
||||
kb_ffi!(agent_eng_preflight, eng::eng_preflight, "agent_eng_preflight");
|
||||
kb_ffi!(agent_eng_reuse, eng::eng_reuse, "agent_eng_reuse");
|
||||
kb_ffi!(agent_eng_bind_execute, eng::eng_bind_execute, "agent_eng_bind_execute");
|
||||
@@ -878,6 +881,8 @@ pub unsafe extern "C" fn marqdo_plugin_init(host: *const MarqdoHostApi) -> c_int
|
||||
("agent_resume_load", "id,resume_dir", agent_resume_load),
|
||||
("agent_resume_clear", "id,resume_dir", agent_resume_clear),
|
||||
("agent_resume_list", "resume_dir", agent_resume_list),
|
||||
("agent_error_make", "code,message,category", agent_error_make),
|
||||
("agent_error_decide_retry", "error,attempts,effect_class", agent_error_decide_retry),
|
||||
("agent_kb_add_alias", "kb_dir,slug,alias", agent_kb_add_alias),
|
||||
("agent_kb_promote", "kb_dir,goal,workbook", agent_kb_promote),
|
||||
("agent_kb_quarantine", "kb_dir,slug", agent_kb_quarantine),
|
||||
|
||||
+283
-16
@@ -1,6 +1,8 @@
|
||||
//! Agent plan/run resume checkpoints under `.marqdo/agent-resume/`.
|
||||
//! M-A1: CheckpointV2 atomic write (temp→fsync→rename); legacy load → manual_review.
|
||||
|
||||
use std::fs;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde_json::{json, Value};
|
||||
@@ -27,7 +29,9 @@ fn resolve_path(rel: &str) -> PathBuf {
|
||||
}
|
||||
|
||||
fn opt_text<'a>(args: &'a Value, key: &str) -> Option<&'a str> {
|
||||
args.get(key).and_then(|v| v.as_str()).filter(|s| !s.is_empty())
|
||||
args.get(key)
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
fn arg_text<'a>(args: &'a Value, key: &str) -> Result<&'a str, String> {
|
||||
@@ -43,9 +47,19 @@ fn opt_i64(args: &Value, key: &str, default: i64) -> i64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn resume_root(args: &Value) -> PathBuf {
|
||||
/// Host-relative resume roots only — reject absolute client paths (M-A1).
|
||||
fn resume_root(args: &Value) -> Result<PathBuf, String> {
|
||||
let rel = opt_text(args, "resume_dir").unwrap_or(DEFAULT_RESUME_DIR);
|
||||
resolve_path(rel)
|
||||
let p = PathBuf::from(rel);
|
||||
if p.is_absolute() {
|
||||
return Err(
|
||||
"resume_dir must be host-relative; absolute client paths are rejected".into(),
|
||||
);
|
||||
}
|
||||
if rel.contains("..") {
|
||||
return Err("resume_dir must not contain '..'".into());
|
||||
}
|
||||
Ok(resolve_path(rel))
|
||||
}
|
||||
|
||||
fn checkpoint_path(root: &Path, id: &str) -> PathBuf {
|
||||
@@ -70,11 +84,106 @@ fn ensure_dir(root: &Path) -> Result<(), String> {
|
||||
fs::create_dir_all(root).map_err(|e| format!("resume mkdir: {e}"))
|
||||
}
|
||||
|
||||
pub fn resume_save(args: &Value) -> Result<Value, String> {
|
||||
let id = arg_text(args, "id")?;
|
||||
let root = resume_root(args);
|
||||
ensure_dir(&root)?;
|
||||
let path = checkpoint_path(&root, id);
|
||||
/// Atomic durable write: unique temp → write → flush/fsync → rename.
|
||||
fn atomic_write_json(path: &Path, doc: &Value) -> Result<(), String> {
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| "checkpoint path has no parent".to_string())?;
|
||||
ensure_dir(parent)?;
|
||||
let tmp_name = format!(
|
||||
".{}.tmp-{}-{}",
|
||||
path.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("ckpt.json"),
|
||||
std::process::id(),
|
||||
now_rfc3339().replace(':', "")
|
||||
);
|
||||
let tmp = parent.join(tmp_name);
|
||||
let text = serde_json::to_string_pretty(doc).map_err(|e| e.to_string())?;
|
||||
{
|
||||
let mut f = OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.open(&tmp)
|
||||
.map_err(|e| format!("resume temp open: {e}"))?;
|
||||
f.write_all(text.as_bytes())
|
||||
.map_err(|e| format!("resume temp write: {e}"))?;
|
||||
f.flush().map_err(|e| format!("resume temp flush: {e}"))?;
|
||||
f.sync_all()
|
||||
.map_err(|e| format!("resume temp fsync: {e}"))?;
|
||||
}
|
||||
// Best-effort directory fsync (platform may ignore).
|
||||
if let Ok(dir) = File::open(parent) {
|
||||
let _ = dir.sync_all();
|
||||
}
|
||||
fs::rename(&tmp, path).map_err(|e| {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
format!("resume atomic rename: {e}")
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn wants_checkpoint_v2(args: &Value) -> bool {
|
||||
matches!(
|
||||
args.get("schema").and_then(|v| v.as_str()),
|
||||
Some("CheckpointV2")
|
||||
) || args.get("schema_version").and_then(|v| v.as_u64()) == Some(2)
|
||||
|| args.get("checkpoint_v2").and_then(|v| v.as_bool()) == Some(true)
|
||||
}
|
||||
|
||||
fn build_checkpoint_v2(args: &Value, id: &str) -> Value {
|
||||
let committed = now_rfc3339();
|
||||
let mut doc = json!({
|
||||
"schema": "CheckpointV2",
|
||||
"schema_version": 2,
|
||||
"checkpoint_id": args.get("checkpoint_id").cloned().unwrap_or_else(|| json!(id)),
|
||||
"logical_run_id": args.get("logical_run_id").cloned().unwrap_or_else(|| json!(id)),
|
||||
"task_id": args.get("task_id").cloned().unwrap_or_else(|| json!(id)),
|
||||
"attempt_id": args.get("attempt_id").cloned().unwrap_or_else(|| json!("attempt_1")),
|
||||
"step_id": args.get("step_id").cloned().unwrap_or(Value::Null),
|
||||
"parent_checkpoint_id": args.get("parent_checkpoint_id").cloned().unwrap_or(Value::Null),
|
||||
"principal_ref": args.get("principal_ref").cloned().unwrap_or(Value::Null),
|
||||
"workspace_id": args.get("workspace_id").cloned().unwrap_or(Value::Null),
|
||||
"session_id": args.get("session_id").cloned().unwrap_or(Value::Null),
|
||||
"completed_steps": args.get("completed_steps").cloned().unwrap_or_else(|| json!([])),
|
||||
"pending_operation": args.get("pending_operation").cloned().unwrap_or(Value::Null),
|
||||
"deadline": args.get("deadline").cloned().unwrap_or(Value::Null),
|
||||
"committed_at": args.get("committed_at").cloned().unwrap_or_else(|| json!(committed)),
|
||||
// Effect watermark — absent/null means no ledger yet (write auto-resume closed).
|
||||
"effect_ledger_watermark": args.get("effect_ledger_watermark").cloned().unwrap_or(Value::Null),
|
||||
"event_seq": args.get("event_seq").cloned().unwrap_or(Value::Null),
|
||||
"manifest_digest": args.get("manifest_digest").cloned().unwrap_or(Value::Null),
|
||||
});
|
||||
if let Some(obj) = doc.as_object_mut() {
|
||||
// Preserve legacy author fields for dual-read.
|
||||
for key in [
|
||||
"goal", "task", "status", "workbook", "events", "route", "result", "round", "meta",
|
||||
"id",
|
||||
] {
|
||||
if let Some(v) = args.get(key) {
|
||||
if !v.is_null() {
|
||||
obj.insert(key.to_string(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
if !obj.contains_key("id") {
|
||||
obj.insert("id".into(), json!(id));
|
||||
}
|
||||
if !obj.contains_key("updated_at") {
|
||||
obj.insert("updated_at".into(), json!(committed.clone()));
|
||||
}
|
||||
if !obj.contains_key("round") {
|
||||
obj.insert("round".into(), json!(opt_i64(args, "round", 0)));
|
||||
}
|
||||
if !obj.contains_key("status") {
|
||||
obj.insert("status".into(), json!("running"));
|
||||
}
|
||||
}
|
||||
doc
|
||||
}
|
||||
|
||||
fn build_legacy_doc(args: &Value, id: &str) -> Value {
|
||||
let mut doc = json!({
|
||||
"id": id,
|
||||
"updated_at": now_rfc3339(),
|
||||
@@ -96,19 +205,61 @@ pub fn resume_save(args: &Value) -> Result<Value, String> {
|
||||
obj.insert("status".into(), json!("running"));
|
||||
}
|
||||
}
|
||||
let text = serde_json::to_string_pretty(&doc).map_err(|e| e.to_string())?;
|
||||
fs::write(&path, text).map_err(|e| format!("resume save: {e}"))?;
|
||||
doc
|
||||
}
|
||||
|
||||
pub fn resume_save(args: &Value) -> Result<Value, String> {
|
||||
let id = arg_text(args, "id")?;
|
||||
let root = resume_root(args)?;
|
||||
ensure_dir(&root)?;
|
||||
let path = checkpoint_path(&root, id);
|
||||
let v2 = wants_checkpoint_v2(args);
|
||||
let doc = if v2 {
|
||||
build_checkpoint_v2(args, id)
|
||||
} else {
|
||||
build_legacy_doc(args, id)
|
||||
};
|
||||
atomic_write_json(&path, &doc)?;
|
||||
Ok(json!({
|
||||
"ok": true,
|
||||
"id": id,
|
||||
"path": path.to_string_lossy(),
|
||||
"updated_at": doc.get("updated_at").cloned().unwrap_or(Value::Null),
|
||||
"schema": doc.get("schema").cloned().unwrap_or(Value::Null),
|
||||
"schema_version": doc.get("schema_version").cloned().unwrap_or(Value::Null),
|
||||
"updated_at": doc.get("updated_at").or_else(|| doc.get("committed_at")).cloned().unwrap_or(Value::Null),
|
||||
"atomic": true,
|
||||
}))
|
||||
}
|
||||
|
||||
fn annotate_recovery(mut doc: Value) -> Value {
|
||||
let is_v2 = doc.get("schema").and_then(|v| v.as_str()) == Some("CheckpointV2")
|
||||
&& doc.get("schema_version").and_then(|v| v.as_u64()) == Some(2);
|
||||
let has_ledger = doc
|
||||
.get("effect_ledger_watermark")
|
||||
.map(|v| !v.is_null())
|
||||
.unwrap_or(false);
|
||||
let (recovery_mode, write_auto_resume) = if is_v2 && has_ledger {
|
||||
("checkpoint_v2", true)
|
||||
} else if is_v2 {
|
||||
// V2 without ledger: read resume OK; write auto-resume still closed.
|
||||
("checkpoint_v2_read_only", false)
|
||||
} else {
|
||||
// Legacy: no effect info → manual_review; never auto redo writes.
|
||||
("manual_review", false)
|
||||
};
|
||||
if let Some(obj) = doc.as_object_mut() {
|
||||
obj.insert("recovery_mode".into(), json!(recovery_mode));
|
||||
obj.insert("write_auto_resume".into(), json!(write_auto_resume));
|
||||
if !is_v2 {
|
||||
obj.insert("legacy_adaptation".into(), json!("manual_review"));
|
||||
}
|
||||
}
|
||||
doc
|
||||
}
|
||||
|
||||
pub fn resume_load(args: &Value) -> Result<Value, String> {
|
||||
let id = arg_text(args, "id")?;
|
||||
let root = resume_root(args);
|
||||
let root = resume_root(args)?;
|
||||
let path = checkpoint_path(&root, id);
|
||||
if !path.is_file() {
|
||||
return Ok(json!({
|
||||
@@ -119,7 +270,31 @@ pub fn resume_load(args: &Value) -> Result<Value, String> {
|
||||
}));
|
||||
}
|
||||
let text = fs::read_to_string(&path).map_err(|e| format!("resume load: {e}"))?;
|
||||
let mut doc: Value = serde_json::from_str(&text).map_err(|e| format!("resume json: {e}"))?;
|
||||
let parsed: Value = match serde_json::from_str(&text) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
return Ok(json!({
|
||||
"ok": false,
|
||||
"found": true,
|
||||
"id": id,
|
||||
"path": path.to_string_lossy(),
|
||||
"error": {
|
||||
"schema": "AgentErrorV1",
|
||||
"schema_version": 1,
|
||||
"code": "checkpoint.corrupt",
|
||||
"category": "unknown_effect",
|
||||
"stage": "checkpoint",
|
||||
"message": format!("corrupt checkpoint json: {e}"),
|
||||
"retryable": false,
|
||||
"effect_state": "unknown",
|
||||
"recovery_action": "manual_review",
|
||||
},
|
||||
"recovery_mode": "manual_review",
|
||||
"write_auto_resume": false,
|
||||
}));
|
||||
}
|
||||
};
|
||||
let mut doc = annotate_recovery(parsed);
|
||||
if let Some(obj) = doc.as_object_mut() {
|
||||
obj.insert("ok".into(), json!(true));
|
||||
obj.insert("found".into(), json!(true));
|
||||
@@ -130,7 +305,7 @@ pub fn resume_load(args: &Value) -> Result<Value, String> {
|
||||
|
||||
pub fn resume_clear(args: &Value) -> Result<Value, String> {
|
||||
let id = arg_text(args, "id")?;
|
||||
let root = resume_root(args);
|
||||
let root = resume_root(args)?;
|
||||
let path = checkpoint_path(&root, id);
|
||||
let existed = path.is_file();
|
||||
if existed {
|
||||
@@ -145,7 +320,7 @@ pub fn resume_clear(args: &Value) -> Result<Value, String> {
|
||||
}
|
||||
|
||||
pub fn resume_list(args: &Value) -> Result<Value, String> {
|
||||
let root = resume_root(args);
|
||||
let root = resume_root(args)?;
|
||||
if !root.is_dir() {
|
||||
return Ok(json!({ "ok": true, "items": [], "resume_dir": root.to_string_lossy() }));
|
||||
}
|
||||
@@ -156,6 +331,14 @@ pub fn resume_list(args: &Value) -> Result<Value, String> {
|
||||
if p.extension().and_then(|e| e.to_str()) != Some("json") {
|
||||
continue;
|
||||
}
|
||||
// Skip leftover temp files from crashed writers.
|
||||
if p.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.map(|s| s.starts_with('.'))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let id = p
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
@@ -179,3 +362,87 @@ pub fn resume_list(args: &Value) -> Result<Value, String> {
|
||||
"resume_dir": root.to_string_lossy(),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn scratch(label: &str) -> PathBuf {
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let dir = std::env::temp_dir().join(format!("marqdo-resume-{label}-{nanos}"));
|
||||
fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absolute_resume_dir_rejected() {
|
||||
let err = resume_root(&json!({"resume_dir": "/tmp/evil"})).unwrap_err();
|
||||
assert!(err.contains("host-relative"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_v2_roundtrip_and_legacy_manual_review() {
|
||||
let dir = scratch("v2");
|
||||
// Use relative path via env cwd — chdir into scratch for this test.
|
||||
let prev = std::env::current_dir().unwrap();
|
||||
std::env::set_current_dir(&dir).unwrap();
|
||||
let rel = "ckpts";
|
||||
let saved = resume_save(&json!({
|
||||
"id": "run1",
|
||||
"schema": "CheckpointV2",
|
||||
"schema_version": 2,
|
||||
"goal": "ping",
|
||||
"status": "running",
|
||||
"round": 1,
|
||||
"resume_dir": rel,
|
||||
"attempt_id": "a1",
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(saved["ok"], true);
|
||||
assert_eq!(saved["atomic"], true);
|
||||
assert_eq!(saved["schema"], "CheckpointV2");
|
||||
|
||||
let loaded = resume_load(&json!({"id": "run1", "resume_dir": rel})).unwrap();
|
||||
assert_eq!(loaded["found"], true);
|
||||
assert_eq!(loaded["schema"], "CheckpointV2");
|
||||
assert_eq!(loaded["recovery_mode"], "checkpoint_v2_read_only");
|
||||
assert_eq!(loaded["write_auto_resume"], false);
|
||||
|
||||
// Legacy document without schema.
|
||||
let _ = resume_save(&json!({
|
||||
"id": "legacy1",
|
||||
"goal": "pong",
|
||||
"status": "interrupted",
|
||||
"round": 2,
|
||||
"resume_dir": rel,
|
||||
}))
|
||||
.unwrap();
|
||||
let legacy = resume_load(&json!({"id": "legacy1", "resume_dir": rel})).unwrap();
|
||||
assert_eq!(legacy["recovery_mode"], "manual_review");
|
||||
assert_eq!(legacy["write_auto_resume"], false);
|
||||
assert_eq!(legacy["legacy_adaptation"], "manual_review");
|
||||
|
||||
std::env::set_current_dir(prev).unwrap();
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_checkpoint_reports_error() {
|
||||
let dir = scratch("corrupt");
|
||||
let prev = std::env::current_dir().unwrap();
|
||||
std::env::set_current_dir(&dir).unwrap();
|
||||
fs::create_dir_all("ckpts").unwrap();
|
||||
fs::write("ckpts/bad.json", "{not-json").unwrap();
|
||||
let loaded = resume_load(&json!({"id": "bad", "resume_dir": "ckpts"})).unwrap();
|
||||
assert_eq!(loaded["found"], true);
|
||||
assert_eq!(loaded["ok"], false);
|
||||
assert_eq!(loaded["error"]["code"], "checkpoint.corrupt");
|
||||
assert_eq!(loaded["recovery_mode"], "manual_review");
|
||||
std::env::set_current_dir(prev).unwrap();
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -117,11 +117,10 @@ pub fn probe_capabilities(params: &Value) -> Value {
|
||||
features.insert("view.document".into(), json!(0));
|
||||
features.insert("view.reference_resolve".into(), json!(0));
|
||||
}
|
||||
// M-A0/M-A3: freeze protocol names. resource.view_v1 flips on when view feature ships
|
||||
// the static ResourceView emitter; other agent bits stay 0 until their emitters land.
|
||||
// M-A0/M-A1/M-A3: protocol capability bits. error/checkpoint open after emitters land.
|
||||
features.insert("agent.error_v1".into(), json!(1));
|
||||
features.insert("agent.checkpoint_v2".into(), json!(1));
|
||||
for (k, v) in [
|
||||
("agent.error_v1", 0),
|
||||
("agent.checkpoint_v2", 0),
|
||||
("agent.tool_schema_v1", 0),
|
||||
("agent.output_channels_v1", 0),
|
||||
("agent.context_pack_v2", 0),
|
||||
@@ -248,8 +247,8 @@ mod tests {
|
||||
assert_eq!(caps["features"]["runtime.event_stream"], 0);
|
||||
assert_eq!(caps["features"]["runtime.cancel"], 0);
|
||||
assert_eq!(caps["features"]["llm.messages"], 0);
|
||||
assert_eq!(caps["features"]["agent.error_v1"], 0);
|
||||
assert_eq!(caps["features"]["agent.checkpoint_v2"], 0);
|
||||
assert_eq!(caps["features"]["agent.error_v1"], 1);
|
||||
assert_eq!(caps["features"]["agent.checkpoint_v2"], 1);
|
||||
assert_eq!(caps["features"]["agent.tool_schema_v1"], 0);
|
||||
assert_eq!(caps["features"]["agent.output_channels_v1"], 0);
|
||||
assert_eq!(caps["features"]["agent.context_pack_v2"], 0);
|
||||
|
||||
@@ -183,18 +183,16 @@ fn ma0_agent_reliability_protocol_freeze() {
|
||||
assert_eq!(caps["ok"], true, "{caps}");
|
||||
let features = &caps["result"]["features"];
|
||||
for bit in [
|
||||
"agent.error_v1",
|
||||
"agent.checkpoint_v2",
|
||||
"agent.tool_schema_v1",
|
||||
"agent.output_channels_v1",
|
||||
"agent.context_pack_v2",
|
||||
] {
|
||||
assert_eq!(features[bit], 0, "{bit} must stay 0 until emitters land");
|
||||
}
|
||||
// M-A3 subset: ResourceView emitter is live under the view feature.
|
||||
// M-A1/M-A3: error + checkpoint + resource view emitters are live.
|
||||
assert_eq!(features["agent.error_v1"], 1);
|
||||
assert_eq!(features["agent.checkpoint_v2"], 1);
|
||||
assert_eq!(features["resource.view_v1"], 1);
|
||||
// Unknown/future schema_version must not flip unimplemented agent bits.
|
||||
assert_eq!(features["agent.error_v1"], 0);
|
||||
|
||||
let formats = caps["result"]["extract_formats"]
|
||||
.as_array()
|
||||
@@ -222,7 +220,8 @@ fn ma0_agent_reliability_protocol_freeze() {
|
||||
"force_cancel_off": true
|
||||
}),
|
||||
);
|
||||
assert_eq!(down["result"]["features"]["agent.error_v1"], 0);
|
||||
assert_eq!(down["result"]["features"]["agent.error_v1"], 1);
|
||||
assert_eq!(down["result"]["features"]["agent.checkpoint_v2"], 1);
|
||||
assert_eq!(down["result"]["features"]["resource.view_v1"], 1);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user