Files
marqdo/tests/isolated_execute_mr01.rs
cflmy 9331af78e2 release(2.8.0): 合入 CHANGELOG/README/发版说明/skills prove 任务;CI 链接指向 doc/releases
- CHANGELOG.md:v2.8.0 条目(中英双语);
- README.md:版本行 2.7.1→2.8.0 + 愿景第 6 条(知识可证明);
- doc/releases/v2.8.0.md:双语发版说明(亮点/用法/已知限制);
- skills/marqdo-dev/index.mq.md:新增 prove 任务(路径/目录模式/
  require_lean/json;参数用 + 行声明升参,check 与 narrative-ratio 通过);
- 设计文档 §12.5 标记已合入;发版链接统一指向 doc/releases/v2.8.0.md。
2026-11-18 10:29:26 +08:00

662 lines
23 KiB
Rust
Executable File

//! MR-01 gold: dual 60s fake SSE execute workers + short reads (ADR 0009).
//! MR-02 (same file): third execute rejected while A continues cleanly.
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
fn ensure_plugins() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let status = Command::new("bash")
.arg(root.join("scripts/build-web-plugin.sh"))
.current_dir(&root)
.status()
.expect("build web plugin");
assert!(status.success(), "web plugin build failed");
let status = Command::new("cargo")
.args(["build", "-p", "marqdo_plugin_llm", "--locked"])
.current_dir(&root)
.status()
.expect("build llm plugin");
assert!(status.success(), "llm plugin build failed");
let web = root.join("plugins/web/build/libweb.so");
let llm = root.join("target/debug/libllm.so");
assert!(web.is_file(), "missing {web:?}");
assert!(llm.is_file(), "missing {llm:?}");
// SAFETY: test process setup before spawning children.
std::env::set_var("MARQDO_WEB_PLUGIN", &web);
std::env::set_var("MARQDO_LLM_PLUGIN", &llm);
std::env::set_var("MARQDO_EXT", root.join("ext"));
}
fn spawn_fake_sse(prefix: &'static str, secs: u64, hz: u64) -> (String, Arc<AtomicBool>, thread::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind fake sse");
listener.set_nonblocking(true).expect("nonblocking");
let addr = listener.local_addr().unwrap();
let stop = Arc::new(AtomicBool::new(false));
let stop_bg = stop.clone();
let handle = thread::spawn(move || {
// Poll accept so a never-connected upstream (e.g. 429 before dial) can stop.
let mut stream = loop {
if stop_bg.load(Ordering::Relaxed) {
return;
}
match listener.accept() {
Ok((s, _)) => break s,
Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(20));
continue;
}
Err(_) => return,
}
};
stream.set_nonblocking(false).ok();
let mut raw = Vec::new();
let mut buf = [0u8; 4096];
// Read request headers (+ body if Content-Length).
loop {
let n = stream.read(&mut buf).unwrap_or(0);
if n == 0 {
break;
}
raw.extend_from_slice(&buf[..n]);
if let Some(pos) = raw.windows(4).position(|w| w == b"\r\n\r\n") {
let header = String::from_utf8_lossy(&raw[..pos]);
let cl = header
.lines()
.find_map(|l| l.strip_prefix("Content-Length:"))
.and_then(|v| v.trim().parse::<usize>().ok())
.unwrap_or(0);
let body_start = pos + 4;
while raw.len() < body_start + cl {
let n = stream.read(&mut buf).unwrap_or(0);
if n == 0 {
break;
}
raw.extend_from_slice(&buf[..n]);
}
break;
}
if raw.len() > 1 << 20 {
break;
}
}
let _ = stream.write_all(
b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\nConnection: close\r\n\r\n",
);
let _ = stream.flush();
let interval = Duration::from_millis(1000 / hz.max(1));
let deadline = Instant::now() + Duration::from_secs(secs);
let mut i = 0u64;
while Instant::now() < deadline && !stop_bg.load(Ordering::Relaxed) {
let token = format!("{prefix}{i}");
let chunk = format!(
"data: {{\"choices\":[{{\"delta\":{{\"content\":\"{token}\"}}}}]}}\n\n"
);
if stream.write_all(chunk.as_bytes()).is_err() {
break;
}
let _ = stream.flush();
i += 1;
thread::sleep(interval);
}
let _ = stream.write_all(b"data: [DONE]\n\n");
let _ = stream.flush();
});
(format!("http://{addr}/v1"), stop, handle)
}
fn wait_http_ok(url: &str, timeout: Duration) -> bool {
let start = Instant::now();
while start.elapsed() < timeout {
let out = Command::new("curl")
.args(["-s", "-o", "/dev/null", "-w", "%{http_code}", url])
.output();
if let Ok(o) = out {
if String::from_utf8_lossy(&o.stdout) == "200" {
return true;
}
}
thread::sleep(Duration::from_millis(200));
}
false
}
fn collect_sse_deltas(url: &str, body: &str, base_url: &str) -> (u16, String, Vec<String>) {
let payload = body.replace("__BASE__", base_url);
let out = Command::new("curl")
.args([
"-s",
"-N",
"--max-time",
"180",
"-w",
"\n__HTTP__%{http_code}",
"-H",
"Content-Type: application/json",
"-d",
&payload,
url,
])
.output()
.expect("curl sse");
let text = String::from_utf8_lossy(&out.stdout);
let (body_text, code_s) = text
.rsplit_once("\n__HTTP__")
.unwrap_or((text.as_ref(), "0"));
let code: u16 = code_s.trim().parse().unwrap_or(0);
let mut deltas = Vec::new();
for line in body_text.lines() {
let line = line.trim();
if let Some(rest) = line.strip_prefix("data: ") {
if rest == "[DONE]" {
continue;
}
if let Ok(v) = serde_json::from_str::<serde_json::Value>(rest) {
if let Some(d) = v
.pointer("/payload/delta")
.and_then(|x| x.as_str())
.or_else(|| v.pointer("/payload/text").and_then(|x| x.as_str()))
{
deltas.push(d.to_string());
}
}
}
}
(code, body_text.to_string(), deltas)
}
fn count_call_lib_path_workers(site_pid: u32) -> usize {
// Walk /proc for children whose cmdline contains --call-lib-path.
let Ok(entries) = std::fs::read_dir("/proc") else {
return 0;
};
let mut n = 0usize;
for ent in entries.flatten() {
let name = ent.file_name();
let pid: u32 = match name.to_str().and_then(|s| s.parse().ok()) {
Some(p) => p,
None => continue,
};
let cmdline = std::fs::read(format!("/proc/{pid}/cmdline")).unwrap_or_default();
let cmd = String::from_utf8_lossy(&cmdline);
if !cmd.contains("--call-lib-path") {
continue;
}
// Prefer workers whose parent is the site (or grandparent via supervisor).
let ppid = std::fs::read_to_string(format!("/proc/{pid}/stat"))
.ok()
.and_then(|s| {
let rparen = s.rfind(')')?;
let rest = s[rparen + 2..].split_whitespace().nth(1)?;
rest.parse::<u32>().ok()
})
.unwrap_or(0);
if ppid == site_pid || cmd.contains("isolated-execute-site") || cmd.contains("serve.mq.md")
{
n += 1;
} else {
// Count any call-lib-path marqdo under this test tree.
n += 1;
}
}
n
}
#[test]
fn mr01_dual_isolated_execute_and_short_reads() {
if TcpListener::bind("127.0.0.1:0").is_err() {
eprintln!("skip MR-01: local sockets unavailable");
return;
}
ensure_plugins();
let secs: u64 = std::env::var("MARQDO_MR01_SECS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(60);
let hz: u64 = 20;
let (base_a, stop_a, join_a) = spawn_fake_sse("A", secs, hz);
let (base_b, stop_b, join_b) = spawn_fake_sse("B", secs, hz);
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let site = root.join("tests/ext/isolated-execute-site");
let runtime_dir = std::env::temp_dir().join(format!("mq-mr01-rt-{}", std::process::id()));
let _ = std::fs::create_dir_all(&runtime_dir);
let bin = env!("CARGO_BIN_EXE_marqdo");
let mut child: Child = Command::new(bin)
.args(["run", "serve.mq.md"])
.current_dir(&site)
.env("MARQDO_EXT", root.join("ext"))
.env("MARQDO_WEB_PLUGIN", std::env::var_os("MARQDO_WEB_PLUGIN").unwrap())
.env("MARQDO_LLM_PLUGIN", std::env::var_os("MARQDO_LLM_PLUGIN").unwrap())
.env("MARQDO_RUNTIME_DIR", &runtime_dir)
.env("MARQDO_SITE_PORT", "18220")
.env("OPENAI_API_KEY", "mr01-key")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("spawn site");
let site_pid = child.id();
let base = "http://127.0.0.1:18220";
if !wait_http_ok(&format!("{base}/v1/models"), Duration::from_secs(30)) {
let _ = child.kill();
let err = child
.stderr
.take()
.map(|mut s| {
let mut t = String::new();
let _ = s.read_to_string(&mut t);
t
})
.unwrap_or_default();
panic!("site did not start; stderr={err}");
}
// Capability honesty: isolated execution is default-on.
let caps = marqdo::workbench::probe_capabilities(&serde_json::json!({}));
assert_eq!(
caps["features"]["runtime.isolated_execution"],
1,
"caps={caps}"
);
let chat_url = format!("{base}/v1/chat/completions");
let body_tmpl = r#"{"model":"mr01","stream":true,"base_url":"__BASE__","messages":[{"role":"user","content":"hi"}]}"#;
let deltas_a = Arc::new(Mutex::new(Vec::new()));
let deltas_b = Arc::new(Mutex::new(Vec::new()));
let code_a = Arc::new(Mutex::new(0u16));
let code_b = Arc::new(Mutex::new(0u16));
let raw_a = Arc::new(Mutex::new(String::new()));
let raw_b = Arc::new(Mutex::new(String::new()));
let da = deltas_a.clone();
let ca = code_a.clone();
let ra = raw_a.clone();
let base_a_c = base_a.clone();
let chat_a = chat_url.clone();
let body_a = body_tmpl.to_string();
let t_a = thread::spawn(move || {
let (code, raw, deltas) = collect_sse_deltas(&chat_a, &body_a, &base_a_c);
*ca.lock().unwrap() = code;
*ra.lock().unwrap() = raw;
*da.lock().unwrap() = deltas;
});
let db = deltas_b.clone();
let cb = code_b.clone();
let rb = raw_b.clone();
let base_b_c = base_b.clone();
let chat_b = chat_url.clone();
let body_b = body_tmpl.to_string();
let t_b = thread::spawn(move || {
let (code, raw, deltas) = collect_sse_deltas(&chat_b, &body_b, &base_b_c);
*cb.lock().unwrap() = code;
*rb.lock().unwrap() = raw;
*db.lock().unwrap() = deltas;
});
// Short reads during the dual stream.
let short_ok = Arc::new(Mutex::new(0u32));
let short_fail = Arc::new(Mutex::new(0u32));
let short_stop = Arc::new(AtomicBool::new(false));
let short_ok_c = short_ok.clone();
let short_fail_c = short_fail.clone();
let short_stop_c = short_stop.clone();
let models_url = format!("{base}/v1/models");
let t_short = thread::spawn(move || {
while !short_stop_c.load(Ordering::Relaxed) {
let out = Command::new("curl")
.args(["-s", "-o", "/dev/null", "-w", "%{http_code}", &models_url])
.output();
match out {
Ok(o) if String::from_utf8_lossy(&o.stdout) == "200" => {
*short_ok_c.lock().unwrap() += 1;
}
_ => {
*short_fail_c.lock().unwrap() += 1;
}
}
thread::sleep(Duration::from_millis(50)); // ~20/s
}
});
// Give workers time to spawn, then assert two --call-lib-path children.
thread::sleep(Duration::from_secs(2));
let workers = count_call_lib_path_workers(site_pid);
assert!(
workers >= 2,
"expected ≥2 --call-lib-path workers, got {workers}"
);
// MR-02 / slot full: third execute must be rejected while A/B hold slots.
let (base_c, stop_c, join_c) = spawn_fake_sse("C", 5, 10);
let third = Command::new("curl")
.args([
"-s",
"-o",
"/tmp/mq-mr01-third.out",
"-w",
"%{http_code}",
"-H",
"Content-Type: application/json",
"-d",
&format!(
r#"{{"model":"mr01","stream":true,"base_url":"{base_c}","messages":[{{"role":"user","content":"x"}}]}}"#
),
&chat_url,
])
.output()
.expect("third curl");
let third_code = String::from_utf8_lossy(&third.stdout).trim().to_string();
assert!(
third_code == "429" || third_code == "503",
"third execute should be rejected, got {third_code}"
);
stop_c.store(true, Ordering::Relaxed);
let _ = join_c.join();
let _ = t_a.join();
let _ = t_b.join();
short_stop.store(true, Ordering::Relaxed);
let _ = t_short.join();
stop_a.store(true, Ordering::Relaxed);
stop_b.store(true, Ordering::Relaxed);
let _ = join_a.join();
let _ = join_b.join();
let _ = child.kill();
let _ = child.wait();
let ca = *code_a.lock().unwrap();
let cb = *code_b.lock().unwrap();
let da = deltas_a.lock().unwrap().clone();
let db = deltas_b.lock().unwrap().clone();
let ra = raw_a.lock().unwrap().clone();
let rb = raw_b.lock().unwrap().clone();
assert_eq!(ca, 200, "stream A http={ca} raw={ra}");
assert_eq!(cb, 200, "stream B http={cb} raw={rb}");
assert!(
!ra.contains("interpreter busy") && !ra.contains("sse bridge in use"),
"A busy: {ra}"
);
assert!(
!rb.contains("interpreter busy") && !rb.contains("sse bridge in use"),
"B busy: {rb}"
);
assert!(!da.is_empty(), "A produced no deltas; raw={ra}");
assert!(!db.is_empty(), "B produced no deltas; raw={rb}");
assert!(
da.iter().all(|t| t.starts_with('A')),
"A mixed foreign tokens: {da:?}"
);
assert!(
db.iter().all(|t| t.starts_with('B')),
"B mixed foreign tokens: {db:?}"
);
assert!(
da.iter().all(|t| !t.starts_with('B')),
"A saw B tokens"
);
assert!(
db.iter().all(|t| !t.starts_with('A')),
"B saw A tokens"
);
let sok = *short_ok.lock().unwrap();
let sfail = *short_fail.lock().unwrap();
assert!(sok > 10, "expected many short 200s, got ok={sok} fail={sfail}");
assert_eq!(sfail, 0, "short reads failed: {sfail}");
assert_eq!(
caps["features"]["runtime.read_lane"],
1,
"read_lane must be on; caps={caps}"
);
let _ = std::fs::remove_dir_all(&runtime_dir);
}
/// Observer disconnect must not cancel the execute task (default).
#[test]
fn disconnect_keeps_running_and_replay() {
if TcpListener::bind("127.0.0.1:0").is_err() {
eprintln!("skip disconnect-keeps-running: local sockets unavailable");
return;
}
ensure_plugins();
let secs: u64 = 6;
let (base_a, stop_a, join_a) = spawn_fake_sse("A", secs, 10);
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let site = root.join("tests/ext/isolated-execute-site");
let runtime_dir = std::env::temp_dir().join(format!("mq-disc-rt-{}", std::process::id()));
let _ = std::fs::create_dir_all(&runtime_dir);
let bin = env!("CARGO_BIN_EXE_marqdo");
let mut child: Child = Command::new(bin)
.args(["run", "serve.mq.md"])
.current_dir(&site)
.env("MARQDO_EXT", root.join("ext"))
.env("MARQDO_WEB_PLUGIN", std::env::var_os("MARQDO_WEB_PLUGIN").unwrap())
.env("MARQDO_LLM_PLUGIN", std::env::var_os("MARQDO_LLM_PLUGIN").unwrap())
.env("MARQDO_RUNTIME_DIR", &runtime_dir)
.env("MARQDO_SITE_PORT", "18221")
.env("OPENAI_API_KEY", "mr01-key")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("spawn site");
let base = "http://127.0.0.1:18221";
if !wait_http_ok(&format!("{base}/v1/models"), Duration::from_secs(45)) {
let _ = child.kill();
let err = child
.stderr
.take()
.map(|mut s| {
let mut t = String::new();
let _ = s.read_to_string(&mut t);
t
})
.unwrap_or_default();
panic!("site did not start for disconnect test; stderr={err}");
}
let chat_url = format!("{base}/v1/chat/completions");
let body = format!(
r#"{{"model":"mr01","stream":true,"base_url":"{base_a}","messages":[{{"role":"user","content":"hi"}}]}}"#
);
// Observer disconnects after ~1.2s; task must keep running.
let out = Command::new("curl")
.args([
"-s",
"-N",
"--max-time",
"1",
"-H",
"Content-Type: application/json",
"-d",
&body,
&chat_url,
])
.output()
.expect("curl disconnect");
let text = String::from_utf8_lossy(&out.stdout);
let mut task_id = String::new();
for line in text.lines() {
let line = line.trim();
if let Some(rest) = line.strip_prefix("data: ") {
if let Ok(v) = serde_json::from_str::<serde_json::Value>(rest) {
if v.get("type").and_then(|t| t.as_str()) == Some("run.started") {
if let Some(tid) = v.pointer("/payload/task_id").and_then(|x| x.as_str()) {
task_id = tid.to_string();
}
}
}
}
}
assert!(!task_id.is_empty(), "missing task_id in early SSE; raw={text}");
// Wait until task finishes (observer already gone).
let mut terminal = false;
let deadline = Instant::now() + Duration::from_secs(secs + 8);
while Instant::now() < deadline {
let get_url = format!("{base}/_runtime/tasks/{task_id}");
let out = Command::new("curl")
.args(["-s", &get_url])
.output()
.expect("runtime get");
let body = String::from_utf8_lossy(&out.stdout);
if body.contains("\"completed\"")
|| body.contains("\"succeeded\"")
|| body.contains("\"failed\"")
|| body.contains("\"cancelled\"")
{
assert!(
!body.contains("\"cancelled\""),
"disconnect must not cancel task: {body}"
);
terminal = true;
break;
}
thread::sleep(Duration::from_millis(200));
}
assert!(terminal, "task {task_id} did not finish after observer disconnect");
// Replay via events (JSON after_seq=0).
let ev_url = format!("{base}/_runtime/tasks/{task_id}/events?after_seq=0");
let ev = Command::new("curl")
.args(["-s", &ev_url])
.output()
.expect("events");
let ev_body = String::from_utf8_lossy(&ev.stdout);
assert!(
ev_body.contains('A') || ev_body.contains("delta") || ev_body.contains("event"),
"replay empty: {ev_body}"
);
stop_a.store(true, Ordering::Relaxed);
let _ = join_a.join();
let _ = child.kill();
let _ = child.wait();
let _ = std::fs::remove_dir_all(&runtime_dir);
}
/// Control + short reads stay 200 while two execute streams hold slots.
#[test]
fn control_under_dual_execute() {
if TcpListener::bind("127.0.0.1:0").is_err() {
eprintln!("skip control-under-dual-execute: local sockets unavailable");
return;
}
ensure_plugins();
let secs: u64 = 8;
let (base_a, stop_a, join_a) = spawn_fake_sse("A", secs, 10);
let (base_b, stop_b, join_b) = spawn_fake_sse("B", secs, 10);
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let site = root.join("tests/ext/isolated-execute-site");
let runtime_dir = std::env::temp_dir().join(format!("mq-ctl-rt-{}", std::process::id()));
let _ = std::fs::create_dir_all(&runtime_dir);
let bin = env!("CARGO_BIN_EXE_marqdo");
let mut child: Child = Command::new(bin)
.args(["run", "serve.mq.md"])
.current_dir(&site)
.env("MARQDO_EXT", root.join("ext"))
.env("MARQDO_WEB_PLUGIN", std::env::var_os("MARQDO_WEB_PLUGIN").unwrap())
.env("MARQDO_LLM_PLUGIN", std::env::var_os("MARQDO_LLM_PLUGIN").unwrap())
.env("MARQDO_RUNTIME_DIR", &runtime_dir)
.env("MARQDO_SITE_PORT", "18222")
.env("OPENAI_API_KEY", "mr01-key")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("spawn site");
let base = "http://127.0.0.1:18222";
if !wait_http_ok(&format!("{base}/v1/models"), Duration::from_secs(45)) {
let _ = child.kill();
let err = child
.stderr
.take()
.map(|mut s| {
let mut t = String::new();
let _ = s.read_to_string(&mut t);
t
})
.unwrap_or_default();
panic!("site did not start for control test; stderr={err}");
}
let chat_url = format!("{base}/v1/chat/completions");
let body_tmpl = r#"{"model":"mr01","stream":true,"base_url":"__BASE__","messages":[{"role":"user","content":"hi"}]}"#;
let t_a = {
let u = chat_url.clone();
let b = body_tmpl.to_string();
let ba = base_a.clone();
thread::spawn(move || {
let _ = collect_sse_deltas(&u, &b, &ba);
})
};
let t_b = {
let u = chat_url.clone();
let b = body_tmpl.to_string();
let bb = base_b.clone();
thread::spawn(move || {
let _ = collect_sse_deltas(&u, &b, &bb);
})
};
thread::sleep(Duration::from_secs(1));
let mut ok_n = 0u32;
let mut fail_n = 0u32;
let start = Instant::now();
while start.elapsed() < Duration::from_secs(3) {
for path in ["/v1/models", "/v1/ping"] {
let url = format!("{base}{path}");
let t0 = Instant::now();
let out = Command::new("curl")
.args(["-s", "-o", "/dev/null", "-w", "%{http_code}", &url])
.output();
let elapsed = t0.elapsed();
match out {
Ok(o) if String::from_utf8_lossy(&o.stdout) == "200" => {
assert!(
elapsed < Duration::from_millis(1500),
"{path} p95 budget; took {elapsed:?}"
);
ok_n += 1;
}
_ => fail_n += 1,
}
}
thread::sleep(Duration::from_millis(80));
}
assert!(ok_n > 4, "expected control/short 200s, ok={ok_n} fail={fail_n}");
assert_eq!(fail_n, 0, "control/short failed under dual execute: {fail_n}");
let _ = t_a.join();
let _ = t_b.join();
stop_a.store(true, Ordering::Relaxed);
stop_b.store(true, Ordering::Relaxed);
let _ = join_a.join();
let _ = join_b.join();
let _ = child.kill();
let _ = child.wait();
let _ = std::fs::remove_dir_all(&runtime_dir);
}