Files
marqdo/plugins/engineering/src/lib.rs
T

612 lines
34 KiB
Rust

//! Native boundaries for executable engineering documents. Business ordering,
//! version decisions and ordinary file reads/writes belong in .mq.md.
//! No Python interpreter, generated shell program or secret-valued tool result.
use std::{
collections::BTreeMap,
ffi::CStr,
fs::{self, File},
io::{Read, Write},
os::raw::{c_char, c_int},
path::{Component, Path, PathBuf},
process::{Command, Stdio},
sync::mpsc,
thread,
time::{Duration, Instant},
};
use hmac::{Hmac, Mac};
use marqdo_plugin_sdk::*;
use sha2::{Digest, Sha256};
use serde_json::{json, Value};
type Answer = Result<Value, String>;
const OUTPUT: usize = 256 * 1024;
const HTTP_MAX: u64 = 16 * 1024 * 1024;
fn field<'a>(v: &'a Value, key: &str) -> Result<&'a str, String> {
v.get(key).and_then(Value::as_str).ok_or_else(|| format!("missing text field {key}"))
}
fn hash(b: &[u8]) -> String { hex(&Sha256::digest(b)) }
fn hex(b: &[u8]) -> String { b.iter().map(|x| format!("{x:02x}")).collect() }
fn read_bounded(mut r: impl Read, max: u64) -> Result<Vec<u8>, String> {
let mut b = Vec::new();
r.by_ref().take(max + 1).read_to_end(&mut b).map_err(|e| format!("read failed ({:?})",e.kind()))?;
if b.len() as u64 > max { return Err("bounded input limit exceeded".into()); }
Ok(b)
}
fn scoped(root: &Path, relative: &str) -> Result<PathBuf, String> {
let p = root.join(relative).canonicalize().map_err(|_| "path does not exist")?;
if !p.starts_with(root) { return Err("artifact path escapes the selected root".into()); }
Ok(p)
}
fn homes(root: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
for key in ["MARQDO_RELEASE_CREDENTIAL_HOME", "USERPROFILE", "HOME"] {
if let Some(s) = std::env::var_os(key) { out.push(PathBuf::from(s)); }
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
for p in std::iter::once(root).chain(root.ancestors().skip(1)) {
if p.join(".git").exists() {
if let Ok(m) = p.metadata() {
unsafe {
let mut entry: libc::passwd = std::mem::zeroed();
let mut result = std::ptr::null_mut();
let mut buf = vec![0u8; 32768];
if libc::getpwuid_r(m.uid(), &mut entry, buf.as_mut_ptr().cast(), buf.len(), &mut result) == 0
&& !result.is_null() && !entry.pw_dir.is_null() {
out.push(PathBuf::from(CStr::from_ptr(entry.pw_dir).to_string_lossy().into_owned()));
}
}
}
}
}
}
out.dedup();
out
}
fn redact(mut text: String, private: &[String]) -> String {
for token in private.iter().filter(|s| !s.is_empty()) { text = text.replace(token, "<secret>"); }
for (k, v) in std::env::vars() {
if v.len() >= 8 && ["TOKEN", "SECRET", "PASSWORD", "ACCESS_KEY"].iter().any(|x| k.contains(x)) {
text = text.replace(&v, "<secret>");
}
}
text
}
struct Output { code: i32, stdout: String, stderr: String, timed_out: bool, truncated: bool }
fn pipe(mut input: impl Read + Send + 'static, limit: usize) -> mpsc::Receiver<(Vec<u8>, bool)> {
let (send, recv) = mpsc::channel();
thread::spawn(move || {
let mut buf = Vec::new(); let mut part = [0u8; 8192]; let mut truncated = false;
while let Ok(n) = input.read(&mut part) {
if n == 0 { break; }
let keep = (limit - buf.len()).min(n);
buf.extend_from_slice(&part[..keep]); truncated |= keep < n;
}
let _ = send.send((buf, truncated));
});
recv
}
fn execute(root: &Path, argv: &[String], input: Option<String>, env: &BTreeMap<String, String>,
timeout_ms: u64, max_output: usize) -> Result<Output, String> {
if argv.is_empty() || argv[0].is_empty() { return Err("argv must not be empty".into()); }
if !(1..=3_600_000).contains(&timeout_ms) || !(1..=16_777_216).contains(&max_output) {
return Err("timeout/output limit outside supported range".into());
}
let mut cmd = Command::new(&argv[0]);
cmd.args(&argv[1..]).current_dir(root).envs(env)
.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped());
#[cfg(unix)] {
use std::os::unix::process::CommandExt;
unsafe { cmd.pre_exec(|| {
if libc::setpgid(0, 0) != 0 { return Err(std::io::Error::last_os_error()); }
Ok(())
}); }
}
let mut child = cmd.spawn().map_err(|_| format!("cannot start {}", argv[0]))?;
let stdout = pipe(child.stdout.take().unwrap(), max_output);
let stderr = pipe(child.stderr.take().unwrap(), max_output);
let mut stdin = child.stdin.take().unwrap();
thread::spawn(move || { if let Some(text) = input { let _ = stdin.write_all(text.as_bytes()); } });
let started = Instant::now(); let mut timed_out = false;
let code = loop {
match child.try_wait().map_err(|_| "cannot observe process")? {
Some(status) => break status.code().unwrap_or(1),
None if started.elapsed() >= Duration::from_millis(timeout_ms) => {
timed_out = true;
#[cfg(unix)] unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL); }
#[cfg(windows)] { let _ = Command::new("taskkill").args(["/PID", &child.id().to_string(), "/T", "/F"]).output(); }
let _ = child.kill(); let _ = child.wait(); break 124;
}
None => thread::sleep(Duration::from_millis(10)),
}
};
// Descendants retaining inherited pipes must not prevent a bounded return.
let (out, out_clip) = stdout.recv_timeout(Duration::from_secs(1)).unwrap_or((Vec::new(), true));
let (err, err_clip) = stderr.recv_timeout(Duration::from_secs(1)).unwrap_or((Vec::new(), true));
Ok(Output { code, stdout: String::from_utf8_lossy(&out).into(), stderr: String::from_utf8_lossy(&err).into(),
timed_out, truncated: out_clip || err_clip })
}
fn github_token(root: &Path) -> Option<String> {
for key in ["GH_TOKEN", "GITHUB_TOKEN"] {
if let Ok(s) = std::env::var(key) { if !s.is_empty() { return Some(s); } }
}
let args = ["git", "-c", &format!("safe.directory={}", root.display()), "credential", "fill"]
.into_iter().map(str::to_string).collect::<Vec<_>>();
let env = BTreeMap::from([("GIT_TERMINAL_PROMPT".into(), "0".into())]);
if let Ok(p) = execute(root, &args, Some("protocol=https\nhost=github.com\n\n".into()), &env, 15_000, OUTPUT) {
if let Some(s) = p.stdout.lines().find_map(|s| s.strip_prefix("password=")) { return Some(s.into()); }
}
for home in homes(root) {
if let Ok(text) = fs::read_to_string(home.join(".config/gh/hosts.yml")) {
if let Some(s) = text.lines().find_map(|s| s.trim().strip_prefix("oauth_token:")) {
return Some(s.trim().trim_matches(['"', '\'']).into());
}
}
}
None
}
fn git_config(env: &mut BTreeMap<String,String>, key: &str, value: String) {
let start = env.get("GIT_CONFIG_COUNT").cloned().or_else(|| std::env::var("GIT_CONFIG_COUNT").ok())
.and_then(|s| s.parse::<usize>().ok()).unwrap_or(0);
env.insert(format!("GIT_CONFIG_KEY_{start}"),key.into());
env.insert(format!("GIT_CONFIG_VALUE_{start}"),value);
env.insert("GIT_CONFIG_COUNT".into(),(start+1).to_string());
}
fn command(root: &Path, p: &Value) -> Answer {
let mut argv = p.get("argv").and_then(Value::as_array).ok_or("argv must be a list")?
.iter().map(|v| v.as_str().map(str::to_string).ok_or("argv items must be text".to_string()))
.collect::<Result<Vec<_>, _>>()?;
let cwd = scoped(root, p.get("cwd").and_then(Value::as_str).unwrap_or("."))?;
let mut env = BTreeMap::new(); let mut private = Vec::new();
// Trust only the explicitly selected workspace, only in this child tree.
// Cargo/Go VCS probes inherit the same scoped Git metadata access.
git_config(&mut env,"safe.directory",root.display().to_string());
let exe = argv.first().and_then(|s| Path::new(s).file_stem()).and_then(|s| s.to_str()).unwrap_or("");
if ["bash", "sh", "pwsh", "powershell"].contains(&exe) && argv.iter().any(|s| ["-c", "-Command", "-EncodedCommand"].contains(&s.as_str())) {
return Err("use argument vectors or maintained external scripts, not generated shell programs".into());
}
if exe == "git" {
if argv.iter().any(|s| ["--force", "-f", "--force-with-lease", "--no-verify"].contains(&s.as_str()) || s.starts_with("--force=")) {
return Err("forced Git updates and hook bypass are forbidden".into());
}
argv.splice(1..1, ["-c".into(), format!("safe.directory={}", cwd.display())]);
env.insert("GIT_TERMINAL_PROMPT".into(), "0".into());
if argv.iter().any(|s| ["fetch", "push", "ls-remote"].contains(&s.as_str())) {
if let Some(token) = github_token(root) {
// URL-scoped headers stay in the child environment, not argv.
let auth = format!("Authorization: Basic {}", base64(format!("x-access-token:{token}").as_bytes()));
for url in ["https://proxy.cflmy.top/github.com/", "https://github.com/"] {
git_config(&mut env,&format!("http.{url}.extraheader"),auth.clone());
}
private.push(token); private.push(auth);
argv.splice(1..1, ["-c".into(), "http.version=HTTP/1.1".into(), "-c".into(), "http.postBuffer=524288000".into()]);
}
}
}
if let Some(e) = p.get("env").and_then(Value::as_object) {
for (k, v) in e { env.insert(k.clone(), v.as_str().ok_or("env values must be text")?.into()); }
}
// Child Git trace output could expose internally attached authentication.
for key in ["GIT_TRACE", "GIT_TRACE_CURL", "GIT_CURL_VERBOSE"] { env.insert(key.into(), "0".into()); }
let timeout = p.get("timeout_ms").and_then(Value::as_u64)
.unwrap_or_else(|| p.get("timeout").and_then(Value::as_u64).unwrap_or(120) * 1000);
let max = p.get("max_output").and_then(Value::as_u64).unwrap_or(OUTPUT as u64) as usize;
let out = execute(&cwd, &argv, p.get("stdin").and_then(Value::as_str).map(str::to_string), &env, timeout, max)?;
Ok(json!({"code":out.code,"stdout":redact(out.stdout,&private),"stderr":redact(out.stderr,&private),
"timed_out":out.timed_out,"truncated":out.truncated}))
}
fn http_json(root: &Path, p: &Value) -> Answer {
let path = field(p, "path")?;
if !path.starts_with("/repos/cflmy/marqdo/") || path.contains("..") { return Err("GitHub path is outside the project".into()); }
let method = p.get("method").and_then(Value::as_str).unwrap_or("GET");
if !["GET", "PATCH", "POST"].contains(&method) { return Err("unsupported GitHub method".into()); }
let bases = std::env::var("MARQDO_RELEASE_API_BASE").map(|s| vec![s])
.unwrap_or_else(|_| vec!["https://proxy.cflmy.top/api.github.com".into(), "https://api.github.com".into()]);
let token = github_token(root);
for (i, base) in bases.iter().enumerate() {
let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(35)).build();
let url = format!("{}{path}", base.trim_end_matches('/'));
let mut req = agent.request(method, &url).set("Accept", "application/vnd.github+json")
.set("User-Agent", "marqdo-engineering/1").set("X-GitHub-Api-Version", "2022-11-28");
if let Some(token) = &token { req = req.set("Authorization", &format!("Bearer {token}")); }
let response = if let Some(body) = p.get("data").filter(|v| !v.is_null()) {
req.set("Content-Type", "application/json").send_string(&body.to_string())
} else { req.call() };
match response {
Ok(r) => {
let body = read_bounded(r.into_reader(), HTTP_MAX)
.and_then(|bytes| serde_json::from_slice(&bytes).map_err(|_| "GitHub response is not JSON".into()));
match body {
Ok(value) => return Ok(value),
Err(_) if method != "POST" && i + 1 < bases.len() => continue,
Err(message) => return Err(message),
}
}
Err(ureq::Error::Status(404,_)) if p.get("missing").and_then(Value::as_bool).unwrap_or(false) => return Ok(Value::Null),
Err(ureq::Error::Status(code,_)) => {
if method == "POST" || i + 1 == bases.len() || ![429,500,502,503,504].contains(&code) {
return Err(format!("GitHub HTTP {code}"));
}
}
Err(_) if method == "POST" || i + 1 == bases.len() => return Err("GitHub network request failed; uncertain mutations are not replayed".into()),
Err(_) => {}
}
}
Err("GitHub request failed after proxy and direct routes".into())
}
fn archive(root: &Path, p: &Value) -> Answer {
let path = scoped(root, field(p,"path")?)?;
let file = File::open(path).map_err(|_|"cannot open archive")?;
let mut zip = zip::ZipArchive::new(file).map_err(|_|"invalid ZIP")?;
if zip.len() > 100000 { return Err("ZIP has too many entries".into()); }
let mut seen=std::collections::HashSet::new();
let mut entries = Vec::new(); let mut metadata = serde_json::Map::new(); let mut total = 0u64; let mut actual = 0u64;
for i in 0..zip.len() {
let mut f = zip.by_index(i).map_err(|_|"cannot read ZIP entry")?;
let name = f.name().replace('\\', "/");
if !seen.insert(name.clone()) || name.as_bytes().get(1)==Some(&b':') {return Err("duplicate or drive-qualified ZIP path".into());}
if Path::new(&name).components().any(|c| matches!(c, Component::ParentDir | Component::RootDir | Component::Prefix(_))) {
return Err("unsafe ZIP path".into());
}
if f.unix_mode().is_some_and(|m|m & 0o170000 == 0o120000) { return Err("ZIP symlink is not supported".into()); }
total = total.checked_add(f.size()).ok_or("ZIP expansion overflow")?;
if total > 2 * 1024 * 1024 * 1024 { return Err("ZIP expands beyond 2 GiB".into()); }
let mut selected = Vec::new();
let keep = ["VERSION","ext/VERSION","Cargo.toml","extension/package.json"].contains(&name.as_str());
let mut buf=[0u8;65536];
loop {
let n=f.read(&mut buf).map_err(|_|"ZIP content/CRC validation failed")?;
if n==0 {break;}
actual = actual.checked_add(n as u64).ok_or("ZIP expansion overflow")?;
if actual > 2 * 1024 * 1024 * 1024 { return Err("ZIP expands beyond 2 GiB".into()); }
if keep && selected.len()<16384 {selected.extend_from_slice(&buf[..n.min(16384-selected.len())]);}
}
if keep { metadata.insert(name.clone(),json!(String::from_utf8(selected).map_err(|_|"ZIP metadata is not UTF-8")?)); }
entries.push(name);
}
Ok(json!({"entries":entries,"metadata":metadata}))
}
fn hmac(key:&[u8], data:&str)->Vec<u8> {
let mut mac=Hmac::<Sha256>::new_from_slice(key).expect("HMAC accepts all key lengths");
mac.update(data.as_bytes());mac.finalize().into_bytes().to_vec()
}
fn r2_settings(root:&Path)->Result<BTreeMap<String,String>,String> {
let mut settings=std::env::vars().filter(|(k,_)|k.starts_with("R2_")).collect::<BTreeMap<_,_>>();
for home in homes(root) {
if let Ok(text)=fs::read_to_string(home.join(".marqdo/r2.env")) {
for line in text.lines().map(str::trim).filter(|s|!s.starts_with('#')) {
if let Some((k,v))=line.split_once('=') {settings.entry(k.trim().into()).or_insert_with(||v.trim().trim_matches(['\'','"']).into());}
}
}
}
for k in ["R2_ENDPOINT","R2_BUCKET","R2_ACCESS_KEY_ID","R2_SECRET_ACCESS_KEY"] {
if settings.get(k).map_or(true, String::is_empty) {return Err(format!("missing {k}; configure ~/.marqdo/r2.env"));}
}
Ok(settings)
}
fn r2_put(root:&Path,p:&Value)->Answer {
let s=r2_settings(root)?;
let key=field(p,"key")?;
if !key.bytes().all(|b|b.is_ascii_alphanumeric() || b"/.-_".contains(&b)) || key.contains("..") {
return Err("unsafe R2 key".into());
}
let body=if let Some(path)=p.get("path").and_then(Value::as_str) {
read_bounded(File::open(scoped(root,path)?).map_err(|_|"cannot open upload")?,256*1024*1024)?
} else {field(p,"text")?.as_bytes().to_vec()};
let digest=hash(&body);
if let Some(expected)=p.get("sha256").and_then(Value::as_str) {
if digest!=expected {return Err("upload bytes changed after verification".into());}
}
let endpoint=s["R2_ENDPOINT"].trim_end_matches('/');
let host=endpoint.split_once("://").ok_or("R2 endpoint needs a scheme")?.1;
if host.contains('/') || !(endpoint.starts_with("https://") || endpoint.starts_with("http://127.0.0.1:")) {
return Err("R2 endpoint must be HTTPS (loopback HTTP is for isolated tests)".into());
}
let uri=format!("/{}/{key}",s["R2_BUCKET"]);
let date=chrono::Utc::now();let amz=date.format("%Y%m%dT%H%M%SZ").to_string();let day=date.format("%Y%m%d").to_string();
let region=s.get("R2_REGION").map(String::as_str).unwrap_or("auto");
let mime=if key.ends_with("/VERSION"){"text/plain; charset=utf-8"}else{"application/zip"};
let headers=format!("content-type:{mime}\nhost:{host}\nx-amz-content-sha256:{digest}\nx-amz-date:{amz}\n");
let signed="content-type;host;x-amz-content-sha256;x-amz-date";
let canonical=format!("PUT\n{uri}\n\n{headers}\n{signed}\n{digest}");
let scope=format!("{day}/{region}/s3/aws4_request");
let string=format!("AWS4-HMAC-SHA256\n{amz}\n{scope}\n{}",hash(canonical.as_bytes()));
let k=hmac(format!("AWS4{}",s["R2_SECRET_ACCESS_KEY"]).as_bytes(),&day);
let k=hmac(&k,region);let k=hmac(&k,"s3");let k=hmac(&k,"aws4_request");
let auth=format!("AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={signed}, Signature={}",s["R2_ACCESS_KEY_ID"],hex(&hmac(&k,&string)));
let response=ureq::AgentBuilder::new().timeout(Duration::from_secs(180)).build()
.put(&format!("{endpoint}{uri}")).set("Content-Type",mime).set("x-amz-content-sha256",&digest)
.set("x-amz-date",&amz).set("Authorization",&auth).send_bytes(&body);
match response {
Ok(r) if [200,201].contains(&r.status()) => Ok(json!({"key":key,"sha256":digest,"size":body.len()})),
Ok(r)=>Err(format!("R2 HTTP {}",r.status())),
Err(ureq::Error::Status(code,_))=>Err(format!("R2 HTTP {code}")),
Err(_)=>Err("R2 upload failed or timed out; verify before retrying".into())
}
}
fn git_inspect(root:&Path)->Answer {
if !root.join(".git").exists() {return Err("inspect requires a Git checkout".into());}
let query=|args:&[&str]|->Result<String,String>{
let argv=std::iter::once("git").chain(args.iter().copied()).map(|s|json!(s)).collect::<Vec<_>>();
let out=command(root,&json!({"argv":argv}))?;
if out["code"]!=0 {return Err("Git metadata query failed".into());}
Ok(out["stdout"].as_str().unwrap_or("").trim().into())
};
Ok(json!({"root":root.display().to_string(),"branch":query(&["branch","--show-current"])?,
"commit":query(&["rev-parse","HEAD"])?,"changes":query(&["status","--porcelain"])?,
"tags":query(&["tag","--list"])?.lines().collect::<Vec<_>>(),
"extension_on_main":query(&["ls-files","vscode-extension"])?,
"origin":query(&["remote","get-url","origin"])?}))
}
fn base64(bytes:&[u8])->String {
const C:&[u8]=b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out=String::new();
for part in bytes.chunks(3) {
let a=part[0] as usize;let b=*part.get(1).unwrap_or(&0) as usize;let c=*part.get(2).unwrap_or(&0) as usize;
out.push(C[a>>2] as char);out.push(C[((a&3)<<4)|(b>>4)] as char);
out.push(if part.len()>1 {C[((b&15)<<2)|(c>>6)] as char}else{'='});
out.push(if part.len()>2 {C[c&63] as char}else{'='});
}
out
}
fn expression(e:&marqdo::ast::Expr)->Option<String> {
use marqdo::ast::{Expr,Literal,InterpPart,UnaryOp,BinaryOp,IndexKey};
Some(match e {
Expr::Literal(Literal::None)=>"None".into(),
Expr::Literal(Literal::Bool(v))=>if *v{"True".into()}else{"False".into()},
Expr::Literal(Literal::Int(v))=>v.to_string(),
Expr::Literal(Literal::Num(v))=>{let s=v.to_string();if s.contains('.')||s.contains('e')||s.contains('E'){s}else{format!("{s}.0")}},
Expr::Literal(Literal::Text(v))=>{
if v.contains('`'){return None;} serde_json::to_string(v).ok()?
},
Expr::Var(v)=>format!("`{v}`"),
Expr::Interp(parts)=>{
let mut text=String::new();
for p in parts {match p {
InterpPart::Lit(v)=>{if v.contains('`'){return None;}text.push_str(v);},
InterpPart::Var(v)=>text.push_str(&format!("`{v}`")),
_=>return None
}}
serde_json::to_string(&text).ok()?
},
Expr::Unary{op,expr}=>match op {UnaryOp::Neg=>format!("-{}",expression(expr)?),UnaryOp::Not=>format!("(not {})",expression(expr)?)},
Expr::Binary{op,left,right}=>format!("({} {} {})",expression(left)?,match op {
BinaryOp::Add=>"+",BinaryOp::Sub=>"-",BinaryOp::Mul=>"*",BinaryOp::Div=>"/",
BinaryOp::Eq=>"==",BinaryOp::Ne=>"!=",BinaryOp::Lt=>"<",BinaryOp::Le=>"<=",
BinaryOp::Gt=>">",BinaryOp::Ge=>">=",BinaryOp::And=>"and",BinaryOp::Or=>"or"},expression(right)?),
Expr::Call(call)=>call_source(call,false)?,
Expr::List(items)=>format!("[{}]",items.iter().map(expression).collect::<Option<Vec<_>>>()?.join(", ")),
// Dynamic map construction is deliberately not emitted as an alternate
// author DSL: keep its real bound GFM table at the structural boundary.
Expr::Map(_)|Expr::Formula(_)|Expr::Code(_)=>return None,
Expr::Index{base,label}=>format!("access({}, {})",expression(base)?,match label {
IndexKey::Lit(v)=>serde_json::to_string(v).ok()?,IndexKey::Var(v)=>format!("`{v}`")}),
Expr::Occluded{label,payload}=>format!("[{}]({})",label,payload.as_deref().map(expression).transpose_option()?),
})
}
trait OptionalText {fn transpose_option(self)->Option<String>;}
impl OptionalText for Option<Option<String>> {
fn transpose_option(self)->Option<String>{match self{None=>Some(String::new()),Some(v)=>v}}
}
fn call_source(call:&marqdo::ast::CallExpr,statement:bool)->Option<String> {
use marqdo::ast::Arg;
let call=call.with_modifiers_expanded().ok()?;
let target=if let Some(path)=&call.path{path.join(".")}
else if let Some(recv)=&call.receiver{format!("`{recv}`.{}",call.callee)}
else{call.callee.clone()};
let args=call.args.iter().map(|a|match a {
Arg::Named{name,value}=>Some(format!("{name}={}",expression(value)?)),
Arg::Positional(value)=>expression(value)
}).collect::<Option<Vec<_>>>()?;
if statement || call.receiver.is_some() {
let target=if target.contains(char::is_whitespace){format!("[{target}]")}else{target};
Some(format!("> {target} {}",args.join(" ")).trim_end().into())
} else {
if target.contains(char::is_whitespace){return None;}
Some(format!("{target}({})",args.join(", ")))
}
}
fn semantic_expr(e:&marqdo::ast::Expr)->Value {
use marqdo::ast::{Expr,Arg};
match e {
Expr::Occluded{payload,..}=>payload.as_deref().map(semantic_expr).unwrap_or(json!(["none"])),
Expr::Call(c)=>{
let c=c.for_evaluation().unwrap_or_else(|_|c.clone());
json!(["call",c.callee,c.path,c.receiver,c.args.iter().map(|a|match a{
Arg::Named{name,value}=>json!(["named",name,semantic_expr(value)]),
Arg::Positional(v)=>json!(["positional",semantic_expr(v)])
}).collect::<Vec<_>>()])
},
Expr::List(v)=>json!(["list",v.iter().map(semantic_expr).collect::<Vec<_>>()]),
Expr::Map(v)=>json!(["map",v.iter().map(|(k,v)|json!([k,semantic_expr(v)])).collect::<Vec<_>>()]),
Expr::Unary{op,expr}=>json!(["unary",format!("{op:?}"),semantic_expr(expr)]),
Expr::Binary{op,left,right}=>json!(["binary",format!("{op:?}"),semantic_expr(left),semantic_expr(right)]),
_=>json!(format!("{e:?}"))
}
}
fn semantic_statements(body:&[marqdo::ast::Stmt])->Vec<Value> {
use marqdo::ast::{Expr,Stmt};
body.iter().map(|s|match s {
Stmt::Assign{name,value,..}=>json!(["assign",name,semantic_expr(value)]),
Stmt::Return{value,..}=>json!(["return",semantic_expr(value)]),
Stmt::Call{call,..}=>json!(["eval",semantic_expr(&Expr::Call(call.clone()))]),
Stmt::Expr{value,..}=>json!(["eval",semantic_expr(value)]),
Stmt::Branch{arms,..}=>json!(["branch",arms.iter().map(|a|json!([a.condition.as_ref().map(semantic_expr),semantic_statements(&a.body)])).collect::<Vec<_>>()]),
Stmt::While{condition,body,..}=>json!(["while",semantic_expr(condition),semantic_statements(body)]),
Stmt::ForEach{item,collection,body,..}=>json!(["foreach",item,collection,semantic_statements(body)])
}).collect()
}
fn statement_locations(body:&[marqdo::ast::Stmt])->Vec<Value> {
use marqdo::ast::Stmt;
let mut out=Vec::new();
for s in body {match s {
Stmt::Assign{name,value,span,end_line}=>out.push(json!({"kind":"assign","name":name,"line":span.line,"end_line":end_line,"rendered":expression(value)})),
Stmt::Return{value,span}=>out.push(json!({"kind":"return","line":span.line,"rendered":expression(value)})),
Stmt::Call{call,span}=>out.push(json!({"kind":"call","line":span.line,"callee":call.path.as_ref().map(|p|p.join(".")).unwrap_or_else(||call.callee.clone()),"rendered":call_source(call,true)})),
Stmt::Expr{value,span}=>out.push(json!({"kind":"eval","line":span.line,"rendered":expression(value)})),
Stmt::Branch{arms,..}=>for arm in arms{out.extend(statement_locations(&arm.body));},
Stmt::While{body,..}|Stmt::ForEach{body,..}=>out.extend(statement_locations(body))
}}
out
}
fn analyze(p:&Value)->Answer {
use marqdo::ast::{Expr,Function,Literal};
fn default(e:&Expr)->Value {match e {
Expr::Literal(Literal::Text(t))=>json!({"text":t}),
Expr::Literal(Literal::None)=>json!({"source":"None"}),
Expr::Literal(Literal::Bool(v))=>json!({"source":if *v{"True"}else{"False"}}),
Expr::Literal(Literal::Int(v))=>json!({"source":v.to_string()}),
Expr::Literal(Literal::Num(v))=>json!({"source":if v.fract()==0.0 { format!("{v:.1}") } else {v.to_string()}}),
_=>json!({"debug":format!("{e:?}")})
}}
fn units(fs:&[Function],body:bool)->Vec<Value> {fs.iter().map(|f|json!({
"name":f.name,"line":f.span.line,"level":f.level,"base":f.base,
"params":f.params.iter().map(|p|json!({"name":p.name,"line":p.span.line,"inferred":p.inferred,"default":p.default.as_ref().map(default)})).collect::<Vec<_>>(),
"contract":f.contract.as_ref().map(|c|json!({"params":c.params.iter().map(|p|json!({"name":p.name,"ty":p.ty.name(),"desc":p.desc})).collect::<Vec<_>>(),"returns":c.returns.as_ref().map(|r|json!({"ty":r.ty.name(),"desc":r.desc}))})),
"children":units(&f.children,body),"semantic":semantic_statements(&f.body),"statements":statement_locations(&f.body),"body":if body{Some(format!("{:?}",f.body))}else{None}
})).collect()}
let source=field(p,"source")?;
let module=marqdo::parse::parse_source(source).map_err(|e|format!("{e:#}"))?;
let diagnostics=marqdo::check::check_module(&module,None).iter().map(|d|d.to_json()).collect::<Vec<_>>();
let ratios=marqdo::narrative_ratio::measure_source_units(source,&module.functions).iter()
.map(|r|json!({"name":r.name,"line":r.line,"narrative":r.narrative,"total":r.total,"ratio":r.ratio})).collect::<Vec<_>>();
Ok(json!({"schema_version":1,"functions":units(&module.functions,p.get("include_body").and_then(Value::as_bool).unwrap_or(false)),
"ratios":ratios,"diagnostics":diagnostics}))
}
fn gzip_text(p: &Value) -> Answer {
use base64::Engine;
let data = p.get("data").ok_or("missing portable bytes")?;
if data.get("_type").and_then(Value::as_str) != Some("bytes") {
return Err("gzip expects portable bytes".into());
}
let encoded = field(data,"base64")?;
if encoded.len() > 24 * 1024 * 1024 { return Err("compressed input exceeds limit".into()); }
let bytes = base64::engine::general_purpose::STANDARD.decode(encoded).map_err(|_| "invalid base64 bytes")?;
let limit = p.get("max_bytes").and_then(Value::as_u64).unwrap_or(16 * 1024 * 1024);
if limit == 0 || limit > 64 * 1024 * 1024 { return Err("invalid gzip output limit".into()); }
let plain = read_bounded(flate2::read::MultiGzDecoder::new(bytes.as_slice()),limit)?;
Ok(json!(String::from_utf8(plain).map_err(|_| "gzip text is not UTF-8")?))
}
fn dispatch(request:&Value)->Answer {
let root=PathBuf::from(request.get("root").and_then(Value::as_str).unwrap_or(".")).canonicalize().map_err(|_|"root does not exist")?;
let op=field(request,"operation")?;
let empty=json!({});let p=request.get("payload").unwrap_or(&empty);
match op {
"command"=>command(&root,p),
"git_inspect"=>git_inspect(&root),
"github"=>http_json(&root,p),
"archive"=>archive(&root,p),
"gzip_text"=>gzip_text(p),
"analyze"=>analyze(p),
"r2_put"=>r2_put(&root,p),
_=>Err("unknown engineering boundary operation".into())
}
}
unsafe extern "C" fn engineering_request(args:*const c_char,out:*mut *mut c_char,err:*mut *mut c_char)->c_int {
let args=match parse_args(args){Ok(v)=>v,Err(e)=>{set_err(err,&e);return 1;}};
let request=args.get("request").unwrap_or(&args);
let result=match dispatch(request) {
Ok(value)=>json!({"ok":true,"value":value,"error":null}),
Err(message)=>json!({"ok":false,"value":null,"error":{"code":"engineering.boundary","message":message,"operation":request.get("operation")}})
};
set_out(out,&result.to_string());0
}
marqdo_plugin!(2,[("engineering_request","request",engineering_request)]);
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize,Ordering};
static NEXT:AtomicUsize=AtomicUsize::new(0);
fn fixture()->PathBuf {
let p=std::env::temp_dir().join(format!("mq-engineering-{}-{}",std::process::id(),NEXT.fetch_add(1,Ordering::Relaxed)));
fs::create_dir_all(&p).unwrap(); p
}
#[cfg(unix)]
#[test]
fn argv_input_is_literal_and_noisy_output_is_bounded() {
let dir=fixture();
let input="quotes \" ' $HOME $(never-run) 😀\n".repeat(1000);
let result=command(&dir,&json!({"argv":["/bin/cat"],"stdin":input})).unwrap();
assert_eq!(result["code"],0);assert_eq!(result["stdout"],input);
let result=command(&dir,&json!({"argv":["/usr/bin/head","-c","65536","/dev/zero"],"max_output":128})).unwrap();
assert_eq!(result["code"],0);assert_eq!(result["stdout"].as_str().unwrap().len(),128);
assert_eq!(result["truncated"],true);
fs::remove_dir_all(dir).unwrap();
}
#[cfg(unix)]
#[test]
fn timeout_and_nonzero_exit_remain_observable() {
let dir=fixture();let started=Instant::now();
let result=command(&dir,&json!({"argv":["/bin/sleep","2"],"timeout_ms":20})).unwrap();
assert_eq!(result["timed_out"],true);assert_eq!(result["code"],124);
assert!(started.elapsed()<Duration::from_secs(2));
let result=command(&dir,&json!({"argv":["/bin/false"]})).unwrap();
assert_eq!(result["code"],1);assert_eq!(result["timed_out"],false);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn unsafe_transport_shortcuts_are_rejected_before_spawn() {
let dir=fixture();
for argv in [json!(["git","push","--force"]),json!(["git","commit","--no-verify"]),json!(["bash","-c","echo forbidden"])] {
assert!(command(&dir,&json!({"argv":argv})).is_err());
}
fs::remove_dir_all(dir).unwrap();
}
fn make_zip(path:&Path,name:&str,payload:&[u8]) {
let mut out=zip::ZipWriter::new(File::create(path).unwrap());
out.start_file(name,zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored)).unwrap();
out.write_all(payload).unwrap();out.finish().unwrap();
}
#[test]
fn zip_inspection_checks_metadata_paths_and_crc_without_extraction() {
let dir=fixture();let path=dir.join("fixture.zip");
make_zip(&path,"ext/VERSION",b"2.6.0\r\n");
let result=archive(&dir,&json!({"path":"fixture.zip"})).unwrap();
assert_eq!(result["metadata"]["ext/VERSION"],"2.6.0\r\n");
assert!(!dir.join("ext").exists());
make_zip(&path,"../escape",b"untrusted");
assert!(archive(&dir,&json!({"path":"fixture.zip"})).is_err());
make_zip(&path,"payload",b"unique CRC fixture payload");
let mut bytes=fs::read(&path).unwrap();
let index=bytes.windows(b"unique CRC fixture payload".len()).position(|x|x==b"unique CRC fixture payload").unwrap();
bytes[index]^=1;fs::write(&path,bytes).unwrap();
assert!(archive(&dir,&json!({"path":"fixture.zip"})).is_err());
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn gzip_text_validates_format_utf8_and_expansion_limit() {
use base64::Engine;
fn compressed(payload:&[u8])->Vec<u8> {
let mut writer=flate2::write::GzEncoder::new(Vec::new(),flate2::Compression::default());
writer.write_all(payload).unwrap();writer.finish().unwrap()
}
let mut bytes=compressed(b"Package: marqdo\n");bytes.extend(compressed(b"Version: 2.6.0\n"));
let data=json!({"_type":"bytes","base64":base64::engine::general_purpose::STANDARD.encode(&bytes)});
assert_eq!(gzip_text(&json!({"data":data})).unwrap(),"Package: marqdo\nVersion: 2.6.0\n");
assert!(gzip_text(&json!({"data":data,"max_bytes":2})).is_err());
assert!(gzip_text(&json!({"data":{"_type":"bytes","base64":"invalid"}})).is_err());
let data=json!({"_type":"bytes","base64":base64::engine::general_purpose::STANDARD.encode(compressed(&[0xff]))});
assert!(gzip_text(&json!({"data":data})).is_err());
}
#[test]
fn analysis_uses_the_real_parser_without_loading_imports() {
let source="---\nmarkup: occlusion\nimport:\n absent: /does/not/exist.mq.md\n---\n## answer\n\n完成计算后返回*42*。\n";
let result=analyze(&json!({"source":source})).unwrap();
assert_eq!(result["functions"][0]["name"],"answer");
assert!(result["diagnostics"].as_array().unwrap().is_empty());
assert_eq!(result["functions"][0]["semantic"][0][0],"return");
}
}