//! Native boundaries for executable engineering documents. Business ordering, //! version decisions and ordinary file reads/writes belong in .mq.md. //! No Python interpreter, generated shell program or secret-valued tool result. use std::{ collections::BTreeMap, ffi::CStr, fs::{self, File}, io::{Read, Write}, os::raw::{c_char, c_int}, path::{Component, Path, PathBuf}, process::{Command, Stdio}, sync::mpsc, thread, time::{Duration, Instant}, }; use hmac::{Hmac, Mac}; use marqdo_plugin_sdk::*; use sha2::{Digest, Sha256}; use serde_json::{json, Value}; type Answer = Result; const OUTPUT: usize = 256 * 1024; const HTTP_MAX: u64 = 16 * 1024 * 1024; fn field<'a>(v: &'a Value, key: &str) -> Result<&'a str, String> { v.get(key).and_then(Value::as_str).ok_or_else(|| format!("missing text field {key}")) } fn hash(b: &[u8]) -> String { hex(&Sha256::digest(b)) } fn hex(b: &[u8]) -> String { b.iter().map(|x| format!("{x:02x}")).collect() } fn read_bounded(mut r: impl Read, max: u64) -> Result, String> { let mut b = Vec::new(); r.by_ref().take(max + 1).read_to_end(&mut b).map_err(|e| format!("read failed ({:?})",e.kind()))?; if b.len() as u64 > max { return Err("bounded input limit exceeded".into()); } Ok(b) } fn scoped(root: &Path, relative: &str) -> Result { let p = root.join(relative).canonicalize().map_err(|_| "path does not exist")?; if !p.starts_with(root) { return Err("artifact path escapes the selected root".into()); } Ok(p) } fn homes(root: &Path) -> Vec { let mut out = Vec::new(); for key in ["MARQDO_RELEASE_CREDENTIAL_HOME", "USERPROFILE", "HOME"] { if let Some(s) = std::env::var_os(key) { out.push(PathBuf::from(s)); } } #[cfg(unix)] { use std::os::unix::fs::MetadataExt; for p in std::iter::once(root).chain(root.ancestors().skip(1)) { if p.join(".git").exists() { if let Ok(m) = p.metadata() { unsafe { let mut entry: libc::passwd = std::mem::zeroed(); let mut result = std::ptr::null_mut(); let mut buf = vec![0u8; 32768]; if libc::getpwuid_r(m.uid(), &mut entry, buf.as_mut_ptr().cast(), buf.len(), &mut result) == 0 && !result.is_null() && !entry.pw_dir.is_null() { out.push(PathBuf::from(CStr::from_ptr(entry.pw_dir).to_string_lossy().into_owned())); } } } } } } out.dedup(); out } fn redact(mut text: String, private: &[String]) -> String { for token in private.iter().filter(|s| !s.is_empty()) { text = text.replace(token, ""); } for (k, v) in std::env::vars() { if v.len() >= 8 && ["TOKEN", "SECRET", "PASSWORD", "ACCESS_KEY"].iter().any(|x| k.contains(x)) { text = text.replace(&v, ""); } } text } struct Output { code: i32, stdout: String, stderr: String, timed_out: bool, truncated: bool } fn pipe(mut input: impl Read + Send + 'static, limit: usize) -> mpsc::Receiver<(Vec, bool)> { let (send, recv) = mpsc::channel(); thread::spawn(move || { let mut buf = Vec::new(); let mut part = [0u8; 8192]; let mut truncated = false; while let Ok(n) = input.read(&mut part) { if n == 0 { break; } let keep = (limit - buf.len()).min(n); buf.extend_from_slice(&part[..keep]); truncated |= keep < n; } let _ = send.send((buf, truncated)); }); recv } fn execute(root: &Path, argv: &[String], input: Option, env: &BTreeMap, timeout_ms: u64, max_output: usize) -> Result { if argv.is_empty() || argv[0].is_empty() { return Err("argv must not be empty".into()); } if !(1..=3_600_000).contains(&timeout_ms) || !(1..=16_777_216).contains(&max_output) { return Err("timeout/output limit outside supported range".into()); } let mut cmd = Command::new(&argv[0]); cmd.args(&argv[1..]).current_dir(root).envs(env) .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()); #[cfg(unix)] { use std::os::unix::process::CommandExt; unsafe { cmd.pre_exec(|| { if libc::setpgid(0, 0) != 0 { return Err(std::io::Error::last_os_error()); } Ok(()) }); } } let mut child = cmd.spawn().map_err(|_| format!("cannot start {}", argv[0]))?; let stdout = pipe(child.stdout.take().unwrap(), max_output); let stderr = pipe(child.stderr.take().unwrap(), max_output); let mut stdin = child.stdin.take().unwrap(); thread::spawn(move || { if let Some(text) = input { let _ = stdin.write_all(text.as_bytes()); } }); let started = Instant::now(); let mut timed_out = false; let code = loop { match child.try_wait().map_err(|_| "cannot observe process")? { Some(status) => break status.code().unwrap_or(1), None if started.elapsed() >= Duration::from_millis(timeout_ms) => { timed_out = true; #[cfg(unix)] unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL); } #[cfg(windows)] { let _ = Command::new("taskkill").args(["/PID", &child.id().to_string(), "/T", "/F"]).output(); } let _ = child.kill(); let _ = child.wait(); break 124; } None => thread::sleep(Duration::from_millis(10)), } }; // Descendants retaining inherited pipes must not prevent a bounded return. let (out, out_clip) = stdout.recv_timeout(Duration::from_secs(1)).unwrap_or((Vec::new(), true)); let (err, err_clip) = stderr.recv_timeout(Duration::from_secs(1)).unwrap_or((Vec::new(), true)); Ok(Output { code, stdout: String::from_utf8_lossy(&out).into(), stderr: String::from_utf8_lossy(&err).into(), timed_out, truncated: out_clip || err_clip }) } fn github_token(root: &Path) -> Option { for key in ["GH_TOKEN", "GITHUB_TOKEN"] { if let Ok(s) = std::env::var(key) { if !s.is_empty() { return Some(s); } } } let args = ["git", "-c", &format!("safe.directory={}", root.display()), "credential", "fill"] .into_iter().map(str::to_string).collect::>(); let env = BTreeMap::from([("GIT_TERMINAL_PROMPT".into(), "0".into())]); if let Ok(p) = execute(root, &args, Some("protocol=https\nhost=github.com\n\n".into()), &env, 15_000, OUTPUT) { if let Some(s) = p.stdout.lines().find_map(|s| s.strip_prefix("password=")) { return Some(s.into()); } } for home in homes(root) { if let Ok(text) = fs::read_to_string(home.join(".config/gh/hosts.yml")) { if let Some(s) = text.lines().find_map(|s| s.trim().strip_prefix("oauth_token:")) { return Some(s.trim().trim_matches(['"', '\'']).into()); } } } None } fn git_config(env: &mut BTreeMap, key: &str, value: String) { let start = env.get("GIT_CONFIG_COUNT").cloned().or_else(|| std::env::var("GIT_CONFIG_COUNT").ok()) .and_then(|s| s.parse::().ok()).unwrap_or(0); env.insert(format!("GIT_CONFIG_KEY_{start}"),key.into()); env.insert(format!("GIT_CONFIG_VALUE_{start}"),value); env.insert("GIT_CONFIG_COUNT".into(),(start+1).to_string()); } fn command(root: &Path, p: &Value) -> Answer { let mut argv = p.get("argv").and_then(Value::as_array).ok_or("argv must be a list")? .iter().map(|v| v.as_str().map(str::to_string).ok_or("argv items must be text".to_string())) .collect::, _>>()?; let cwd = scoped(root, p.get("cwd").and_then(Value::as_str).unwrap_or("."))?; let mut env = BTreeMap::new(); let mut private = Vec::new(); // Trust only the explicitly selected workspace, only in this child tree. // Cargo/Go VCS probes inherit the same scoped Git metadata access. git_config(&mut env,"safe.directory",root.display().to_string()); let exe = argv.first().and_then(|s| Path::new(s).file_stem()).and_then(|s| s.to_str()).unwrap_or(""); if ["bash", "sh", "pwsh", "powershell"].contains(&exe) && argv.iter().any(|s| ["-c", "-Command", "-EncodedCommand"].contains(&s.as_str())) { return Err("use argument vectors or maintained external scripts, not generated shell programs".into()); } if exe == "git" { if argv.iter().any(|s| ["--force", "-f", "--force-with-lease", "--no-verify"].contains(&s.as_str()) || s.starts_with("--force=")) { return Err("forced Git updates and hook bypass are forbidden".into()); } argv.splice(1..1, ["-c".into(), format!("safe.directory={}", cwd.display())]); env.insert("GIT_TERMINAL_PROMPT".into(), "0".into()); if argv.iter().any(|s| ["fetch", "push", "ls-remote"].contains(&s.as_str())) { if let Some(token) = github_token(root) { // URL-scoped headers stay in the child environment, not argv. let auth = format!("Authorization: Basic {}", base64(format!("x-access-token:{token}").as_bytes())); for url in ["https://proxy.cflmy.top/github.com/", "https://github.com/"] { git_config(&mut env,&format!("http.{url}.extraheader"),auth.clone()); } private.push(token); private.push(auth); argv.splice(1..1, ["-c".into(), "http.version=HTTP/1.1".into(), "-c".into(), "http.postBuffer=524288000".into()]); } } } if let Some(e) = p.get("env").and_then(Value::as_object) { for (k, v) in e { env.insert(k.clone(), v.as_str().ok_or("env values must be text")?.into()); } } // Child Git trace output could expose internally attached authentication. for key in ["GIT_TRACE", "GIT_TRACE_CURL", "GIT_CURL_VERBOSE"] { env.insert(key.into(), "0".into()); } let timeout = p.get("timeout_ms").and_then(Value::as_u64) .unwrap_or_else(|| p.get("timeout").and_then(Value::as_u64).unwrap_or(120) * 1000); let max = p.get("max_output").and_then(Value::as_u64).unwrap_or(OUTPUT as u64) as usize; let out = execute(&cwd, &argv, p.get("stdin").and_then(Value::as_str).map(str::to_string), &env, timeout, max)?; Ok(json!({"code":out.code,"stdout":redact(out.stdout,&private),"stderr":redact(out.stderr,&private), "timed_out":out.timed_out,"truncated":out.truncated})) } fn http_json(root: &Path, p: &Value) -> Answer { let path = field(p, "path")?; if !path.starts_with("/repos/cflmy/marqdo/") || path.contains("..") { return Err("GitHub path is outside the project".into()); } let method = p.get("method").and_then(Value::as_str).unwrap_or("GET"); if !["GET", "PATCH", "POST"].contains(&method) { return Err("unsupported GitHub method".into()); } let bases = std::env::var("MARQDO_RELEASE_API_BASE").map(|s| vec![s]) .unwrap_or_else(|_| vec!["https://proxy.cflmy.top/api.github.com".into(), "https://api.github.com".into()]); let token = github_token(root); for (i, base) in bases.iter().enumerate() { let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(35)).build(); let url = format!("{}{path}", base.trim_end_matches('/')); let mut req = agent.request(method, &url).set("Accept", "application/vnd.github+json") .set("User-Agent", "marqdo-engineering/1").set("X-GitHub-Api-Version", "2022-11-28"); if let Some(token) = &token { req = req.set("Authorization", &format!("Bearer {token}")); } let response = if let Some(body) = p.get("data").filter(|v| !v.is_null()) { req.set("Content-Type", "application/json").send_string(&body.to_string()) } else { req.call() }; match response { Ok(r) => { let body = read_bounded(r.into_reader(), HTTP_MAX) .and_then(|bytes| serde_json::from_slice(&bytes).map_err(|_| "GitHub response is not JSON".into())); match body { Ok(value) => return Ok(value), Err(_) if method != "POST" && i + 1 < bases.len() => continue, Err(message) => return Err(message), } } Err(ureq::Error::Status(404,_)) if p.get("missing").and_then(Value::as_bool).unwrap_or(false) => return Ok(Value::Null), Err(ureq::Error::Status(code,_)) => { if method == "POST" || i + 1 == bases.len() || ![429,500,502,503,504].contains(&code) { return Err(format!("GitHub HTTP {code}")); } } Err(_) if method == "POST" || i + 1 == bases.len() => return Err("GitHub network request failed; uncertain mutations are not replayed".into()), Err(_) => {} } } Err("GitHub request failed after proxy and direct routes".into()) } fn archive(root: &Path, p: &Value) -> Answer { let path = scoped(root, field(p,"path")?)?; let file = File::open(path).map_err(|_|"cannot open archive")?; let mut zip = zip::ZipArchive::new(file).map_err(|_|"invalid ZIP")?; if zip.len() > 100000 { return Err("ZIP has too many entries".into()); } let mut seen=std::collections::HashSet::new(); let mut entries = Vec::new(); let mut metadata = serde_json::Map::new(); let mut total = 0u64; let mut actual = 0u64; for i in 0..zip.len() { let mut f = zip.by_index(i).map_err(|_|"cannot read ZIP entry")?; let name = f.name().replace('\\', "/"); if !seen.insert(name.clone()) || name.as_bytes().get(1)==Some(&b':') {return Err("duplicate or drive-qualified ZIP path".into());} if Path::new(&name).components().any(|c| matches!(c, Component::ParentDir | Component::RootDir | Component::Prefix(_))) { return Err("unsafe ZIP path".into()); } if f.unix_mode().is_some_and(|m|m & 0o170000 == 0o120000) { return Err("ZIP symlink is not supported".into()); } total = total.checked_add(f.size()).ok_or("ZIP expansion overflow")?; if total > 2 * 1024 * 1024 * 1024 { return Err("ZIP expands beyond 2 GiB".into()); } let mut selected = Vec::new(); let keep = ["VERSION","ext/VERSION","Cargo.toml","extension/package.json"].contains(&name.as_str()); let mut buf=[0u8;65536]; loop { let n=f.read(&mut buf).map_err(|_|"ZIP content/CRC validation failed")?; if n==0 {break;} actual = actual.checked_add(n as u64).ok_or("ZIP expansion overflow")?; if actual > 2 * 1024 * 1024 * 1024 { return Err("ZIP expands beyond 2 GiB".into()); } if keep && selected.len()<16384 {selected.extend_from_slice(&buf[..n.min(16384-selected.len())]);} } if keep { metadata.insert(name.clone(),json!(String::from_utf8(selected).map_err(|_|"ZIP metadata is not UTF-8")?)); } entries.push(name); } Ok(json!({"entries":entries,"metadata":metadata})) } fn hmac(key:&[u8], data:&str)->Vec { let mut mac=Hmac::::new_from_slice(key).expect("HMAC accepts all key lengths"); mac.update(data.as_bytes());mac.finalize().into_bytes().to_vec() } fn r2_settings(root:&Path)->Result,String> { let mut settings=std::env::vars().filter(|(k,_)|k.starts_with("R2_")).collect::>(); for home in homes(root) { if let Ok(text)=fs::read_to_string(home.join(".marqdo/r2.env")) { for line in text.lines().map(str::trim).filter(|s|!s.starts_with('#')) { if let Some((k,v))=line.split_once('=') {settings.entry(k.trim().into()).or_insert_with(||v.trim().trim_matches(['\'','"']).into());} } } } for k in ["R2_ENDPOINT","R2_BUCKET","R2_ACCESS_KEY_ID","R2_SECRET_ACCESS_KEY"] { if settings.get(k).map_or(true, String::is_empty) {return Err(format!("missing {k}; configure ~/.marqdo/r2.env"));} } Ok(settings) } fn r2_put(root:&Path,p:&Value)->Answer { let s=r2_settings(root)?; let key=field(p,"key")?; if !key.bytes().all(|b|b.is_ascii_alphanumeric() || b"/.-_".contains(&b)) || key.contains("..") { return Err("unsafe R2 key".into()); } let body=if let Some(path)=p.get("path").and_then(Value::as_str) { read_bounded(File::open(scoped(root,path)?).map_err(|_|"cannot open upload")?,256*1024*1024)? } else {field(p,"text")?.as_bytes().to_vec()}; let digest=hash(&body); if let Some(expected)=p.get("sha256").and_then(Value::as_str) { if digest!=expected {return Err("upload bytes changed after verification".into());} } let endpoint=s["R2_ENDPOINT"].trim_end_matches('/'); let host=endpoint.split_once("://").ok_or("R2 endpoint needs a scheme")?.1; if host.contains('/') || !(endpoint.starts_with("https://") || endpoint.starts_with("http://127.0.0.1:")) { return Err("R2 endpoint must be HTTPS (loopback HTTP is for isolated tests)".into()); } let uri=format!("/{}/{key}",s["R2_BUCKET"]); let date=chrono::Utc::now();let amz=date.format("%Y%m%dT%H%M%SZ").to_string();let day=date.format("%Y%m%d").to_string(); let region=s.get("R2_REGION").map(String::as_str).unwrap_or("auto"); let mime=if key.ends_with("/VERSION"){"text/plain; charset=utf-8"}else{"application/zip"}; let headers=format!("content-type:{mime}\nhost:{host}\nx-amz-content-sha256:{digest}\nx-amz-date:{amz}\n"); let signed="content-type;host;x-amz-content-sha256;x-amz-date"; let canonical=format!("PUT\n{uri}\n\n{headers}\n{signed}\n{digest}"); let scope=format!("{day}/{region}/s3/aws4_request"); let string=format!("AWS4-HMAC-SHA256\n{amz}\n{scope}\n{}",hash(canonical.as_bytes())); let k=hmac(format!("AWS4{}",s["R2_SECRET_ACCESS_KEY"]).as_bytes(),&day); let k=hmac(&k,region);let k=hmac(&k,"s3");let k=hmac(&k,"aws4_request"); let auth=format!("AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={signed}, Signature={}",s["R2_ACCESS_KEY_ID"],hex(&hmac(&k,&string))); let response=ureq::AgentBuilder::new().timeout(Duration::from_secs(180)).build() .put(&format!("{endpoint}{uri}")).set("Content-Type",mime).set("x-amz-content-sha256",&digest) .set("x-amz-date",&amz).set("Authorization",&auth).send_bytes(&body); match response { Ok(r) if [200,201].contains(&r.status()) => Ok(json!({"key":key,"sha256":digest,"size":body.len()})), Ok(r)=>Err(format!("R2 HTTP {}",r.status())), Err(ureq::Error::Status(code,_))=>Err(format!("R2 HTTP {code}")), Err(_)=>Err("R2 upload failed or timed out; verify before retrying".into()) } } fn git_inspect(root:&Path)->Answer { if !root.join(".git").exists() {return Err("inspect requires a Git checkout".into());} let query=|args:&[&str]|->Result{ let argv=std::iter::once("git").chain(args.iter().copied()).map(|s|json!(s)).collect::>(); let out=command(root,&json!({"argv":argv}))?; if out["code"]!=0 {return Err("Git metadata query failed".into());} Ok(out["stdout"].as_str().unwrap_or("").trim().into()) }; Ok(json!({"root":root.display().to_string(),"branch":query(&["branch","--show-current"])?, "commit":query(&["rev-parse","HEAD"])?,"changes":query(&["status","--porcelain"])?, "tags":query(&["tag","--list"])?.lines().collect::>(), "extension_on_main":query(&["ls-files","vscode-extension"])?, "origin":query(&["remote","get-url","origin"])?})) } fn base64(bytes:&[u8])->String { const C:&[u8]=b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; let mut out=String::new(); for part in bytes.chunks(3) { let a=part[0] as usize;let b=*part.get(1).unwrap_or(&0) as usize;let c=*part.get(2).unwrap_or(&0) as usize; out.push(C[a>>2] as char);out.push(C[((a&3)<<4)|(b>>4)] as char); out.push(if part.len()>1 {C[((b&15)<<2)|(c>>6)] as char}else{'='}); out.push(if part.len()>2 {C[c&63] as char}else{'='}); } out } fn expression(e:&marqdo::ast::Expr)->Option { use marqdo::ast::{Expr,Literal,InterpPart,UnaryOp,BinaryOp,IndexKey}; Some(match e { Expr::Literal(Literal::None)=>"None".into(), Expr::Literal(Literal::Bool(v))=>if *v{"True".into()}else{"False".into()}, Expr::Literal(Literal::Int(v))=>v.to_string(), Expr::Literal(Literal::Num(v))=>{let s=v.to_string();if s.contains('.')||s.contains('e')||s.contains('E'){s}else{format!("{s}.0")}}, Expr::Literal(Literal::Text(v))=>{ if v.contains('`'){return None;} serde_json::to_string(v).ok()? }, Expr::Var(v)=>format!("`{v}`"), Expr::Interp(parts)=>{ let mut text=String::new(); for p in parts {match p { InterpPart::Lit(v)=>{if v.contains('`'){return None;}text.push_str(v);}, InterpPart::Var(v)=>text.push_str(&format!("`{v}`")), _=>return None }} serde_json::to_string(&text).ok()? }, Expr::Unary{op,expr}=>match op {UnaryOp::Neg=>format!("-{}",expression(expr)?),UnaryOp::Not=>format!("(not {})",expression(expr)?)}, Expr::Binary{op,left,right}=>format!("({} {} {})",expression(left)?,match op { BinaryOp::Add=>"+",BinaryOp::Sub=>"-",BinaryOp::Mul=>"*",BinaryOp::Div=>"/", BinaryOp::Eq=>"==",BinaryOp::Ne=>"!=",BinaryOp::Lt=>"<",BinaryOp::Le=>"<=", BinaryOp::Gt=>">",BinaryOp::Ge=>">=",BinaryOp::And=>"and",BinaryOp::Or=>"or"},expression(right)?), Expr::Call(call)=>call_source(call,false)?, Expr::List(items)=>format!("[{}]",items.iter().map(expression).collect::>>()?.join(", ")), // Dynamic map construction is deliberately not emitted as an alternate // author DSL: keep its real bound GFM table at the structural boundary. Expr::Map(_)|Expr::Formula(_)|Expr::Code(_)=>return None, Expr::Index{base,label}=>format!("access({}, {})",expression(base)?,match label { IndexKey::Lit(v)=>serde_json::to_string(v).ok()?,IndexKey::Var(v)=>format!("`{v}`")}), Expr::Occluded{label,payload}=>format!("[{}]({})",label,payload.as_deref().map(expression).transpose_option()?), }) } trait OptionalText {fn transpose_option(self)->Option;} impl OptionalText for Option> { fn transpose_option(self)->Option{match self{None=>Some(String::new()),Some(v)=>v}} } fn call_source(call:&marqdo::ast::CallExpr,statement:bool)->Option { use marqdo::ast::Arg; let call=call.with_modifiers_expanded().ok()?; let target=if let Some(path)=&call.path{path.join(".")} else if let Some(recv)=&call.receiver{format!("`{recv}`.{}",call.callee)} else{call.callee.clone()}; let args=call.args.iter().map(|a|match a { Arg::Named{name,value}=>Some(format!("{name}={}",expression(value)?)), Arg::Positional(value)=>expression(value) }).collect::>>()?; if statement || call.receiver.is_some() { let target=if target.contains(char::is_whitespace){format!("[{target}]")}else{target}; Some(format!("> {target} {}",args.join(" ")).trim_end().into()) } else { if target.contains(char::is_whitespace){return None;} Some(format!("{target}({})",args.join(", "))) } } fn semantic_expr(e:&marqdo::ast::Expr)->Value { use marqdo::ast::{Expr,Arg}; match e { Expr::Occluded{payload,..}=>payload.as_deref().map(semantic_expr).unwrap_or(json!(["none"])), Expr::Call(c)=>{ let c=c.for_evaluation().unwrap_or_else(|_|c.clone()); json!(["call",c.callee,c.path,c.receiver,c.args.iter().map(|a|match a{ Arg::Named{name,value}=>json!(["named",name,semantic_expr(value)]), Arg::Positional(v)=>json!(["positional",semantic_expr(v)]) }).collect::>()]) }, Expr::List(v)=>json!(["list",v.iter().map(semantic_expr).collect::>()]), Expr::Map(v)=>json!(["map",v.iter().map(|(k,v)|json!([k,semantic_expr(v)])).collect::>()]), Expr::Unary{op,expr}=>json!(["unary",format!("{op:?}"),semantic_expr(expr)]), Expr::Binary{op,left,right}=>json!(["binary",format!("{op:?}"),semantic_expr(left),semantic_expr(right)]), _=>json!(format!("{e:?}")) } } fn semantic_statements(body:&[marqdo::ast::Stmt])->Vec { use marqdo::ast::{Expr,Stmt}; body.iter().map(|s|match s { Stmt::Assign{name,value,..}=>json!(["assign",name,semantic_expr(value)]), Stmt::Return{value,..}=>json!(["return",semantic_expr(value)]), Stmt::Call{call,..}=>json!(["eval",semantic_expr(&Expr::Call(call.clone()))]), Stmt::Expr{value,..}=>json!(["eval",semantic_expr(value)]), Stmt::Branch{arms,..}=>json!(["branch",arms.iter().map(|a|json!([a.condition.as_ref().map(semantic_expr),semantic_statements(&a.body)])).collect::>()]), Stmt::While{condition,body,..}=>json!(["while",semantic_expr(condition),semantic_statements(body)]), Stmt::ForEach{item,collection,body,..}=>json!(["foreach",item,collection,semantic_statements(body)]) }).collect() } fn statement_locations(body:&[marqdo::ast::Stmt])->Vec { use marqdo::ast::Stmt; let mut out=Vec::new(); for s in body {match s { Stmt::Assign{name,value,span,end_line}=>out.push(json!({"kind":"assign","name":name,"line":span.line,"end_line":end_line,"rendered":expression(value)})), Stmt::Return{value,span}=>out.push(json!({"kind":"return","line":span.line,"rendered":expression(value)})), Stmt::Call{call,span}=>out.push(json!({"kind":"call","line":span.line,"callee":call.path.as_ref().map(|p|p.join(".")).unwrap_or_else(||call.callee.clone()),"rendered":call_source(call,true)})), Stmt::Expr{value,span}=>out.push(json!({"kind":"eval","line":span.line,"rendered":expression(value)})), Stmt::Branch{arms,..}=>for arm in arms{out.extend(statement_locations(&arm.body));}, Stmt::While{body,..}|Stmt::ForEach{body,..}=>out.extend(statement_locations(body)) }} out } fn analyze(p:&Value)->Answer { use marqdo::ast::{Expr,Function,Literal}; fn default(e:&Expr)->Value {match e { Expr::Literal(Literal::Text(t))=>json!({"text":t}), Expr::Literal(Literal::None)=>json!({"source":"None"}), Expr::Literal(Literal::Bool(v))=>json!({"source":if *v{"True"}else{"False"}}), Expr::Literal(Literal::Int(v))=>json!({"source":v.to_string()}), Expr::Literal(Literal::Num(v))=>json!({"source":if v.fract()==0.0 { format!("{v:.1}") } else {v.to_string()}}), _=>json!({"debug":format!("{e:?}")}) }} fn units(fs:&[Function],body:bool)->Vec {fs.iter().map(|f|json!({ "name":f.name,"line":f.span.line,"level":f.level,"base":f.base, "params":f.params.iter().map(|p|json!({"name":p.name,"line":p.span.line,"inferred":p.inferred,"default":p.default.as_ref().map(default)})).collect::>(), "contract":f.contract.as_ref().map(|c|json!({"params":c.params.iter().map(|p|json!({"name":p.name,"ty":p.ty.name(),"desc":p.desc})).collect::>(),"returns":c.returns.as_ref().map(|r|json!({"ty":r.ty.name(),"desc":r.desc}))})), "children":units(&f.children,body),"semantic":semantic_statements(&f.body),"statements":statement_locations(&f.body),"body":if body{Some(format!("{:?}",f.body))}else{None} })).collect()} let source=field(p,"source")?; let module=marqdo::parse::parse_source(source).map_err(|e|format!("{e:#}"))?; let diagnostics=marqdo::check::check_module(&module,None).iter().map(|d|d.to_json()).collect::>(); let ratios=marqdo::narrative_ratio::measure_source_units(source,&module.functions).iter() .map(|r|json!({"name":r.name,"line":r.line,"narrative":r.narrative,"total":r.total,"ratio":r.ratio})).collect::>(); Ok(json!({"schema_version":1,"functions":units(&module.functions,p.get("include_body").and_then(Value::as_bool).unwrap_or(false)), "ratios":ratios,"diagnostics":diagnostics})) } fn gzip_text(p: &Value) -> Answer { use base64::Engine; let data = p.get("data").ok_or("missing portable bytes")?; if data.get("_type").and_then(Value::as_str) != Some("bytes") { return Err("gzip expects portable bytes".into()); } let encoded = field(data,"base64")?; if encoded.len() > 24 * 1024 * 1024 { return Err("compressed input exceeds limit".into()); } let bytes = base64::engine::general_purpose::STANDARD.decode(encoded).map_err(|_| "invalid base64 bytes")?; let limit = p.get("max_bytes").and_then(Value::as_u64).unwrap_or(16 * 1024 * 1024); if limit == 0 || limit > 64 * 1024 * 1024 { return Err("invalid gzip output limit".into()); } let plain = read_bounded(flate2::read::MultiGzDecoder::new(bytes.as_slice()),limit)?; Ok(json!(String::from_utf8(plain).map_err(|_| "gzip text is not UTF-8")?)) } fn dispatch(request:&Value)->Answer { let root=PathBuf::from(request.get("root").and_then(Value::as_str).unwrap_or(".")).canonicalize().map_err(|_|"root does not exist")?; let op=field(request,"operation")?; let empty=json!({});let p=request.get("payload").unwrap_or(&empty); match op { "command"=>command(&root,p), "git_inspect"=>git_inspect(&root), "github"=>http_json(&root,p), "archive"=>archive(&root,p), "gzip_text"=>gzip_text(p), "analyze"=>analyze(p), "r2_put"=>r2_put(&root,p), _=>Err("unknown engineering boundary operation".into()) } } unsafe extern "C" fn engineering_request(args:*const c_char,out:*mut *mut c_char,err:*mut *mut c_char)->c_int { let args=match parse_args(args){Ok(v)=>v,Err(e)=>{set_err(err,&e);return 1;}}; let request=args.get("request").unwrap_or(&args); let result=match dispatch(request) { Ok(value)=>json!({"ok":true,"value":value,"error":null}), Err(message)=>json!({"ok":false,"value":null,"error":{"code":"engineering.boundary","message":message,"operation":request.get("operation")}}) }; set_out(out,&result.to_string());0 } marqdo_plugin!(2,[("engineering_request","request",engineering_request)]); #[cfg(test)] mod tests { use super::*; use std::sync::atomic::{AtomicUsize,Ordering}; static NEXT:AtomicUsize=AtomicUsize::new(0); fn fixture()->PathBuf { let p=std::env::temp_dir().join(format!("mq-engineering-{}-{}",std::process::id(),NEXT.fetch_add(1,Ordering::Relaxed))); fs::create_dir_all(&p).unwrap(); p } #[cfg(unix)] #[test] fn argv_input_is_literal_and_noisy_output_is_bounded() { let dir=fixture(); let input="quotes \" ' $HOME $(never-run) 😀\n".repeat(1000); let result=command(&dir,&json!({"argv":["/bin/cat"],"stdin":input})).unwrap(); assert_eq!(result["code"],0);assert_eq!(result["stdout"],input); let result=command(&dir,&json!({"argv":["/usr/bin/head","-c","65536","/dev/zero"],"max_output":128})).unwrap(); assert_eq!(result["code"],0);assert_eq!(result["stdout"].as_str().unwrap().len(),128); assert_eq!(result["truncated"],true); fs::remove_dir_all(dir).unwrap(); } #[cfg(unix)] #[test] fn timeout_and_nonzero_exit_remain_observable() { let dir=fixture();let started=Instant::now(); let result=command(&dir,&json!({"argv":["/bin/sleep","2"],"timeout_ms":20})).unwrap(); assert_eq!(result["timed_out"],true);assert_eq!(result["code"],124); assert!(started.elapsed()Vec { let mut writer=flate2::write::GzEncoder::new(Vec::new(),flate2::Compression::default()); writer.write_all(payload).unwrap();writer.finish().unwrap() } let mut bytes=compressed(b"Package: marqdo\n");bytes.extend(compressed(b"Version: 2.6.0\n")); let data=json!({"_type":"bytes","base64":base64::engine::general_purpose::STANDARD.encode(&bytes)}); assert_eq!(gzip_text(&json!({"data":data})).unwrap(),"Package: marqdo\nVersion: 2.6.0\n"); assert!(gzip_text(&json!({"data":data,"max_bytes":2})).is_err()); assert!(gzip_text(&json!({"data":{"_type":"bytes","base64":"invalid"}})).is_err()); let data=json!({"_type":"bytes","base64":base64::engine::general_purpose::STANDARD.encode(compressed(&[0xff]))}); assert!(gzip_text(&json!({"data":data})).is_err()); } #[test] fn analysis_uses_the_real_parser_without_loading_imports() { let source="---\nmarkup: occlusion\nimport:\n absent: /does/not/exist.mq.md\n---\n## answer\n\n完成计算后返回*42*。\n"; let result=analyze(&json!({"source":source})).unwrap(); assert_eq!(result["functions"][0]["name"],"answer"); assert!(result["diagnostics"].as_array().unwrap().is_empty()); assert_eq!(result["functions"][0]["semantic"][0][0],"return"); } }