diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ccbb34..f3a3b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,12 @@ ## Unreleased ### Added +- **`sys.exec` / `系统.执行`**:可选 `capture=` / `捕获=`;为真时返回 `{code, stdout, stderr}`(GAP-09)。 ### Fixed +- **GAP-06/07**:`*…*` 分类对反引号/字符串内的 `*` 不再误判;错误收尾的赋值行不再吞掉后续 `##` 体(`>` / 合法 `*…*` 可脱出注释段);命名参数边界尊重 `[…]`/`{…}`。 +- **GAP-08**:`app.invoke` 对 `Null`/`None` 返回不再伪装成 `{"ok":true}`,改为 500 + `{"ok":false,"error":"null result"}`。 +- **GAP-10**:`app.proxy` `stream=true` 响应补齐 `Cache-Control: no-cache, no-transform` 与 `X-Accel-Buffering: no`。 ### Changed diff --git a/ext/web/web.mq.md b/ext/web/web.mq.md index c23a9c4..28b7a35 100644 --- a/ext/web/web.mq.md +++ b/ext/web/web.mq.md @@ -671,7 +671,7 @@ Tables stay as data; `configure` assembles them. 配置即数据、装配即函 + `headers_from_env`=None + `timeout_ms`=120000 -Same-origin reverse proxy to an upstream HTTP(S) URL. When `stream` is True (default), the response body — including `text/event-stream` SSE — is piped without buffering the full payload. `upstream` may contain `$ENV` / `${ENV}`. `headers_from_env` is `ENV=Header-Name` (comma-separated); secrets stay on the server. Prefer this over a side Python LLM proxy. +Same-origin reverse proxy to an upstream HTTP(S) URL. When `stream` is True (default), the response body — including `text/event-stream` SSE — is piped without buffering the full payload, and responses include `Cache-Control: no-cache, no-transform` plus `X-Accel-Buffering: no` for Nginx/CDN. `upstream` may contain `$ENV` / `${ENV}`. `headers_from_env` is `ENV=Header-Name` (comma-separated); secrets stay on the server. Prefer this over a side Python LLM proxy. **> web_app_proxy app=`self` path=`path` upstream=`upstream` stream=`stream` strip_prefix=`strip_prefix` methods=`methods` headers_from_env=`headers_from_env` timeout_ms=`timeout_ms`** diff --git a/lib/sys.mq.md b/lib/sys.mq.md index da48739..9a8fabe 100644 --- a/lib/sys.mq.md +++ b/lib/sys.mq.md @@ -37,8 +37,15 @@ Load `.env` (optional named arg `path=`). Existing process env is not overridden ## exec + `cmd` + `args`=None + + `capture`=False -**> host_exec cmd=`cmd` args=`args`** +Run a process. Default return is the exit code (int). + +With `capture=True`, return a map `{code, stdout, stderr}` (stdout/stderr may be truncated for huge output). + +Inside `*…*` prefer bare `args=args` (variable); `` args=`args` `` is also accepted. + +**> host_exec cmd=`cmd` args=`args` capture=`capture`** ## stream_publish + `event` diff --git a/lib/系统.mq.md b/lib/系统.mq.md index 3abd13a..74c3675 100644 --- a/lib/系统.mq.md +++ b/lib/系统.mq.md @@ -37,8 +37,15 @@ description: 中文系统包装(加载环境 可选 路径=) ## 执行 + `命令` + `参数`=None + + `捕获`=False -**> host_exec cmd=`命令` args=`参数`** +运行子进程。默认返回退出码(整数)。 + +`捕获=True` 时返回映射 `{code, stdout, stderr}`(过大输出可能截断)。 + +在 `*…*` 内优先写裸变量 `参数=参数`;`` 参数=`参数` `` 亦可。 + +**> host_exec cmd=`命令` args=`参数` capture=`捕获`** ## 发布流事件 + `事件` diff --git a/plugins/web/src/invoke.rs b/plugins/web/src/invoke.rs index 9181839..02408da 100644 --- a/plugins/web/src/invoke.rs +++ b/plugins/web/src/invoke.rs @@ -243,6 +243,9 @@ async fn invoke_handler( match route.return_mode.as_str() { "text" => { + if matches!(value, Value::Null) { + return err_json(StatusCode::INTERNAL_SERVER_ERROR, "null result"); + } let text = match &value { Value::String(s) => s.clone(), other => other.to_string(), @@ -257,16 +260,37 @@ async fn invoke_handler( ) .into_response() } - "status" => StatusCode::NO_CONTENT.into_response(), - _ => { - // Wrap non-objects so clients always get JSON object when possible. - let body = match value { - Value::Null => json!({ "ok": true }), - Value::Object(_) => value, - other => json!({ "ok": true, "result": other }), - }; - JsonBody(body).into_response() + "status" => { + if matches!(value, Value::Null) { + return err_json(StatusCode::INTERNAL_SERVER_ERROR, "null result"); + } + StatusCode::NO_CONTENT.into_response() } + _ => { + let body = match json_wrap_invoke_value(value) { + Ok((status, body)) => { + let mut resp = JsonBody(body).into_response(); + *resp.status_mut() = status; + resp + } + Err(resp) => resp, + }; + body + } + } +} + +/// Map host return value to invoke JSON. `Null` must not look like success (GAP-08). +pub(crate) fn json_wrap_invoke_value( + value: Value, +) -> Result<(StatusCode, Value), Response> { + match value { + Value::Null => Err(err_json( + StatusCode::INTERNAL_SERVER_ERROR, + "null result", + )), + Value::Object(_) => Ok((StatusCode::OK, value)), + other => Ok((StatusCode::OK, json!({ "ok": true, "result": other }))), } } @@ -330,3 +354,20 @@ fn err_json(status: StatusCode, msg: &str) -> Response { ) .into_response() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn null_result_is_not_fake_ok() { + let err = json_wrap_invoke_value(Value::Null).expect_err("null must fail"); + assert_eq!(err.status(), StatusCode::INTERNAL_SERVER_ERROR); + let ok = json_wrap_invoke_value(json!({ "hits": [] })).expect("object ok"); + assert_eq!(ok.0, StatusCode::OK); + let wrap = json_wrap_invoke_value(json!(1)).expect("scalar wrap"); + assert_eq!(wrap.0, StatusCode::OK); + assert_eq!(wrap.1["ok"], true); + assert_eq!(wrap.1["result"], 1); + } +} diff --git a/plugins/web/src/proxy.rs b/plugins/web/src/proxy.rs index e38e2e9..1f43014 100644 --- a/plugins/web/src/proxy.rs +++ b/plugins/web/src/proxy.rs @@ -364,6 +364,15 @@ async fn proxy_handler(route: ProxyRoute, req: Request) -> Response { } if route.stream { + // Industry SSE / reverse-proxy practice: disable buffering at CDN/Nginx. + out_headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static("no-cache, no-transform"), + ); + out_headers.insert( + HeaderName::from_static("x-accel-buffering"), + HeaderValue::from_static("no"), + ); let stream = upstream.bytes_stream().map_err(|e| { std::io::Error::new(std::io::ErrorKind::Other, e) }); diff --git a/src/aliases.rs b/src/aliases.rs index cf72387..acc33ac 100644 --- a/src/aliases.rs +++ b/src/aliases.rs @@ -87,6 +87,7 @@ pub fn canonical_param(builtin: &str, param: &str) -> String { ("host_dotenv_load" | "load_dotenv", "path" | "路径") => "path".into(), ("host_exec" | "exec", "cmd" | "命令") => "cmd".into(), ("host_exec" | "exec", "args" | "参数") => "args".into(), + ("host_exec" | "exec", "capture" | "捕获") => "capture".into(), ( "host_stream_publish" | "stream_publish", "event" | "事件", diff --git a/src/host/dispatch.rs b/src/host/dispatch.rs index 804c585..8bd3e45 100644 --- a/src/host/dispatch.rs +++ b/src/host/dispatch.rs @@ -910,7 +910,7 @@ impl HostFn { Self::OpenAiSseParse => &["echo"], Self::HttpRequest => &["body", "content_type", "headers"], Self::DotenvLoad => &["path"], - Self::Exec => &["args"], + Self::Exec => &["args", "capture"], Self::Solve => &["min", "max"], Self::Plot => &["steps", "path", "derivative", "grid"], Self::PlotPoints => &["path", "grid"], @@ -981,7 +981,12 @@ pub fn call_host( HostFn::Args => sys::args(ctx), HostFn::Cwd => sys::cwd(ctx), HostFn::Exit => sys::exit(ctx, require(bound, "code")?), - HostFn::Exec => sys::exec(ctx, require(bound, "cmd")?, bound.get("args")), + HostFn::Exec => sys::exec( + ctx, + require(bound, "cmd")?, + bound.get("args"), + bound.get("capture"), + ), HostFn::JsonParse => json::parse(require(bound, "text")?), HostFn::JsonStringify => { json::stringify(require(bound, "value")?, bound.get("indent")) diff --git a/src/host/sys.rs b/src/host/sys.rs index 681a96b..85298e2 100644 --- a/src/host/sys.rs +++ b/src/host/sys.rs @@ -108,18 +108,28 @@ pub fn exit(ctx: &HostContext, code: &Value) -> Result { } } -pub fn exec(ctx: &HostContext, cmd: &Value, args: Option<&Value>) -> Result { +pub fn exec( + ctx: &HostContext, + cmd: &Value, + args: Option<&Value>, + capture: Option<&Value>, +) -> Result { #[cfg(not(feature = "exec-host"))] { - let _ = (ctx, cmd, args); + let _ = (ctx, cmd, args, capture); return Err("exec unavailable in browser wasm".into()); } #[cfg(feature = "exec-host")] { + use std::process::Stdio; if !ctx.allow_exec() { return Err("exec denied by host policy".into()); } let cmd = as_text(cmd, "cmd")?; + let capture = match capture { + None | Some(Value::None) => false, + Some(v) => crate::host::math::as_bool(v), + }; let mut command = Command::new(cmd); command.current_dir(&ctx.cwd); match args { @@ -134,10 +144,38 @@ pub fn exec(ctx: &HostContext, cmd: &Value, args: Option<&Value>) -> Result return Err("exec args must be list or text".into()), } - let status = command - .status() - .map_err(|e| format!("exec `{cmd}`: {e}"))?; - Ok(Value::Int(status.code().unwrap_or(1) as i64)) + if capture { + command.stdout(Stdio::piped()); + command.stderr(Stdio::piped()); + let output = command + .output() + .map_err(|e| format!("exec `{cmd}`: {e}"))?; + let code = output.status.code().unwrap_or(1) as i64; + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + Ok(Value::Map(vec![ + ("code".into(), Value::Int(code)), + ("stdout".into(), Value::Text(clip_exec_io(&stdout))), + ("stderr".into(), Value::Text(clip_exec_io(&stderr))), + ])) + } else { + let status = command + .status() + .map_err(|e| format!("exec `{cmd}`: {e}"))?; + Ok(Value::Int(status.code().unwrap_or(1) as i64)) + } + } +} + +const EXEC_IO_CAP: usize = 256 * 1024; + +fn clip_exec_io(s: &str) -> String { + let mut it = s.chars(); + let head: String = it.by_ref().take(EXEC_IO_CAP).collect(); + if it.next().is_some() { + format!("{head}\n…(truncated)") + } else { + head } } diff --git a/src/lex/mod.rs b/src/lex/mod.rs index 5b99c9a..a50f50d 100644 --- a/src/lex/mod.rs +++ b/src/lex/mod.rs @@ -44,6 +44,8 @@ fn is_code_starter(c: char) -> bool { } /// `*` / `**` lines that are Marqdo executable (statement / return / else), not narrative Markdown. +/// +/// Closing `*` / `**` must not be taken from inside `` `…` `` or `"…"` (GAP-06/07 nested markers). fn is_marqdo_star_line(trimmed: &str) -> bool { if trimmed == "*" || trimmed == "****" { return true; @@ -52,14 +54,126 @@ fn is_marqdo_star_line(trimmed: &str) -> bool { if trimmed.trim() == "** **" { return true; } - if let Some(rest) = trimmed.strip_prefix("**") { - if let Some(pos) = rest.find("**") { - return rest[pos + 2..].trim().is_empty(); - } - } + return matching_bold_close(trimmed).is_some(); + } + if !trimmed.starts_with('*') || trimmed.len() < 2 { return false; } - trimmed.starts_with('*') && trimmed.ends_with('*') && trimmed.len() >= 2 + matching_italic_close(trimmed).is_some() +} + +/// Index of the closing `*` for a single-star statement, skipping strings and backticks. +fn matching_italic_close(trimmed: &str) -> Option { + if trimmed.starts_with("**") { + return None; + } + let bytes = trimmed.as_bytes(); + let mut i = 1usize; // skip opening * + let mut in_bt = false; + let mut in_str = false; + while i < bytes.len() { + let c = bytes[i]; + if in_str { + if c == b'\\' && i + 1 < bytes.len() { + i += 2; + continue; + } + if c == b'"' { + in_str = false; + } + i += 1; + continue; + } + if in_bt { + if c == b'`' { + in_bt = false; + } + i += 1; + continue; + } + match c { + b'`' => { + in_bt = true; + i += 1; + } + b'"' => { + in_str = true; + i += 1; + } + b'*' => { + // Closing * must be the last non-ws on the line. + if trimmed[i + 1..].trim().is_empty() { + return Some(i); + } + i += 1; + } + _ => i += 1, + } + } + None +} + +/// Closing `**` for a bold return, skipping strings and backticks. +fn matching_bold_close(trimmed: &str) -> Option { + let Some(rest) = trimmed.strip_prefix("**") else { + return None; + }; + let bytes = rest.as_bytes(); + let mut i = 0usize; + let mut in_bt = false; + let mut in_str = false; + while i + 1 < bytes.len() { + let c = bytes[i]; + if in_str { + if c == b'\\' && i + 1 < bytes.len() { + i += 2; + continue; + } + if c == b'"' { + in_str = false; + } + i += 1; + continue; + } + if in_bt { + if c == b'`' { + in_bt = false; + } + i += 1; + continue; + } + match c { + b'`' => { + in_bt = true; + i += 1; + } + b'"' => { + in_str = true; + i += 1; + } + b'*' if bytes[i + 1] == b'*' => { + let after = i + 2; + if rest[after..].trim().is_empty() { + // Position relative to full `trimmed` (after opening `**`). + return Some(2 + i); + } + i += 2; + } + _ => i += 1, + } + } + None +} + +/// Headings / calls / finished star lines should escape a narrative paragraph (GAP-06/07 recovery). +fn breaks_comment_paragraph(trimmed: &str) -> bool { + if is_structural_code_line(trimmed) { + return true; + } + if trimmed.starts_with('#') || trimmed.starts_with('>') { + return true; + } + is_marqdo_star_line(trimmed) } /// Classify a single logical line (no trailing newline required). @@ -141,7 +255,7 @@ pub fn classify_source(source: &str) -> Vec { in_comment_paragraph = true; LineKind::Comment } - LineKind::Code if structural => { + LineKind::Code if structural || breaks_comment_paragraph(trimmed) => { in_comment_paragraph = false; LineKind::Code } @@ -298,8 +412,8 @@ mod tests { fn star_line_classification_is_star_terminated() { // A `*…*` Marqdo statement line must END with `*` to be treated as code. // If it accidentally ends with a backtick (e.g. `table="t"`), the line - // falls back to narrative Comment and swallows the following paragraph. - // Regression: `store.count` calls after a blank line were being dropped. + // falls back to narrative Comment — but later executable lines recover + // (GAP-06/07: do not swallow the rest of a `##` body). let star_term = "*c = > `store`.count table=\"t\"*"; assert!( is_marqdo_star_line(star_term), @@ -307,6 +421,21 @@ mod tests { ); assert_eq!(classify_line(star_term), LineKind::Code); + let nested_bt = "*code = > sys.exec cmd=\"python3\" args=`args`*"; + assert!( + is_marqdo_star_line(nested_bt), + "args=`args`* must still be a star line" + ); + let nested_json = "*args = > json.parse text=[\"scripts/legacy/web_search.py\"]*"; + assert!( + is_marqdo_star_line(nested_json), + "text=[…] * must still be a star line" + ); + assert!( + is_marqdo_star_line("*x = > f text=`a*b`*"), + "* inside backticks must not steal the closer" + ); + let backtick_term = "*c = > `store`.count table=\"t\"`"; assert!( !is_marqdo_star_line(backtick_term), @@ -314,9 +443,11 @@ mod tests { ); assert_eq!(classify_line(backtick_term), LineKind::Comment); - // In paragraph context the misplaced line must not stay Code. - let src = "> `store`.insert table=t rows=`数据`\n\n*c = > `store`.count table=\"t\"`\n> print text=count1-done\n"; + // Bad terminator → comment, but following `>` / `*…*` escape the paragraph. + let src = "> `store`.insert table=t rows=`数据`\n\n*c = > `store`.count table=\"t\"`\n> print text=count1-done\n*y = 1*\n"; let lines = classify_source(src); assert_eq!(lines[2].kind, LineKind::Comment, "bad terminator → comment"); + assert_eq!(lines[3].kind, LineKind::Code, "call recovers after bad line"); + assert_eq!(lines[4].kind, LineKind::Code, "assign recovers after bad line"); } } diff --git a/src/parse/expr.rs b/src/parse/expr.rs index e01a0fa..39b157d 100644 --- a/src/parse/expr.rs +++ b/src/parse/expr.rs @@ -924,9 +924,12 @@ fn split_first_token(s: &str) -> (&str, &str) { } fn find_next_arg_boundary(after_eq: &str) -> usize { - // Look for ` =` pattern not inside backticks or quoted strings. + // Look for ` =` pattern not inside backticks, quoted strings, or + // JSON-ish `[…]` / `{…}` (GAP-07: `text=["a"] other=…`). let mut in_bt = false; let mut in_str = false; + let mut bracket_depth: i32 = 0; + let mut brace_depth: i32 = 0; let mut chars = after_eq.char_indices().peekable(); while let Some((i, c)) = chars.next() { if in_str { @@ -945,12 +948,34 @@ fn find_next_arg_boundary(after_eq: &str) -> usize { } continue; } - if c == '`' { - in_bt = true; - continue; + match c { + '`' => { + in_bt = true; + continue; + } + '"' => { + in_str = true; + continue; + } + '[' => { + bracket_depth += 1; + continue; + } + ']' => { + bracket_depth = bracket_depth.saturating_sub(1); + continue; + } + '{' => { + brace_depth += 1; + continue; + } + '}' => { + brace_depth = brace_depth.saturating_sub(1); + continue; + } + _ => {} } - if c == '"' { - in_str = true; + if bracket_depth > 0 || brace_depth > 0 { continue; } if c.is_whitespace() { diff --git a/tests/gold.rs b/tests/gold.rs index 4896b8b..c804c5b 100644 --- a/tests/gold.rs +++ b/tests/gold.rs @@ -189,18 +189,16 @@ sk-live } #[test] -fn structure_bare_id_italic_bold() { +fn structure_gap_star_json_body() { assert_out( - "tests/structure/bare-id-italic-bold.mq.md", - "2 -2 -quoted -7 -ok -hi", + "tests/structure/gap-star-json-body.mq.md", + "1 +1 +7", ); } + #[test] fn structure_footnote_index() { assert_out( @@ -1323,6 +1321,16 @@ fn lib_sys_cwd() { assert_out("tests/lib/sys-cwd.mq.md", "cwd-ok"); } +#[test] +fn lib_sys_exec_capture() { + assert_out( + "tests/lib/sys-exec-capture.mq.md", + "exec-code-ok +capture-code-ok +capture-stdout-ok", + ); +} + #[test] fn lib_sys_dotenv() { assert_out("tests/lib/sys-dotenv.mq.md", "loaded"); diff --git a/tests/lib/sys-exec-capture.mq.md b/tests/lib/sys-exec-capture.mq.md new file mode 100644 index 0000000..a570167 --- /dev/null +++ b/tests/lib/sys-exec-capture.mq.md @@ -0,0 +1,28 @@ +--- +title: sys.exec capture=True returns stdout map +description: GAP-09 optional capture. +import sys:lib/sys.mq.md +import json:lib/json.mq.md +--- + +# main + +*code = > sys.exec cmd="true"* +1. `code` == 0 + > print text=exec-code-ok +2. * + > print text=exec-code-fail + +*args = > json.parse text=["-c","printf hi-capture"]* +*out = > sys.exec cmd="sh" args=args capture=True* +*c = out[^code]* +*s = out[^stdout]* +1. `c` == 0 + > print text=capture-code-ok +2. * + > print text=capture-code-fail + +1. `s` == "hi-capture" + > print text=capture-stdout-ok +2. * + > print text=capture-stdout-fail diff --git a/tests/structure/gap-star-json-body.mq.md b/tests/structure/gap-star-json-body.mq.md new file mode 100644 index 0000000..239ec63 --- /dev/null +++ b/tests/structure/gap-star-json-body.mq.md @@ -0,0 +1,40 @@ +--- +title: GAP-06/07 star + JSON array body keep following stmts +description: Nested args=`args` and text=["…"] must not truncate the ## body. +import json:lib/json.mq.md +--- + +# main + +*a = > case_backtick* +*b = > case_json_array* +*c = > case_recover* +> print text=`a` +> print text=`b` +> print text=`c` + +## case_backtick + +`args` = + +| x | +|---| +| ok | + +*code = > len value=`args`* +*y = 1* +**y** + +## case_json_array + +*xs = > json.parse text=["scripts/legacy/web_search.py"]* +*n = > len value=`xs`* +**n** + +## case_recover + +故意少写闭合星号,后续语句仍须可执行: + +*c = > json.parse text=[]` +*z = 7* +**z**