feat(foreign): support raw stdin and per-cell inputs

Session eval now carries raw stdin, and the Python/Node workers temporarily replace program stdin during evaluation. foreign.session accepts an inputs table; view exposes per-cell stdin and Run All sends a per-cell input map. Stateless and persistent cells share the same input semantics.
This commit is contained in:
cflmy committed 2026-10-18 14:36:47 +08:00
1 parent a7266185b2
commit c85f1a2829
8 files changed
+303 -19

No files matched your search

+6 -2
View File
@@ -153,6 +153,7 @@ in input order. Raises with the failing index after all workers finish.
+ `kernels`=None
+ `cells`=None
+ `values`=None
+ `inputs`=None
Declare a document view session. `kernels` and `cells` are ordinary tables:
kernels are `|名称|语言|命令|` (ZH or EN), cells are `|单元|内核|`.
@@ -164,6 +165,7 @@ view has no persistent kernel state.
**p = > table.put in=`p` at="kernels" value=`kernels`**
**p = > table.put in=`p` at="cells" value=`cells`**
**p = > table.put in=`p` at="values" value=`values`**
**p = > table.put in=`p` at="inputs" value=`inputs`**
*p*
## session_open
@@ -183,12 +185,14 @@ default per-eval timeout.
+ `code`=None
+ `source`=None
+ `input`=None
+ `stdin`=None
+ `timeout`=None
Evaluate code (a code value) or source text in session id; returns a record
with id/alive. `input` is JSON-encoded for the kernel.
with id/alive. `input` is JSON-encoded for the kernel; `stdin` is raw program
stdin for this evaluation.
*> host_foreign_session_eval id=`id` code=`code` source=`source` input=`input` timeout=`timeout`*
*> host_foreign_session_eval id=`id` code=`code` source=`source` input=`input` stdin=`stdin` timeout=`timeout`*
## session_set
+ `id`
+4 -1
View File
@@ -151,6 +151,7 @@ JSON 进 / JSON 出:把 `输入` 编码到 stdin,解析 stdout JSON 返回
+ `内核表`=None
+ `单元表`=None
+ `值表`=None
+ `输入表`=None
声明文档视图会话。内核表和单元表都是普通表格;返回一个 foreign 会话句柄,
供 view 消费。没有这个句柄时,view 的 cells 是无状态的。
@@ -160,6 +161,7 @@ JSON 进 / JSON 出:把 `输入` 编码到 stdin,解析 stdout JSON 返回
**p = > table.put in=`p` at="kernels" value=`内核表`**
**p = > table.put in=`p` at="cells" value=`单元表`**
**p = > table.put in=`p` at="values" value=`值表`**
**p = > table.put in=`p` at="inputs" value=`输入表`**
*p*
## 会话开启
@@ -178,12 +180,13 @@ JSON 进 / JSON 出:把 `输入` 编码到 stdin,解析 stdout JSON 返回
+ `代码`=None
+ `源码`=None
+ `输入`=None
+ `标准输入`=None
+ `超时`=None
在会话中求值代码(code 值)或源码文本;返回带 id/alive 的记录。
`输入` 会编码为 JSON 交给内核。
*> host_foreign_session_eval id=`标识` code=`代码` source=`源码` input=`输入` timeout=`超时`*
*> host_foreign_session_eval id=`标识` code=`代码` source=`源码` input=`输入` stdin=`标准输入` timeout=`超时`*
## 会话设值
+ `标识`
+3 -2
View File
@@ -1053,7 +1053,7 @@ impl HostFn {
"concurrency",
],
Self::ForeignSessionOpen => &["cmd", "env", "cwd", "timeout"],
Self::ForeignSessionEval => &["code", "source", "input", "timeout"],
Self::ForeignSessionEval => &["code", "source", "input", "stdin", "timeout"],
Self::ForeignSessionSet
| Self::ForeignSessionGet
| Self::ForeignSessionNames
@@ -1512,12 +1512,13 @@ pub fn call_host(
HostFn::ForeignSessionEval => {
#[cfg(feature = "exec-host")]
{
crate::host::foreign_session::eval(
crate::host::foreign_session::eval_with_stdin(
ctx,
require(bound, "id")?,
bound.get("code"),
bound.get("source"),
bound.get("input"),
bound.get("stdin").and_then(|v| v.reveal_text()),
bound.get("timeout"),
)
}
+210 -7
View File
@@ -48,6 +48,8 @@ pub struct Cell {
pub line_end: u32,
pub source_hash: String,
pub kernel: String,
/// Per-cell raw stdin from foreign.session inputs=.
pub stdin: String,
pub listed: bool,
pub status: CellStatus,
pub runs: u64,
@@ -170,16 +172,17 @@ fn plan_tables(
Option<Vec<HashMap<String, Value>>>,
Option<Vec<HashMap<String, Value>>>,
Option<Vec<HashMap<String, Value>>>,
Option<Vec<HashMap<String, Value>>>,
),
String,
> {
let Some(plan) = bindings.get("会话") else {
return Ok((None, None, None));
return Ok((None, None, None, None));
};
let Value::Map(fields) = plan else {
// `会话` is also a common low-level session id binding (e.g. from
// foreign.session_open). It is not a view plan; leave cells stateless.
return Ok((None, None, None));
return Ok((None, None, None, None));
};
let field = |name: &str| {
fields
@@ -203,7 +206,11 @@ fn plan_tables(
Some(Value::None) | None => None,
Some(v) => Some(table_rows(&v)?),
};
Ok((kernels, cells, values))
let inputs = match field("inputs") {
Some(Value::None) | None => None,
Some(v) => Some(table_rows(&v)?),
};
Ok((kernels, cells, values, inputs))
}
fn col_value<'a>(row: &'a HashMap<String, Value>, names: &[&str]) -> Option<&'a Value> {
@@ -272,6 +279,7 @@ fn collect_cells(stmts: &[crate::ast::Stmt], out: &mut Vec<Cell>, seen: &mut Has
line_end: *end_line,
source_hash: hash_source(&c.source),
kernel: String::new(),
stdin: String::new(),
listed: false,
status: CellStatus::Never,
runs: 0,
@@ -322,7 +330,7 @@ impl DocumentSession {
collect_cells(&entry.body, &mut cells, &mut seen)?;
let declarations = capture_bindings(path, root, &module, &source)?;
let (plan_kernels, plan_cells, plan_values) = plan_tables(&declarations)?;
let (plan_kernels, plan_cells, plan_values, plan_inputs) = plan_tables(&declarations)?;
let persistent = matches!(
declarations.get("会话"),
Some(Value::Map(fields))
@@ -455,6 +463,18 @@ impl DocumentSession {
)?;
}
}
if let Some(rows) = plan_inputs {
for row in rows {
let cell = col_text(&row, &["单元", "Cell", "cell", "单元名"])
.ok_or_else(|| "foreign.session inputs: missing 单元/Cell".to_string())?;
let raw = col_value(&row, &["stdin", "标准输入", "输入"])
.map(value_text)
.unwrap_or_default();
if let Some(i) = session.cell_index(&cell) {
session.cells[i].stdin = raw;
}
}
}
Ok(session)
}
@@ -614,6 +634,11 @@ impl DocumentSession {
if !self.cells[ci].listed && kernel_override.is_none() {
return Err(format!("cell `{cell_name}` is not listed in the foreign session"));
}
let stdin_owned = stdin.map(str::to_string).or_else(|| {
let s = self.cells[ci].stdin.clone();
if s.is_empty() { None } else { Some(s) }
});
let stdin = stdin_owned.as_deref();
if !self.persistent {
self.cells[ci].kernel = "one-shot".into();
self.cells[ci].status = CellStatus::Running;
@@ -666,12 +691,13 @@ impl DocumentSession {
let result = (|| -> Result<Value, String> {
let id = self.ensure_kernel_session(&kernel)?;
let source = Value::Text(self.cells[ci].source.clone());
let record = foreign_session::eval(
let record = foreign_session::eval_with_stdin(
&mut self.host,
&Value::Text(id),
None,
Some(&source),
stdin.map(|s| Value::Text(s.to_string())).as_ref(),
None,
stdin,
None,
)?;
self.record_result(cell_name, &kernel, &record)?;
@@ -717,6 +743,15 @@ impl DocumentSession {
&mut self,
restart: bool,
stdin: Option<&str>,
) -> Result<Value, String> {
self.run_all_with_inputs(restart, stdin, None)
}
pub fn run_all_with_inputs(
&mut self,
restart: bool,
stdin: Option<&str>,
inputs: Option<&HashMap<String, String>>,
) -> Result<Value, String> {
if restart {
self.restart_all()?;
@@ -725,7 +760,11 @@ impl DocumentSession {
let mut records = Vec::new();
let mut failed = 0i64;
for name in self.order.clone() {
let record = match self.run_cell_with_stdin(&name, None, stdin) {
let cell_stdin = inputs
.and_then(|m| m.get(&name))
.map(String::as_str)
.or(stdin);
let record = match self.run_cell_with_stdin(&name, None, cell_stdin) {
Ok(record) => record,
Err(error) => {
if let Some(i) = self.cell_index(&name) {
@@ -885,6 +924,7 @@ impl DocumentSession {
("last_ms".into(), Value::Int(c.last_ms as i64)),
("last_ok".into(), Value::Bool(c.last_ok)),
("last_error".into(), Value::Text(c.last_error.clone())),
("stdin".into(), Value::Text(c.stdin.clone())),
])
})
.collect();
@@ -1339,6 +1379,169 @@ console.log(globalThis.kept);
let _ = std::fs::remove_file(&path);
}
#[test]
fn plan_inputs_feed_each_cell() {
let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let path = root.join(format!(
"target/view-plan-input-{}-{}.mq.md",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0)
));
let source = r#"---
title: plan inputs test
import foreign:lib/foreign.mq.md
---
# main
`内核` =
| 名称 | 语言 |
|------|------|
| py | python |
`单元` =
| 单元 | 内核 |
|------|------|
| a | py |
`输入` =
| 单元 | stdin |
|------|-------|
| a | hello plan |
`会话` = > foreign.session kernels=`内核` cells=`单元` inputs=`输入`
`a` =
```python
import sys
print(sys.stdin.read().strip())
```
"#;
std::fs::write(&path, source).unwrap();
let mut s = DocumentSession::open(&path, &root).unwrap();
let summary = s.run_all_with_stdin(false, None).unwrap();
let records = match &summary {
Value::Map(entries) => entries.iter().find(|(k, _)| k == "records").map(|(_, v)| v.clone()),
_ => None,
};
let Some(Value::List(records)) = records else { panic!("no records") };
assert_eq!(
field(&records[0], "stdout").and_then(|v| v.reveal_text().map(|s| s.to_string())),
Some("hello plan\n".to_string())
);
s.close();
let _ = std::fs::remove_file(&path);
}
#[test]
fn persistent_node_cell_accepts_raw_stdin() {
if !node_available() {
return;
}
let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let path = root.join(format!(
"target/view-node-stdin-{}-{}.mq.md",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0)
));
let source = r#"---
title: persistent node stdin test
import foreign:lib/foreign.mq.md
---
# main
`内核` =
| 名称 | 语言 |
|------|------|
| js | node |
`单元` =
| 单元 | 内核 |
|------|------|
| a | js |
`会话` = > foreign.session kernels=`内核` cells=`单元`
`a` =
```node
(async () => {
await new Promise((resolve) => {
let input = "";
process.stdin.on("data", (chunk) => { input += chunk; });
process.stdin.on("end", () => { console.log(input.trim()); resolve(); });
});
})();
```
"#;
std::fs::write(&path, source).unwrap();
let mut s = DocumentSession::open(&path, &root).unwrap();
let record = s
.run_cell_with_stdin("a", None, Some("hello node"))
.unwrap();
assert_eq!(
field(&record, "stdout").and_then(|v| v.reveal_text().map(|s| s.to_string())),
Some("hello node\n".to_string())
);
s.close();
let _ = std::fs::remove_file(&path);
}
#[test]
fn persistent_python_cell_accepts_raw_stdin() {
let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let path = root.join(format!(
"target/view-persistent-stdin-{}-{}.mq.md",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0)
));
let source = r#"---
title: persistent stdin test
import foreign:lib/foreign.mq.md
---
# main
`内核` =
| 名称 | 语言 |
|------|------|
| py | python |
`单元` =
| 单元 | 内核 |
|------|------|
| a | py |
`会话` = > foreign.session kernels=`内核` cells=`单元`
`a` =
```python
import sys
print(sys.stdin.read().strip())
```
"#;
std::fs::write(&path, source).unwrap();
let mut s = DocumentSession::open(&path, &root).unwrap();
let record = s
.run_cell_with_stdin("a", None, Some("hello persistent"))
.unwrap();
assert_eq!(
field(&record, "stdout").and_then(|v| v.reveal_text().map(|s| s.to_string())),
Some("hello persistent\n".to_string())
);
s.close();
let _ = std::fs::remove_file(&path);
}
#[test]
fn stateless_cell_accepts_stdin() {
if !node_available() {
+32
View File
@@ -53,6 +53,9 @@ for _line in sys.stdin:
continue
_code = req.get("code", "")
_g["marqdo_input"] = req.get("input")
_old_stdin = sys.stdin
if req.get("stdin") is not None:
sys.stdin = io.StringIO(str(req.get("stdin")))
_out_buf = io.StringIO()
_err_buf = io.StringIO()
try:
@@ -69,6 +72,8 @@ for _line in sys.stdin:
_send({"id": rid, "ok": True, "stdout": _out_buf.getvalue(), "stderr": _err_buf.getvalue(), "result": _result, "error": ""})
except BaseException:
_send({"id": rid, "ok": False, "stdout": _out_buf.getvalue(), "stderr": _err_buf.getvalue(), "result": None, "error": traceback.format_exc(limit=8)})
finally:
sys.stdin = _old_stdin
"#;
const JS_BOOTSTRAP: &str = r#"const readline = require("readline");
@@ -113,7 +118,21 @@ rl.on("line", async (line) => {
return;
}
globalThis.marqdo_input = req.input === undefined ? null : req.input;
const oldStdin = process.stdin;
let stdinPatched = false;
if (req.stdin !== undefined && req.stdin !== null) {
try {
const { PassThrough } = require("stream");
const ps = new PassThrough();
ps.end(String(req.stdin));
Object.defineProperty(process, "stdin", { value: ps, configurable: true, writable: true });
stdinPatched = true;
} catch (e) {}
}
const cap = await capture(async () => vm.runInThisContext(req.code || "", { filename: "<marqdo>" }));
if (stdinPatched) {
try { Object.defineProperty(process, "stdin", { value: oldStdin, configurable: true, writable: true }); } catch (e) {}
}
send({ id: rid, ok: !cap.error, stdout: cap.logs, stderr: cap.errs, result: toJSON(cap.result), error: cap.error });
});
"#;
@@ -425,6 +444,18 @@ pub fn eval(
source: Option<&Value>,
input: Option<&Value>,
timeout: Option<&Value>,
) -> Result<Value, String> {
eval_with_stdin(ctx, id, code, source, input, None, timeout)
}
pub fn eval_with_stdin(
ctx: &mut HostContext,
id: &Value,
code: Option<&Value>,
source: Option<&Value>,
input: Option<&Value>,
stdin: Option<&str>,
timeout: Option<&Value>,
) -> Result<Value, String> {
ensure_config(ctx);
let id = as_text(id, "id")?.to_string();
@@ -462,6 +493,7 @@ pub fn eval(
"op": "eval",
"code": code,
"input": input_json,
"stdin": stdin,
});
let start = Instant::now();
let (lang, cmd) = {
+40 -5
View File
@@ -3290,8 +3290,18 @@ body.notebook-page .code-head .nb-kernel {
font: 0.66rem/1.2 var(--sans);
cursor: pointer;
}
body.notebook-page .code-head .nb-clean {
body.notebook-page .code-head .nb-cell-stdin {
order: 4;
width: 8rem;
padding: 0.16rem 0.3rem;
border: 1px solid var(--nb-border);
border-radius: 3px;
background: var(--nb-bg);
color: var(--nb-text);
font: 0.62rem/1.2 var(--mono);
}
body.notebook-page .code-head .nb-clean {
order: 5;
padding: 0.14rem 0.4rem;
border: 1px solid var(--nb-border);
border-radius: 3px;
@@ -3305,7 +3315,7 @@ body.notebook-page .code-head .nb-clean:hover {
color: var(--nb-strong);
}
body.notebook-page .code-head .nb-badge {
order: 5;
order: 6;
padding: 0.14rem 0.34rem;
border: 1px solid var(--nb-border);
border-radius: 3px;
@@ -3317,7 +3327,7 @@ body.notebook-page .code-head .nb-badge {
body.notebook-page .nb-badge.ok { color: var(--nb-success); }
body.notebook-page .nb-badge.bad { color: var(--nb-error); }
body.notebook-page .code-head .code-actions {
order: 6;
order: 7;
display: flex;
align-items: center;
gap: 0.3rem;
@@ -3637,6 +3647,22 @@ body.notebook-page .wb-source-text { color: var(--nb-text); }
}
return sel;
}
function ensureCellStdin(el, c) {
var inp = el.querySelector(".nb-cell-stdin");
if (!inp) {
inp = document.createElement("input");
inp.className = "nb-cell-stdin";
inp.placeholder = "stdin";
inp.autocomplete = "off";
var head = el.querySelector(".code-head");
if (head) head.appendChild(inp);
}
if (c && inp.dataset.loaded !== "1") {
inp.value = c.stdin || "";
inp.dataset.loaded = "1";
}
return inp;
}
function clearCard(el) {
var out = el.querySelector(".code-out");
if (out) {
@@ -3729,6 +3755,7 @@ body.notebook-page .wb-source-text { color: var(--nb-text); }
var el = card(c.name);
if (!el) return;
badge(el, c.status, c.status === "clean");
ensureCellStdin(el, c);
if (!state.persistent) {
ensureKernel(el, false);
return;
@@ -3758,7 +3785,9 @@ body.notebook-page .wb-source-text { color: var(--nb-text); }
});
}
function runOne(name) {
return post("api/session/cell/run", { cell: name, stdin: cellStdin() }).then(function (r) {
var inputEl = document.querySelector('.code-card[data-code-name="' + name + '"] .nb-cell-stdin');
var stdinValue = inputEl && inputEl.value ? inputEl.value : cellStdin();
return post("api/session/cell/run", { cell: name, stdin: stdinValue }).then(function (r) {
if (r.ok) { renderRecord(name, r.record); if (r.session) renderSummary(r.session); }
else status(r.error || "run failed");
});
@@ -3782,7 +3811,13 @@ body.notebook-page .wb-source-text { color: var(--nb-text); }
}).catch(function (e) { status(String(e)); });
return;
}
post("api/session/run_all", { restart: !!restart, stdin: cellStdin() }).then(function (r) {
var perCell = {};
document.querySelectorAll(".code-card").forEach(function (el) {
var inp = el.querySelector(".nb-cell-stdin");
var name = el.getAttribute("data-code-name");
if (inp && name && inp.value) perCell[name] = inp.value;
});
post("api/session/run_all", { restart: !!restart, stdin: cellStdin(), inputs: perCell }).then(function (r) {
if (!r.ok) { status(r.error || "run all failed"); return; }
if (r.session) renderSummary(r.session);
var recs = (r.summary && r.summary.records) || [];
+6 -1
View File
@@ -204,7 +204,12 @@ pub fn api_run_all(root: &Path, body: &str) -> serde_json::Value {
let v = body_value(body);
let restart = v.get("restart").and_then(|x| x.as_bool()).unwrap_or(false);
let stdin = v.get("stdin").and_then(|x| x.as_str());
with_session(root, body, "summary", |s| s.run_all_with_stdin(restart, stdin))
let inputs: Option<HashMap<String, String>> = v.get("inputs").and_then(|x| x.as_object()).map(|m| {
m.iter()
.filter_map(|(k, val)| val.as_str().map(|s| (k.clone(), s.to_string())))
.collect()
});
with_session(root, body, "summary", |s| s.run_all_with_inputs(restart, stdin, inputs.as_ref()))
}
pub fn api_kernel_restart(root: &Path, body: &str) -> serde_json::Value {
+2 -1
View File
@@ -59,7 +59,8 @@ fn static_page_data(
let Ok(mut session) = DocumentSession::open(abs, &root.root) else {
return (out, None);
};
let Ok(summary) = session.run_all_with_stdin(false, Some(stdin)) else {
let page_stdin = if stdin.trim().is_empty() { None } else { Some(stdin) };
let Ok(summary) = session.run_all_with_stdin(false, page_stdin) else {
session.close();
return (out, None);
};