Default execute/sse-live/stream to Supervisor worker pool; MR-01 gold; /_runtime/*; bump CLI/wasm/ext to 2.1.4. Co-authored-by: Cursor <cursoragent@cursor.com>
488 lines
17 KiB
Rust
488 lines
17 KiB
Rust
//! Marqdo reference interpreter (Phase I + M5 bytecode prototype).
|
|
//!
|
|
//! Pipeline: load → line classify → parse → tree-walk eval | bytecode VM.
|
|
//! CLI also offers `view` for AST-backed browsing.
|
|
|
|
pub mod aliases;
|
|
pub mod artifact;
|
|
pub mod label;
|
|
pub mod migrate;
|
|
pub mod ast;
|
|
pub mod binding;
|
|
pub mod browser_session;
|
|
pub mod builtin;
|
|
pub mod bytecode;
|
|
pub mod capture;
|
|
#[cfg(feature = "cli")]
|
|
pub mod catalog;
|
|
#[cfg(feature = "cli")]
|
|
pub mod knowledge;
|
|
pub mod check;
|
|
pub mod contract;
|
|
pub mod debug;
|
|
pub mod diagnostics;
|
|
pub mod document_extract;
|
|
pub mod embedded_lib;
|
|
#[cfg(feature = "cli")]
|
|
pub mod ext_cli;
|
|
#[cfg(all(feature = "cli", feature = "net-host"))]
|
|
pub mod ext_fetch;
|
|
pub mod foreign;
|
|
pub mod formula;
|
|
pub mod host;
|
|
pub mod inherit;
|
|
pub mod input_feed;
|
|
pub mod interp;
|
|
pub mod lex;
|
|
pub mod linalg_ops;
|
|
pub mod load;
|
|
pub mod mlsp;
|
|
pub mod runtime;
|
|
pub mod workbench;
|
|
pub mod parse;
|
|
pub mod project;
|
|
pub mod repair;
|
|
pub mod sema;
|
|
pub mod value;
|
|
#[cfg(feature = "net-host")]
|
|
pub mod version_check;
|
|
#[cfg(feature = "view")]
|
|
pub mod view;
|
|
#[cfg(feature = "cli")]
|
|
pub mod wasm_cli;
|
|
|
|
use std::path::Path;
|
|
|
|
use anyhow::{bail, Result};
|
|
|
|
use crate::host::{flush_auto_plots, HostCaps, HostContext};
|
|
use crate::ast::format_ast_dump;
|
|
use crate::debug::format_lines_dump_v03;
|
|
use crate::bytecode::{compile_module, Vm};
|
|
use crate::capture::RunCapture;
|
|
use crate::interp::Interpreter;
|
|
use crate::lex::classify_source;
|
|
use crate::load::{load_module_from_source, load_module_with_ctx};
|
|
use crate::binding::{cwd_for_path, BindingContext};
|
|
|
|
pub use browser_session::{value_as_json, BrowserSession};
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
|
pub enum Backend {
|
|
#[default]
|
|
Tree,
|
|
Bytecode,
|
|
}
|
|
|
|
impl Backend {
|
|
pub fn parse(s: &str) -> Result<Self> {
|
|
match s {
|
|
"tree" => Ok(Self::Tree),
|
|
"bytecode" | "bc" => Ok(Self::Bytecode),
|
|
other => bail!("unknown backend `{other}` (expected tree|bytecode)"),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Options for a single `run` / dump invocation.
|
|
#[derive(Debug, Clone)]
|
|
pub struct RunOptions {
|
|
pub dump_lines: bool,
|
|
pub dump_tokens: bool,
|
|
pub dump_ast: bool,
|
|
pub dump_sema: bool,
|
|
pub dump_bytecode: bool,
|
|
pub trace_eval: bool,
|
|
pub backend: Backend,
|
|
/// Preset lines consumed by `input` (from `--stdin-file` or view).
|
|
pub stdin_lines: Vec<String>,
|
|
pub allow_fs_write: bool,
|
|
pub allow_exec: bool,
|
|
pub allow_net: bool,
|
|
/// Native `plugin.load` (default true for CLI; false inside `sandbox.run`).
|
|
pub allow_plugin: bool,
|
|
/// Exposed via `host_args` / `args`.
|
|
pub argv: Vec<String>,
|
|
/// Override filesystem sandbox root (default: source file's directory).
|
|
pub fs_root: Option<std::path::PathBuf>,
|
|
/// Sleep clamp when capturing (None = host default; view export uses `Some(0)`).
|
|
pub sleep_limit_ms: Option<u64>,
|
|
/// When set, write `# main` return value as JSON to this path (file subtasks).
|
|
pub emit_result: Option<std::path::PathBuf>,
|
|
/// CLI `--bind KEY=VALUE` pairs for Artifact Metadata Binding (`arg` + overlay).
|
|
pub binds: Vec<(String, String)>,
|
|
/// Attach CoW session for `ext/ai/sandbox` nested runs.
|
|
pub sandbox_session: Option<String>,
|
|
/// Isolated worker: call `lib.member` on the site entry instead of `# main`.
|
|
pub call_lib_path: Option<String>,
|
|
/// JSON object of named args for [`Self::call_lib_path`] (file path or inline).
|
|
pub args_json: Option<std::path::PathBuf>,
|
|
}
|
|
|
|
impl Default for RunOptions {
|
|
fn default() -> Self {
|
|
Self {
|
|
dump_lines: false,
|
|
dump_tokens: false,
|
|
dump_ast: false,
|
|
dump_sema: false,
|
|
dump_bytecode: false,
|
|
trace_eval: false,
|
|
backend: Backend::default(),
|
|
stdin_lines: Vec::new(),
|
|
allow_fs_write: true,
|
|
allow_exec: true,
|
|
allow_net: true,
|
|
allow_plugin: true,
|
|
argv: Vec::new(),
|
|
fs_root: None,
|
|
sleep_limit_ms: None,
|
|
emit_result: None,
|
|
binds: Vec::new(),
|
|
sandbox_session: None,
|
|
call_lib_path: None,
|
|
args_json: None,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl RunOptions {
|
|
pub fn dump_all() -> Self {
|
|
Self {
|
|
dump_lines: true,
|
|
dump_tokens: true,
|
|
dump_ast: true,
|
|
dump_sema: true,
|
|
dump_bytecode: true,
|
|
trace_eval: true,
|
|
..Self::default()
|
|
}
|
|
}
|
|
|
|
pub fn any_dump(&self) -> bool {
|
|
self.dump_lines
|
|
|| self.dump_tokens
|
|
|| self.dump_ast
|
|
|| self.dump_sema
|
|
|| self.dump_bytecode
|
|
|| self.trace_eval
|
|
}
|
|
|
|
pub fn host_caps(&self) -> HostCaps {
|
|
HostCaps {
|
|
fs_write: self.allow_fs_write,
|
|
exec: self.allow_exec,
|
|
net: self.allow_net,
|
|
plugin: self.allow_plugin,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Run a `.mq.md` program (prints to real stdout).
|
|
pub fn run_file(path: &Path, opts: &RunOptions) -> Result<i32> {
|
|
let source = std::fs::read_to_string(path)
|
|
.map_err(|e| anyhow::anyhow!("failed to read {}: {e}", path.display()))?;
|
|
|
|
if source.trim().is_empty() {
|
|
bail!("{} is empty", path.display());
|
|
}
|
|
|
|
let stdin_lines = crate::input_feed::effective_stdin(&source, &opts.stdin_lines);
|
|
let path_label = path.display().to_string();
|
|
|
|
if opts.dump_lines {
|
|
let lines = classify_source(&source);
|
|
print!("{}", format_lines_dump_v03(&path_label, &lines));
|
|
}
|
|
if opts.dump_tokens {
|
|
println!("=== marqdo: tokens ({path_label}) ===");
|
|
println!("(fine-grained tokens not implemented yet — use --dump-lines)");
|
|
println!("=== marqdo: end tokens ===");
|
|
}
|
|
|
|
let bind_ctx = BindingContext::from_binds(&opts.binds, cwd_for_path(Some(path)));
|
|
let module = load_module_with_ctx(path, &bind_ctx)?;
|
|
|
|
if opts.dump_ast {
|
|
print!("{}", format_ast_dump(&path_label, &module));
|
|
}
|
|
if opts.dump_sema {
|
|
println!("=== marqdo: sema ({path_label}) ===");
|
|
println!(
|
|
"functions: {:?}",
|
|
module
|
|
.functions
|
|
.iter()
|
|
.map(|f| f.name.as_str())
|
|
.collect::<Vec<_>>()
|
|
);
|
|
println!("imports: {:?}", module.imports);
|
|
println!("metadata: {:?}", module.metadata.iter().map(|(k, v)| (k, v.as_display())).collect::<Vec<_>>());
|
|
println!("=== marqdo: end sema ===");
|
|
}
|
|
|
|
match opts.backend {
|
|
Backend::Tree => {
|
|
let mut host = HostContext::for_run(Some(path), opts.host_caps(), opts.argv.clone());
|
|
host.set_entry_source(Some(path), &source);
|
|
host.entry_metadata = module.metadata.clone();
|
|
if let Some(root) = &opts.fs_root {
|
|
host.fs_root = Some(root.clone());
|
|
host.cwd = root.clone();
|
|
}
|
|
let mut interp = Interpreter::new(Some(path), opts.trace_eval)
|
|
.with_stdin(stdin_lines.clone())
|
|
.with_host(host);
|
|
if opts.trace_eval {
|
|
eprintln!("=== marqdo: trace-eval ({path_label}) ===");
|
|
}
|
|
let value = if let Some(lib_path) = &opts.call_lib_path {
|
|
run_call_lib_path(&mut interp, &module, lib_path, opts)?
|
|
} else {
|
|
interp.run_module(&module)?
|
|
};
|
|
if opts.trace_eval {
|
|
eprintln!("=== marqdo: end trace-eval ===");
|
|
}
|
|
let plots = interp.take_plots();
|
|
flush_auto_plots(Some(path), &plots).map_err(|e| anyhow::anyhow!(e))?;
|
|
emit_result_file(opts, &value)?;
|
|
}
|
|
Backend::Bytecode => {
|
|
let program = compile_module(Some(path), &module)?;
|
|
if opts.dump_bytecode {
|
|
print!("{}", program.disassemble());
|
|
}
|
|
let mut host = HostContext::for_run(Some(path), opts.host_caps(), opts.argv.clone());
|
|
host.set_entry_source(Some(path), &source);
|
|
host.entry_metadata = module.metadata.clone();
|
|
if let Some(root) = &opts.fs_root {
|
|
host.fs_root = Some(root.clone());
|
|
host.cwd = root.clone();
|
|
}
|
|
let mut vm = Vm::new(Some(path))
|
|
.with_stdin(stdin_lines)
|
|
.with_trace(opts.trace_eval)
|
|
.with_host(host);
|
|
if opts.trace_eval {
|
|
eprintln!("=== marqdo: trace-eval ({path_label}) ===");
|
|
}
|
|
let value = vm.run(&program)?;
|
|
if opts.trace_eval {
|
|
eprintln!("=== marqdo: end trace-eval ===");
|
|
}
|
|
let plots = vm.take_plots();
|
|
flush_auto_plots(Some(path), &plots).map_err(|e| anyhow::anyhow!(e))?;
|
|
emit_result_file(opts, &value)?;
|
|
}
|
|
}
|
|
|
|
// dump bytecode even on tree backend if requested
|
|
if opts.backend == Backend::Tree && opts.dump_bytecode {
|
|
let program = compile_module(Some(path), &module)?;
|
|
print!("{}", program.disassemble());
|
|
}
|
|
|
|
Ok(0)
|
|
}
|
|
|
|
/// Worker mode: resolve `lib.member` on the loaded site module (imports already wired).
|
|
fn run_call_lib_path(
|
|
interp: &mut Interpreter,
|
|
module: &crate::ast::Module,
|
|
lib_path: &str,
|
|
opts: &RunOptions,
|
|
) -> Result<crate::value::Value> {
|
|
let args_val = load_args_json(opts)?;
|
|
// Keep site_module pinned for the duration of the lib call (plugins may re-enter).
|
|
let value = interp
|
|
.run_site_lib_path(module, lib_path, args_val.as_ref())
|
|
.map_err(|e| anyhow::anyhow!(e))?;
|
|
Ok(value)
|
|
}
|
|
|
|
fn load_args_json(opts: &RunOptions) -> Result<Option<crate::value::Value>> {
|
|
let Some(path) = &opts.args_json else {
|
|
return Ok(None);
|
|
};
|
|
let text = if path.as_os_str() == "-" {
|
|
use std::io::Read;
|
|
let mut buf = String::new();
|
|
std::io::stdin()
|
|
.read_to_string(&mut buf)
|
|
.map_err(|e| anyhow::anyhow!("args-json stdin: {e}"))?;
|
|
buf
|
|
} else {
|
|
std::fs::read_to_string(path)
|
|
.map_err(|e| anyhow::anyhow!("args-json {}: {e}", path.display()))?
|
|
};
|
|
if text.trim().is_empty() {
|
|
return Ok(None);
|
|
}
|
|
let json: serde_json::Value =
|
|
serde_json::from_str(&text).map_err(|e| anyhow::anyhow!("args-json parse: {e}"))?;
|
|
let v = crate::host::json::json_to_value(&json).map_err(|e| anyhow::anyhow!(e))?;
|
|
Ok(Some(v))
|
|
}
|
|
|
|
fn emit_result_file(opts: &RunOptions, value: &crate::value::Value) -> Result<()> {
|
|
let Some(path) = &opts.emit_result else {
|
|
return Ok(());
|
|
};
|
|
let json = crate::host::json::value_to_json(value).map_err(|e| anyhow::anyhow!(e))?;
|
|
let text = serde_json::to_string(&json).map_err(|e| anyhow::anyhow!(e))?;
|
|
if let Some(parent) = path.parent() {
|
|
std::fs::create_dir_all(parent).map_err(|e| anyhow::anyhow!(e))?;
|
|
}
|
|
std::fs::write(path, text).map_err(|e| anyhow::anyhow!("emit-result {}: {e}", path.display()))?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Run and capture stdout (used by `view` and tests).
|
|
pub fn run_file_capture(path: &Path, opts: &RunOptions) -> Result<RunCapture> {
|
|
let source = std::fs::read_to_string(path)
|
|
.map_err(|e| anyhow::anyhow!("failed to read {}: {e}", path.display()))?;
|
|
let stdin_lines = crate::input_feed::effective_stdin(&source, &opts.stdin_lines);
|
|
let bind_ctx = BindingContext::from_binds(&opts.binds, cwd_for_path(Some(path)));
|
|
let module = load_module_with_ctx(path, &bind_ctx)?;
|
|
match opts.backend {
|
|
Backend::Tree => {
|
|
let mut host = HostContext::for_capture(Some(path), opts.host_caps());
|
|
host.set_entry_source(Some(path), &source);
|
|
host.argv = opts.argv.clone();
|
|
host.entry_metadata = module.metadata.clone();
|
|
host.sandbox_session = opts.sandbox_session.clone();
|
|
if let Some(root) = &opts.fs_root {
|
|
host.fs_root = Some(root.clone());
|
|
host.cwd = root.clone();
|
|
}
|
|
if let Some(lim) = opts.sleep_limit_ms {
|
|
host.sleep_limit_ms = Some(lim);
|
|
}
|
|
let mut interp = Interpreter::with_capture(Some(path), false)
|
|
.with_stdin(stdin_lines)
|
|
.with_host(host);
|
|
let value = interp.run_module(&module)?;
|
|
let plots = interp.take_plots();
|
|
Ok(RunCapture {
|
|
stdout: interp.captured_stdout,
|
|
value,
|
|
plots,
|
|
bindings: interp.entry_bindings,
|
|
view_trace: interp.view_trace,
|
|
})
|
|
}
|
|
Backend::Bytecode => {
|
|
let program = compile_module(Some(path), &module)?;
|
|
let mut host = HostContext::for_capture(Some(path), opts.host_caps());
|
|
host.set_entry_source(Some(path), &source);
|
|
host.argv = opts.argv.clone();
|
|
host.entry_metadata = module.metadata.clone();
|
|
host.sandbox_session = opts.sandbox_session.clone();
|
|
if let Some(root) = &opts.fs_root {
|
|
host.fs_root = Some(root.clone());
|
|
host.cwd = root.clone();
|
|
}
|
|
if let Some(lim) = opts.sleep_limit_ms {
|
|
host.sleep_limit_ms = Some(lim);
|
|
}
|
|
let mut vm = Vm::with_capture(Some(path))
|
|
.with_stdin(stdin_lines)
|
|
.with_host(host);
|
|
let value = vm.run(&program)?;
|
|
let plots = vm.take_plots();
|
|
Ok(RunCapture {
|
|
stdout: vm.captured_stdout,
|
|
value,
|
|
plots,
|
|
bindings: Default::default(),
|
|
view_trace: Default::default(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Run source text in-memory (browser wasm / tests). Imports resolve via embedded `lib/` only.
|
|
pub fn run_source(source: &str, opts: &RunOptions) -> Result<RunCapture> {
|
|
if source.trim().is_empty() {
|
|
bail!("source is empty");
|
|
}
|
|
let stdin_lines = crate::input_feed::effective_stdin(source, &opts.stdin_lines);
|
|
let module = load_module_from_source(source)?;
|
|
match opts.backend {
|
|
Backend::Tree => {
|
|
let mut host = HostContext::for_capture(None, opts.host_caps());
|
|
host.set_entry_source(None, source);
|
|
host.argv = opts.argv.clone();
|
|
host.entry_metadata = module.metadata.clone();
|
|
host.sandbox_session = opts.sandbox_session.clone();
|
|
if let Some(root) = &opts.fs_root {
|
|
host.fs_root = Some(root.clone());
|
|
host.cwd = root.clone();
|
|
}
|
|
if let Some(lim) = opts.sleep_limit_ms {
|
|
host.sleep_limit_ms = Some(lim);
|
|
}
|
|
let mut interp = Interpreter::with_capture(None, false)
|
|
.with_stdin(stdin_lines)
|
|
.with_host(host);
|
|
let value = interp.run_module(&module)?;
|
|
let plots = interp.take_plots();
|
|
Ok(RunCapture {
|
|
stdout: interp.captured_stdout,
|
|
value,
|
|
plots,
|
|
bindings: interp.entry_bindings,
|
|
view_trace: interp.view_trace,
|
|
})
|
|
}
|
|
Backend::Bytecode => {
|
|
let program = compile_module(None, &module)?;
|
|
let mut host = HostContext::for_capture(None, opts.host_caps());
|
|
host.set_entry_source(None, source);
|
|
host.argv = opts.argv.clone();
|
|
host.entry_metadata = module.metadata.clone();
|
|
host.sandbox_session = opts.sandbox_session.clone();
|
|
if let Some(root) = &opts.fs_root {
|
|
host.fs_root = Some(root.clone());
|
|
host.cwd = root.clone();
|
|
}
|
|
if let Some(lim) = opts.sleep_limit_ms {
|
|
host.sleep_limit_ms = Some(lim);
|
|
}
|
|
let mut vm = Vm::with_capture(None)
|
|
.with_stdin(stdin_lines)
|
|
.with_host(host);
|
|
let value = vm.run(&program)?;
|
|
let plots = vm.take_plots();
|
|
Ok(RunCapture {
|
|
stdout: vm.captured_stdout,
|
|
value,
|
|
plots,
|
|
bindings: Default::default(),
|
|
view_trace: Default::default(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn run_source_hello() {
|
|
let cap = run_source("# main\n\n> print text=Hello World!\n", &RunOptions::default())
|
|
.expect("run_source");
|
|
assert_eq!(cap.stdout, "Hello World!\n");
|
|
}
|
|
|
|
#[test]
|
|
fn run_source_hello_bytecode() {
|
|
let mut opts = RunOptions::default();
|
|
opts.backend = Backend::Bytecode;
|
|
let cap = run_source("# main\n\n> print text=Hello World!\n", &opts).expect("bc");
|
|
assert_eq!(cap.stdout, "Hello World!\n");
|
|
}
|
|
}
|