Files
marqdo/plugins/web/internal/httpx/routes_invoke.go
T
cflmy 9331af78e2 release(2.8.0): 合入 CHANGELOG/README/发版说明/skills prove 任务;CI 链接指向 doc/releases
- CHANGELOG.md:v2.8.0 条目(中英双语);
- README.md:版本行 2.7.1→2.8.0 + 愿景第 6 条(知识可证明);
- doc/releases/v2.8.0.md:双语发版说明(亮点/用法/已知限制);
- skills/marqdo-dev/index.mq.md:新增 prove 任务(路径/目录模式/
  require_lean/json;参数用 + 行声明升参,check 与 narrative-ratio 通过);
- 设计文档 §12.5 标记已合入;发版链接统一指向 doc/releases/v2.8.0.md。
2026-11-18 10:29:26 +08:00

763 lines
21 KiB
Go
Executable File

package httpx
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/marqdo/marqdo/plugins/web/internal/plugin"
)
// legacySSEBridgeMu serializes the process-global MARQDO_EVENT_FORWARD /
// MARQDO_CANCEL_TOKEN bridge for the legacy_serial path only. Isolated execute
// never acquires this lock (ADR 0009).
var legacySSEBridgeMu sync.Mutex
// legacySSEBridgeActive is 1 while a holder has mutated process env for the bridge.
var legacySSEBridgeActive atomic.Int32
func tryAcquireLegacySSEBridge() bool {
return legacySSEBridgeMu.TryLock()
}
func releaseLegacySSEBridge() {
legacySSEBridgeActive.Store(0)
legacySSEBridgeMu.Unlock()
}
type invokeRoute struct {
Method string
FnPath string
BodyMode string
ReturnMode string
Class string
Path string
}
func parseInvokeRoute(v any) invokeRoute {
m, _ := v.(map[string]any)
method := strings.ToUpper(strOpt(m, "method", "POST"))
body := strings.ToLower(strOpt(m, "body", "json"))
ret := strings.ToLower(strOpt(m, "return", "json"))
fn := strOpt(m, "fn", "")
if fn == "" {
fn = strOpt(m, "function", "")
}
class := strings.ToLower(strOpt(m, "class", ""))
if class == "" {
class = strings.ToLower(strOpt(m, "类别", ""))
}
return invokeRoute{
Method: method,
FnPath: fn,
BodyMode: body,
ReturnMode: ret,
Class: class,
}
}
func (st *state) mountInvokeRoutes(mux routeRegistrar) {
for _, key := range sortedKeys(st.invokeRoutes) {
cfg := parseInvokeRoute(st.invokeRoutes[key])
path := routePathKey(key)
cfg.Path = path
method := cfg.Method
if method == "" {
method = http.MethodPost
}
pattern := method + " " + path
routeCfg := cfg
mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
if r.Method != method {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
st.handleInvoke(w, r, routeCfg)
})
}
}
func (st *state) handleInvoke(w http.ResponseWriter, r *http.Request, route invokeRoute) {
queryMap := queryToMap(r.URL.RawQuery)
var payload map[string]any
switch route.BodyMode {
case "query":
payload = queryMap
case "raw":
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 8<<20))
if err != nil {
jsonErr(w, http.StatusBadRequest, fmt.Sprintf("body: %v", err))
return
}
payload = map[string]any{"payload": string(body)}
case "form":
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 8<<20))
if err != nil {
jsonErr(w, http.StatusBadRequest, fmt.Sprintf("body: %v", err))
return
}
payload = formToMap(body)
default:
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 8<<20))
if err != nil {
jsonErr(w, http.StatusBadRequest, fmt.Sprintf("body: %v", err))
return
}
if len(body) == 0 {
payload = queryMap
} else {
var raw map[string]any
if err := json.Unmarshal(body, &raw); err != nil {
jsonErr(w, http.StatusBadRequest, fmt.Sprintf("json: %v", err))
return
}
if raw == nil {
payload = queryMap
} else {
for k, v := range queryMap {
if _, exists := raw[k]; !exists {
raw[k] = v
}
}
payload = raw
}
}
}
if auth := strings.TrimSpace(r.Header.Get("Authorization")); auth != "" {
if _, exists := payload["authorization"]; !exists {
payload["authorization"] = auth
}
}
if key := strings.TrimSpace(r.Header.Get("X-Api-Key")); key != "" {
if _, exists := payload["api_key"]; !exists {
payload["api_key"] = key
}
}
if last := strings.TrimSpace(r.Header.Get("Last-Event-ID")); last != "" {
if _, exists := payload["last_event_id"]; !exists {
payload["last_event_id"] = last
}
}
// Merge Go ServeMux path wildcards (e.g. /turns/{id}/events → id / turn_id).
if route.Path != "" {
pathVals := pathValuesOf(r, route.Path)
for k, v := range pathVals {
if _, exists := payload[k]; !exists {
payload[k] = v
}
}
if id, ok := pathVals["id"]; ok {
if _, exists := payload["turn_id"]; !exists {
payload["turn_id"] = id
}
}
}
args := invokeArgsFromPayload(payload)
liveSSE := route.ReturnMode == "sse-live" || route.ReturnMode == "sse_live"
if !liveSSE && payloadWantsStream(payload) {
liveSSE = true
}
class := resolveInvokeClass(route, liveSSE)
switch class {
case "execute":
// Isolated workers: do not take parent INVOKE_LOCK (ADR 0009).
st.handleInvokeIsolated(w, r, route, args, liveSSE, class)
return
case "short", "read", "control":
// Stay on parent: execute already runs off-process, so parent lock is free.
// Putting short/read/control on the 2-slot execute worker pool caused
// systemic no_slot (and zombies held slots after observer disconnect).
// Product cancel/events prefer /_runtime/* (lock-free).
st.handleInvokeParent(w, r, route, args)
return
default:
// legacy_serial: parent CallLibPath (ADR 0008). Never isolate undeclared non-stream.
st.handleInvokeParent(w, r, route, args)
}
}
func resolveInvokeClass(route invokeRoute, liveSSE bool) string {
c := strings.ToLower(strings.TrimSpace(route.Class))
switch c {
case "execute", "short", "read", "control":
return c
case "legacy", "legacy_serial":
return "legacy_serial"
}
if liveSSE {
return "execute"
}
return "legacy_serial"
}
func (st *state) handleInvokeParent(w http.ResponseWriter, r *http.Request, route invokeRoute, args map[string]any) {
_ = r
value, err := plugin.CallLibPath(route.FnPath, args)
if err != nil {
code := http.StatusInternalServerError
msg := err.Error()
if strings.Contains(msg, "interpreter busy") {
w.Header().Set("Retry-After", "2")
jsonErr(w, http.StatusServiceUnavailable, msg)
return
}
if strings.Contains(msg, "unknown library") ||
strings.Contains(msg, "call_lib_path: need") ||
strings.Contains(msg, "not allowed") ||
strings.Contains(msg, "no site") ||
strings.Contains(msg, "no active host") {
code = http.StatusBadRequest
}
if strings.Contains(msg, "unauthorized") || strings.Contains(msg, "Unauthorized") {
code = http.StatusUnauthorized
}
jsonErr(w, code, msg)
return
}
if m, ok := value.(map[string]any); ok {
if s, ok := m["sse"].(string); ok && strings.TrimSpace(s) != "" {
writeSSE(w, m)
return
}
}
switch route.ReturnMode {
case "text", "html", "text/html":
if value == nil {
jsonErr(w, http.StatusInternalServerError, "null result")
return
}
text := fmt.Sprint(value)
if s, ok := value.(string); ok {
text = s
}
ct := "text/plain; charset=utf-8"
if route.ReturnMode == "html" || route.ReturnMode == "text/html" {
ct = "text/html; charset=utf-8"
}
w.Header().Set("Content-Type", ct)
w.WriteHeader(http.StatusOK)
_, _ = io.WriteString(w, text)
case "sse", "event-stream", "text/event-stream":
writeSSE(w, value)
case "status":
if value == nil {
jsonErr(w, http.StatusInternalServerError, "null result")
return
}
w.WriteHeader(http.StatusNoContent)
default:
code, body, err := jsonWrapInvokeValue(value)
if err != nil {
jsonErr(w, code, err.Error())
return
}
writeJSON(w, code, body)
}
}
// handleInvokeIsolated runs admits on a Supervisor worker (no parent
// CallLibPath / Setenv / legacySSEBridgeMu). admissionClass is execute|short|read|control.
func (st *state) handleInvokeIsolated(w http.ResponseWriter, r *http.Request, route invokeRoute, args map[string]any, liveSSE bool, admissionClass string) {
if admissionClass == "" {
admissionClass = "execute"
}
taskID := fmt.Sprintf("exec_%d", time.Now().UnixNano())
adm, err := runtimeQuery("admit", map[string]any{
"task_id": taskID,
"idempotency_key": taskID,
"class": admissionClass,
"payload": map[string]any{"fn": route.FnPath},
})
if err != nil {
jsonErr(w, http.StatusInternalServerError, fmt.Sprintf("admit: %v", err))
return
}
if ok, _ := adm["ok"].(bool); !ok {
status := http.StatusServiceUnavailable
if s, ok := adm["status"].(float64); ok && s >= 400 {
status = int(s)
}
w.Header().Set("Retry-After", "2")
jsonErr(w, status, fmt.Sprint(adm["error"]))
return
}
cancelTok, _ := adm["cancel_token"].(string)
eventPath, _ := adm["event_path"].(string)
entryModule, fsRoot, err := resolveSiteEntry(st)
if err != nil {
jsonErr(w, http.StatusInternalServerError, err.Error())
return
}
argsFile, err := materializeArgsJSON(args)
if err != nil {
jsonErr(w, http.StatusInternalServerError, fmt.Sprintf("args: %v", err))
return
}
defer os.Remove(argsFile)
disp, err := runtimeQuery("dispatch", map[string]any{
"task_id": taskID,
"entry": map[string]any{
"module": entryModule,
"function": route.FnPath,
},
"fs_root": fsRoot,
"args_json": argsFile,
})
if err != nil {
_, _ = runtimeQuery("cancel", map[string]any{"task_id": taskID})
jsonErr(w, http.StatusInternalServerError, fmt.Sprintf("dispatch: %v", err))
return
}
if ok, _ := disp["ok"].(bool); !ok {
status := http.StatusTooManyRequests
if s, ok := disp["status"].(float64); ok && s >= 400 {
status = int(s)
}
if ra, ok := disp["Retry-After"]; ok {
w.Header().Set("Retry-After", fmt.Sprint(ra))
} else {
w.Header().Set("Retry-After", "2")
}
jsonErr(w, status, fmt.Sprint(disp["error"]))
return
}
if ep, ok := disp["event_path"].(string); ok && ep != "" {
eventPath = ep
}
if tok, ok := disp["cancel_token"].(string); ok && tok != "" {
cancelTok = tok
}
emitResult, _ := disp["emit_result"].(string)
if liveSSE {
st.streamIsolatedSSE(w, r, route, taskID, eventPath, cancelTok, emitResult)
return
}
st.waitIsolatedJSON(w, r, route, taskID, eventPath, cancelTok, emitResult)
}
// cancelOnDisconnect reports whether the client opted into legacy
// disconnect=cancel (header X-Marqdo-Cancel-On-Disconnect: 1|true).
// Default is false: subscription close must not cancel the task (ADR 0009 / design §7.2).
func cancelOnDisconnect(r *http.Request) bool {
v := strings.TrimSpace(strings.ToLower(r.Header.Get("X-Marqdo-Cancel-On-Disconnect")))
return v == "1" || v == "true" || v == "yes"
}
func (st *state) streamIsolatedSSE(w http.ResponseWriter, r *http.Request, route invokeRoute, taskID, eventPath, cancelTok, emitResult string) {
w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.WriteHeader(http.StatusOK)
flusher, _ := w.(http.Flusher)
writeEvent := func(payload any) {
b, err := json.Marshal(payload)
if err != nil {
return
}
_, _ = io.WriteString(w, "data: "+string(b)+"\n\n")
if flusher != nil {
flusher.Flush()
}
}
writeEvent(map[string]any{
"schema": "RuntimeEventV1",
"schema_version": 1,
"type": "run.started",
"payload": map[string]any{
"fn": route.FnPath,
"bridge": "isolated",
"task_id": taskID,
},
})
offset := int64(0)
carry := ""
seq := uint64(1)
poll := time.NewTicker(25 * time.Millisecond)
defer poll.Stop()
ctx := r.Context()
finish := func(clientGone bool) {
offset, carry = drainForwardCarry(eventPath, offset, carry, func(line string) {
seq++
writeEvent(normalizeLiveSSELine(seq, line))
})
if !clientGone {
result := readEmitResult(emitResult)
if result == nil {
if g, err := runtimeQuery("get", map[string]any{"task_id": taskID}); err == nil {
if t, ok := g["task"].(map[string]any); ok {
result = t["result"]
}
}
}
writeEvent(map[string]any{
"schema": "RuntimeEventV1", "schema_version": 1, "type": "run.completed",
"payload": map[string]any{"result": result, "task_id": taskID},
})
_, _ = io.WriteString(w, "data: [DONE]\n\n")
if flusher != nil {
flusher.Flush()
}
}
}
doCancelOnDisconnect := cancelOnDisconnect(r)
for {
select {
case <-ctx.Done():
// Default: observer gone ≠ cancel. Task keeps running; client may
// reattach via GET /_runtime/tasks/{id}/events (Last-Event-ID).
if doCancelOnDisconnect {
_, _ = runtimeQuery("cancel", map[string]any{"task_id": taskID})
_ = writeCancelToken(cancelTok)
writeEvent(map[string]any{
"schema": "RuntimeEventV1", "schema_version": 1, "type": "run.interrupted",
"payload": map[string]any{
"reason": "client_disconnect",
"cancel_propagated": cancelTok != "",
"status": "stopping",
"task_id": taskID,
},
})
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if workerDone(taskID, emitResult) {
break
}
time.Sleep(50 * time.Millisecond)
}
} else {
writeEvent(map[string]any{
"schema": "RuntimeEventV1", "schema_version": 1, "type": "run.interrupted",
"payload": map[string]any{
"reason": "client_disconnect",
"cancel_propagated": false,
"status": "observer_closed",
"task_id": taskID,
"resume": "/_runtime/tasks/" + taskID + "/events",
},
})
// Observer gone ≠ cancel, but still reap when the worker exits so
// execute slots are not held forever by zombies.
go reapIsolatedTask(taskID, emitResult, 10*time.Minute)
}
finish(true)
return
case <-poll.C:
offset, carry = drainForwardCarry(eventPath, offset, carry, func(line string) {
seq++
writeEvent(normalizeLiveSSELine(seq, line))
})
if workerDone(taskID, emitResult) {
finish(false)
return
}
}
}
}
func (st *state) waitIsolatedJSON(w http.ResponseWriter, r *http.Request, route invokeRoute, taskID, eventPath, cancelTok, emitResult string) {
_ = eventPath
poll := time.NewTicker(25 * time.Millisecond)
defer poll.Stop()
ctx := r.Context()
for {
select {
case <-ctx.Done():
_, _ = runtimeQuery("cancel", map[string]any{"task_id": taskID})
_ = writeCancelToken(cancelTok)
jsonErr(w, http.StatusRequestTimeout, "client disconnected")
return
case <-poll.C:
if !workerDone(taskID, emitResult) {
continue
}
value := readEmitResult(emitResult)
if value == nil {
jsonErr(w, http.StatusInternalServerError, "worker produced no result")
return
}
if m, ok := value.(map[string]any); ok {
if s, ok := m["sse"].(string); ok && strings.TrimSpace(s) != "" {
writeSSE(w, m)
return
}
}
switch route.ReturnMode {
case "text", "html", "text/html":
text := fmt.Sprint(value)
if s, ok := value.(string); ok {
text = s
}
ct := "text/plain; charset=utf-8"
if route.ReturnMode == "html" || route.ReturnMode == "text/html" {
ct = "text/html; charset=utf-8"
}
w.Header().Set("Content-Type", ct)
w.WriteHeader(http.StatusOK)
_, _ = io.WriteString(w, text)
case "status":
w.WriteHeader(http.StatusNoContent)
default:
code, body, err := jsonWrapInvokeValue(value)
if err != nil {
jsonErr(w, code, err.Error())
return
}
writeJSON(w, code, body)
}
return
}
}
}
// reapIsolatedTask polls runtime get until the worker exits (or timeout),
// so supervisor/scheduler slots are released after observer disconnect.
func reapIsolatedTask(taskID, emitResult string, budget time.Duration) {
deadline := time.Now().Add(budget)
for time.Now().Before(deadline) {
if workerDone(taskID, emitResult) {
return
}
time.Sleep(200 * time.Millisecond)
}
// Last-chance cancel so slots can free.
_, _ = runtimeQuery("cancel", map[string]any{"task_id": taskID})
for i := 0; i < 25; i++ {
if workerDone(taskID, emitResult) {
return
}
time.Sleep(200 * time.Millisecond)
}
}
func workerDone(taskID, emitResult string) bool {
// Always runtime.get when probing completion: get runs try_reap and
// releases execute slots. Do not return early on emit_result alone —
// that left zombie workers holding both pool slots (no_slot).
if taskID == "" {
return false
}
g, err := runtimeQuery("get", map[string]any{"task_id": taskID})
if err != nil {
// Fallback: result file present (get may fail during early boot).
if emitResult != "" {
if st, err := os.Stat(emitResult); err == nil && st.Size() > 0 {
return true
}
}
if dir := os.Getenv("MARQDO_RUNTIME_DIR"); dir != "" {
p := filepath.Join(dir, "events", taskID+".result.json")
if st, err := os.Stat(p); err == nil && st.Size() > 0 {
return true
}
}
return false
}
if _, ok := g["exit_code"]; ok {
return true
}
if t, ok := g["task"].(map[string]any); ok {
switch fmt.Sprint(t["state"]) {
case "succeeded", "failed", "cancelled", "interrupted":
return true
}
}
return false
}
func readEmitResult(path string) any {
if path == "" {
return nil
}
b, err := os.ReadFile(path)
if err != nil || len(b) == 0 {
return nil
}
var v any
if err := json.Unmarshal(b, &v); err != nil {
return string(b)
}
return v
}
func materializeArgsJSON(args map[string]any) (string, error) {
f, err := os.CreateTemp("", "mq-args-*.json")
if err != nil {
return "", err
}
path := f.Name()
enc := json.NewEncoder(f)
if err := enc.Encode(args); err != nil {
_ = f.Close()
_ = os.Remove(path)
return "", err
}
if err := f.Close(); err != nil {
_ = os.Remove(path)
return "", err
}
return path, nil
}
func resolveSiteEntry(st *state) (module string, fsRoot string, err error) {
fsRoot = st.entryDir
if raw, qerr := plugin.Query("entry_dir", "{}"); qerr == nil {
var s string
if json.Unmarshal([]byte(raw), &s) == nil && s != "" {
fsRoot = s
}
}
if raw, qerr := plugin.Query("entry_path", "{}"); qerr == nil {
var s string
if json.Unmarshal([]byte(raw), &s) == nil && s != "" {
module = s
}
}
if module == "" {
// Fallback: common serve entry names under fs_root.
for _, name := range []string{"serve.mq.md", "index.mq.md", "main.mq.md"} {
p := filepath.Join(fsRoot, name)
if st, e := os.Stat(p); e == nil && !st.IsDir() {
module = p
break
}
}
}
if module == "" {
return "", "", fmt.Errorf("isolated execute: cannot resolve site entry module")
}
if !filepath.IsAbs(module) {
abs, aerr := filepath.Abs(module)
if aerr == nil {
module = abs
}
}
if fsRoot == "" {
fsRoot = filepath.Dir(module)
}
if !filepath.IsAbs(fsRoot) {
abs, aerr := filepath.Abs(fsRoot)
if aerr == nil {
fsRoot = abs
}
}
return module, fsRoot, nil
}
// normalizeLiveSSELine maps Marqdo EventBus frames onto RuntimeEventV1.
func normalizeLiveSSELine(seq uint64, line string) map[string]any {
base := map[string]any{
"schema": "RuntimeEventV1", "schema_version": 1, "seq": seq,
}
var obj map[string]any
if err := json.Unmarshal([]byte(line), &obj); err != nil || obj == nil {
base["type"] = "message.delta"
base["payload"] = map[string]any{"raw": line}
return base
}
if t, _ := obj["type"].(string); t == "message.delta" || t == "turn.started" || t == "model.resolved" || t == "usage.updated" || t == "turn.completed" || t == "turn.failed" {
for k, v := range obj {
base[k] = v
}
base["seq"] = seq
return base
}
if t, _ := obj["type"].(string); t == "delta" || t == "plan" || t == "reasoning" {
text, _ := obj["text"].(string)
base["type"] = "message.delta"
base["payload"] = map[string]any{"delta": text, "kind": t}
return base
}
if t, _ := obj["type"].(string); t == "decompose" || t == "decision" || t == "await" || t == "round" || t == "done" || t == "tool_start" || t == "tool_end" || t == "plan_act" {
base["type"] = "message.delta"
base["payload"] = obj
return base
}
if text, ok := obj["text"].(string); ok && text != "" {
base["type"] = "message.delta"
base["payload"] = map[string]any{"delta": text}
return base
}
base["type"] = "message.delta"
base["payload"] = obj
return base
}
func drainForward(path string, offset int64, onLine func(string)) int64 {
off, _ := drainForwardCarry(path, offset, "", onLine)
return off
}
const drainForwardMaxLine = 1024 * 1024
func drainForwardCarry(path string, offset int64, carry string, onLine func(string)) (int64, string) {
f, err := os.Open(path)
if err != nil {
return offset, carry
}
defer f.Close()
if _, err := f.Seek(offset, io.SeekStart); err != nil {
return offset, carry
}
chunk, err := io.ReadAll(io.LimitReader(f, drainForwardMaxLine+1))
if err != nil && err != io.EOF {
onLine(fmt.Sprintf(`{"type":"protocol.error","error":%q}`, err.Error()))
return offset, carry
}
if len(chunk) == 0 {
return offset, carry
}
offset += int64(len(chunk))
buf := carry + string(chunk)
for {
i := strings.IndexByte(buf, '\n')
if i < 0 {
if len(buf) > drainForwardMaxLine {
onLine(`{"type":"protocol.error","error":"ndjson line exceeds 1MiB"}`)
return offset, ""
}
return offset, buf
}
lineBytes := buf[:i]
line := strings.TrimSpace(strings.TrimSuffix(lineBytes, "\r"))
buf = buf[i+1:]
if line == "" {
continue
}
onLine(line)
}
}
func legacySSEBridgeSnapshot() (active bool, forward string, cancelTok string) {
return legacySSEBridgeActive.Load() != 0, os.Getenv("MARQDO_EVENT_FORWARD"), os.Getenv("MARQDO_CANCEL_TOKEN")
}
func writeCancelToken(path string) error {
if path == "" {
return nil
}
return os.WriteFile(path, []byte("cancelled"), 0o600)
}