diff --git a/.cursor/skills/marqdo-release/reference.md b/.cursor/skills/marqdo-release/reference.md index 59bb8a1..4214206 100644 --- a/.cursor/skills/marqdo-release/reference.md +++ b/.cursor/skills/marqdo-release/reference.md @@ -33,6 +33,8 @@ Produced by `.github/workflows/release.yml` on **windows-latest**: | `marqdo-VER-x86_64-pc-windows-msvc.zip` | `marqdo.exe` + `lib/` | | `marqdo-VER-stdlib.zip` | `lib/` only | | `marqdo-VER-ext.zip` | Official `ext/` L1 | +| `marqdo-VER-native-x86_64-pc-windows-msvc.zip` | Prebuilt Windows plugins (`native/*.dll`) | +| `marqdo-VER-native-x86_64-unknown-linux-gnu.zip` | Prebuilt Linux plugins (`native/lib*.so`) | | `marqdo-VER-source.zip` | `git archive` of tagged commit | | `marqdo-VER.vsix` | VS Code / Cursor (from branch `vscode-extension`) | | `marqdo-VER-public.zip` | Static user docs (`marqdo view output public`) | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e3cdf7f..fa638e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,8 +25,10 @@ jobs: with: node-version: "20" - - name: Build release - run: cargo build --release + - name: Build release CLI + official plugins + run: | + cargo build --release + cargo build --release -p marqdo_plugin_web -p marqdo_plugin_agent -p marqdo_plugin_quantum - name: Checkout extension sources (branch vscode-extension only) run: git fetch origin vscode-extension && git checkout origin/vscode-extension -- vscode-extension @@ -55,6 +57,7 @@ jobs: $vsix = "marqdo-$ver.vsix" $public = "marqdo-$ver-public.zip" $extZip = "marqdo-$ver-ext.zip" + $nativeZip = "marqdo-$ver-native-$target.zip" Copy-Item target/release/marqdo.exe "dist/$exe" git archive --format=zip -o "dist/$source" HEAD ./target/release/marqdo view output public -o public @@ -65,11 +68,25 @@ jobs: Copy-Item ext "$stageExt/ext" -Recurse if (Test-Path "dist/$extZip") { Remove-Item "dist/$extZip" -Force } Compress-Archive -Path "$stageExt/*" -DestinationPath "dist/$extZip" -Force + # Prebuilt native plugins (no Rust required for `marqdo ext add`) + $stageNative = "dist/stage-native" + if (Test-Path $stageNative) { Remove-Item $stageNative -Recurse -Force } + New-Item -ItemType Directory -Force -Path "$stageNative/native" | Out-Null + Copy-Item target/release/web.dll "$stageNative/native/web.dll" + Copy-Item target/release/agent.dll "$stageNative/native/agent.dll" + Copy-Item target/release/quantum.dll "$stageNative/native/quantum.dll" + if (Test-Path "dist/$nativeZip") { Remove-Item "dist/$nativeZip" -Force } + Compress-Archive -Path "$stageNative/*" -DestinationPath "dist/$nativeZip" -Force $stage = "dist/stage-bundle" if (Test-Path $stage) { Remove-Item $stage -Recurse -Force } New-Item -ItemType Directory -Force -Path $stage | Out-Null Copy-Item target/release/marqdo.exe "$stage/marqdo.exe" Copy-Item lib "$stage/lib" -Recurse + Copy-Item ext "$stage/ext" -Recurse + New-Item -ItemType Directory -Force -Path "$stage/ext/native" | Out-Null + Copy-Item target/release/web.dll "$stage/ext/native/web.dll" + Copy-Item target/release/agent.dll "$stage/ext/native/agent.dll" + Copy-Item target/release/quantum.dll "$stage/ext/native/quantum.dll" if (Test-Path "dist/$zip") { Remove-Item "dist/$zip" -Force } Compress-Archive -Path "$stage/*" -DestinationPath "dist/$zip" -Force $stageLib = "dist/stage-stdlib" @@ -85,6 +102,7 @@ jobs: echo "SOURCE=$source" >> $env:GITHUB_ENV echo "PUBLIC=$public" >> $env:GITHUB_ENV echo "EXT=$extZip" >> $env:GITHUB_ENV + echo "NATIVE=$nativeZip" >> $env:GITHUB_ENV - name: Upload release assets uses: softprops/action-gh-release@v2 @@ -92,24 +110,21 @@ jobs: body: | ## Marqdo ${{ github.ref_name }} - ### 亮点 / Highlights — v0.3.4 - - **`ext/web` 定制 C0–C4**:admin 前缀可让出、段边界 gates、shell/layout、样式 strict、脚本 defer/version、条件导航 when/media - - **ext/web customization C0–C4**: configurable admin/gates/shell/layout/CSS strict/script lifecycle/conditional nav - - Agent Wave B0 demos + writeback shadow fix - - Install: `marqdo ext list` / `marqdo ext add llm|agent|web|quantum` - - Native plugins: `cargo build --release -p marqdo_plugin_*` then `ext add` - - Details: [CHANGELOG](https://github.com/cflmy/marqdo/blob/main/CHANGELOG.md) · [README](https://github.com/cflmy/marqdo/blob/main/README.md) - - Also: embedded stdlib, writeback/subtask, Variables modal in `view`, VSIX from branch **`vscode-extension`**. + ### 亮点 / Highlights + - **`marqdo ext add` 无需本机 Rust**:发布附带预编译 native 插件;`ext add` 可自动下载 + - **Prebuilt native plugins**: `marqdo-*-native-*.zip`; Windows zip includes `ext/native/` + - Install: `marqdo ext add web|agent|quantum|llm`(零源码编译) + - Details: [CHANGELOG](https://github.com/cflmy/marqdo/blob/main/CHANGELOG.md) · [ext-cli.md](https://github.com/cflmy/marqdo/blob/main/doc/design/ext-cli.md) ### Downloads | Asset | Contents | |-------|----------| | `marqdo-*-windows-msvc.exe` | CLI with **embedded** official stdlib | - | `marqdo-*-windows-msvc.zip` | `marqdo.exe` + `lib/` (optional disk override) | + | `marqdo-*-windows-msvc.zip` | `marqdo.exe` + `lib/` + `ext/`(含 native DLL) | + | `marqdo-*-native-*.zip` | 预编译 `web`/`agent`/`quantum` 插件 | | `marqdo-*-stdlib.zip` | `lib/` only | - | `marqdo-*-ext.zip` | Official `ext/` L1 sources (for `MARQDO_EXT_SOURCE` / local `./ext`) | + | `marqdo-*-ext.zip` | Official `ext/` L1 sources | | `marqdo-*-source.zip` | Source snapshot (`git archive`) | | `marqdo-*.vsix` | VS Code / Cursor extension | | `marqdo-*-public.zip` | Static user docs site | @@ -120,9 +135,41 @@ jobs: dist/${{ env.ZIP }} dist/${{ env.STDLIB }} dist/${{ env.EXT }} + dist/${{ env.NATIVE }} dist/${{ env.VSIX }} dist/${{ env.SOURCE }} dist/${{ env.PUBLIC }} fail_on_unmatched_files: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + linux-native: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build official plugins (release) + run: cargo build --release -p marqdo_plugin_web -p marqdo_plugin_agent -p marqdo_plugin_quantum + + - name: Package native zip + run: | + set -euo pipefail + VER="${GITHUB_REF_NAME#v}" + TARGET="x86_64-unknown-linux-gnu" + mkdir -p dist/stage-native/native + cp target/release/libweb.so dist/stage-native/native/libweb.so + cp target/release/libagent.so dist/stage-native/native/libagent.so + cp target/release/libquantum.so dist/stage-native/native/libquantum.so + (cd dist/stage-native && zip -r "../marqdo-${VER}-native-${TARGET}.zip" .) + echo "NATIVE=marqdo-${VER}-native-${TARGET}.zip" >> "$GITHUB_ENV" + + - name: Upload Linux native plugins + uses: softprops/action-gh-release@v2 + with: + files: dist/${{ env.NATIVE }} + fail_on_unmatched_files: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index d784f01..c9164cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,12 @@ ## Unreleased ### Added +- **`marqdo ext add` 预编译原生插件**:Release 发布 `marqdo-*-native-{windows,linux}.zip`;Windows 便携 zip 内含 `ext/native/*.dll`;无本地 `target/` 时自动从 GitHub Release 下载 L1(`*-ext.zip`)与 native(可用 `MARQDO_EXT_NO_DOWNLOAD=1` 关闭)。 ### Fixed ### Changed +- 文档:[ext-cli.md](doc/design/ext-cli.md) 以「无需本机 Rust」为默认安装路径;[08-ext-deploy-coupling](doc/design/ext-web-customization/08-ext-deploy-coupling.md) 对齐发布资产。 ## v0.3.4 — 2026-09-04 diff --git a/Cargo.lock b/Cargo.lock index 7459ea8..df909cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -463,6 +463,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1018,7 +1024,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "marqdo" -version = "0.3.3" +version = "0.3.4" dependencies = [ "anyhow", "assert_cmd", @@ -1038,11 +1044,12 @@ dependencies = [ "tokio-tungstenite 0.24.0", "ureq", "zeroize", + "zip", ] [[package]] name = "marqdo-wasm" -version = "0.3.3" +version = "0.3.4" dependencies = [ "marqdo", "serde_json", @@ -2562,6 +2569,18 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 7448509..ab4af5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ default = ["native"] native = ["cli", "view", "net-host", "plugin-host", "exec-host", "fs-host", "tty"] cli = ["dep:clap"] view = ["dep:tiny_http"] -net-host = ["dep:ureq", "dep:idna_adapter", "dep:zeroize"] +net-host = ["dep:ureq", "dep:idna_adapter", "dep:zeroize", "dep:zip"] plugin-host = ["dep:libloading"] exec-host = [] fs-host = [] @@ -36,6 +36,8 @@ tiny_http = { version = "0.12", optional = true } ureq = { version = "=2.9.7", default-features = false, features = ["tls", "gzip"], optional = true } idna_adapter = { version = "=1.1.0", optional = true } zeroize = { version = "=1.8.1", optional = true } +# Extract GitHub Release ext/native zips (`marqdo ext add` without local Rust). +zip = { version = "=0.6.6", default-features = false, features = ["deflate"], optional = true } libloading = { version = "=0.8.6", optional = true } include_dir = "0.7" libc = { version = "0.2", optional = true } diff --git a/README.md b/README.md index 1177745..697af83 100644 --- a/README.md +++ b/README.md @@ -138,12 +138,13 @@ marqdo view public --no-open marqdo debug public --no-open marqdo catalog public -o .marqdo -# 3) 安装官方扩展(需先编原生插件) -cargo build --release -p marqdo_plugin_web -p marqdo_plugin_agent -p marqdo_plugin_quantum -marqdo ext add web # 或:网页 +# 3) 安装官方扩展(Release 用户:无需本机 Rust) +marqdo ext add web # 或:网页 — 自动下载 L1 + 预编译 native marqdo ext add agent # 或:智能体 marqdo ext add quantum # 或:量子 -marqdo ext add llm # 或:大模型 +marqdo ext add llm # 或:大模型(纯 .mq.md) +# 开发者也可:cargo build --release -p marqdo_plugin_* 后在本机 target/ 安装 +# 离线:解压 Windows zip(已含 ext/native)或手动放下 native zip;MARQDO_EXT_NO_DOWNLOAD=1 # 4) 动态站示例(扩展装好后) marqdo run examples/marqdo-blog/index.mq.md diff --git a/doc/design/ext-cli.md b/doc/design/ext-cli.md index 0533634..e04eb94 100644 --- a/doc/design/ext-cli.md +++ b/doc/design/ext-cli.md @@ -2,9 +2,9 @@ | | | |---|---| -| Status | **Accepted (v0.1)** | -| Date | 2026-08-07 | -| Related | [ext-llm.md](ext-llm.md) · [ext-agent.md](ext-agent.md) · [ext-abi.md](ext-abi.md) | +| Status | **Accepted (v0.3.5 prep)** — prebuilt native download | +| Date | 2026-09-04 | +| Related | [ext-llm.md](ext-llm.md) · [ext-agent.md](ext-agent.md) · [ext-abi.md](ext-abi.md) · [08-ext-deploy-coupling.md](ext-web-customization/08-ext-deploy-coupling.md) | ## Commands @@ -12,8 +12,8 @@ marqdo ext list marqdo ext add llm marqdo ext add agent +marqdo ext add web marqdo ext remove llm -marqdo ext remove agent ``` ## Install root @@ -23,57 +23,88 @@ marqdo ext remove agent Imports use paths like `import llm:ext/ai/llm.mq.md` (see [`load.rs`](../../src/load.rs)). -## Source for `add` +## Goal: no local Rust required + +End users who install Marqdo from **GitHub Releases** should run: + +```bash +marqdo ext add web +``` + +and get both L1 `.mq.md` **and** the matching native plugin (`.dll` / `.so`) **without** `cargo build`. + +### Resolution order (`add` with native crate) + +1. Local artifact (`MARQDO_*_PLUGIN`, `target/{debug,release}`, beside `marqdo`, `MARQDO_EXT_SOURCE/native`, …) +2. **Download** `marqdo-{VER}-native-{target}.zip` from the GitHub Release for this CLI version → cache under `~/.marqdo/cache/` +3. If Rust/`cargo` is available → `cargo build -p marqdo_plugin_*` (developers) +4. Else fail with a clear message pointing at the Release assets + +L1 sources: + +1. `MARQDO_EXT_SOURCE` / repo `./ext` / beside binary +2. Else download `marqdo-{VER}-ext.zip` into `~/.marqdo/cache/ext-src-v{VER}/` + +### Env knobs + +| Variable | Meaning | +|----------|---------| +| `MARQDO_EXT_VERSION` | Override SemVer used for Release asset names (default: CLI `CARGO_PKG_VERSION`) | +| `MARQDO_EXT_NO_DOWNLOAD=1` | Never hit the network (CI / air-gap); local/cargo only | +| `MARQDO_WEB_PLUGIN` / `MARQDO_AGENT_PLUGIN` / `MARQDO_QUANTUM_PLUGIN` | Explicit native path | + +Windows Release **zip** also ships `ext/` + `ext/native/*.dll` next to `marqdo.exe` so a portable unzip works offline after one download. + +## Source for `add` (developers) 1. `MARQDO_EXT_SOURCE` — directory containing `ai/llm.mq.md` … -2. Else repo `./ext` near cwd / binary +2. Else repo `./ext` near cwd / binary +3. Else Release `marqdo-*-ext.zip` (see above) ## Layout (`ext/`) -Official extensions live under **`ext/ai/`** (LLM + agent framework only). Do not add flat `.mq.md` files at `ext/` root — keeps the tree from accumulating unrelated modules. - ```text ext/ - ai/ - llm.mq.md - 大模型.mq.md - agent.mq.md - 智能体.mq.md + ai/ # llm + agent L1 + web/ + quantum/ ``` -Future optional ext domains get their own subdirs (e.g. `ext/web/`, `ext/quantum/`), not mixed into `ai/`. +Installed tree (`~/.marqdo/ext` or `MARQDO_EXT`): + +```text +web/web.mq.md +web/网页.mq.md +native/libweb.so # or web.dll / libweb.dylib +web.plugin # absolute path hint +``` ## Catalog | Id | Installs under `MARQDO_EXT` | |----|-----------------------------| | `llm` | `ai/llm.mq.md`, `ai/大模型.mq.md` | -| `agent` | `ai/agent.mq.md`, `ai/智能体.mq.md`, **`native/libagent.so`** (or `.dylib`) | -| `web` | `web/web.mq.md`, `web/网页.mq.md`, **`native/libweb.so`** — 见 [ext-web.md](ext-web.md) · [roadmap](../roadmap/ext-web.md) | -| `quantum` | `quantum/quantum.mq.md`, `quantum/量子.mq.md`, **`native/libquantum.so`** — 见 [ext-quantum.md](ext-quantum.md) · [roadmap](../roadmap/ext-quantum.md) | +| `agent` | `ai/agent.mq.md`, `ai/智能体.mq.md`, **native `agent`** | +| `web` | `web/web.mq.md`, `web/网页.mq.md`, **native `web`** | +| `quantum` | `quantum/quantum.mq.md`, `quantum/量子.mq.md`, **native `quantum`** | -`add agent` / `add web` / `add quantum` **先定位原生插件**,再复制 `.mq.md`: +## Release assets (native) -1. 在 `CARGO_TARGET_DIR`、`target/{debug,release}`、源码 `ext` 旁的 `target/`、可执行文件旁等路径查找 `lib{name}.so`(或 `.dll` / `.dylib`) -2. 找不到则自动执行 `cargo build -p marqdo_plugin_{name}`,再安装到 `MARQDO_EXT/native/` -3. 写入 `{name}.plugin` 绝对路径提示,供 `plugin.native_path` 解析 +| Asset | Platform | +|-------|----------| +| `marqdo-VER-native-x86_64-pc-windows-msvc.zip` | Windows x64 (`native/*.dll`) | +| `marqdo-VER-native-x86_64-unknown-linux-gnu.zip` | Linux x64 (`native/lib*.so`) | -这样不会出现「只装了 `.mq.md`、运行时报 native plugin not found」的半安装状态。 - -```bash -# 推荐:在仓库根目录 -marqdo ext add web -# 等价于找到或编好 libweb.so 后拷入 ~/.marqdo/ext/native/ -``` - -Without a prior install, runtime still resolves via `plugin.native_path name=web` (see [ext-abi.md](ext-abi.md)): `MARQDO_WEB_PLUGIN` → `CARGO_TARGET_DIR/{debug,release}/` → cwd `target/…` → beside `marqdo` binary → `MARQDO_EXT/native/`. +macOS: use local `cargo build` until CI adds Darwin natives (same zip naming pattern reserved). ## Tests -`tests/gold.rs`: `ext_cli_add_list_remove_llm`, `ext_cli_add_agent`, `ext_agent_framework_smoke`, `ext_agent_run_live` ([`tests/ext/agent-run-live.mq.md`](../../tests/ext/agent-run-live.mq.md); needs `tests/ext/.env` + built `libagent`). +`tests/gold.rs`: `ext_cli_add_list_remove_llm`, `ext_cli_add_agent`, `ext_cli_add_web` (local build path; set `MARQDO_EXT_NO_DOWNLOAD=1` in sandboxes if needed). + +Unit: `ext_fetch` URL / triple tests. ## Non-goals - Third-party registry -- Merging into `lib/` -- Bundling domain/task helpers in official ext +- Merging domain helpers into `lib/` +- Guaranteeing plugin **source** ABI stability (author `.mq.md` API is the contract) diff --git a/doc/design/ext-web-customization/08-ext-deploy-coupling.md b/doc/design/ext-web-customization/08-ext-deploy-coupling.md index 1f9d292..3a9c6e5 100644 --- a/doc/design/ext-web-customization/08-ext-deploy-coupling.md +++ b/doc/design/ext-web-customization/08-ext-deploy-coupling.md @@ -55,6 +55,14 @@ --- -## 6. 过渡期 workaround +## 6. 过渡期 → 预编译路径(进行中) -锁定 Marqdo 与插件同版本镜像;本地 `MARQDO_EXT` 指向仓库构建产物;定制需求提 issue 对照本目录解决文,避免沉默 fork 漂移。 +**目标:** 用户 `marqdo ext add web` **不必**本机 Rust。 + +| 交付 | 说明 | +|------|------| +| Release `marqdo-*-native-*.zip` | Windows / Linux 预编译 `web`/`agent`/`quantum` | +| Windows portable zip | 内含 `ext/` + `ext/native/*.dll` | +| `ext add` 下载回退 | 本地无 `.so`/`.dll` 时从 GitHub Release 拉取(`MARQDO_EXT_NO_DOWNLOAD=1` 可关) | + +仍保留开发者 `cargo build -p marqdo_plugin_*` 路径。详见 [ext-cli.md](../ext-cli.md)。 diff --git a/src/ext_cli.rs b/src/ext_cli.rs index 7e35312..fe30080 100644 --- a/src/ext_cli.rs +++ b/src/ext_cli.rs @@ -173,8 +173,24 @@ fn find_source_file(name: &str) -> Result { return Ok(p); } } + #[cfg(feature = "net-host")] + { + if crate::ext_fetch::downloads_enabled() { + match crate::ext_fetch::ensure_ext_source_tree() { + Ok(root) => { + let p = root.join(name); + if p.is_file() { + return Ok(p); + } + } + Err(e) => { + eprintln!("note: could not download ext sources: {e:#}"); + } + } + } + } bail!( - "cannot find source `{name}` for install (set MARQDO_EXT_SOURCE to an ext/ directory, or run from the Marqdo repo)" + "cannot find source `{name}` for install (set MARQDO_EXT_SOURCE to an ext/ directory, run from the Marqdo repo, or use a release that publishes marqdo-*-ext.zip)" ); } @@ -240,31 +256,58 @@ fn find_native_plugin(short: &str) -> Result { ) } -/// Locate a built native plugin, or `cargo build -p …` once then look again. +/// Locate a built native plugin: local artifact → GitHub Release prebuild → optional cargo build. fn ensure_native_plugin(crate_name: &str) -> Result { let short = native_short_name(crate_name); if let Ok(p) = find_native_plugin(short) { return Ok(p); } - println!( - "native plugin for `{short}` not found; building `{crate_name}` (debug)…" - ); - let status = std::process::Command::new("cargo") - .args(["build", "-p", crate_name]) - .status() - .with_context(|| format!("spawn cargo build -p {crate_name}"))?; - if !status.success() { - bail!( - "cargo build -p {crate_name} failed (status {status}); build the plugin then re-run `marqdo ext add {short}`" - ); + let lib_name = native_lib_filename(short); + + #[cfg(feature = "net-host")] + { + match crate::ext_fetch::download_native_plugin(short, &lib_name) { + Ok(p) => return Ok(p), + Err(e) => { + eprintln!("note: prebuilt native download skipped/failed: {e:#}"); + } + } } - find_native_plugin(short).with_context(|| { - format!( - "built `{crate_name}` but still cannot find {}; set {} to the .so/.dll path", - native_lib_filename(short), - native_env_var(short) - ) - }) + + #[cfg(feature = "net-host")] + let have_cargo = crate::ext_fetch::cargo_available(); + #[cfg(not(feature = "net-host"))] + let have_cargo = true; + + if have_cargo { + println!( + "native plugin for `{short}` not found; building `{crate_name}` (debug)…" + ); + let status = std::process::Command::new("cargo") + .args(["build", "-p", crate_name]) + .status() + .with_context(|| format!("spawn cargo build -p {crate_name}"))?; + if !status.success() { + bail!( + "cargo build -p {crate_name} failed (status {status}); \ + install a prebuilt release asset or build the plugin then re-run `marqdo ext add {short}`" + ); + } + return find_native_plugin(short).with_context(|| { + format!( + "built `{crate_name}` but still cannot find {}; set {} to the .so/.dll path", + lib_name, + native_env_var(short) + ) + }); + } + + bail!( + "cannot find native plugin `{lib_name}` and Rust/cargo is not available. \ + Install Marqdo from a GitHub Release that includes `marqdo-*-native-*.zip`, \ + or set {} to a built plugin path.", + native_env_var(short) + ) } pub fn add_ext(id: &str) -> Result<()> { diff --git a/src/ext_fetch.rs b/src/ext_fetch.rs new file mode 100644 index 0000000..b213bbc --- /dev/null +++ b/src/ext_fetch.rs @@ -0,0 +1,307 @@ +//! Download official `ext/` L1 sources and native plugins from GitHub Releases. +//! +//! Enables `marqdo ext add web` without a local Rust toolchain or repo checkout. +//! Disable with `MARQDO_EXT_NO_DOWNLOAD=1`. Override tag with `MARQDO_EXT_VERSION=0.3.4`. + +use std::env; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; + +const REPO: &str = "cflmy/marqdo"; + +pub fn downloads_enabled() -> bool { + match env::var("MARQDO_EXT_NO_DOWNLOAD") { + Ok(v) => { + let v = v.trim(); + !(v == "1" || v.eq_ignore_ascii_case("true") || v.eq_ignore_ascii_case("yes")) + } + Err(_) => true, + } +} + +/// SemVer without leading `v` (matches release asset names). +pub fn release_version() -> String { + if let Ok(v) = env::var("MARQDO_EXT_VERSION") { + let v = v.trim().trim_start_matches('v'); + if !v.is_empty() { + return v.to_string(); + } + } + env!("CARGO_PKG_VERSION").to_string() +} + +pub fn host_target_triple() -> Option<&'static str> { + if cfg!(all(target_os = "windows", target_arch = "x86_64")) { + Some("x86_64-pc-windows-msvc") + } else if cfg!(all(target_os = "linux", target_arch = "x86_64")) { + Some("x86_64-unknown-linux-gnu") + } else if cfg!(all(target_os = "macos", target_arch = "aarch64")) { + Some("aarch64-apple-darwin") + } else if cfg!(all(target_os = "macos", target_arch = "x86_64")) { + Some("x86_64-apple-darwin") + } else { + None + } +} + +fn cache_root() -> Result { + let home = env::var_os("USERPROFILE") + .or_else(|| env::var_os("HOME")) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".")); + let dir = home.join(".marqdo").join("cache"); + fs::create_dir_all(&dir)?; + Ok(dir) +} + +fn user_agent() -> String { + format!("marqdo/{}", env!("CARGO_PKG_VERSION")) +} + +fn download_to_file(url: &str, dest: &Path) -> Result<()> { + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent)?; + } + let tmp = dest.with_extension("download-tmp"); + let resp = ureq::get(url) + .set("User-Agent", &user_agent()) + .set("Accept", "application/octet-stream") + .call() + .with_context(|| format!("GET {url}"))?; + if !(200..300).contains(&resp.status()) { + bail!("download {url} returned HTTP {}", resp.status()); + } + let mut reader = resp.into_reader(); + let mut out = File::create(&tmp).with_context(|| format!("create {}", tmp.display()))?; + io::copy(&mut reader, &mut out).with_context(|| format!("write {}", tmp.display()))?; + out.flush()?; + drop(out); + if dest.exists() { + let _ = fs::remove_file(dest); + } + fs::rename(&tmp, dest).with_context(|| format!("rename {}", dest.display()))?; + Ok(()) +} + +fn extract_zip(zip_path: &Path, dest_dir: &Path) -> Result<()> { + fs::create_dir_all(dest_dir)?; + let file = File::open(zip_path).with_context(|| format!("open {}", zip_path.display()))?; + let mut archive = zip::ZipArchive::new(file) + .with_context(|| format!("read zip {}", zip_path.display()))?; + for i in 0..archive.len() { + let mut entry = archive + .by_index(i) + .with_context(|| format!("zip entry {i}"))?; + let name = entry.name().to_string(); + if name.is_empty() { + continue; + } + if Path::new(&name) + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + bail!("refusing zip entry with ..: {name}"); + } + let out_path = dest_dir.join(Path::new(&name)); + if entry.is_dir() || name.ends_with('/') { + fs::create_dir_all(&out_path)?; + continue; + } + if let Some(parent) = out_path.parent() { + fs::create_dir_all(parent)?; + } + let mut outfile = + File::create(&out_path).with_context(|| format!("create {}", out_path.display()))?; + io::copy(&mut entry, &mut outfile) + .with_context(|| format!("extract {name}"))?; + } + Ok(()) +} + +fn release_asset_url(ver: &str, filename: &str) -> String { + format!("https://github.com/{REPO}/releases/download/v{ver}/{filename}") +} + +/// Ensure L1 `ext/` tree is available (repo checkout or downloaded Release zip). +/// Returns a directory that contains `web/web.mq.md`, `ai/…`, etc. +pub fn ensure_ext_source_tree() -> Result { + let ver = release_version(); + let cache = cache_root()?.join(format!("ext-src-v{ver}")); + let marker = cache.join("web").join("web.mq.md"); + if marker.is_file() { + return Ok(cache); + } + // Zip layout from CI: top-level `ext/…` + let nested = cache.join("ext").join("web").join("web.mq.md"); + if nested.is_file() { + return Ok(cache.join("ext")); + } + if !downloads_enabled() { + bail!("extension sources not found and downloads disabled (MARQDO_EXT_NO_DOWNLOAD)"); + } + let zip_name = format!("marqdo-{ver}-ext.zip"); + let url = release_asset_url(&ver, &zip_name); + let zip_path = cache_root()?.join(&zip_name); + println!("downloading official ext sources ({url})…"); + download_to_file(&url, &zip_path)?; + let stage = cache_root()?.join(format!("ext-src-v{ver}-extract")); + if stage.exists() { + let _ = fs::remove_dir_all(&stage); + } + extract_zip(&zip_path, &stage)?; + // Prefer `ext/` child if present + let src = if stage.join("ext").join("web").join("web.mq.md").is_file() { + stage.join("ext") + } else if stage.join("web").join("web.mq.md").is_file() { + stage.clone() + } else { + bail!( + "downloaded {zip_name} but could not find web/web.mq.md inside; check release assets" + ); + }; + if cache.exists() { + let _ = fs::remove_dir_all(&cache); + } + // Move/copy tree into stable cache path + copy_dir_recursive(&src, &cache)?; + let _ = fs::remove_dir_all(&stage); + if !cache.join("web").join("web.mq.md").is_file() { + bail!("failed to cache ext sources under {}", cache.display()); + } + println!("cached ext sources at {}", cache.display()); + Ok(cache) +} + +fn copy_dir_recursive(src: &Path, dest: &Path) -> Result<()> { + fs::create_dir_all(dest)?; + for entry in fs::read_dir(src).with_context(|| format!("read {}", src.display()))? { + let entry = entry?; + let ty = entry.file_type()?; + let to = dest.join(entry.file_name()); + if ty.is_dir() { + copy_dir_recursive(&entry.path(), &to)?; + } else if ty.is_file() { + if let Some(parent) = to.parent() { + fs::create_dir_all(parent)?; + } + fs::copy(entry.path(), &to) + .with_context(|| format!("copy {} → {}", entry.path().display(), to.display()))?; + } + } + Ok(()) +} + +/// Download prebuilt native plugins zip for this host; return path to `lib` file for `short`. +pub fn download_native_plugin(short: &str, lib_name: &str) -> Result { + if !downloads_enabled() { + bail!("native plugin not found and downloads disabled (MARQDO_EXT_NO_DOWNLOAD)"); + } + let Some(triple) = host_target_triple() else { + bail!( + "no prebuilt native plugins for this platform; install Rust and run `cargo build -p marqdo_plugin_{short}`" + ); + }; + let ver = release_version(); + let zip_name = format!("marqdo-{ver}-native-{triple}.zip"); + let url = release_asset_url(&ver, &zip_name); + let cache = cache_root()?.join(format!("native-v{ver}-{triple}")); + let lib_in_cache = cache.join("native").join(lib_name); + if lib_in_cache.is_file() { + return Ok(lib_in_cache); + } + // Also accept flat layout + let flat = cache.join(lib_name); + if flat.is_file() { + return Ok(flat); + } + let zip_path = cache_root()?.join(&zip_name); + println!("downloading prebuilt native plugins ({url})…"); + download_to_file(&url, &zip_path).with_context(|| { + format!( + "failed to download prebuilt plugins for {triple}. \ + Build locally with `cargo build --release -p marqdo_plugin_{short}`, \ + or check https://github.com/{REPO}/releases/tag/v{ver}" + ) + })?; + if cache.exists() { + let _ = fs::remove_dir_all(&cache); + } + fs::create_dir_all(&cache)?; + extract_zip(&zip_path, &cache)?; + if lib_in_cache.is_file() { + println!("cached {}", lib_in_cache.display()); + return Ok(lib_in_cache); + } + if flat.is_file() { + return Ok(flat); + } + // Search recursively for the lib name + if let Some(found) = find_named_file(&cache, lib_name)? { + return Ok(found); + } + bail!( + "downloaded {zip_name} but missing {lib_name}; expected native/{lib_name} in the archive" + ) +} + +fn find_named_file(root: &Path, name: &str) -> Result> { + fn walk(dir: &Path, name: &str, out: &mut Option) -> Result<()> { + if out.is_some() { + return Ok(()); + } + for entry in fs::read_dir(dir)? { + let entry = entry?; + let p = entry.path(); + if entry.file_type()?.is_dir() { + walk(&p, name, out)?; + } else if entry.file_name() == name { + *out = Some(p); + } + } + Ok(()) + } + let mut found = None; + walk(root, name, &mut found)?; + Ok(found) +} + +pub fn cargo_available() -> bool { + std::process::Command::new("cargo") + .arg("-V") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn asset_url_shape() { + let u = release_asset_url("0.3.4", "marqdo-0.3.4-ext.zip"); + assert_eq!( + u, + "https://github.com/cflmy/marqdo/releases/download/v0.3.4/marqdo-0.3.4-ext.zip" + ); + } + + #[test] + fn triple_known_on_ci_hosts() { + // At least one of these builds is what we ship. + let t = host_target_triple(); + if cfg!(any( + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "windows", target_arch = "x86_64"), + all(target_os = "macos", target_arch = "aarch64"), + all(target_os = "macos", target_arch = "x86_64"), + )) { + assert!(t.is_some()); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index c8cda78..b5d0589 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,8 @@ pub mod diagnostics; pub mod embedded_lib; #[cfg(feature = "cli")] pub mod ext_cli; +#[cfg(all(feature = "cli", feature = "net-host"))] +pub mod ext_fetch; pub mod foreign; pub mod formula; pub mod host; diff --git a/tests/gold.rs b/tests/gold.rs index a6611a5..ccbcee7 100644 --- a/tests/gold.rs +++ b/tests/gold.rs @@ -1261,6 +1261,7 @@ fn ext_cli_add_agent() { .args(["ext", "add", "agent"]) .env("MARQDO_EXT", &tmp) .env("MARQDO_EXT_SOURCE", &src) + .env("MARQDO_EXT_NO_DOWNLOAD", "1") .current_dir(env!("CARGO_MANIFEST_DIR")) .status() .expect("ext add agent"); @@ -1297,6 +1298,7 @@ fn ext_cli_add_web() { .args(["ext", "add", "web"]) .env("MARQDO_EXT", &tmp) .env("MARQDO_EXT_SOURCE", &src) + .env("MARQDO_EXT_NO_DOWNLOAD", "1") .current_dir(env!("CARGO_MANIFEST_DIR")) .status() .expect("ext add web");