From 54c7b43d1565d162dc6e0031e746d3716d3cbf5d Mon Sep 17 00:00:00 2001 From: cflmy Date: Fri, 20 Nov 2026 11:49:05 +0800 Subject: [PATCH] feat(m0.6-r5): pinned Git packages and materialized environments - add content-addressed Git checkout cache for exact revisions; no hooks, submodules or floating branches - resolve transitive @alias imports from each package's own manifest - materialize locked path/Git packages into .marqdo/env atomically and prefer a matching env copy in the resolver - make sync a real environment sync while keeping lock immutable - kill Unix foreign-kernel process groups so grandchildren do not survive - add Git/env/transitive integration tests and update package/security specs --- Cargo.lock | 2 + Cargo.toml | 1 + crates/cli/src/commands.rs | 15 +- crates/cli/src/loader.rs | 2 +- crates/cli/tests/package_env.rs | 207 ++++++++++ crates/marqdo-foreign/Cargo.toml | 3 + crates/marqdo-foreign/src/lib.rs | 27 ++ crates/marqdo-package/Cargo.toml | 1 + crates/marqdo-package/src/lib.rs | 652 +++++++++++++++++++++++++++---- docs/STATUS.md | 4 +- spec/package-format.md | 58 +-- spec/security.md | 8 +- 12 files changed, 862 insertions(+), 118 deletions(-) create mode 100644 crates/cli/tests/package_env.rs diff --git a/Cargo.lock b/Cargo.lock index ad535f0..f0b7fb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -360,6 +360,7 @@ dependencies = [ name = "marqdo-foreign" version = "0.1.0" dependencies = [ + "libc", "marqdo-span", "marqdo-value", "serde_json", @@ -437,6 +438,7 @@ version = "0.1.0" dependencies = [ "hex", "serde", + "serde_json", "serde_yaml", "sha2", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 810a4cf..f4f24e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,5 +55,6 @@ thiserror = "2" clap = { version = "4", features = ["derive"] } walkdir = "2" glob = "0.3" +libc = "0.2" unicode-normalization = "0.1" unicode-ident = "1" diff --git a/crates/cli/src/commands.rs b/crates/cli/src/commands.rs index 1e5daa9..5decf53 100644 --- a/crates/cli/src/commands.rs +++ b/crates/cli/src/commands.rs @@ -1111,6 +1111,13 @@ pub fn sync(locked: bool, json: bool) -> i32 { ); return 1; } + let resolver = + marqdo_package::PackageResolver::new(root, Some(manifest.clone()), Some(lockfile)) + .with_fetch(true); + if let Err(error) = resolver.materialize_env(&manifest, &expected) { + eprintln!("error: cannot materialize project environment: {error}"); + return 1; + } if json { output::print_json(&json!({ "command": "sync", @@ -1120,9 +1127,10 @@ pub fn sync(locked: bool, json: bool) -> i32 { })); } else { println!( - "synchronized {} package(s){}", + "synchronized {} package(s){} into {}", expected.packages.len(), - if locked { " (locked)" } else { "" } + if locked { " (locked)" } else { "" }, + resolver.env_root().display() ); } 0 @@ -1143,7 +1151,8 @@ fn build_lockfile( root: &Path, manifest: &marqdo_package::Manifest, ) -> Result { - let resolver = marqdo_package::PackageResolver::new(root, Some(manifest.clone()), None); + let resolver = + marqdo_package::PackageResolver::new(root, Some(manifest.clone()), None).with_fetch(true); let mut packages = Vec::new(); for name in manifest.dependencies.keys() { let resolved = resolver.resolve_alias(name)?; diff --git a/crates/cli/src/loader.rs b/crates/cli/src/loader.rs index ec137c7..3f5109c 100644 --- a/crates/cli/src/loader.rs +++ b/crates/cli/src/loader.rs @@ -254,7 +254,7 @@ impl Loader { for (alias, target, span) in imports { let resolved = if let Some(alias_name) = target.strip_prefix('@') { - self.resolver.resolve_alias(alias_name) + self.resolver.resolve_alias_from(package, alias_name) } else { self.resolver.resolve_relative(package, &path, &target) }; diff --git a/crates/cli/tests/package_env.rs b/crates/cli/tests/package_env.rs new file mode 100644 index 0000000..eda13ee --- /dev/null +++ b/crates/cli/tests/package_env.rs @@ -0,0 +1,207 @@ +//! Package instance identity, transitive imports, pinned Git and environment +//! materialization (M0.6 R1/R5). +#![cfg(unix)] + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicU32, Ordering}; + +static COUNTER: AtomicU32 = AtomicU32::new(0); + +struct TempDir(PathBuf); + +impl TempDir { + fn new(tag: &str) -> Self { + let id = COUNTER.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "marqdo-package-test-{}-{id}-{tag}", + std::process::id() + )); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn binary() -> &'static str { + env!("CARGO_BIN_EXE_marqdo") +} + +fn cli(dir: &Path, args: &[&str]) -> Output { + Command::new(binary()) + .args(args) + .current_dir(dir) + .output() + .expect("run marqdo") +} + +fn write(path: &Path, text: &str) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, text).unwrap(); +} + +fn have_git() -> bool { + Command::new("git") + .arg("--version") + .output() + .map(|output| output.status.success()) + .unwrap_or(false) +} + +fn git(dir: &Path, args: &[&str]) { + let status = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .status() + .expect("git"); + assert!(status.success(), "git {args:?} failed"); +} + +#[test] +fn transitive_path_dependency_resolves_from_its_own_manifest() { + let dir = TempDir::new("transitive"); + let root = dir.path(); + write( + &root.join("deps/a/marqdo.yaml"), + "project:\n name: a\n version: 0.1.0\ndependencies:\n b:\n path: ./b\n", + ); + write( + &root.join("deps/a/index.mq.md"), + "---\nimport:\n b: \"@b\"\n---\n\n## outer\n\n*b.inner()*\n", + ); + write( + &root.join("deps/a/b/marqdo.yaml"), + "project:\n name: b\n version: 0.1.0\n", + ); + write(&root.join("deps/a/b/index.mq.md"), "## inner\n\n*42*\n"); + write( + &root.join("marqdo.yaml"), + "project:\n name: root\n version: 0.1.0\ndependencies:\n a:\n path: ./deps/a\n", + ); + write( + &root.join("index.mq.md"), + "---\nimport:\n a: \"@a\"\n---\n\n## main\n\n> 打印(a.outer())\n", + ); + let output = cli(root, &["run", "index.mq.md"]); + assert_eq!( + output.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), "42\n"); + assert!( + !root.join(".marqdo").exists(), + "run must not materialize env" + ); +} + +#[test] +fn pinned_git_dependency_locks_syncs_and_runs() { + if !have_git() { + eprintln!("skipping: git is unavailable"); + return; + } + let dir = TempDir::new("git"); + let root = dir.path().join("project"); + let dep = dir.path().join("dep"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::create_dir_all(&dep).unwrap(); + write( + &dep.join("marqdo.yaml"), + "project:\n name: remote\n version: 0.3.0\n", + ); + write(&dep.join("index.mq.md"), "## hello\n\n*7*\n"); + git(&dep, &["init", "-q"]); + git(&dep, &["add", "."]); + git(&dep, &["commit", "-q", "-m", "initial"]); + let rev = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&dep) + .output() + .unwrap() + .stdout, + ) + .unwrap(); + write( + &root.join("marqdo.yaml"), + &format!( + "project:\n name: root\n version: 0.1.0\ndependencies:\n remote:\n git: {}\n rev: {}\n", + dep.display(), + rev.trim() + ), + ); + write( + &root.join("index.mq.md"), + "---\nimport:\n remote: \"@remote\"\n---\n\n## main\n\n> 打印(remote.hello())\n", + ); + + let lock = cli(&root, &["lock"]); + assert_eq!( + lock.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&lock.stderr) + ); + let lock_text = std::fs::read_to_string(root.join("marqdo.lock")).unwrap(); + assert!(lock_text.contains("lock_version: 2")); + assert!(lock_text.contains("instance_id: dep:remote@0.3.0#")); + assert!(lock_text.contains("source: git:")); + + let sync = cli(&root, &["sync", "--locked"]); + assert_eq!( + sync.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&sync.stderr) + ); + let env_json = std::fs::read_to_string(root.join(".marqdo/env/env.json")).unwrap(); + assert!(env_json.contains("dep:remote@0.3.0#")); + let run = cli(&root, &["run", "index.mq.md"]); + assert_eq!( + run.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&run.stderr) + ); + assert_eq!(String::from_utf8_lossy(&run.stdout), "7\n"); +} + +#[test] +fn check_does_not_fetch_a_missing_git_revision() { + if !have_git() { + eprintln!("skipping: git is unavailable"); + return; + } + let dir = TempDir::new("git-missing"); + let root = dir.path(); + write( + &root.join("marqdo.yaml"), + "project:\n name: root\n version: 0.1.0\ndependencies:\n remote:\n git: /nonexistent/repo\n rev: 0000000000000000000000000000000000000000\n", + ); + write( + &root.join("index.mq.md"), + "---\nimport:\n remote: \"@remote\"\n---\n\n## main\n\n*1*\n", + ); + let output = cli(root, &["check", "index.mq.md"]); + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("not cached") || stderr.contains("cannot open dependency")); + assert!(!root.join(".marqdo/cache/git").exists()); +} diff --git a/crates/marqdo-foreign/Cargo.toml b/crates/marqdo-foreign/Cargo.toml index ce90118..fa2a0ec 100644 --- a/crates/marqdo-foreign/Cargo.toml +++ b/crates/marqdo-foreign/Cargo.toml @@ -17,3 +17,6 @@ marqdo-value = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +[target.'cfg(unix)'.dependencies] +libc = { workspace = true } + diff --git a/crates/marqdo-foreign/src/lib.rs b/crates/marqdo-foreign/src/lib.rs index 997b3d9..8aed993 100644 --- a/crates/marqdo-foreign/src/lib.rs +++ b/crates/marqdo-foreign/src/lib.rs @@ -827,6 +827,20 @@ fn spawn_and_capture( }) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + // SAFETY: setpgid is async-signal-safe and only touches the child + // process between fork and exec. + unsafe { + command.pre_exec(|| { + if libc::setpgid(0, 0) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + } let start = Instant::now(); let mut child = command.spawn().map_err(|error| ForeignError::Spawn { @@ -970,6 +984,19 @@ fn collect_reader(state: ReaderState) -> Vec { } fn kill_and_reap(child: &mut Child) { + #[cfg(unix)] + { + // The child is its own process-group leader (setpgid in pre_exec), so + // the negative pid terminates surviving grandchildren too. + let pid = child.id() as i32; + if pid > 0 { + // SAFETY: kill(2) with a negative pid targets that process group; + // failure is harmless (the child may already be gone). + unsafe { + libc::kill(-pid, libc::SIGKILL); + } + } + } let _ = child.kill(); let _ = child.wait(); } diff --git a/crates/marqdo-package/Cargo.toml b/crates/marqdo-package/Cargo.toml index a623642..19e42b1 100644 --- a/crates/marqdo-package/Cargo.toml +++ b/crates/marqdo-package/Cargo.toml @@ -11,6 +11,7 @@ authors.workspace = true [dependencies] serde = { workspace = true } serde_yaml = { workspace = true } +serde_json = { workspace = true } sha2 = { workspace = true } walkdir = { workspace = true } hex = { workspace = true } diff --git a/crates/marqdo-package/src/lib.rs b/crates/marqdo-package/src/lib.rs index 66b437b..16bae99 100644 --- a/crates/marqdo-package/src/lib.rs +++ b/crates/marqdo-package/src/lib.rs @@ -321,28 +321,61 @@ pub fn hash_tree(root: &Path) -> Result { Ok(hex::encode(hasher.finalize())) } +/// Sanitize a logical instance id into one filesystem component. +pub fn env_dir_name(instance: &str) -> String { + instance + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() + || character == '.' + || character == '-' + || character == '_' + { + character + } else { + '_' + } + }) + .collect() +} + /// Resolve entry files and explicit `@dependency` imports. #[derive(Debug, Clone)] pub struct PackageResolver { root: PathBuf, manifest: Option, lock: Option, + cache_root: PathBuf, + env_root: PathBuf, + fetch: bool, } impl PackageResolver { - /// Build a resolver for a project root. + /// Build a resolver for a project root. Fetching is disabled; `check` and + /// `run` only read path sources, the Git cache and the materialized env. pub fn new( root: impl Into, manifest: Option, lock: Option, ) -> Self { + let root = root.into(); Self { - root: root.into(), + cache_root: root.join(".marqdo/cache"), + env_root: root.join(".marqdo/env"), + root, manifest, lock, + fetch: false, } } + /// Allow `lock`/`sync` to fetch pinned Git revisions (never install + /// scripts, never submodules). + pub fn with_fetch(mut self, fetch: bool) -> Self { + self.fetch = fetch; + self + } + /// Project root. pub fn root(&self) -> &Path { &self.root @@ -353,6 +386,16 @@ impl PackageResolver { self.manifest.as_ref() } + /// Content-addressed cache root. + pub fn cache_root(&self) -> &Path { + &self.cache_root + } + + /// Materialized environment root. + pub fn env_root(&self) -> &Path { + &self.env_root + } + /// Resolve the package that owns `entry`. pub fn resolve_entry(&self, entry: &Path) -> Result { let entry = entry @@ -366,102 +409,165 @@ impl PackageResolver { ) })?; Ok(ResolvedModule { - package: ResolvedPackage { - instance_id: root_instance(self.manifest.as_ref()), - name: self - .manifest - .as_ref() - .map(|manifest| manifest.project.name.clone()) - .unwrap_or_else(|| "default".to_string()), - version: self - .manifest - .as_ref() - .map(|manifest| manifest.project.version.clone()) - .unwrap_or_else(|| "0.0.0".to_string()), - origin: PackageOrigin::Root, - revision: None, - content_digest: None, - module_root: self.root.clone(), - entry_module: normalize_relative(relative)?, - manifest: self.manifest.clone(), - }, + package: self.root_package()?, relative_path: normalize_relative(relative)?, }) } - /// Resolve a declared `@alias` dependency. + fn root_package(&self) -> Result { + Ok(ResolvedPackage { + instance_id: root_instance(self.manifest.as_ref()), + name: self + .manifest + .as_ref() + .map(|manifest| manifest.project.name.clone()) + .unwrap_or_else(|| "default".to_string()), + version: self + .manifest + .as_ref() + .map(|manifest| manifest.project.version.clone()) + .unwrap_or_else(|| "0.0.0".to_string()), + origin: PackageOrigin::Root, + revision: None, + content_digest: None, + module_root: self.root.clone(), + entry_module: "index.mq.md".to_string(), + manifest: self.manifest.clone(), + }) + } + + /// Resolve a declared `@alias` dependency from the root project. pub fn resolve_alias(&self, alias: &str) -> Result { - let manifest = self + self.resolve_alias_from(&self.root_package()?, alias) + } + + /// Resolve `@alias` from the manifest of the package containing the + /// importing module. This supports transitive dependencies without + /// implicit searching. + pub fn resolve_alias_from( + &self, + from: &ResolvedPackage, + alias: &str, + ) -> Result { + let manifest = from .manifest .as_ref() - .ok_or_else(|| "no marqdo.yaml is available for dependency imports".to_string())?; + .ok_or_else(|| format!("package `{}` has no marqdo.yaml", from.instance_id))?; let dependency = manifest .dependencies .get(alias) - .ok_or_else(|| format!("unknown dependency `{alias}`"))?; - let declared = dependency.path.as_ref().ok_or_else(|| { - format!("dependency `{alias}` has no local path source in this build") - })?; - let declared_path = self.root.join(declared); - let canonical = declared_path.canonicalize().map_err(|error| { - format!( - "cannot open dependency `{alias}` at `{}`: {error}", - declared_path.display() - ) - })?; - let (module_root, entry_module) = if canonical.is_dir() { - let entry = canonical.join("index.mq.md"); - if !entry.is_file() { - return Err(format!( - "dependency `{alias}` directory `{}` has no index.mq.md", - canonical.display() - )); - } - (canonical, "index.mq.md".to_string()) - } else { - let parent = canonical - .parent() - .ok_or_else(|| format!("dependency `{alias}` has no parent directory"))? - .to_path_buf(); - let name = canonical - .file_name() - .map(|name| name.to_string_lossy().to_string()) - .ok_or_else(|| format!("dependency `{alias}` has no file name"))?; - (parent, name) - }; - let dependency_manifest = if module_root.join(MANIFEST_FILE).is_file() { - Some(Manifest::load(&module_root.join(MANIFEST_FILE))?) - } else { - None - }; - let name = dependency_manifest - .as_ref() - .map(|manifest| manifest.project.name.clone()) - .unwrap_or_else(|| alias.to_string()); - let version = dependency_manifest - .as_ref() - .map(|manifest| manifest.project.version.clone()) - .unwrap_or_else(|| "0.0.0".to_string()); - let digest = hash_tree(&module_root)?; - let instance_id = dependency_instance_id(&name, &version, Some(&digest)); - if let Some(lock) = &self.lock { - if let Some(locked) = lock.package(alias) { - if locked.sha256.as_deref().is_some_and(|sha| sha != digest) { - return Err(format!( - "dependency `{alias}` content changed since marqdo.lock; run `marqdo lock`" - )); + .ok_or_else(|| format!("unknown dependency `{alias}` in package `{}`", from.name))?; + self.resolve_dependency( + &from.module_root, + manifest, + alias, + dependency, + &from.instance_id, + ) + } + + fn resolve_dependency( + &self, + base: &Path, + _manifest: &Manifest, + alias: &str, + dependency: &Dependency, + from_instance: &str, + ) -> Result { + if let Some(path) = &dependency.path { + let declared_path = base.join(path); + let canonical = declared_path.canonicalize().map_err(|error| { + format!( + "cannot open dependency `{alias}` at `{}`: {error}", + declared_path.display() + ) + })?; + let (module_root, entry_module) = package_root_for(&canonical, alias)?; + let dependency_manifest = load_optional_manifest(&module_root)?; + let (name, version) = package_name_version(&dependency_manifest, alias); + let digest = hash_tree(&module_root)?; + let instance_id = dependency_instance_id(&name, &version, Some(&digest)); + self.check_lock( + alias, + &PackageOrigin::Path { + declared: path.clone(), } - } + .source(), + Some(&digest), + )?; + // Prefer a materialized env copy only when it is the same content. + let env_candidate = self.env_root.join(env_dir_name(&instance_id)); + let module_root = if env_candidate.is_dir() + && hash_tree(&env_candidate).ok().as_deref() == Some(&digest) + { + env_candidate + } else { + module_root + }; + return Ok(ResolvedModule { + package: ResolvedPackage { + instance_id, + name, + version, + origin: PackageOrigin::Path { + declared: path.clone(), + }, + revision: None, + content_digest: Some(digest), + module_root, + entry_module: entry_module.clone(), + manifest: dependency_manifest, + }, + relative_path: entry_module, + }); } + let (url, rev) = match (&dependency.git, &dependency.rev) { + (Some(url), Some(rev)) => (url.clone(), rev.clone()), + (Some(_), None) => { + return Err(format!( + "dependency `{alias}` uses git without an exact rev" + )) + } + _ => return Err(format!("dependency `{alias}` has no source")), + }; + let checkout = self.git_checkout(&url, &rev, from_instance)?; + let dependency_manifest = load_optional_manifest(&checkout)?; + let (name, version) = package_name_version(&dependency_manifest, alias); + let digest = hash_tree(&checkout)?; + let instance_id = dependency_instance_id(&name, &version, Some(&digest)); + // Prefer a materialized env copy when sync produced one. + let env_candidate = self.env_root.join(env_dir_name(&instance_id)); + let module_root = if env_candidate.is_dir() { + env_candidate + } else { + checkout + }; + let entry_module = "index.mq.md".to_string(); + if !module_root.join(&entry_module).is_file() { + return Err(format!( + "dependency `{alias}` has no {entry_module} at `{}`", + module_root.display() + )); + } + self.check_lock( + alias, + &PackageOrigin::Git { + url: url.clone(), + rev: rev.clone(), + } + .source(), + Some(&digest), + )?; Ok(ResolvedModule { package: ResolvedPackage { instance_id, name, version, - origin: PackageOrigin::Path { - declared: declared.clone(), + origin: PackageOrigin::Git { + url, + rev: rev.clone(), }, - revision: None, + revision: Some(rev), content_digest: Some(digest), module_root, entry_module: entry_module.clone(), @@ -471,6 +577,172 @@ impl PackageResolver { }) } + fn check_lock(&self, alias: &str, source: &str, digest: Option<&str>) -> Result<(), String> { + let Some(lock) = &self.lock else { + return Ok(()); + }; + let Some(locked) = lock.package(alias) else { + return Ok(()); + }; + if locked.source != source { + return Err(format!( + "dependency `{alias}` source changed from `{}` to `{source}` since marqdo.lock", + locked.source + )); + } + if let (Some(expected), Some(actual)) = (locked.sha256.as_deref(), digest) { + if expected != actual { + return Err(format!( + "dependency `{alias}` content changed since marqdo.lock; run `marqdo lock`" + )); + } + } + Ok(()) + } + + fn git_checkout(&self, url: &str, rev: &str, _from_instance: &str) -> Result { + let digest = { + let mut hasher = Sha256::new(); + hasher.update(url.as_bytes()); + hex::encode(hasher.finalize()) + }; + let target = + self.cache_root + .join("git") + .join(format!("{}-{}", &digest[..16], safe_component(rev))); + if target.is_dir() { + return Ok(target); + } + if !self.fetch { + return Err(format!( + "Git dependency `{url}` at `{rev}` is not cached; run `marqdo sync` (or `marqdo lock`) first" + )); + } + let parent = target + .parent() + .ok_or_else(|| "invalid Git cache path".to_string())?; + std::fs::create_dir_all(parent) + .map_err(|error| format!("cannot create Git cache: {error}"))?; + let staging = parent.join(format!( + ".tmp-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0) + )); + let _ = std::fs::remove_dir_all(&staging); + let result = (|| -> Result<(), String> { + let status = std::process::Command::new("git") + .args([ + "clone", + "--quiet", + "--no-checkout", + "--config", + "core.hooksPath=/dev/null", + url, + ]) + .arg(&staging) + .env("GIT_TERMINAL_PROMPT", "0") + .status() + .map_err(|error| format!("cannot execute git: {error}"))?; + if !status.success() { + return Err(format!("git clone failed for `{url}`")); + } + let status = std::process::Command::new("git") + .args(["checkout", "--quiet", "--detach", rev]) + .current_dir(&staging) + .env("GIT_TERMINAL_PROMPT", "0") + .status() + .map_err(|error| format!("cannot execute git checkout: {error}"))?; + if !status.success() { + return Err(format!( + "git checkout of revision `{rev}` failed for `{url}`" + )); + } + let _ = std::fs::remove_dir_all(staging.join(".git")); + std::fs::rename(&staging, &target) + .map_err(|error| format!("cannot publish Git cache: {error}"))?; + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_dir_all(&staging); + } + result.map(|()| target) + } + + /// Materialize the locked path/Git packages into `.marqdo/env`. + pub fn materialize_env(&self, manifest: &Manifest, lock: &LockFile) -> Result<(), String> { + let fetch_resolver = PackageResolver { + fetch: true, + lock: Some(lock.clone()), + ..self.clone() + }; + let mut packages = Vec::new(); + for name in manifest.dependencies.keys() { + let resolved = fetch_resolver.resolve_alias(name)?; + packages.push((name.clone(), resolved.package)); + } + let staging = self.env_root.with_file_name(format!( + "env.tmp-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0) + )); + let _ = std::fs::remove_dir_all(&staging); + std::fs::create_dir_all(&staging) + .map_err(|error| format!("cannot create environment staging: {error}"))?; + let result = (|| -> Result<(), String> { + let mut records = Vec::new(); + for (alias, package) in &packages { + let directory = staging.join(env_dir_name(&package.instance_id)); + copy_tree(&package.module_root, &directory)?; + records.push(serde_json::json!({ + "alias": alias, + "instance_id": package.instance_id, + "source": package.origin.source(), + "sha256": package.content_digest, + })); + } + let index = serde_json::json!({ + "schema": "marqdo.env", + "schema_version": 1, + "packages": records, + }); + std::fs::write( + staging.join("env.json"), + serde_json::to_vec_pretty(&index).unwrap_or_default(), + ) + .map_err(|error| format!("cannot write env index: {error}"))?; + let backup = self.env_root.with_file_name(format!( + "env.old-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0) + )); + if self.env_root.exists() { + std::fs::rename(&self.env_root, &backup) + .map_err(|error| format!("cannot replace old environment: {error}"))?; + } + if let Err(error) = std::fs::rename(&staging, &self.env_root) { + if backup.exists() { + let _ = std::fs::rename(&backup, &self.env_root); + } + return Err(format!("cannot publish environment: {error}")); + } + let _ = std::fs::remove_dir_all(&backup); + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_dir_all(&staging); + } + result + } + /// Resolve a path relative to an already resolved package, rejecting /// traversal outside that package's module root. pub fn resolve_relative( @@ -499,6 +771,102 @@ impl PackageResolver { } } +fn package_root_for(canonical: &Path, alias: &str) -> Result<(PathBuf, String), String> { + if canonical.is_dir() { + let entry = canonical.join("index.mq.md"); + if !entry.is_file() { + return Err(format!( + "dependency `{alias}` directory `{}` has no index.mq.md", + canonical.display() + )); + } + Ok((canonical.to_path_buf(), "index.mq.md".to_string())) + } else { + let parent = canonical + .parent() + .ok_or_else(|| format!("dependency `{alias}` has no parent directory"))? + .to_path_buf(); + let name = canonical + .file_name() + .map(|name| name.to_string_lossy().to_string()) + .ok_or_else(|| format!("dependency `{alias}` has no file name"))?; + Ok((parent, name)) + } +} + +fn load_optional_manifest(root: &Path) -> Result, String> { + if root.join(MANIFEST_FILE).is_file() { + Ok(Some(Manifest::load(&root.join(MANIFEST_FILE))?)) + } else { + Ok(None) + } +} + +fn package_name_version(manifest: &Option, alias: &str) -> (String, String) { + ( + manifest + .as_ref() + .map(|manifest| manifest.project.name.clone()) + .unwrap_or_else(|| alias.to_string()), + manifest + .as_ref() + .map(|manifest| manifest.project.version.clone()) + .unwrap_or_else(|| "0.0.0".to_string()), + ) +} + +fn safe_component(value: &str) -> String { + value + .chars() + .filter(|character| { + character.is_ascii_alphanumeric() || *character == '-' || *character == '_' + }) + .collect() +} + +fn copy_tree(source: &Path, target: &Path) -> Result<(), String> { + if !source.is_dir() { + return Err(format!( + "package source `{}` is not a directory", + source.display() + )); + } + for entry in walkdir::WalkDir::new(source).follow_links(false) { + let entry = entry.map_err(|error| format!("cannot walk package: {error}"))?; + let relative = entry + .path() + .strip_prefix(source) + .map_err(|_| "package path escaped its root".to_string())?; + if relative.as_os_str() == ".git" || relative.starts_with(".git/") { + continue; + } + let destination = target.join(relative); + if entry.file_type().is_symlink() { + return Err(format!( + "package contains a symlink `{}`; refusing to materialize it", + entry.path().display() + )); + } + if entry.file_type().is_dir() { + std::fs::create_dir_all(&destination) + .map_err(|error| format!("cannot create `{}`: {error}", destination.display()))?; + } else if entry.file_type().is_file() { + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("cannot create `{}`: {error}", parent.display()))?; + } + std::fs::copy(entry.path(), &destination).map_err(|error| { + format!( + "cannot copy `{}` to `{}`: {error}", + entry.path().display(), + destination.display() + ) + })?; + } + } + Ok(()) +} + /// One pinned dependency entry in `marqdo.lock`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct LockedPackage { @@ -554,8 +922,6 @@ impl LockFile { lock.lock_version )); } - // v1 had no logical instance id; derive one so callers always see the - // package-aware shape. Writers emit v2. for package in &mut lock.packages { package.migrate_instance_id(); } @@ -732,6 +1098,128 @@ capabilities: assert!(Manifest::from_yaml(yaml).is_err()); } + fn have_git() -> bool { + std::process::Command::new("git") + .arg("--version") + .output() + .map(|output| output.status.success()) + .unwrap_or(false) + } + + fn git(dir: &Path, args: &[&str]) { + let status = std::process::Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .status() + .expect("git"); + assert!(status.success(), "git {args:?} failed"); + } + + #[test] + fn pinned_git_dependency_resolves_and_reuses_cache() { + if !have_git() { + eprintln!("skipping: git is unavailable"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + let dep = dir.path().join("dep"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::create_dir_all(&dep).unwrap(); + std::fs::write( + dep.join(MANIFEST_FILE), + "project:\n name: remote\n version: 0.2.0\n", + ) + .unwrap(); + std::fs::write(dep.join("index.mq.md"), "## hello\n\n*1*\n").unwrap(); + git(&dep, &["init", "-q"]); + git(&dep, &["add", "."]); + git(&dep, &["commit", "-q", "-m", "initial"]); + let rev = String::from_utf8( + std::process::Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&dep) + .output() + .unwrap() + .stdout, + ) + .unwrap(); + std::fs::write( + root.join(MANIFEST_FILE), + format!( + "project:\n name: p\n version: 0.1.0\ndependencies:\n remote:\n git: {}\n rev: {}\n", + dep.display(), + rev.trim() + ), + ) + .unwrap(); + let manifest = Manifest::load(&root.join(MANIFEST_FILE)).unwrap(); + let resolver = + PackageResolver::new(root.clone(), Some(manifest.clone()), None).with_fetch(true); + let resolved = resolver.resolve_alias("remote").unwrap(); + assert_eq!(resolved.package.name, "remote"); + assert_eq!(resolved.package.version, "0.2.0"); + assert!(resolved + .package + .instance_id + .starts_with("dep:remote@0.2.0#")); + assert_eq!(resolved.package.revision.as_deref(), Some(rev.trim())); + assert!(resolved.package.module_root.join("index.mq.md").is_file()); + + // A later resolver without fetch reads the content-addressed cache. + let offline = PackageResolver::new(root, Some(manifest), None); + let again = offline.resolve_alias("remote").unwrap(); + assert_eq!(again.package.instance_id, resolved.package.instance_id); + } + + #[test] + fn sync_materializes_a_content_checked_environment() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + let dep = root.join("deps/geometry"); + std::fs::create_dir_all(&dep).unwrap(); + std::fs::write( + dep.join(MANIFEST_FILE), + "project:\n name: geometry\n version: 0.1.0\n", + ) + .unwrap(); + std::fs::write(dep.join("index.mq.md"), "## area\n\n*1*\n").unwrap(); + std::fs::write( + root.join(MANIFEST_FILE), + "project:\n name: p\n version: 0.1.0\ndependencies:\n geometry:\n path: ./deps/geometry\n", + ) + .unwrap(); + let manifest = Manifest::load(&root.join(MANIFEST_FILE)).unwrap(); + let resolver = + PackageResolver::new(root.clone(), Some(manifest.clone()), None).with_fetch(true); + let resolved = resolver.resolve_alias("geometry").unwrap(); + let lock = LockFile { + lock_version: 2, + packages: vec![LockedPackage { + name: "geometry".into(), + instance_id: resolved.package.instance_id.clone(), + source: resolved.package.origin.source(), + sha256: resolved.package.content_digest.clone(), + }], + }; + resolver.materialize_env(&manifest, &lock).unwrap(); + let env = root.join(".marqdo/env"); + assert!(env.join("env.json").is_file()); + assert!(env + .join(env_dir_name(&resolved.package.instance_id)) + .join("index.mq.md") + .is_file()); + // A resolver with the lock reads the materialized copy. + let read = PackageResolver::new(root, Some(manifest), Some(lock)); + let again = read.resolve_alias("geometry").unwrap(); + assert_eq!(again.package.instance_id, resolved.package.instance_id); + assert!(again.package.module_root.starts_with(env)); + } + #[test] fn discovers_project_root() { let dir = tempfile::tempdir().unwrap(); diff --git a/docs/STATUS.md b/docs/STATUS.md index 224961a..7b8fd0c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -44,7 +44,7 @@ providers). | R2 ABI/Kernel/Broker | **In progress** | `KernelRegistry` external providers; complete-name plugin export routing; `CapabilityBroker` with vendor policy; static `check --plugin` signatures; multi-export C test; provider tests. Isolated worker and handle table still pending | | R3 Tool Protocol | **Done** | `tool install/list/remove`; unknown-command fallback; atomic local install; SHA/tamper checks; black-box tests for arbitrary names, Unicode args, conflicts, traversal and PATH hijacking | | R4 Analysis/Trace v1 | **Done** | `check --emit` Analysis v1; `run/call --trace-file` Trace v1; deterministic schema tests; independent Python consumer; stale-artifact removal; value redaction | -| R5 environment/isolation | Not started | — | +| R5 environment/isolation | **Partial** | Pinned Git fetch/cache, transitive `@alias` resolution, atomic `.marqdo/env` materialization, foreign process-group cleanup. Strict OS isolation remains explicitly unavailable (fail-closed) | | R6 release/consumers | Not started | — | Reproduce the evidence: @@ -66,7 +66,7 @@ cargo clippy --workspace --all-targets -- -D warnings | **P0** Formal specs + parser goldens | **Done** | `spec/markdown-profile.md`, `spec/mapping.md`, `spec/expression.ebnf`, `spec/evaluation.md`; frozen profile test `crates/marqdo-markdown/src/profile.rs`; 150 AST fixtures + goldens in `tests/conformance/ast/`; 48 compile-error fixtures in `tests/negative/`; `inspect --ast`; `conformance_ast.rs`/`negative.rs` harnesses. | | **P1** Interpreter, expressions, control flow | **Done** | `marqdo-expression`, `marqdo-ir`, `marqdo-frontend`, `marqdo-runtime`; factorial/countdown examples run without Python (`examples/factorial.mq.md`, `examples/hello.mq.md`); 36 execution goldens in `tests/conformance/semantics/`; runtime tests `crates/marqdo-runtime/tests/interpreter.rs`; prose invariance `crates/cli/tests/properties.rs`. | | **P2** Modules, references, CLI experience | **Partial** | Imports/ModuleId/UnitId/anchor/cycle detection + `run/call/check/inspect/fmt` (`crates/cli`), multi-module example and `multi_module_imports_and_cycles` CLI test; `check --refs` reference validation (all loaded documents, on-demand static analysis of link targets) and `inspect --refs` static graph. Missing: cross-module link calls, rename/interface-lock tooling, canonical AST-preserving `fmt` (see §3). | -| **P3** Package environment | **Partial** | `marqdo.yaml`/`marqdo.lock` types+validation (`marqdo-package`), `init`, `add`/`remove` (manifest editing), deterministic `lock`, `sync --locked`; CLI test `lock_and_sync_locked_detect_dependency_tampering`; `examples/multi-module`. Missing: git resolution, transitive deps, registry, `tree/update/vendor`, `--offline`, environment materialization. | +| **P3** Package environment | **Partial** | `marqdo.yaml`/`marqdo.lock` types+validation, `init`, `add`/`remove`, deterministic `lock`, `sync --locked`; M0.6 adds pinned Git fetch/cache, transitive `@alias` resolution and atomic `.marqdo/env` materialization. Missing: registry, `tree/update/vendor`, `--offline`, full conflict solver. | | **P4** Security core and isolation | **Partial** | `CapabilityHost`, capabilities at execution points (console and math included), project-rooted FS with traversal rejection, budgets, effect trace, pure/project modes, **strict refuses without OS isolation** (`marqdo-sandbox`); CLI test `strict_mode_refuses_without_isolation`; host unit tests. Missing: OS isolation adapter, plugin worker, network enforcement. | | **P5** Math and Foreign | **Partial (documented subset)** | `marqdo-math` exact-rational formula kernel for the grammar in `spec/math.md`; `math.parse/eval/substitute` host ops; `marqdo-foreign` Python/Node one-shot kernels with typed `foreign.call`; tests `crates/marqdo-math/tests/kernel.rs`, `crates/marqdo-foreign/tests/kernels.rs`; examples `math.mq.md`, `python-foreign.mq.md`. Missing: `simplify`/`differentiate`, matrices/series/symbolic algebra, foreign sessions, Python environment management. | | **P6** Plugin ABI and developer experience | **Partial** | Frozen `include/marqdo_plugin_abi_v1.h`; C ABI mirrors/layout assertions, version negotiation, `plugin.yaml` validation, target triple + sha256 verification, wire envelopes, native loader (`marqdo-plugin-abi`); Rust SDK macro (`marqdo-plugin-sdk`); C demo (`examples/plugin-demo`); CLI `plugin inspect` and explicitly trusted in-process invocation (`--plugin`, `--trust-native-plugin`, `--allow-plugin`); tests `plugin_inspect_and_explicit_trusted_invocation`, `crates/marqdo-plugin-abi/tests/loader_unix.rs`. Missing: isolated worker transport, plugin capability bridge, installation, Rust plugin example crate. | diff --git a/spec/package-format.md b/spec/package-format.md index c2bb178..64e9730 100644 --- a/spec/package-format.md +++ b/spec/package-format.md @@ -120,8 +120,8 @@ order-insensitive. ### 3.2 Tree hash -For a path dependency, `lock` hashes the dependency tree -(`crates/cli/src/commands.rs::hash_tree`): +For a path or Git dependency, `lock` hashes the resolved package tree +(`marqdo_package::hash_tree`): 1. walk the dependency directory with `walkdir` (`sort_by_file_name`), keeping regular files only (symlinks are not followed and are skipped); @@ -141,15 +141,18 @@ detects content changes and additions/removals of regular files. 1. discovers the project root from the cwd; 2. loads and validates `marqdo.yaml`; 3. for each manifest dependency, in map (sorted) order: - * a git source → error "git dependencies are not supported by this build - (path dependencies only)"; no lock is written; - * no path source → error; - * the dependency path must exist; the tree is hashed; -4. writes `marqdo.lock` (exit 0, `locked N package(s)`). + * a `path` source: the directory/file must exist; its tree is hashed; + * a `git` source: an exact `rev` is required. `lock` clones it into + `.marqdo/cache/git/-`, checks out that revision, strips + `.git`, and hashes the tree. Install hooks and submodules are not run; + * no source → error; +4. records `instance_id`, `source` and `sha256` for each package; +5. writes `marqdo.lock` (exit 0, `locked N package(s)`). -What it does **not** do: resolve semver ranges, traverse a dependency's own -`marqdo.yaml`/transitive dependencies, clone git repositories, download -registries, verify package signatures, or install anything. +What it does **not** do: resolve SemVer ranges, download a registry, verify +package signatures, install packages or run dependency scripts. A dependency's +own `marqdo.yaml` is used for transitive `@alias` imports (R1/R5); the lock is +still a flat direct-dependency record. ## 4. `sync` @@ -157,18 +160,22 @@ registries, verify package signatures, or install anything. 1. discovers the root; loads the manifest and the existing `marqdo.lock` (missing lock → error "run `marqdo lock` first", exit 1); -2. recomputes the expected lock file exactly as `lock` would (including tree - hashes); +2. recomputes the expected lock file exactly as `lock` would (including Git + cache resolution and tree hashes); 3. with `--locked`, compares the expected lock with the on-disk lock; a - difference (manifest change or dependency content change) prints - "lockfile is out of date or dependency contents changed; run `marqdo lock`" - and exits 1; -4. without `--locked`, prints `synchronized N package(s)` and exits 0. + difference (manifest change, source change or dependency content change) + prints "lockfile is out of date or dependency contents changed; run + `marqdo lock`" and exits 1; +4. materializes the locked packages into a project-private + `.marqdo/env//` tree, writes `.marqdo/env/env.json`, and + publishes it atomically (staging + rename, old env restored on failure); +5. without `--locked`, also refreshes the env; with `--locked`, the materialized + content must match the lock. `run`/`check` prefer a materialized copy only + when its tree digest matches the resolved package digest. -Current behavior: `sync` **never rewrites** the lock file and never materializes -a `.marqdo/env` directory. It is a verification command only. There is no -`--offline` flag and no `lock --locked` flag (the blueprint example -`marqdo lock --locked` is not implemented; use `sync --locked`). +`sync` never rewrites the lock file. There is no `--offline` flag yet; a +missing Git cache causes a clear "not cached; run sync/lock" error rather than +an implicit network fetch during `check`/`run`. ## 5. `init` @@ -215,8 +222,7 @@ stale lock entry). They do not fetch, install, resolve or authorize anything. `marqdo add --git --rev `: validate the dependency shape (same rules as §2.2), refuse an existing name (exit 1), insert it, write the manifest, and tell the user to run - `marqdo lock`. Adding a git dependency is allowed at the manifest level even - though `lock` will refuse it later. + `marqdo lock`. A pinned Git dependency is lockable/resolvable in R5. * `marqdo remove `: remove the dependency from the manifest (exit 1 if absent), and drop a same-named entry from an existing lock file. It does not re-hash or re-lock the rest of the project. @@ -245,12 +251,12 @@ inconsistency to fix). | Capability | Status | |---|---| -| Git dependency resolution (clone/fetch/checkout) | Not implemented. Manifest accepts `git`+`rev`; `lock` refuses it. | +| Git dependency resolution (pinned rev, clone/fetch/checkout) | Implemented for exact revisions; no floating branches, submodules or dependency scripts. | | Registry / package publishing / SemVer resolution | Not implemented. | -| Transitive dependency graph and conflict resolution | Not implemented; only direct manifest dependencies are hashed. | +| Transitive dependency graph and conflict resolution | Transitive `@alias` imports resolve from the dependency's own manifest; the lock remains flat and a full conflict solver is not implemented. | | `tree`, `update`, `vendor`, `package` commands | Not implemented. | -| `--offline`, `--locked` on `lock` | Not implemented. | -| Environment materialization (`.marqdo/env`, `cache`, `kernels`, `sessions`) | Not implemented; `init` only writes `.marqdo/` to `.gitignore`. | +| `--offline`, `--locked` on `lock` | Not implemented; missing cache fails instead of fetching during check/run. | +| Environment materialization (`.marqdo/env`, `cache`) | Implemented: `sync` writes `.marqdo/env` atomically and `lock` populates the Git cache. Kernels/sessions are not materialized. | | Dependency signatures / trusted publishers / supply-chain scanning | Not implemented. | | Multiple/nested project boundary diagnostics | Not implemented; nearest `marqdo.yaml` wins. | | Lock file migration between `lock_version`s | v1 is read and migrated; writers emit v2. | diff --git a/spec/security.md b/spec/security.md index 226a10d..8af883c 100644 --- a/spec/security.md +++ b/spec/security.md @@ -185,10 +185,10 @@ overflow). In-process math and native plugins are not OS-limited. * direct `argv` exec (no shell, no interpolation); stdin carries the versioned request envelope; stdout/stderr are always piped and captured; * the environment is cleared except `PATH`; -* the wall-clock deadline is enforced by polling `try_wait` and killing the - direct child; output flooding kills the child once the combined byte budget - is exceeded; a 2-second reader grace prevents a grandchild holding the pipes - from hanging the host; +* on Unix the child is a process-group leader (`setpgid`) and the wall-clock + deadline/output overflow kills the whole group, including grandchildren; a + 2-second reader grace prevents a surviving pipe holder from hanging the + host. Non-Unix targets only kill the direct child; * a non-zero exit is data (`RunStatus::Failed`, stderr preserved); a timeout is `TimedOut` for `run` and an error for the typed `call`.