feat(m0.6-r5): pinned Git packages and materialized environments

- add content-addressed Git checkout cache for exact revisions; no hooks,
  submodules or floating branches
- resolve transitive @alias imports from each package's own manifest
- materialize locked path/Git packages into .marqdo/env atomically and prefer
  a matching env copy in the resolver
- make sync a real environment sync while keeping lock immutable
- kill Unix foreign-kernel process groups so grandchildren do not survive
- add Git/env/transitive integration tests and update package/security specs
This commit is contained in:
cflmy committed 2026-11-20 11:49:05 +08:00
1 parent b5a28a9825
commit 54c7b43d15
12 files changed
+862 -118

No files matched your search

Generated
+2
View File
@@ -360,6 +360,7 @@ dependencies = [
name = "marqdo-foreign"
version = "0.1.0"
dependencies = [
"libc",
"marqdo-span",
"marqdo-value",
"serde_json",
@@ -437,6 +438,7 @@ version = "0.1.0"
dependencies = [
"hex",
"serde",
"serde_json",
"serde_yaml",
"sha2",
"tempfile",
+1
View File
@@ -55,5 +55,6 @@ thiserror = "2"
clap = { version = "4", features = ["derive"] }
walkdir = "2"
glob = "0.3"
libc = "0.2"
unicode-normalization = "0.1"
unicode-ident = "1"
+12 -3
View File
@@ -1111,6 +1111,13 @@ pub fn sync(locked: bool, json: bool) -> i32 {
);
return 1;
}
let resolver =
marqdo_package::PackageResolver::new(root, Some(manifest.clone()), Some(lockfile))
.with_fetch(true);
if let Err(error) = resolver.materialize_env(&manifest, &expected) {
eprintln!("error: cannot materialize project environment: {error}");
return 1;
}
if json {
output::print_json(&json!({
"command": "sync",
@@ -1120,9 +1127,10 @@ pub fn sync(locked: bool, json: bool) -> i32 {
}));
} else {
println!(
"synchronized {} package(s){}",
"synchronized {} package(s){} into {}",
expected.packages.len(),
if locked { " (locked)" } else { "" }
if locked { " (locked)" } else { "" },
resolver.env_root().display()
);
}
0
@@ -1143,7 +1151,8 @@ fn build_lockfile(
root: &Path,
manifest: &marqdo_package::Manifest,
) -> Result<marqdo_package::LockFile, String> {
let resolver = marqdo_package::PackageResolver::new(root, Some(manifest.clone()), None);
let resolver =
marqdo_package::PackageResolver::new(root, Some(manifest.clone()), None).with_fetch(true);
let mut packages = Vec::new();
for name in manifest.dependencies.keys() {
let resolved = resolver.resolve_alias(name)?;
+1 -1
View File
@@ -254,7 +254,7 @@ impl Loader {
for (alias, target, span) in imports {
let resolved = if let Some(alias_name) = target.strip_prefix('@') {
self.resolver.resolve_alias(alias_name)
self.resolver.resolve_alias_from(package, alias_name)
} else {
self.resolver.resolve_relative(package, &path, &target)
};
+207
View File
@@ -0,0 +1,207 @@
//! Package instance identity, transitive imports, pinned Git and environment
//! materialization (M0.6 R1/R5).
#![cfg(unix)]
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use std::sync::atomic::{AtomicU32, Ordering};
static COUNTER: AtomicU32 = AtomicU32::new(0);
struct TempDir(PathBuf);
impl TempDir {
fn new(tag: &str) -> Self {
let id = COUNTER.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"marqdo-package-test-{}-{id}-{tag}",
std::process::id()
));
std::fs::create_dir_all(&path).unwrap();
Self(path)
}
fn path(&self) -> &Path {
&self.0
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn binary() -> &'static str {
env!("CARGO_BIN_EXE_marqdo")
}
fn cli(dir: &Path, args: &[&str]) -> Output {
Command::new(binary())
.args(args)
.current_dir(dir)
.output()
.expect("run marqdo")
}
fn write(path: &Path, text: &str) {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(path, text).unwrap();
}
fn have_git() -> bool {
Command::new("git")
.arg("--version")
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
fn git(dir: &Path, args: &[&str]) {
let status = Command::new("git")
.args(args)
.current_dir(dir)
.env("GIT_AUTHOR_NAME", "test")
.env("GIT_AUTHOR_EMAIL", "test@example.invalid")
.env("GIT_COMMITTER_NAME", "test")
.env("GIT_COMMITTER_EMAIL", "test@example.invalid")
.status()
.expect("git");
assert!(status.success(), "git {args:?} failed");
}
#[test]
fn transitive_path_dependency_resolves_from_its_own_manifest() {
let dir = TempDir::new("transitive");
let root = dir.path();
write(
&root.join("deps/a/marqdo.yaml"),
"project:\n name: a\n version: 0.1.0\ndependencies:\n b:\n path: ./b\n",
);
write(
&root.join("deps/a/index.mq.md"),
"---\nimport:\n b: \"@b\"\n---\n\n## outer\n\n*b.inner()*\n",
);
write(
&root.join("deps/a/b/marqdo.yaml"),
"project:\n name: b\n version: 0.1.0\n",
);
write(&root.join("deps/a/b/index.mq.md"), "## inner\n\n*42*\n");
write(
&root.join("marqdo.yaml"),
"project:\n name: root\n version: 0.1.0\ndependencies:\n a:\n path: ./deps/a\n",
);
write(
&root.join("index.mq.md"),
"---\nimport:\n a: \"@a\"\n---\n\n## main\n\n> 打印(a.outer())\n",
);
let output = cli(root, &["run", "index.mq.md"]);
assert_eq!(
output.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(String::from_utf8_lossy(&output.stdout), "42\n");
assert!(
!root.join(".marqdo").exists(),
"run must not materialize env"
);
}
#[test]
fn pinned_git_dependency_locks_syncs_and_runs() {
if !have_git() {
eprintln!("skipping: git is unavailable");
return;
}
let dir = TempDir::new("git");
let root = dir.path().join("project");
let dep = dir.path().join("dep");
std::fs::create_dir_all(&root).unwrap();
std::fs::create_dir_all(&dep).unwrap();
write(
&dep.join("marqdo.yaml"),
"project:\n name: remote\n version: 0.3.0\n",
);
write(&dep.join("index.mq.md"), "## hello\n\n*7*\n");
git(&dep, &["init", "-q"]);
git(&dep, &["add", "."]);
git(&dep, &["commit", "-q", "-m", "initial"]);
let rev = String::from_utf8(
Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(&dep)
.output()
.unwrap()
.stdout,
)
.unwrap();
write(
&root.join("marqdo.yaml"),
&format!(
"project:\n name: root\n version: 0.1.0\ndependencies:\n remote:\n git: {}\n rev: {}\n",
dep.display(),
rev.trim()
),
);
write(
&root.join("index.mq.md"),
"---\nimport:\n remote: \"@remote\"\n---\n\n## main\n\n> 打印(remote.hello())\n",
);
let lock = cli(&root, &["lock"]);
assert_eq!(
lock.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&lock.stderr)
);
let lock_text = std::fs::read_to_string(root.join("marqdo.lock")).unwrap();
assert!(lock_text.contains("lock_version: 2"));
assert!(lock_text.contains("instance_id: dep:remote@0.3.0#"));
assert!(lock_text.contains("source: git:"));
let sync = cli(&root, &["sync", "--locked"]);
assert_eq!(
sync.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&sync.stderr)
);
let env_json = std::fs::read_to_string(root.join(".marqdo/env/env.json")).unwrap();
assert!(env_json.contains("dep:remote@0.3.0#"));
let run = cli(&root, &["run", "index.mq.md"]);
assert_eq!(
run.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&run.stderr)
);
assert_eq!(String::from_utf8_lossy(&run.stdout), "7\n");
}
#[test]
fn check_does_not_fetch_a_missing_git_revision() {
if !have_git() {
eprintln!("skipping: git is unavailable");
return;
}
let dir = TempDir::new("git-missing");
let root = dir.path();
write(
&root.join("marqdo.yaml"),
"project:\n name: root\n version: 0.1.0\ndependencies:\n remote:\n git: /nonexistent/repo\n rev: 0000000000000000000000000000000000000000\n",
);
write(
&root.join("index.mq.md"),
"---\nimport:\n remote: \"@remote\"\n---\n\n## main\n\n*1*\n",
);
let output = cli(root, &["check", "index.mq.md"]);
assert_eq!(output.status.code(), Some(1));
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("not cached") || stderr.contains("cannot open dependency"));
assert!(!root.join(".marqdo/cache/git").exists());
}
+3
View File
@@ -17,3 +17,6 @@ marqdo-value = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
[target.'cfg(unix)'.dependencies]
libc = { workspace = true }
+27
View File
@@ -827,6 +827,20 @@ fn spawn_and_capture(
})
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
{
use std::os::unix::process::CommandExt;
// SAFETY: setpgid is async-signal-safe and only touches the child
// process between fork and exec.
unsafe {
command.pre_exec(|| {
if libc::setpgid(0, 0) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
}
let start = Instant::now();
let mut child = command.spawn().map_err(|error| ForeignError::Spawn {
@@ -970,6 +984,19 @@ fn collect_reader(state: ReaderState) -> Vec<u8> {
}
fn kill_and_reap(child: &mut Child) {
#[cfg(unix)]
{
// The child is its own process-group leader (setpgid in pre_exec), so
// the negative pid terminates surviving grandchildren too.
let pid = child.id() as i32;
if pid > 0 {
// SAFETY: kill(2) with a negative pid targets that process group;
// failure is harmless (the child may already be gone).
unsafe {
libc::kill(-pid, libc::SIGKILL);
}
}
}
let _ = child.kill();
let _ = child.wait();
}
+1
View File
@@ -11,6 +11,7 @@ authors.workspace = true
[dependencies]
serde = { workspace = true }
serde_yaml = { workspace = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
walkdir = { workspace = true }
hex = { workspace = true }
+570 -82
View File
@@ -321,28 +321,61 @@ pub fn hash_tree(root: &Path) -> Result<String, String> {
Ok(hex::encode(hasher.finalize()))
}
/// Sanitize a logical instance id into one filesystem component.
pub fn env_dir_name(instance: &str) -> String {
instance
.chars()
.map(|character| {
if character.is_ascii_alphanumeric()
|| character == '.'
|| character == '-'
|| character == '_'
{
character
} else {
'_'
}
})
.collect()
}
/// Resolve entry files and explicit `@dependency` imports.
#[derive(Debug, Clone)]
pub struct PackageResolver {
root: PathBuf,
manifest: Option<Manifest>,
lock: Option<LockFile>,
cache_root: PathBuf,
env_root: PathBuf,
fetch: bool,
}
impl PackageResolver {
/// Build a resolver for a project root.
/// Build a resolver for a project root. Fetching is disabled; `check` and
/// `run` only read path sources, the Git cache and the materialized env.
pub fn new(
root: impl Into<PathBuf>,
manifest: Option<Manifest>,
lock: Option<LockFile>,
) -> Self {
let root = root.into();
Self {
root: root.into(),
cache_root: root.join(".marqdo/cache"),
env_root: root.join(".marqdo/env"),
root,
manifest,
lock,
fetch: false,
}
}
/// Allow `lock`/`sync` to fetch pinned Git revisions (never install
/// scripts, never submodules).
pub fn with_fetch(mut self, fetch: bool) -> Self {
self.fetch = fetch;
self
}
/// Project root.
pub fn root(&self) -> &Path {
&self.root
@@ -353,6 +386,16 @@ impl PackageResolver {
self.manifest.as_ref()
}
/// Content-addressed cache root.
pub fn cache_root(&self) -> &Path {
&self.cache_root
}
/// Materialized environment root.
pub fn env_root(&self) -> &Path {
&self.env_root
}
/// Resolve the package that owns `entry`.
pub fn resolve_entry(&self, entry: &Path) -> Result<ResolvedModule, String> {
let entry = entry
@@ -366,102 +409,165 @@ impl PackageResolver {
)
})?;
Ok(ResolvedModule {
package: ResolvedPackage {
instance_id: root_instance(self.manifest.as_ref()),
name: self
.manifest
.as_ref()
.map(|manifest| manifest.project.name.clone())
.unwrap_or_else(|| "default".to_string()),
version: self
.manifest
.as_ref()
.map(|manifest| manifest.project.version.clone())
.unwrap_or_else(|| "0.0.0".to_string()),
origin: PackageOrigin::Root,
revision: None,
content_digest: None,
module_root: self.root.clone(),
entry_module: normalize_relative(relative)?,
manifest: self.manifest.clone(),
},
package: self.root_package()?,
relative_path: normalize_relative(relative)?,
})
}
/// Resolve a declared `@alias` dependency.
fn root_package(&self) -> Result<ResolvedPackage, String> {
Ok(ResolvedPackage {
instance_id: root_instance(self.manifest.as_ref()),
name: self
.manifest
.as_ref()
.map(|manifest| manifest.project.name.clone())
.unwrap_or_else(|| "default".to_string()),
version: self
.manifest
.as_ref()
.map(|manifest| manifest.project.version.clone())
.unwrap_or_else(|| "0.0.0".to_string()),
origin: PackageOrigin::Root,
revision: None,
content_digest: None,
module_root: self.root.clone(),
entry_module: "index.mq.md".to_string(),
manifest: self.manifest.clone(),
})
}
/// Resolve a declared `@alias` dependency from the root project.
pub fn resolve_alias(&self, alias: &str) -> Result<ResolvedModule, String> {
let manifest = self
self.resolve_alias_from(&self.root_package()?, alias)
}
/// Resolve `@alias` from the manifest of the package containing the
/// importing module. This supports transitive dependencies without
/// implicit searching.
pub fn resolve_alias_from(
&self,
from: &ResolvedPackage,
alias: &str,
) -> Result<ResolvedModule, String> {
let manifest = from
.manifest
.as_ref()
.ok_or_else(|| "no marqdo.yaml is available for dependency imports".to_string())?;
.ok_or_else(|| format!("package `{}` has no marqdo.yaml", from.instance_id))?;
let dependency = manifest
.dependencies
.get(alias)
.ok_or_else(|| format!("unknown dependency `{alias}`"))?;
let declared = dependency.path.as_ref().ok_or_else(|| {
format!("dependency `{alias}` has no local path source in this build")
})?;
let declared_path = self.root.join(declared);
let canonical = declared_path.canonicalize().map_err(|error| {
format!(
"cannot open dependency `{alias}` at `{}`: {error}",
declared_path.display()
)
})?;
let (module_root, entry_module) = if canonical.is_dir() {
let entry = canonical.join("index.mq.md");
if !entry.is_file() {
return Err(format!(
"dependency `{alias}` directory `{}` has no index.mq.md",
canonical.display()
));
}
(canonical, "index.mq.md".to_string())
} else {
let parent = canonical
.parent()
.ok_or_else(|| format!("dependency `{alias}` has no parent directory"))?
.to_path_buf();
let name = canonical
.file_name()
.map(|name| name.to_string_lossy().to_string())
.ok_or_else(|| format!("dependency `{alias}` has no file name"))?;
(parent, name)
};
let dependency_manifest = if module_root.join(MANIFEST_FILE).is_file() {
Some(Manifest::load(&module_root.join(MANIFEST_FILE))?)
} else {
None
};
let name = dependency_manifest
.as_ref()
.map(|manifest| manifest.project.name.clone())
.unwrap_or_else(|| alias.to_string());
let version = dependency_manifest
.as_ref()
.map(|manifest| manifest.project.version.clone())
.unwrap_or_else(|| "0.0.0".to_string());
let digest = hash_tree(&module_root)?;
let instance_id = dependency_instance_id(&name, &version, Some(&digest));
if let Some(lock) = &self.lock {
if let Some(locked) = lock.package(alias) {
if locked.sha256.as_deref().is_some_and(|sha| sha != digest) {
return Err(format!(
"dependency `{alias}` content changed since marqdo.lock; run `marqdo lock`"
));
.ok_or_else(|| format!("unknown dependency `{alias}` in package `{}`", from.name))?;
self.resolve_dependency(
&from.module_root,
manifest,
alias,
dependency,
&from.instance_id,
)
}
fn resolve_dependency(
&self,
base: &Path,
_manifest: &Manifest,
alias: &str,
dependency: &Dependency,
from_instance: &str,
) -> Result<ResolvedModule, String> {
if let Some(path) = &dependency.path {
let declared_path = base.join(path);
let canonical = declared_path.canonicalize().map_err(|error| {
format!(
"cannot open dependency `{alias}` at `{}`: {error}",
declared_path.display()
)
})?;
let (module_root, entry_module) = package_root_for(&canonical, alias)?;
let dependency_manifest = load_optional_manifest(&module_root)?;
let (name, version) = package_name_version(&dependency_manifest, alias);
let digest = hash_tree(&module_root)?;
let instance_id = dependency_instance_id(&name, &version, Some(&digest));
self.check_lock(
alias,
&PackageOrigin::Path {
declared: path.clone(),
}
}
.source(),
Some(&digest),
)?;
// Prefer a materialized env copy only when it is the same content.
let env_candidate = self.env_root.join(env_dir_name(&instance_id));
let module_root = if env_candidate.is_dir()
&& hash_tree(&env_candidate).ok().as_deref() == Some(&digest)
{
env_candidate
} else {
module_root
};
return Ok(ResolvedModule {
package: ResolvedPackage {
instance_id,
name,
version,
origin: PackageOrigin::Path {
declared: path.clone(),
},
revision: None,
content_digest: Some(digest),
module_root,
entry_module: entry_module.clone(),
manifest: dependency_manifest,
},
relative_path: entry_module,
});
}
let (url, rev) = match (&dependency.git, &dependency.rev) {
(Some(url), Some(rev)) => (url.clone(), rev.clone()),
(Some(_), None) => {
return Err(format!(
"dependency `{alias}` uses git without an exact rev"
))
}
_ => return Err(format!("dependency `{alias}` has no source")),
};
let checkout = self.git_checkout(&url, &rev, from_instance)?;
let dependency_manifest = load_optional_manifest(&checkout)?;
let (name, version) = package_name_version(&dependency_manifest, alias);
let digest = hash_tree(&checkout)?;
let instance_id = dependency_instance_id(&name, &version, Some(&digest));
// Prefer a materialized env copy when sync produced one.
let env_candidate = self.env_root.join(env_dir_name(&instance_id));
let module_root = if env_candidate.is_dir() {
env_candidate
} else {
checkout
};
let entry_module = "index.mq.md".to_string();
if !module_root.join(&entry_module).is_file() {
return Err(format!(
"dependency `{alias}` has no {entry_module} at `{}`",
module_root.display()
));
}
self.check_lock(
alias,
&PackageOrigin::Git {
url: url.clone(),
rev: rev.clone(),
}
.source(),
Some(&digest),
)?;
Ok(ResolvedModule {
package: ResolvedPackage {
instance_id,
name,
version,
origin: PackageOrigin::Path {
declared: declared.clone(),
origin: PackageOrigin::Git {
url,
rev: rev.clone(),
},
revision: None,
revision: Some(rev),
content_digest: Some(digest),
module_root,
entry_module: entry_module.clone(),
@@ -471,6 +577,172 @@ impl PackageResolver {
})
}
fn check_lock(&self, alias: &str, source: &str, digest: Option<&str>) -> Result<(), String> {
let Some(lock) = &self.lock else {
return Ok(());
};
let Some(locked) = lock.package(alias) else {
return Ok(());
};
if locked.source != source {
return Err(format!(
"dependency `{alias}` source changed from `{}` to `{source}` since marqdo.lock",
locked.source
));
}
if let (Some(expected), Some(actual)) = (locked.sha256.as_deref(), digest) {
if expected != actual {
return Err(format!(
"dependency `{alias}` content changed since marqdo.lock; run `marqdo lock`"
));
}
}
Ok(())
}
fn git_checkout(&self, url: &str, rev: &str, _from_instance: &str) -> Result<PathBuf, String> {
let digest = {
let mut hasher = Sha256::new();
hasher.update(url.as_bytes());
hex::encode(hasher.finalize())
};
let target =
self.cache_root
.join("git")
.join(format!("{}-{}", &digest[..16], safe_component(rev)));
if target.is_dir() {
return Ok(target);
}
if !self.fetch {
return Err(format!(
"Git dependency `{url}` at `{rev}` is not cached; run `marqdo sync` (or `marqdo lock`) first"
));
}
let parent = target
.parent()
.ok_or_else(|| "invalid Git cache path".to_string())?;
std::fs::create_dir_all(parent)
.map_err(|error| format!("cannot create Git cache: {error}"))?;
let staging = parent.join(format!(
".tmp-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0)
));
let _ = std::fs::remove_dir_all(&staging);
let result = (|| -> Result<(), String> {
let status = std::process::Command::new("git")
.args([
"clone",
"--quiet",
"--no-checkout",
"--config",
"core.hooksPath=/dev/null",
url,
])
.arg(&staging)
.env("GIT_TERMINAL_PROMPT", "0")
.status()
.map_err(|error| format!("cannot execute git: {error}"))?;
if !status.success() {
return Err(format!("git clone failed for `{url}`"));
}
let status = std::process::Command::new("git")
.args(["checkout", "--quiet", "--detach", rev])
.current_dir(&staging)
.env("GIT_TERMINAL_PROMPT", "0")
.status()
.map_err(|error| format!("cannot execute git checkout: {error}"))?;
if !status.success() {
return Err(format!(
"git checkout of revision `{rev}` failed for `{url}`"
));
}
let _ = std::fs::remove_dir_all(staging.join(".git"));
std::fs::rename(&staging, &target)
.map_err(|error| format!("cannot publish Git cache: {error}"))?;
Ok(())
})();
if result.is_err() {
let _ = std::fs::remove_dir_all(&staging);
}
result.map(|()| target)
}
/// Materialize the locked path/Git packages into `.marqdo/env`.
pub fn materialize_env(&self, manifest: &Manifest, lock: &LockFile) -> Result<(), String> {
let fetch_resolver = PackageResolver {
fetch: true,
lock: Some(lock.clone()),
..self.clone()
};
let mut packages = Vec::new();
for name in manifest.dependencies.keys() {
let resolved = fetch_resolver.resolve_alias(name)?;
packages.push((name.clone(), resolved.package));
}
let staging = self.env_root.with_file_name(format!(
"env.tmp-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0)
));
let _ = std::fs::remove_dir_all(&staging);
std::fs::create_dir_all(&staging)
.map_err(|error| format!("cannot create environment staging: {error}"))?;
let result = (|| -> Result<(), String> {
let mut records = Vec::new();
for (alias, package) in &packages {
let directory = staging.join(env_dir_name(&package.instance_id));
copy_tree(&package.module_root, &directory)?;
records.push(serde_json::json!({
"alias": alias,
"instance_id": package.instance_id,
"source": package.origin.source(),
"sha256": package.content_digest,
}));
}
let index = serde_json::json!({
"schema": "marqdo.env",
"schema_version": 1,
"packages": records,
});
std::fs::write(
staging.join("env.json"),
serde_json::to_vec_pretty(&index).unwrap_or_default(),
)
.map_err(|error| format!("cannot write env index: {error}"))?;
let backup = self.env_root.with_file_name(format!(
"env.old-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0)
));
if self.env_root.exists() {
std::fs::rename(&self.env_root, &backup)
.map_err(|error| format!("cannot replace old environment: {error}"))?;
}
if let Err(error) = std::fs::rename(&staging, &self.env_root) {
if backup.exists() {
let _ = std::fs::rename(&backup, &self.env_root);
}
return Err(format!("cannot publish environment: {error}"));
}
let _ = std::fs::remove_dir_all(&backup);
Ok(())
})();
if result.is_err() {
let _ = std::fs::remove_dir_all(&staging);
}
result
}
/// Resolve a path relative to an already resolved package, rejecting
/// traversal outside that package's module root.
pub fn resolve_relative(
@@ -499,6 +771,102 @@ impl PackageResolver {
}
}
fn package_root_for(canonical: &Path, alias: &str) -> Result<(PathBuf, String), String> {
if canonical.is_dir() {
let entry = canonical.join("index.mq.md");
if !entry.is_file() {
return Err(format!(
"dependency `{alias}` directory `{}` has no index.mq.md",
canonical.display()
));
}
Ok((canonical.to_path_buf(), "index.mq.md".to_string()))
} else {
let parent = canonical
.parent()
.ok_or_else(|| format!("dependency `{alias}` has no parent directory"))?
.to_path_buf();
let name = canonical
.file_name()
.map(|name| name.to_string_lossy().to_string())
.ok_or_else(|| format!("dependency `{alias}` has no file name"))?;
Ok((parent, name))
}
}
fn load_optional_manifest(root: &Path) -> Result<Option<Manifest>, String> {
if root.join(MANIFEST_FILE).is_file() {
Ok(Some(Manifest::load(&root.join(MANIFEST_FILE))?))
} else {
Ok(None)
}
}
fn package_name_version(manifest: &Option<Manifest>, alias: &str) -> (String, String) {
(
manifest
.as_ref()
.map(|manifest| manifest.project.name.clone())
.unwrap_or_else(|| alias.to_string()),
manifest
.as_ref()
.map(|manifest| manifest.project.version.clone())
.unwrap_or_else(|| "0.0.0".to_string()),
)
}
fn safe_component(value: &str) -> String {
value
.chars()
.filter(|character| {
character.is_ascii_alphanumeric() || *character == '-' || *character == '_'
})
.collect()
}
fn copy_tree(source: &Path, target: &Path) -> Result<(), String> {
if !source.is_dir() {
return Err(format!(
"package source `{}` is not a directory",
source.display()
));
}
for entry in walkdir::WalkDir::new(source).follow_links(false) {
let entry = entry.map_err(|error| format!("cannot walk package: {error}"))?;
let relative = entry
.path()
.strip_prefix(source)
.map_err(|_| "package path escaped its root".to_string())?;
if relative.as_os_str() == ".git" || relative.starts_with(".git/") {
continue;
}
let destination = target.join(relative);
if entry.file_type().is_symlink() {
return Err(format!(
"package contains a symlink `{}`; refusing to materialize it",
entry.path().display()
));
}
if entry.file_type().is_dir() {
std::fs::create_dir_all(&destination)
.map_err(|error| format!("cannot create `{}`: {error}", destination.display()))?;
} else if entry.file_type().is_file() {
if let Some(parent) = destination.parent() {
std::fs::create_dir_all(parent)
.map_err(|error| format!("cannot create `{}`: {error}", parent.display()))?;
}
std::fs::copy(entry.path(), &destination).map_err(|error| {
format!(
"cannot copy `{}` to `{}`: {error}",
entry.path().display(),
destination.display()
)
})?;
}
}
Ok(())
}
/// One pinned dependency entry in `marqdo.lock`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LockedPackage {
@@ -554,8 +922,6 @@ impl LockFile {
lock.lock_version
));
}
// v1 had no logical instance id; derive one so callers always see the
// package-aware shape. Writers emit v2.
for package in &mut lock.packages {
package.migrate_instance_id();
}
@@ -732,6 +1098,128 @@ capabilities:
assert!(Manifest::from_yaml(yaml).is_err());
}
fn have_git() -> bool {
std::process::Command::new("git")
.arg("--version")
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
fn git(dir: &Path, args: &[&str]) {
let status = std::process::Command::new("git")
.args(args)
.current_dir(dir)
.env("GIT_AUTHOR_NAME", "test")
.env("GIT_AUTHOR_EMAIL", "test@example.invalid")
.env("GIT_COMMITTER_NAME", "test")
.env("GIT_COMMITTER_EMAIL", "test@example.invalid")
.status()
.expect("git");
assert!(status.success(), "git {args:?} failed");
}
#[test]
fn pinned_git_dependency_resolves_and_reuses_cache() {
if !have_git() {
eprintln!("skipping: git is unavailable");
return;
}
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("root");
let dep = dir.path().join("dep");
std::fs::create_dir_all(&root).unwrap();
std::fs::create_dir_all(&dep).unwrap();
std::fs::write(
dep.join(MANIFEST_FILE),
"project:\n name: remote\n version: 0.2.0\n",
)
.unwrap();
std::fs::write(dep.join("index.mq.md"), "## hello\n\n*1*\n").unwrap();
git(&dep, &["init", "-q"]);
git(&dep, &["add", "."]);
git(&dep, &["commit", "-q", "-m", "initial"]);
let rev = String::from_utf8(
std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(&dep)
.output()
.unwrap()
.stdout,
)
.unwrap();
std::fs::write(
root.join(MANIFEST_FILE),
format!(
"project:\n name: p\n version: 0.1.0\ndependencies:\n remote:\n git: {}\n rev: {}\n",
dep.display(),
rev.trim()
),
)
.unwrap();
let manifest = Manifest::load(&root.join(MANIFEST_FILE)).unwrap();
let resolver =
PackageResolver::new(root.clone(), Some(manifest.clone()), None).with_fetch(true);
let resolved = resolver.resolve_alias("remote").unwrap();
assert_eq!(resolved.package.name, "remote");
assert_eq!(resolved.package.version, "0.2.0");
assert!(resolved
.package
.instance_id
.starts_with("dep:remote@0.2.0#"));
assert_eq!(resolved.package.revision.as_deref(), Some(rev.trim()));
assert!(resolved.package.module_root.join("index.mq.md").is_file());
// A later resolver without fetch reads the content-addressed cache.
let offline = PackageResolver::new(root, Some(manifest), None);
let again = offline.resolve_alias("remote").unwrap();
assert_eq!(again.package.instance_id, resolved.package.instance_id);
}
#[test]
fn sync_materializes_a_content_checked_environment() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("root");
let dep = root.join("deps/geometry");
std::fs::create_dir_all(&dep).unwrap();
std::fs::write(
dep.join(MANIFEST_FILE),
"project:\n name: geometry\n version: 0.1.0\n",
)
.unwrap();
std::fs::write(dep.join("index.mq.md"), "## area\n\n*1*\n").unwrap();
std::fs::write(
root.join(MANIFEST_FILE),
"project:\n name: p\n version: 0.1.0\ndependencies:\n geometry:\n path: ./deps/geometry\n",
)
.unwrap();
let manifest = Manifest::load(&root.join(MANIFEST_FILE)).unwrap();
let resolver =
PackageResolver::new(root.clone(), Some(manifest.clone()), None).with_fetch(true);
let resolved = resolver.resolve_alias("geometry").unwrap();
let lock = LockFile {
lock_version: 2,
packages: vec![LockedPackage {
name: "geometry".into(),
instance_id: resolved.package.instance_id.clone(),
source: resolved.package.origin.source(),
sha256: resolved.package.content_digest.clone(),
}],
};
resolver.materialize_env(&manifest, &lock).unwrap();
let env = root.join(".marqdo/env");
assert!(env.join("env.json").is_file());
assert!(env
.join(env_dir_name(&resolved.package.instance_id))
.join("index.mq.md")
.is_file());
// A resolver with the lock reads the materialized copy.
let read = PackageResolver::new(root, Some(manifest), Some(lock));
let again = read.resolve_alias("geometry").unwrap();
assert_eq!(again.package.instance_id, resolved.package.instance_id);
assert!(again.package.module_root.starts_with(env));
}
#[test]
fn discovers_project_root() {
let dir = tempfile::tempdir().unwrap();
+2 -2
View File
@@ -44,7 +44,7 @@ providers).
| R2 ABI/Kernel/Broker | **In progress** | `KernelRegistry` external providers; complete-name plugin export routing; `CapabilityBroker` with vendor policy; static `check --plugin` signatures; multi-export C test; provider tests. Isolated worker and handle table still pending |
| R3 Tool Protocol | **Done** | `tool install/list/remove`; unknown-command fallback; atomic local install; SHA/tamper checks; black-box tests for arbitrary names, Unicode args, conflicts, traversal and PATH hijacking |
| R4 Analysis/Trace v1 | **Done** | `check --emit` Analysis v1; `run/call --trace-file` Trace v1; deterministic schema tests; independent Python consumer; stale-artifact removal; value redaction |
| R5 environment/isolation | Not started | — |
| R5 environment/isolation | **Partial** | Pinned Git fetch/cache, transitive `@alias` resolution, atomic `.marqdo/env` materialization, foreign process-group cleanup. Strict OS isolation remains explicitly unavailable (fail-closed) |
| R6 release/consumers | Not started | — |
Reproduce the evidence:
@@ -66,7 +66,7 @@ cargo clippy --workspace --all-targets -- -D warnings
| **P0** Formal specs + parser goldens | **Done** | `spec/markdown-profile.md`, `spec/mapping.md`, `spec/expression.ebnf`, `spec/evaluation.md`; frozen profile test `crates/marqdo-markdown/src/profile.rs`; 150 AST fixtures + goldens in `tests/conformance/ast/`; 48 compile-error fixtures in `tests/negative/`; `inspect --ast`; `conformance_ast.rs`/`negative.rs` harnesses. |
| **P1** Interpreter, expressions, control flow | **Done** | `marqdo-expression`, `marqdo-ir`, `marqdo-frontend`, `marqdo-runtime`; factorial/countdown examples run without Python (`examples/factorial.mq.md`, `examples/hello.mq.md`); 36 execution goldens in `tests/conformance/semantics/`; runtime tests `crates/marqdo-runtime/tests/interpreter.rs`; prose invariance `crates/cli/tests/properties.rs`. |
| **P2** Modules, references, CLI experience | **Partial** | Imports/ModuleId/UnitId/anchor/cycle detection + `run/call/check/inspect/fmt` (`crates/cli`), multi-module example and `multi_module_imports_and_cycles` CLI test; `check --refs` reference validation (all loaded documents, on-demand static analysis of link targets) and `inspect --refs` static graph. Missing: cross-module link calls, rename/interface-lock tooling, canonical AST-preserving `fmt` (see §3). |
| **P3** Package environment | **Partial** | `marqdo.yaml`/`marqdo.lock` types+validation (`marqdo-package`), `init`, `add`/`remove` (manifest editing), deterministic `lock`, `sync --locked`; CLI test `lock_and_sync_locked_detect_dependency_tampering`; `examples/multi-module`. Missing: git resolution, transitive deps, registry, `tree/update/vendor`, `--offline`, environment materialization. |
| **P3** Package environment | **Partial** | `marqdo.yaml`/`marqdo.lock` types+validation, `init`, `add`/`remove`, deterministic `lock`, `sync --locked`; M0.6 adds pinned Git fetch/cache, transitive `@alias` resolution and atomic `.marqdo/env` materialization. Missing: registry, `tree/update/vendor`, `--offline`, full conflict solver. |
| **P4** Security core and isolation | **Partial** | `CapabilityHost`, capabilities at execution points (console and math included), project-rooted FS with traversal rejection, budgets, effect trace, pure/project modes, **strict refuses without OS isolation** (`marqdo-sandbox`); CLI test `strict_mode_refuses_without_isolation`; host unit tests. Missing: OS isolation adapter, plugin worker, network enforcement. |
| **P5** Math and Foreign | **Partial (documented subset)** | `marqdo-math` exact-rational formula kernel for the grammar in `spec/math.md`; `math.parse/eval/substitute` host ops; `marqdo-foreign` Python/Node one-shot kernels with typed `foreign.call`; tests `crates/marqdo-math/tests/kernel.rs`, `crates/marqdo-foreign/tests/kernels.rs`; examples `math.mq.md`, `python-foreign.mq.md`. Missing: `simplify`/`differentiate`, matrices/series/symbolic algebra, foreign sessions, Python environment management. |
| **P6** Plugin ABI and developer experience | **Partial** | Frozen `include/marqdo_plugin_abi_v1.h`; C ABI mirrors/layout assertions, version negotiation, `plugin.yaml` validation, target triple + sha256 verification, wire envelopes, native loader (`marqdo-plugin-abi`); Rust SDK macro (`marqdo-plugin-sdk`); C demo (`examples/plugin-demo`); CLI `plugin inspect` and explicitly trusted in-process invocation (`--plugin`, `--trust-native-plugin`, `--allow-plugin`); tests `plugin_inspect_and_explicit_trusted_invocation`, `crates/marqdo-plugin-abi/tests/loader_unix.rs`. Missing: isolated worker transport, plugin capability bridge, installation, Rust plugin example crate. |
+32 -26
View File
@@ -120,8 +120,8 @@ order-insensitive.
### 3.2 Tree hash
For a path dependency, `lock` hashes the dependency tree
(`crates/cli/src/commands.rs::hash_tree`):
For a path or Git dependency, `lock` hashes the resolved package tree
(`marqdo_package::hash_tree`):
1. walk the dependency directory with `walkdir` (`sort_by_file_name`), keeping
regular files only (symlinks are not followed and are skipped);
@@ -141,15 +141,18 @@ detects content changes and additions/removals of regular files.
1. discovers the project root from the cwd;
2. loads and validates `marqdo.yaml`;
3. for each manifest dependency, in map (sorted) order:
* a git source → error "git dependencies are not supported by this build
(path dependencies only)"; no lock is written;
* no path source → error;
* the dependency path must exist; the tree is hashed;
4. writes `marqdo.lock` (exit 0, `locked N package(s)`).
* a `path` source: the directory/file must exist; its tree is hashed;
* a `git` source: an exact `rev` is required. `lock` clones it into
`.marqdo/cache/git/<url-digest>-<rev>`, checks out that revision, strips
`.git`, and hashes the tree. Install hooks and submodules are not run;
* no source → error;
4. records `instance_id`, `source` and `sha256` for each package;
5. writes `marqdo.lock` (exit 0, `locked N package(s)`).
What it does **not** do: resolve semver ranges, traverse a dependency's own
`marqdo.yaml`/transitive dependencies, clone git repositories, download
registries, verify package signatures, or install anything.
What it does **not** do: resolve SemVer ranges, download a registry, verify
package signatures, install packages or run dependency scripts. A dependency's
own `marqdo.yaml` is used for transitive `@alias` imports (R1/R5); the lock is
still a flat direct-dependency record.
## 4. `sync`
@@ -157,18 +160,22 @@ registries, verify package signatures, or install anything.
1. discovers the root; loads the manifest and the existing `marqdo.lock`
(missing lock → error "run `marqdo lock` first", exit 1);
2. recomputes the expected lock file exactly as `lock` would (including tree
hashes);
2. recomputes the expected lock file exactly as `lock` would (including Git
cache resolution and tree hashes);
3. with `--locked`, compares the expected lock with the on-disk lock; a
difference (manifest change or dependency content change) prints
"lockfile is out of date or dependency contents changed; run `marqdo lock`"
and exits 1;
4. without `--locked`, prints `synchronized N package(s)` and exits 0.
difference (manifest change, source change or dependency content change)
prints "lockfile is out of date or dependency contents changed; run
`marqdo lock`" and exits 1;
4. materializes the locked packages into a project-private
`.marqdo/env/<instance-id>/` tree, writes `.marqdo/env/env.json`, and
publishes it atomically (staging + rename, old env restored on failure);
5. without `--locked`, also refreshes the env; with `--locked`, the materialized
content must match the lock. `run`/`check` prefer a materialized copy only
when its tree digest matches the resolved package digest.
Current behavior: `sync` **never rewrites** the lock file and never materializes
a `.marqdo/env` directory. It is a verification command only. There is no
`--offline` flag and no `lock --locked` flag (the blueprint example
`marqdo lock --locked` is not implemented; use `sync --locked`).
`sync` never rewrites the lock file. There is no `--offline` flag yet; a
missing Git cache causes a clear "not cached; run sync/lock" error rather than
an implicit network fetch during `check`/`run`.
## 5. `init`
@@ -215,8 +222,7 @@ stale lock entry). They do not fetch, install, resolve or authorize anything.
`marqdo add <name> --git <url> --rev <sha>`:
validate the dependency shape (same rules as §2.2), refuse an existing name
(exit 1), insert it, write the manifest, and tell the user to run
`marqdo lock`. Adding a git dependency is allowed at the manifest level even
though `lock` will refuse it later.
`marqdo lock`. A pinned Git dependency is lockable/resolvable in R5.
* `marqdo remove <name>`: remove the dependency from the manifest (exit 1 if
absent), and drop a same-named entry from an existing lock file. It does not
re-hash or re-lock the rest of the project.
@@ -245,12 +251,12 @@ inconsistency to fix).
| Capability | Status |
|---|---|
| Git dependency resolution (clone/fetch/checkout) | Not implemented. Manifest accepts `git`+`rev`; `lock` refuses it. |
| Git dependency resolution (pinned rev, clone/fetch/checkout) | Implemented for exact revisions; no floating branches, submodules or dependency scripts. |
| Registry / package publishing / SemVer resolution | Not implemented. |
| Transitive dependency graph and conflict resolution | Not implemented; only direct manifest dependencies are hashed. |
| Transitive dependency graph and conflict resolution | Transitive `@alias` imports resolve from the dependency's own manifest; the lock remains flat and a full conflict solver is not implemented. |
| `tree`, `update`, `vendor`, `package` commands | Not implemented. |
| `--offline`, `--locked` on `lock` | Not implemented. |
| Environment materialization (`.marqdo/env`, `cache`, `kernels`, `sessions`) | Not implemented; `init` only writes `.marqdo/` to `.gitignore`. |
| `--offline`, `--locked` on `lock` | Not implemented; missing cache fails instead of fetching during check/run. |
| Environment materialization (`.marqdo/env`, `cache`) | Implemented: `sync` writes `.marqdo/env` atomically and `lock` populates the Git cache. Kernels/sessions are not materialized. |
| Dependency signatures / trusted publishers / supply-chain scanning | Not implemented. |
| Multiple/nested project boundary diagnostics | Not implemented; nearest `marqdo.yaml` wins. |
| Lock file migration between `lock_version`s | v1 is read and migrated; writers emit v2. |
+4 -4
View File
@@ -185,10 +185,10 @@ overflow). In-process math and native plugins are not OS-limited.
* direct `argv` exec (no shell, no interpolation); stdin carries the versioned
request envelope; stdout/stderr are always piped and captured;
* the environment is cleared except `PATH`;
* the wall-clock deadline is enforced by polling `try_wait` and killing the
direct child; output flooding kills the child once the combined byte budget
is exceeded; a 2-second reader grace prevents a grandchild holding the pipes
from hanging the host;
* on Unix the child is a process-group leader (`setpgid`) and the wall-clock
deadline/output overflow kills the whole group, including grandchildren; a
2-second reader grace prevents a surviving pipe holder from hanging the
host. Non-Unix targets only kill the direct child;
* a non-zero exit is data (`RunStatus::Failed`, stderr preserved); a timeout is
`TimedOut` for `run` and an error for the typed `call`.