From 5877d9aa5ea098668fbe81820d22fc6b87ae5042 Mon Sep 17 00:00:00 2001 From: cflmy Date: Fri, 20 Nov 2026 11:37:16 +0800 Subject: [PATCH] feat(m0.6-r1): package-aware module identity and resolver - replace path-only ModuleId with {instance, relative_path} and canonical :: encoding - add PackageResolver/ResolvedPackage in marqdo-package for path sources; hash package trees deterministically for content-addressed instance ids - move import resolution out of the CLI loader; reject package escapes - emit lock_version 2 with instance ids and migrate v1 lockfiles - update references/package-format specs and the function-ref golden - add package resolver tests for stable identity across path moves --- Cargo.lock | 3 + crates/cli/src/commands.rs | 87 +--- crates/cli/src/loader.rs | 155 +++++-- crates/cli/src/output.rs | 2 +- crates/cli/tests/conformance_semantics.rs | 2 +- crates/cli/tests/negative.rs | 2 +- crates/cli/tests/properties.rs | 2 +- crates/marqdo-ir/src/lib.rs | 82 +++- crates/marqdo-package/Cargo.toml | 3 + crates/marqdo-package/src/lib.rs | 403 +++++++++++++++++- crates/marqdo-runtime/src/builtins_impl.rs | 2 +- crates/marqdo-runtime/src/interpreter.rs | 4 +- docs/STATUS.md | 12 + spec/package-format.md | 26 +- spec/references.md | 33 +- .../14-function-ref-value.expected.json | 2 +- 16 files changed, 665 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd40092..e4fe5ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -434,9 +434,12 @@ dependencies = [ name = "marqdo-package" version = "0.1.0" dependencies = [ + "hex", "serde", "serde_yaml", + "sha2", "tempfile", + "walkdir", ] [[package]] diff --git a/crates/cli/src/commands.rs b/crates/cli/src/commands.rs index c8f7429..a6d4c0c 100644 --- a/crates/cli/src/commands.rs +++ b/crates/cli/src/commands.rs @@ -4,12 +4,10 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; use marqdo_host::{Capabilities, Host, StdHost}; -use marqdo_ir::ModuleId; use marqdo_runtime::{run, EvaluatedArg, RuntimeError, Value}; use marqdo_sandbox::{PlatformCapabilities, SandboxMode}; use marqdo_span::Span; use serde_json::json; -use sha2::{Digest, Sha256}; use crate::args; use crate::loader::{self, FileDiagnostic, LoadedProject}; @@ -215,7 +213,7 @@ pub fn inspect(file: Option<&Path>, flags: &InspectFlags, json: bool) -> i32 { if json { let mut object = serde_json::Map::new(); object.insert("command".into(), json!("inspect")); - object.insert("module".into(), json!(entry.module_id.0)); + object.insert("module".into(), json!(entry.module_id.to_string())); if !selected_any { object.insert("summary".into(), summary_json(&project)); } @@ -253,7 +251,7 @@ pub fn inspect(file: Option<&Path>, flags: &InspectFlags, json: bool) -> i32 { return 0; } if !selected_any { - println!("module: {}", entry.module_id.0); + println!("module: {}", entry.module_id); println!("path: {}", entry.path.display()); println!("project root: {}", project.root.display()); println!("functions:"); @@ -396,7 +394,7 @@ fn refs_json(project: &LoadedProject) -> serde_json::Value { let (resolved, target_module, target_unit) = resolve_link(project, &entry.analysis.units, parent, url); edges.push(json!({ - "source": entry.module_id.0, + "source": entry.module_id.to_string(), "url": url, "span": node.span, "resolved": resolved, @@ -422,7 +420,11 @@ fn resolve_link( .map(|unit| &unit.id) }); return match unit { - Some(unit) => (true, Some(project.entry.0.clone()), Some(unit.to_string())), + Some(unit) => ( + true, + Some(project.entry.to_string()), + Some(unit.to_string()), + ), None => (false, None, None), }; } @@ -437,16 +439,9 @@ fn resolve_link( let Ok(candidate) = candidate.canonicalize() else { return (false, None, None); }; - let Ok(relative) = candidate.strip_prefix(&project.root) else { + let Some(module_id) = project.module_id_for_path(&candidate) else { return (false, None, None); }; - let module_id = ModuleId( - relative - .components() - .map(|component| component.as_os_str().to_string_lossy().to_string()) - .collect::>() - .join("/"), - ); let analyzed; let target_units = match project.files.get(&module_id) { Some(target_file) => &target_file.analysis.units, @@ -456,12 +451,12 @@ fn resolve_link( let Ok(source) = std::fs::read_to_string(&candidate) else { return (false, None, None); }; - analyzed = marqdo_frontend::analyze_source(&module_id.0, &source); + analyzed = marqdo_frontend::analyze_source(&module_id.to_string(), &source); &analyzed.units } }; let Some(fragment) = fragment else { - return (true, Some(module_id.0), None); + return (true, Some(module_id.to_string()), None); }; let resolved = target_units.anchors.get(fragment).or_else(|| { target_units @@ -471,8 +466,8 @@ fn resolve_link( .map(|unit| &unit.id) }); match resolved { - Some(unit) => (true, Some(module_id.0), Some(unit.to_string())), - None => (false, Some(module_id.0), None), + Some(unit) => (true, Some(module_id.to_string()), Some(unit.to_string())), + None => (false, Some(module_id.to_string()), None), } } @@ -950,66 +945,24 @@ fn build_lockfile( root: &Path, manifest: &marqdo_package::Manifest, ) -> Result { + let resolver = marqdo_package::PackageResolver::new(root, Some(manifest.clone()), None); let mut packages = Vec::new(); - for (name, dependency) in &manifest.dependencies { - if dependency.git.is_some() { - return Err(format!( - "dependency `{name}` uses a git source; git dependencies are not supported by this build (path dependencies only)" - )); - } - let Some(path) = &dependency.path else { - return Err(format!("dependency `{name}` has no path source")); - }; - let dependency_root = root.join(path); - let hash = hash_tree(&dependency_root)?; + for name in manifest.dependencies.keys() { + let resolved = resolver.resolve_alias(name)?; packages.push(marqdo_package::LockedPackage { name: name.clone(), - source: format!("path:{}", path.display()), - sha256: Some(hash), + instance_id: resolved.package.instance_id, + source: resolved.package.origin.source(), + sha256: resolved.package.content_digest, }); } packages.sort_by(|a, b| a.name.cmp(&b.name)); Ok(marqdo_package::LockFile { - lock_version: 1, + lock_version: 2, packages, }) } -fn hash_tree(root: &Path) -> Result { - if !root.exists() { - return Err(format!( - "dependency path `{}` does not exist", - root.display() - )); - } - let mut files = Vec::new(); - for entry in walkdir::WalkDir::new(root) - .sort_by_file_name() - .into_iter() - .filter_map(Result::ok) - { - if entry.file_type().is_file() { - let relative = entry - .path() - .strip_prefix(root) - .map(|path| path.to_string_lossy().replace('\\', "/")) - .unwrap_or_default(); - files.push((relative, entry.path().to_path_buf())); - } - } - files.sort_by(|a, b| a.0.cmp(&b.0)); - let mut hasher = Sha256::new(); - for (relative, path) in files { - hasher.update(relative.as_bytes()); - hasher.update([0u8]); - let bytes = std::fs::read(&path) - .map_err(|error| format!("cannot read `{}`: {error}", path.display()))?; - hasher.update(&bytes); - hasher.update([0u8]); - } - Ok(hex::encode(hasher.finalize())) -} - /// `marqdo add --path [--git URL --rev SHA]`. pub fn add( name: &str, diff --git a/crates/cli/src/loader.rs b/crates/cli/src/loader.rs index fbbf369..ec137c7 100644 --- a/crates/cli/src/loader.rs +++ b/crates/cli/src/loader.rs @@ -4,18 +4,25 @@ //! come only from explicit YAML front matter and are resolved exactly as //! declared (`@dependency` or a relative `.mq.md` path); nothing is searched //! implicitly. +//! +//! Module identity is package-aware (ADR-0008): a module is +//! `PackageInstanceId::relative_path`, never a physical cache path. The +//! package layer resolves path/Git sources; this loader consumes +//! `ResolvedPackage` and does not branch on the source kind. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use marqdo_frontend::{analyze_source, AnalysisResult}; use marqdo_ir::{ModuleId, Program, ResolvedImport}; -use marqdo_package::{discover_root, Manifest, MANIFEST_FILE}; +use marqdo_package::{ + discover_root, LockFile, Manifest, PackageResolver, ResolvedPackage, LOCK_FILE, MANIFEST_FILE, +}; use marqdo_value::{codes, Diagnostic}; /// One analysed source file. pub struct LoadedFile { - /// Canonical module identity (project-relative path). + /// Canonical module identity (package instance + package-relative path). pub module_id: ModuleId, /// Absolute path on disk. pub path: PathBuf, @@ -66,6 +73,8 @@ pub struct LoadedProject { pub program: Program, /// Analysed files keyed by module id. pub files: BTreeMap, + /// Package root per logical instance. + pub packages: BTreeMap, /// All diagnostics in deterministic order. pub diagnostics: Vec, } @@ -82,11 +91,34 @@ impl LoadedProject { pub fn entry_file(&self) -> &LoadedFile { &self.files[&self.entry] } + + /// Resolve a physical path to its loaded or statically addressable module. + pub fn module_id_for_path(&self, path: &Path) -> Option { + let canonical = path.canonicalize().ok()?; + let mut best: Option<(&String, &PathBuf)> = None; + for (instance, root) in &self.packages { + if canonical.starts_with(root) + && best + .as_ref() + .map(|(_, current)| root.components().count() > current.components().count()) + .unwrap_or(true) + { + best = Some((instance, root)); + } + } + let (instance, root) = best?; + let relative = canonical.strip_prefix(root).ok()?; + let mut parts = Vec::new(); + for component in relative.components() { + parts.push(component.as_os_str().to_string_lossy().to_string()); + } + Some(ModuleId::new(instance.clone(), parts.join("/"))) + } } struct Loader { - root: PathBuf, - manifest: Option, + resolver: PackageResolver, + packages: BTreeMap, files: BTreeMap, program: Program, diagnostics: Vec, @@ -113,15 +145,22 @@ pub fn load(entry: &Path) -> Result { } else { None }; + let lock = if root.join(LOCK_FILE).is_file() { + Some(LockFile::load(&root.join(LOCK_FILE))?) + } else { + None + }; + let resolver = PackageResolver::new(root.clone(), manifest, lock); + let entry_module = resolver.resolve_entry(&entry)?; let mut loader = Loader { - root, - manifest, + resolver, + packages: BTreeMap::new(), files: BTreeMap::new(), program: Program::default(), diagnostics: Vec::new(), stack: Vec::new(), }; - let entry_id = loader.load_module(&entry)?; + let entry_id = loader.load_module(&entry_module.package, &entry_module.relative_path)?; loader.program.main_module = Some(entry_id.clone()); for (module_id, diagnostic) in marqdo_frontend::calls::validate_program(&loader.program) { if let Some(file) = loader.files.get(&module_id) { @@ -134,31 +173,47 @@ pub fn load(entry: &Path) -> Result { } } Ok(LoadedProject { - root: loader.root, + root: loader.resolver.root().to_path_buf(), entry: entry_id, program: loader.program, files: loader.files, + packages: loader.packages, diagnostics: loader.diagnostics, }) } impl Loader { - fn module_id_for(&self, path: &Path) -> Option { - let relative = path.strip_prefix(&self.root).ok()?; + fn load_module( + &mut self, + package: &ResolvedPackage, + relative_path: &str, + ) -> Result { + let package_root = package + .module_root + .canonicalize() + .unwrap_or_else(|_| package.module_root.clone()); + let path = package_root + .join(relative_path) + .canonicalize() + .map_err(|error| format!("cannot open `{relative_path}`: {error}"))?; + if !path.starts_with(&package_root) { + return Err(format!( + "module `{relative_path}` escapes package `{}`", + package.instance_id + )); + } + let relative = path.strip_prefix(&package_root).map_err(|_| { + format!( + "module `{}` is outside package `{}`", + path.display(), + package.instance_id + ) + })?; let mut parts = Vec::new(); for component in relative.components() { parts.push(component.as_os_str().to_string_lossy().to_string()); } - Some(ModuleId(parts.join("/"))) - } - - fn load_module(&mut self, path: &Path) -> Result { - let path = path - .canonicalize() - .map_err(|error| format!("cannot open `{}`: {error}", path.display()))?; - let module_id = self - .module_id_for(&path) - .ok_or_else(|| format!("`{}` is outside the project root", path.display()))?; + let module_id = ModuleId::new(&package.instance_id, parts.join("/")); if self.files.contains_key(&module_id) { return Ok(module_id); } @@ -169,9 +224,12 @@ impl Loader { )); } self.stack.push(path.clone()); + self.packages + .entry(package.instance_id.clone()) + .or_insert_with(|| package_root.clone()); let source = std::fs::read_to_string(&path) .map_err(|error| format!("cannot read `{}`: {error}", path.display()))?; - let analysis = analyze_source(&module_id.0, &source); + let analysis = analyze_source(&module_id.to_string(), &source); let imports: Vec<(String, String, marqdo_span::Span)> = analysis .document .frontmatter() @@ -195,8 +253,34 @@ impl Loader { .unwrap_or_default(); for (alias, target, span) in imports { - match self.resolve_import(&path, &target) { - Some(target_path) => { + let resolved = if let Some(alias_name) = target.strip_prefix('@') { + self.resolver.resolve_alias(alias_name) + } else { + self.resolver.resolve_relative(package, &path, &target) + }; + match resolved { + Ok(resolved) => { + let target_path = match resolved + .package + .module_root + .join(&resolved.relative_path) + .canonicalize() + { + Ok(path) => path, + Err(error) => { + self.diagnostics.push(FileDiagnostic { + module_id: module_id.clone(), + path: path.clone(), + source: source.clone(), + diagnostic: Diagnostic::error( + codes::MODULE_UNRESOLVED, + Some(span), + format!("cannot open imported module `{target}`: {error}"), + ), + }); + continue; + } + }; if self.stack.contains(&target_path) { self.diagnostics.push(FileDiagnostic { module_id: module_id.clone(), @@ -211,7 +295,7 @@ impl Loader { }); continue; } - match self.load_module(&target_path) { + match self.load_module(&resolved.package, &resolved.relative_path) { Ok(target_id) => { self.program.resolved_imports.push(ResolvedImport { from: module_id.clone(), @@ -231,7 +315,7 @@ impl Loader { }), } } - None => { + Err(error) => { self.diagnostics.push(FileDiagnostic { module_id: module_id.clone(), path: path.clone(), @@ -239,7 +323,7 @@ impl Loader { diagnostic: Diagnostic::error( codes::MODULE_UNRESOLVED, Some(span), - format!("cannot resolve imported module `{target}`"), + format!("cannot resolve imported module `{target}`: {error}"), ) .with_hint( "use `@dependency` for marqdo.yaml dependencies or `./file.mq.md` for project files", @@ -267,7 +351,6 @@ impl Loader { analysis, }, ); - // Move the analysed module into the program. if let Some(file) = self.files.get(&module_id) { self.program .modules @@ -275,22 +358,4 @@ impl Loader { } Ok(module_id) } - - fn resolve_import(&self, importing: &Path, target: &str) -> Option { - if let Some(alias) = target.strip_prefix('@') { - let manifest = self.manifest.as_ref()?; - let dependency = manifest.dependencies.get(alias)?; - let path = dependency.path.as_ref()?; - let dependency_path = self.root.join(path); - let candidate = if dependency_path.is_dir() { - dependency_path.join("index.mq.md") - } else { - dependency_path - }; - return candidate.canonicalize().ok(); - } - let base = importing.parent()?; - let candidate = base.join(target); - candidate.canonicalize().ok() - } } diff --git a/crates/cli/src/output.rs b/crates/cli/src/output.rs index 61cc9ad..fc628c4 100644 --- a/crates/cli/src/output.rs +++ b/crates/cli/src/output.rs @@ -66,7 +66,7 @@ pub fn diagnostic_json(item: &FileDiagnostic) -> Json { .unwrap_or((Json::Null, Json::Null)); json!({ "file": item.path.display().to_string(), - "module": item.module_id.0, + "module": item.module_id.to_string(), "line": line, "column": column, "severity": severity_name(diagnostic), diff --git a/crates/cli/tests/conformance_semantics.rs b/crates/cli/tests/conformance_semantics.rs index 89d8cfb..16df02e 100644 --- a/crates/cli/tests/conformance_semantics.rs +++ b/crates/cli/tests/conformance_semantics.rs @@ -156,7 +156,7 @@ fn semantic_goldens_match_end_to_end_execution() { let source = fs::read_to_string(fixture) .unwrap_or_else(|error| panic!("cannot read {}: {error}", fixture.display())); let module_id = ModuleId::from("index.mq.md"); - let analysis = marqdo_frontend::analyze_source(&module_id.0, &source); + let analysis = marqdo_frontend::analyze_source(&module_id.to_string(), &source); if analysis.has_errors() { panic!( "semantic fixture `{}` must compile without errors:\n{}", diff --git a/crates/cli/tests/negative.rs b/crates/cli/tests/negative.rs index 23c833a..e9f483f 100644 --- a/crates/cli/tests/negative.rs +++ b/crates/cli/tests/negative.rs @@ -209,7 +209,7 @@ fn runtime_negative_goldens() { let source = fs::read_to_string(fixture) .unwrap_or_else(|error| panic!("cannot read {}: {error}", fixture.display())); let module_id = ModuleId::from("index.mq.md"); - let analysis = marqdo_frontend::analyze_source(&module_id.0, &source); + let analysis = marqdo_frontend::analyze_source(&module_id.to_string(), &source); assert!( !analysis.has_errors(), "runtime fixture `{}` must compile; diagnostics: {:?}", diff --git a/crates/cli/tests/properties.rs b/crates/cli/tests/properties.rs index 30c13ba..12a2d94 100644 --- a/crates/cli/tests/properties.rs +++ b/crates/cli/tests/properties.rs @@ -379,7 +379,7 @@ fn narration_removal_preserves_execution() { let mut values = Vec::new(); for source in [base.source, clean.as_str()] { - let analysis = marqdo_frontend::analyze_source(&module_id.0, source); + let analysis = marqdo_frontend::analyze_source(&module_id.to_string(), source); assert!(!analysis.has_errors(), "{:?}", analysis.diagnostics); let mut program = marqdo_ir::Program::default(); program.insert(analysis.module.clone()); diff --git a/crates/marqdo-ir/src/lib.rs b/crates/marqdo-ir/src/lib.rs index 661d4e9..e7c7930 100644 --- a/crates/marqdo-ir/src/lib.rs +++ b/crates/marqdo-ir/src/lib.rs @@ -18,19 +18,89 @@ use serde::{Deserialize, Serialize}; pub mod builtins; -/// Canonical identity of a module: project-relative path with `/` separators. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -pub struct ModuleId(pub String); +/// Canonical logical identity of a module: package instance plus a path +/// relative to that package's module root. +/// +/// The instance prefix is **not** a physical cache path (ADR-0008). The display +/// and wire form is `::`; a bare path is accepted for +/// standalone tests and documents as `root:default`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ModuleId { + /// Logical package instance (`root:...` or `dep:...`). + pub instance: String, + /// Path relative to the package module root, with `/` separators. + pub relative_path: String, +} + +impl ModuleId { + /// Construct a module id from an explicit instance and relative path. + pub fn new(instance: impl Into, relative_path: impl Into) -> Self { + Self { + instance: instance.into(), + relative_path: relative_path.into(), + } + } + + /// Construct a module id for the default/root package. + pub fn root_default(relative_path: impl Into) -> Self { + Self::new("root:default", relative_path) + } + + /// Parse the canonical `::` form, accepting a bare path as + /// `root:default` for compatibility with standalone documents/tests. + pub fn parse(value: &str) -> Result { + if value.is_empty() { + return Err("module id must not be empty".to_string()); + } + match value.split_once("::") { + Some((instance, path)) if !instance.is_empty() && !path.is_empty() => { + Ok(Self::new(instance, path)) + } + Some(_) => Err(format!("invalid module id `{value}`")), + None => Ok(Self::root_default(value)), + } + } + + /// The canonical string form. + pub fn canonical(&self) -> String { + format!("{}::{}", self.instance, self.relative_path) + } +} impl fmt::Display for ModuleId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) + write!(f, "{}::{}", self.instance, self.relative_path) } } impl From<&str> for ModuleId { fn from(value: &str) -> Self { - ModuleId(value.to_string()) + ModuleId::parse(value).unwrap_or_else(|_| ModuleId::root_default(value)) + } +} + +impl From for ModuleId { + fn from(value: String) -> Self { + ModuleId::from(value.as_str()) + } +} + +impl Serialize for ModuleId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.canonical()) + } +} + +impl<'de> Deserialize<'de> for ModuleId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + ModuleId::parse(&value).map_err(serde::de::Error::custom) } } @@ -465,7 +535,7 @@ mod tests { }; assert!(module.entry().is_none()); assert_eq!(module.import("geometry").unwrap().target, "@geometry"); - assert_eq!(module.id.to_string(), "index.mq.md"); + assert_eq!(module.id.to_string(), "root:default::index.mq.md"); } #[test] diff --git a/crates/marqdo-package/Cargo.toml b/crates/marqdo-package/Cargo.toml index 1a389d5..a623642 100644 --- a/crates/marqdo-package/Cargo.toml +++ b/crates/marqdo-package/Cargo.toml @@ -11,6 +11,9 @@ authors.workspace = true [dependencies] serde = { workspace = true } serde_yaml = { workspace = true } +sha2 = { workspace = true } +walkdir = { workspace = true } +hex = { workspace = true } diff --git a/crates/marqdo-package/src/lib.rs b/crates/marqdo-package/src/lib.rs index fae8134..66b437b 100644 --- a/crates/marqdo-package/src/lib.rs +++ b/crates/marqdo-package/src/lib.rs @@ -9,6 +9,10 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Canonical instance id used by a standalone document without a manifest. +pub const DEFAULT_INSTANCE: &str = "root:default"; /// Manifest file name. pub const MANIFEST_FILE: &str = "marqdo.yaml"; @@ -182,18 +186,345 @@ impl Manifest { } } +/// How a resolved package was obtained. Identity does not depend on this. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PackageOrigin { + /// The project entry package. + Root, + /// A project-relative path dependency. + Path { + /// The declared path from `marqdo.yaml` (may be relative). + declared: PathBuf, + }, + /// A pinned Git revision (implemented in the environment milestone). + Git { + /// Repository URL. + url: String, + /// Exact revision. + rev: String, + }, +} + +impl PackageOrigin { + /// Stable lockfile source string. + pub fn source(&self) -> String { + match self { + PackageOrigin::Root => "root".to_string(), + PackageOrigin::Path { declared } => format!("path:{}", declared.display()), + PackageOrigin::Git { url, rev } => format!("git:{url}@{rev}"), + } + } +} + +/// A dependency resolved from the project manifest. +/// +/// `module_root` is a physical location; the `instance_id` is the logical +/// identity used by [`PackageInstance`]/`ModuleId` and by Analysis v1. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedPackage { + /// Logical package instance (`root:...` or `dep:...`). + pub instance_id: String, + /// Package name (from the dependency manifest or the manifest alias). + pub name: String, + /// Package version (from the dependency manifest or `0.0.0`). + pub version: String, + /// How the package was obtained. + pub origin: PackageOrigin, + /// Exact Git revision, when applicable. + pub revision: Option, + /// Content digest of the package tree, when computable. + pub content_digest: Option, + /// Physical directory containing the package sources. + pub module_root: PathBuf, + /// Entry module relative to `module_root` (`@alias` default). + pub entry_module: String, + /// The dependency's own manifest, if present. + pub manifest: Option, +} + +/// A module resolved inside a package. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedModule { + /// Owning package. + pub package: ResolvedPackage, + /// Path relative to the package's `module_root`. + pub relative_path: String, +} + +/// Resolve the logical instance id of a root project. +pub fn root_instance(manifest: Option<&Manifest>) -> String { + match manifest { + Some(manifest) => format!( + "root:{}@{}", + manifest.project.name, manifest.project.version + ), + None => DEFAULT_INSTANCE.to_string(), + } +} + +/// Derive a content-addressed dependency instance id. +pub fn dependency_instance_id(name: &str, version: &str, digest: Option<&str>) -> String { + let suffix = digest + .filter(|digest| !digest.is_empty()) + .unwrap_or("unlocked"); + let prefix: String = suffix.chars().take(12).collect(); + format!("dep:{name}@{version}#{prefix}") +} + +fn normalize_relative(path: &Path) -> Result { + let mut parts = Vec::new(); + for component in path.components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::Normal(part) => { + parts.push(part.to_string_lossy().to_string()); + } + _ => return Err(format!("unsafe package-relative path `{}`", path.display())), + } + } + Ok(parts.join("/")) +} + +/// Deterministic content digest of a package tree (relative path + bytes). +pub fn hash_tree(root: &Path) -> Result { + if !root.exists() { + return Err(format!( + "dependency path `{}` does not exist", + root.display() + )); + } + let mut files = Vec::new(); + for entry in walkdir::WalkDir::new(root) + .sort_by_file_name() + .into_iter() + .filter_map(Result::ok) + { + if entry.file_type().is_file() { + let relative = entry + .path() + .strip_prefix(root) + .map(|path| path.to_string_lossy().replace('\\', "/")) + .unwrap_or_default(); + files.push((relative, entry.path().to_path_buf())); + } + } + files.sort_by(|a, b| a.0.cmp(&b.0)); + let mut hasher = Sha256::new(); + for (relative, path) in files { + hasher.update(relative.as_bytes()); + hasher.update([0u8]); + let bytes = std::fs::read(&path) + .map_err(|error| format!("cannot read `{}`: {error}", path.display()))?; + hasher.update(&bytes); + hasher.update([0u8]); + } + Ok(hex::encode(hasher.finalize())) +} + +/// Resolve entry files and explicit `@dependency` imports. +#[derive(Debug, Clone)] +pub struct PackageResolver { + root: PathBuf, + manifest: Option, + lock: Option, +} + +impl PackageResolver { + /// Build a resolver for a project root. + pub fn new( + root: impl Into, + manifest: Option, + lock: Option, + ) -> Self { + Self { + root: root.into(), + manifest, + lock, + } + } + + /// Project root. + pub fn root(&self) -> &Path { + &self.root + } + + /// The project manifest, if the entry is part of a project. + pub fn manifest(&self) -> Option<&Manifest> { + self.manifest.as_ref() + } + + /// Resolve the package that owns `entry`. + pub fn resolve_entry(&self, entry: &Path) -> Result { + let entry = entry + .canonicalize() + .map_err(|error| format!("cannot open `{}`: {error}", entry.display()))?; + let relative = entry.strip_prefix(&self.root).map_err(|_| { + format!( + "entry `{}` is outside the project root `{}`", + entry.display(), + self.root.display() + ) + })?; + Ok(ResolvedModule { + package: ResolvedPackage { + instance_id: root_instance(self.manifest.as_ref()), + name: self + .manifest + .as_ref() + .map(|manifest| manifest.project.name.clone()) + .unwrap_or_else(|| "default".to_string()), + version: self + .manifest + .as_ref() + .map(|manifest| manifest.project.version.clone()) + .unwrap_or_else(|| "0.0.0".to_string()), + origin: PackageOrigin::Root, + revision: None, + content_digest: None, + module_root: self.root.clone(), + entry_module: normalize_relative(relative)?, + manifest: self.manifest.clone(), + }, + relative_path: normalize_relative(relative)?, + }) + } + + /// Resolve a declared `@alias` dependency. + pub fn resolve_alias(&self, alias: &str) -> Result { + let manifest = self + .manifest + .as_ref() + .ok_or_else(|| "no marqdo.yaml is available for dependency imports".to_string())?; + let dependency = manifest + .dependencies + .get(alias) + .ok_or_else(|| format!("unknown dependency `{alias}`"))?; + let declared = dependency.path.as_ref().ok_or_else(|| { + format!("dependency `{alias}` has no local path source in this build") + })?; + let declared_path = self.root.join(declared); + let canonical = declared_path.canonicalize().map_err(|error| { + format!( + "cannot open dependency `{alias}` at `{}`: {error}", + declared_path.display() + ) + })?; + let (module_root, entry_module) = if canonical.is_dir() { + let entry = canonical.join("index.mq.md"); + if !entry.is_file() { + return Err(format!( + "dependency `{alias}` directory `{}` has no index.mq.md", + canonical.display() + )); + } + (canonical, "index.mq.md".to_string()) + } else { + let parent = canonical + .parent() + .ok_or_else(|| format!("dependency `{alias}` has no parent directory"))? + .to_path_buf(); + let name = canonical + .file_name() + .map(|name| name.to_string_lossy().to_string()) + .ok_or_else(|| format!("dependency `{alias}` has no file name"))?; + (parent, name) + }; + let dependency_manifest = if module_root.join(MANIFEST_FILE).is_file() { + Some(Manifest::load(&module_root.join(MANIFEST_FILE))?) + } else { + None + }; + let name = dependency_manifest + .as_ref() + .map(|manifest| manifest.project.name.clone()) + .unwrap_or_else(|| alias.to_string()); + let version = dependency_manifest + .as_ref() + .map(|manifest| manifest.project.version.clone()) + .unwrap_or_else(|| "0.0.0".to_string()); + let digest = hash_tree(&module_root)?; + let instance_id = dependency_instance_id(&name, &version, Some(&digest)); + if let Some(lock) = &self.lock { + if let Some(locked) = lock.package(alias) { + if locked.sha256.as_deref().is_some_and(|sha| sha != digest) { + return Err(format!( + "dependency `{alias}` content changed since marqdo.lock; run `marqdo lock`" + )); + } + } + } + Ok(ResolvedModule { + package: ResolvedPackage { + instance_id, + name, + version, + origin: PackageOrigin::Path { + declared: declared.clone(), + }, + revision: None, + content_digest: Some(digest), + module_root, + entry_module: entry_module.clone(), + manifest: dependency_manifest, + }, + relative_path: entry_module, + }) + } + + /// Resolve a path relative to an already resolved package, rejecting + /// traversal outside that package's module root. + pub fn resolve_relative( + &self, + package: &ResolvedPackage, + importing: &Path, + target: &str, + ) -> Result { + let base = importing + .parent() + .ok_or_else(|| "importing module has no parent directory".to_string())?; + let candidate = base.join(target); + let canonical = candidate + .canonicalize() + .map_err(|error| format!("cannot open `{}`: {error}", candidate.display()))?; + let relative = canonical.strip_prefix(&package.module_root).map_err(|_| { + format!( + "import `{target}` escapes package `{}`", + package.instance_id + ) + })?; + Ok(ResolvedModule { + package: package.clone(), + relative_path: normalize_relative(relative)?, + }) + } +} + /// One pinned dependency entry in `marqdo.lock`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct LockedPackage { /// Local dependency name. pub name: String, + /// Logical package instance id (lock format v2). + #[serde(default)] + pub instance_id: String, /// Source description (`path:...` or `git:...@rev`). pub source: String, - /// SHA-256 of the resolved module tree, when computable. + /// SHA-256 of the resolved package tree, when computable. #[serde(skip_serializing_if = "Option::is_none")] pub sha256: Option, } +impl LockedPackage { + /// Fill a v1 entry's missing instance id deterministically. + fn migrate_instance_id(&mut self) { + if self.instance_id.is_empty() { + let digest = self.sha256.as_deref().unwrap_or("unlocked"); + let prefix: String = digest.chars().take(12).collect(); + self.instance_id = format!("dep:{}@0.0.0#{prefix}", self.name); + } + } +} + /// The `marqdo.lock` lockfile. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct LockFile { @@ -206,7 +537,7 @@ pub struct LockFile { impl Default for LockFile { fn default() -> Self { Self { - lock_version: 1, + lock_version: 2, packages: Vec::new(), } } @@ -217,12 +548,18 @@ impl LockFile { pub fn from_yaml(yaml: &str) -> Result { let mut lock: LockFile = serde_yaml::from_str(yaml).map_err(|error| format!("invalid {LOCK_FILE}: {error}"))?; - if lock.lock_version != 1 { + if lock.lock_version != 1 && lock.lock_version != 2 { return Err(format!( - "unsupported lock_version {} (expected 1)", + "unsupported lock_version {} (expected 1 or 2)", lock.lock_version )); } + // v1 had no logical instance id; derive one so callers always see the + // package-aware shape. Writers emit v2. + for package in &mut lock.packages { + package.migrate_instance_id(); + } + lock.lock_version = 2; lock.packages.sort_by(|a, b| a.name.cmp(&b.name)); Ok(lock) } @@ -311,15 +648,17 @@ capabilities: #[test] fn lockfile_serialization_is_stable_and_sorted() { let lock = LockFile { - lock_version: 1, + lock_version: 2, packages: vec![ LockedPackage { name: "z".into(), + instance_id: "dep:z@0.0.0#unlocked".into(), source: "path:z".into(), sha256: None, }, LockedPackage { name: "a".into(), + instance_id: "dep:a@0.0.0#abc".into(), source: "path:a".into(), sha256: Some("abc".into()), }, @@ -330,9 +669,63 @@ capabilities: assert_eq!(a, b); assert!(a.find("name: a").unwrap() < a.find("name: z").unwrap()); let parsed = LockFile::from_yaml(&a).unwrap(); + assert_eq!(parsed.lock_version, 2); assert_eq!(parsed.packages.len(), 2); } + #[test] + fn v1_lock_migrates_to_logical_instance_ids() { + let v1 = "lock_version: 1\npackages:\n- name: geometry\n source: path:./geometry\n sha256: abcdef0123456789\n"; + let lock = LockFile::from_yaml(v1).unwrap(); + assert_eq!(lock.lock_version, 2); + assert_eq!( + lock.packages[0].instance_id, + "dep:geometry@0.0.0#abcdef012345" + ); + } + + #[test] + fn package_resolution_is_stable_across_path_moves() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join(MANIFEST_FILE), + "project:\n name: p\n version: 0.1.0\ndependencies:\n geometry:\n path: ./deps/geometry\n", + ) + .unwrap(); + let first = root.join("deps/geometry"); + std::fs::create_dir_all(&first).unwrap(); + std::fs::write( + first.join(MANIFEST_FILE), + "project:\n name: geometry\n version: 0.1.0\n", + ) + .unwrap(); + std::fs::write(first.join("index.mq.md"), "## f\n\n*1*\n").unwrap(); + let manifest = Manifest::load(&root.join(MANIFEST_FILE)).unwrap(); + let resolver = PackageResolver::new(root.clone(), Some(manifest.clone()), None); + let before = resolver + .resolve_alias("geometry") + .unwrap() + .package + .instance_id; + + let second = root.join("moved/geometry"); + std::fs::create_dir_all(second.parent().unwrap()).unwrap(); + std::fs::rename(&first, &second).unwrap(); + let mut manifest = manifest; + manifest.dependencies.get_mut("geometry").unwrap().path = Some("./moved/geometry".into()); + std::fs::write(root.join(MANIFEST_FILE), manifest.to_yaml()).unwrap(); + let resolver = PackageResolver::new(root, Some(manifest), None); + let after = resolver + .resolve_alias("geometry") + .unwrap() + .package + .instance_id; + assert_eq!(before, after); + assert!(after.starts_with("dep:geometry@0.1.0#")); + } + #[test] fn rejects_absolute_dependency_paths() { let yaml = "project:\n name: p\n version: 0.1.0\ndependencies:\n x:\n path: /tmp/x\n"; diff --git a/crates/marqdo-runtime/src/builtins_impl.rs b/crates/marqdo-runtime/src/builtins_impl.rs index a4f46bd..bc74dad 100644 --- a/crates/marqdo-runtime/src/builtins_impl.rs +++ b/crates/marqdo-runtime/src/builtins_impl.rs @@ -329,7 +329,7 @@ fn resolve_ref( 0 => { if let Some(function) = module.functions.iter().find(|f| f.unit == unit_id) { return Ok(Value::FunctionRef(FunctionRef { - module: function.id.module.0.clone(), + module: function.id.module.to_string(), name: function.id.name.clone(), })); } diff --git a/crates/marqdo-runtime/src/interpreter.rs b/crates/marqdo-runtime/src/interpreter.rs index f842620..0a46a9b 100644 --- a/crates/marqdo-runtime/src/interpreter.rs +++ b/crates/marqdo-runtime/src/interpreter.rs @@ -551,7 +551,7 @@ impl<'a> Interpreter<'a> { match callee { Value::FunctionRef(reference) => { let id = FunctionId { - module: ModuleId(reference.module), + module: ModuleId::from(reference.module), name: reference.name, }; let Some(function) = program.function(&id) else { @@ -596,7 +596,7 @@ impl<'a> Interpreter<'a> { span: Option, module: &ModuleId, ) -> Result { - let context = ExecutionContext::new(module.0.clone(), span); + let context = ExecutionContext::new(module.to_string(), span); self.host.invoke(op, args, &context) } } diff --git a/docs/STATUS.md b/docs/STATUS.md index 6e37877..79e7a22 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -35,6 +35,18 @@ example smoke tests. The R0 ADRs are `docs/adr/0008`–`0012` (package identity, callable resolution, plugin export routing, capability broker, kernel providers). +### M0.6 implementation progress + +| Step | State | Evidence | +|---|---|---| +| R0 baseline + ADRs | **Done** | this section; `docs/adr/0008`–`0012`; full workspace gate | +| R1 package-aware identity | **In progress** | `ModuleId { instance, relative_path }`; `PackageResolver` path sources; `lock_version: 2` with v1 migration; tests in `marqdo-package` | +| R2 ABI/Kernel/Broker | Not started | — | +| R3 Tool Protocol | Not started | — | +| R4 Analysis/Trace v1 | Not started | — | +| R5 environment/isolation | Not started | — | +| R6 release/consumers | Not started | — | + Reproduce the evidence: ```bash diff --git a/spec/package-format.md b/spec/package-format.md index f40eccf..c2bb178 100644 --- a/spec/package-format.md +++ b/spec/package-format.md @@ -1,8 +1,9 @@ -# Marqdo Project Package Format v1 +# Marqdo Project Package Format v2 -Status: **v1 format as implemented in M0.5.** Git resolution, registries, -transitive dependency solving and environment materialization are **not** -implemented; see §8. +Status: **lock_version 2 as implemented by M0.6 R1.** Path sources and +package-instance identity are implemented; pinned Git sources, registries, +transitive dependency solving and environment materialization are added in R5 +or later; see §8. Implementation: `crates/marqdo-package/src/lib.rs` (manifest/lock types and validation), `crates/cli/src/commands.rs` (`init`, `add`, `remove`, `lock`, @@ -15,8 +16,9 @@ contains `marqdo.yaml` (`marqdo_package::discover_root`). If no manifest exists anywhere above the entry, the entry file's parent directory is used as the root (a document can be checked/run without a project). -`ModuleId`s are project-root-relative paths (`spec/references.md` §2.1), so the -root choice affects import identity. `add`/`remove`/`lock`/`sync` discover the +`ModuleId`s are package-instance + package-relative paths (`spec/references.md` +§2.1), so the project root selects the entry package instance; physical paths do +not form module identity. `add`/`remove`/`lock`/`sync` discover the root from the **current working directory**, not from a document path. There is no `--project` flag in this build. @@ -89,21 +91,23 @@ grants (`spec/security.md` §2). `net` has no implementing host op, and ## 3. `marqdo.lock` -`lock_version: 1`, packages sorted by name: +`lock_version: 2`, packages sorted by name: ```yaml -lock_version: 1 +lock_version: 2 packages: - name: geo + instance_id: dep:geo@0.1.0#101a6f6db4cc source: path:./packages/geo sha256: 101a6f6db4cc48d8c138b4716028da9bc8934b5f02b2235e020fbcabba15b757 ``` | Field | Semantics | |---|---| -| `lock_version` | must be `1`; any other value is rejected ("unsupported lock_version N (expected 1)") | +| `lock_version` | writers emit `2`; readers accept `1` and `2` and migrate v1 entries by deriving an instance id | | `packages[].name` | dependency name from the manifest | -| `packages[].source` | `path:`; a git source would be `git:...@rev`, but git is not implemented in this build | +| `packages[].instance_id` | logical `dep:@#` identity (ADR-0008) | +| `packages[].source` | `path:`; a Git source is `git:@` | | `packages[].sha256` | optional; lowercase hex SHA-256 of the dependency tree | ### 3.1 Deterministic serialization @@ -249,7 +253,7 @@ inconsistency to fix). | Environment materialization (`.marqdo/env`, `cache`, `kernels`, `sessions`) | Not implemented; `init` only writes `.marqdo/` to `.gitignore`. | | Dependency signatures / trusted publishers / supply-chain scanning | Not implemented. | | Multiple/nested project boundary diagnostics | Not implemented; nearest `marqdo.yaml` wins. | -| Lock file migration between `lock_version`s | Only version 1 is accepted. | +| Lock file migration between `lock_version`s | v1 is read and migrated; writers emit v2. | | Plugin installation / plugin dependency management | Not implemented; `plugins:` in the manifest is declarative only. | The lock/package format is deliberately small and deterministic so these can diff --git a/spec/references.md b/spec/references.md index 71701bf..3992df9 100644 --- a/spec/references.md +++ b/spec/references.md @@ -46,21 +46,28 @@ These types live in `crates/marqdo-ir/src/lib.rs`. ### 2.1 `ModuleId` ```rust -pub struct ModuleId(pub String); +pub struct ModuleId { + pub instance: String, // logical package instance + pub relative_path: String, // path inside the package module root +} ``` -* A canonical **project-root-relative path with `/` separators**, for example - `index.mq.md`, `packages/geometry/index.mq.md`. -* Computed in `crates/cli/src/loader.rs::Loader::module_id_for` by taking the - canonicalized file path, stripping the project root, and joining the path - components with `/`. -* There is no "module name" identity separate from the path: the module id is - the file location inside the project. Imported `@dependency` modules keep - their project-root-relative path (for example - `packages/geometry/index.mq.md`). -* A file that canonicalizes outside the project root is rejected with - "outside the project root" (entry: fatal load error; imported module: - `module.unresolved` diagnostic). +* Canonical display/wire form: `::`, for example + `root:default::index.mq.md` or + `dep:geometry@0.1.0#0123456789ab::index.mq.md`. +* A bare path without `::` is accepted by the parser as `root:default` for + standalone documents and tests, but writers always emit the canonical form. +* The instance is a **logical** identity (ADR-0008): + `root:@`, `root:default` without a manifest, or + `dep:@#` for a resolved dependency. + Physical cache paths and source kinds (`path`/`git`) are not part of identity; + moving a cache does not change a module id, while a different locked version + or content does. +* `crates/marqdo-package::PackageResolver` resolves path/Git sources and returns + `ResolvedPackage`; `crates/cli/src/loader.rs` consumes that result and rejects + paths outside the package's `module_root` (entry: fatal load error; imported + module: `module.unresolved` diagnostic). The loader never branches on + `path`/`git` itself. * Module ids use `BTreeMap` ordering, so program iteration and JSON output are deterministic. diff --git a/tests/conformance/semantics/14-function-ref-value.expected.json b/tests/conformance/semantics/14-function-ref-value.expected.json index 7f5c09d..9f1fbf6 100644 --- a/tests/conformance/semantics/14-function-ref-value.expected.json +++ b/tests/conformance/semantics/14-function-ref-value.expected.json @@ -3,7 +3,7 @@ "function": "main", "stdout": "", "value": { - "module": "index.mq.md", + "module": "root:default::index.mq.md", "name": "加一", "type": "function_ref" }