feat(m0.6-r1): package-aware module identity and resolver

- replace path-only ModuleId with {instance, relative_path} and canonical
  <instance>::<path> encoding
- add PackageResolver/ResolvedPackage in marqdo-package for path sources;
  hash package trees deterministically for content-addressed instance ids
- move import resolution out of the CLI loader; reject package escapes
- emit lock_version 2 with instance ids and migrate v1 lockfiles
- update references/package-format specs and the function-ref golden
- add package resolver tests for stable identity across path moves
This commit is contained in:
cflmy committed 2026-11-20 11:37:16 +08:00
1 parent 66476fbbfb
commit 5877d9aa5e
16 files changed
+665 -155

No files matched your search

+20 -67
View File
@@ -4,12 +4,10 @@ use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use marqdo_host::{Capabilities, Host, StdHost};
use marqdo_ir::ModuleId;
use marqdo_runtime::{run, EvaluatedArg, RuntimeError, Value};
use marqdo_sandbox::{PlatformCapabilities, SandboxMode};
use marqdo_span::Span;
use serde_json::json;
use sha2::{Digest, Sha256};
use crate::args;
use crate::loader::{self, FileDiagnostic, LoadedProject};
@@ -215,7 +213,7 @@ pub fn inspect(file: Option<&Path>, flags: &InspectFlags, json: bool) -> i32 {
if json {
let mut object = serde_json::Map::new();
object.insert("command".into(), json!("inspect"));
object.insert("module".into(), json!(entry.module_id.0));
object.insert("module".into(), json!(entry.module_id.to_string()));
if !selected_any {
object.insert("summary".into(), summary_json(&project));
}
@@ -253,7 +251,7 @@ pub fn inspect(file: Option<&Path>, flags: &InspectFlags, json: bool) -> i32 {
return 0;
}
if !selected_any {
println!("module: {}", entry.module_id.0);
println!("module: {}", entry.module_id);
println!("path: {}", entry.path.display());
println!("project root: {}", project.root.display());
println!("functions:");
@@ -396,7 +394,7 @@ fn refs_json(project: &LoadedProject) -> serde_json::Value {
let (resolved, target_module, target_unit) =
resolve_link(project, &entry.analysis.units, parent, url);
edges.push(json!({
"source": entry.module_id.0,
"source": entry.module_id.to_string(),
"url": url,
"span": node.span,
"resolved": resolved,
@@ -422,7 +420,11 @@ fn resolve_link(
.map(|unit| &unit.id)
});
return match unit {
Some(unit) => (true, Some(project.entry.0.clone()), Some(unit.to_string())),
Some(unit) => (
true,
Some(project.entry.to_string()),
Some(unit.to_string()),
),
None => (false, None, None),
};
}
@@ -437,16 +439,9 @@ fn resolve_link(
let Ok(candidate) = candidate.canonicalize() else {
return (false, None, None);
};
let Ok(relative) = candidate.strip_prefix(&project.root) else {
let Some(module_id) = project.module_id_for_path(&candidate) else {
return (false, None, None);
};
let module_id = ModuleId(
relative
.components()
.map(|component| component.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>()
.join("/"),
);
let analyzed;
let target_units = match project.files.get(&module_id) {
Some(target_file) => &target_file.analysis.units,
@@ -456,12 +451,12 @@ fn resolve_link(
let Ok(source) = std::fs::read_to_string(&candidate) else {
return (false, None, None);
};
analyzed = marqdo_frontend::analyze_source(&module_id.0, &source);
analyzed = marqdo_frontend::analyze_source(&module_id.to_string(), &source);
&analyzed.units
}
};
let Some(fragment) = fragment else {
return (true, Some(module_id.0), None);
return (true, Some(module_id.to_string()), None);
};
let resolved = target_units.anchors.get(fragment).or_else(|| {
target_units
@@ -471,8 +466,8 @@ fn resolve_link(
.map(|unit| &unit.id)
});
match resolved {
Some(unit) => (true, Some(module_id.0), Some(unit.to_string())),
None => (false, Some(module_id.0), None),
Some(unit) => (true, Some(module_id.to_string()), Some(unit.to_string())),
None => (false, Some(module_id.to_string()), None),
}
}
@@ -950,66 +945,24 @@ fn build_lockfile(
root: &Path,
manifest: &marqdo_package::Manifest,
) -> Result<marqdo_package::LockFile, String> {
let resolver = marqdo_package::PackageResolver::new(root, Some(manifest.clone()), None);
let mut packages = Vec::new();
for (name, dependency) in &manifest.dependencies {
if dependency.git.is_some() {
return Err(format!(
"dependency `{name}` uses a git source; git dependencies are not supported by this build (path dependencies only)"
));
}
let Some(path) = &dependency.path else {
return Err(format!("dependency `{name}` has no path source"));
};
let dependency_root = root.join(path);
let hash = hash_tree(&dependency_root)?;
for name in manifest.dependencies.keys() {
let resolved = resolver.resolve_alias(name)?;
packages.push(marqdo_package::LockedPackage {
name: name.clone(),
source: format!("path:{}", path.display()),
sha256: Some(hash),
instance_id: resolved.package.instance_id,
source: resolved.package.origin.source(),
sha256: resolved.package.content_digest,
});
}
packages.sort_by(|a, b| a.name.cmp(&b.name));
Ok(marqdo_package::LockFile {
lock_version: 1,
lock_version: 2,
packages,
})
}
fn hash_tree(root: &Path) -> Result<String, String> {
if !root.exists() {
return Err(format!(
"dependency path `{}` does not exist",
root.display()
));
}
let mut files = Vec::new();
for entry in walkdir::WalkDir::new(root)
.sort_by_file_name()
.into_iter()
.filter_map(Result::ok)
{
if entry.file_type().is_file() {
let relative = entry
.path()
.strip_prefix(root)
.map(|path| path.to_string_lossy().replace('\\', "/"))
.unwrap_or_default();
files.push((relative, entry.path().to_path_buf()));
}
}
files.sort_by(|a, b| a.0.cmp(&b.0));
let mut hasher = Sha256::new();
for (relative, path) in files {
hasher.update(relative.as_bytes());
hasher.update([0u8]);
let bytes = std::fs::read(&path)
.map_err(|error| format!("cannot read `{}`: {error}", path.display()))?;
hasher.update(&bytes);
hasher.update([0u8]);
}
Ok(hex::encode(hasher.finalize()))
}
/// `marqdo add <name> --path <dir> [--git URL --rev SHA]`.
pub fn add(
name: &str,
+110 -45
View File
@@ -4,18 +4,25 @@
//! come only from explicit YAML front matter and are resolved exactly as
//! declared (`@dependency` or a relative `.mq.md` path); nothing is searched
//! implicitly.
//!
//! Module identity is package-aware (ADR-0008): a module is
//! `PackageInstanceId::relative_path`, never a physical cache path. The
//! package layer resolves path/Git sources; this loader consumes
//! `ResolvedPackage` and does not branch on the source kind.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use marqdo_frontend::{analyze_source, AnalysisResult};
use marqdo_ir::{ModuleId, Program, ResolvedImport};
use marqdo_package::{discover_root, Manifest, MANIFEST_FILE};
use marqdo_package::{
discover_root, LockFile, Manifest, PackageResolver, ResolvedPackage, LOCK_FILE, MANIFEST_FILE,
};
use marqdo_value::{codes, Diagnostic};
/// One analysed source file.
pub struct LoadedFile {
/// Canonical module identity (project-relative path).
/// Canonical module identity (package instance + package-relative path).
pub module_id: ModuleId,
/// Absolute path on disk.
pub path: PathBuf,
@@ -66,6 +73,8 @@ pub struct LoadedProject {
pub program: Program,
/// Analysed files keyed by module id.
pub files: BTreeMap<ModuleId, LoadedFile>,
/// Package root per logical instance.
pub packages: BTreeMap<String, PathBuf>,
/// All diagnostics in deterministic order.
pub diagnostics: Vec<FileDiagnostic>,
}
@@ -82,11 +91,34 @@ impl LoadedProject {
pub fn entry_file(&self) -> &LoadedFile {
&self.files[&self.entry]
}
/// Resolve a physical path to its loaded or statically addressable module.
pub fn module_id_for_path(&self, path: &Path) -> Option<ModuleId> {
let canonical = path.canonicalize().ok()?;
let mut best: Option<(&String, &PathBuf)> = None;
for (instance, root) in &self.packages {
if canonical.starts_with(root)
&& best
.as_ref()
.map(|(_, current)| root.components().count() > current.components().count())
.unwrap_or(true)
{
best = Some((instance, root));
}
}
let (instance, root) = best?;
let relative = canonical.strip_prefix(root).ok()?;
let mut parts = Vec::new();
for component in relative.components() {
parts.push(component.as_os_str().to_string_lossy().to_string());
}
Some(ModuleId::new(instance.clone(), parts.join("/")))
}
}
struct Loader {
root: PathBuf,
manifest: Option<Manifest>,
resolver: PackageResolver,
packages: BTreeMap<String, PathBuf>,
files: BTreeMap<ModuleId, LoadedFile>,
program: Program,
diagnostics: Vec<FileDiagnostic>,
@@ -113,15 +145,22 @@ pub fn load(entry: &Path) -> Result<LoadedProject, String> {
} else {
None
};
let lock = if root.join(LOCK_FILE).is_file() {
Some(LockFile::load(&root.join(LOCK_FILE))?)
} else {
None
};
let resolver = PackageResolver::new(root.clone(), manifest, lock);
let entry_module = resolver.resolve_entry(&entry)?;
let mut loader = Loader {
root,
manifest,
resolver,
packages: BTreeMap::new(),
files: BTreeMap::new(),
program: Program::default(),
diagnostics: Vec::new(),
stack: Vec::new(),
};
let entry_id = loader.load_module(&entry)?;
let entry_id = loader.load_module(&entry_module.package, &entry_module.relative_path)?;
loader.program.main_module = Some(entry_id.clone());
for (module_id, diagnostic) in marqdo_frontend::calls::validate_program(&loader.program) {
if let Some(file) = loader.files.get(&module_id) {
@@ -134,31 +173,47 @@ pub fn load(entry: &Path) -> Result<LoadedProject, String> {
}
}
Ok(LoadedProject {
root: loader.root,
root: loader.resolver.root().to_path_buf(),
entry: entry_id,
program: loader.program,
files: loader.files,
packages: loader.packages,
diagnostics: loader.diagnostics,
})
}
impl Loader {
fn module_id_for(&self, path: &Path) -> Option<ModuleId> {
let relative = path.strip_prefix(&self.root).ok()?;
fn load_module(
&mut self,
package: &ResolvedPackage,
relative_path: &str,
) -> Result<ModuleId, String> {
let package_root = package
.module_root
.canonicalize()
.unwrap_or_else(|_| package.module_root.clone());
let path = package_root
.join(relative_path)
.canonicalize()
.map_err(|error| format!("cannot open `{relative_path}`: {error}"))?;
if !path.starts_with(&package_root) {
return Err(format!(
"module `{relative_path}` escapes package `{}`",
package.instance_id
));
}
let relative = path.strip_prefix(&package_root).map_err(|_| {
format!(
"module `{}` is outside package `{}`",
path.display(),
package.instance_id
)
})?;
let mut parts = Vec::new();
for component in relative.components() {
parts.push(component.as_os_str().to_string_lossy().to_string());
}
Some(ModuleId(parts.join("/")))
}
fn load_module(&mut self, path: &Path) -> Result<ModuleId, String> {
let path = path
.canonicalize()
.map_err(|error| format!("cannot open `{}`: {error}", path.display()))?;
let module_id = self
.module_id_for(&path)
.ok_or_else(|| format!("`{}` is outside the project root", path.display()))?;
let module_id = ModuleId::new(&package.instance_id, parts.join("/"));
if self.files.contains_key(&module_id) {
return Ok(module_id);
}
@@ -169,9 +224,12 @@ impl Loader {
));
}
self.stack.push(path.clone());
self.packages
.entry(package.instance_id.clone())
.or_insert_with(|| package_root.clone());
let source = std::fs::read_to_string(&path)
.map_err(|error| format!("cannot read `{}`: {error}", path.display()))?;
let analysis = analyze_source(&module_id.0, &source);
let analysis = analyze_source(&module_id.to_string(), &source);
let imports: Vec<(String, String, marqdo_span::Span)> = analysis
.document
.frontmatter()
@@ -195,8 +253,34 @@ impl Loader {
.unwrap_or_default();
for (alias, target, span) in imports {
match self.resolve_import(&path, &target) {
Some(target_path) => {
let resolved = if let Some(alias_name) = target.strip_prefix('@') {
self.resolver.resolve_alias(alias_name)
} else {
self.resolver.resolve_relative(package, &path, &target)
};
match resolved {
Ok(resolved) => {
let target_path = match resolved
.package
.module_root
.join(&resolved.relative_path)
.canonicalize()
{
Ok(path) => path,
Err(error) => {
self.diagnostics.push(FileDiagnostic {
module_id: module_id.clone(),
path: path.clone(),
source: source.clone(),
diagnostic: Diagnostic::error(
codes::MODULE_UNRESOLVED,
Some(span),
format!("cannot open imported module `{target}`: {error}"),
),
});
continue;
}
};
if self.stack.contains(&target_path) {
self.diagnostics.push(FileDiagnostic {
module_id: module_id.clone(),
@@ -211,7 +295,7 @@ impl Loader {
});
continue;
}
match self.load_module(&target_path) {
match self.load_module(&resolved.package, &resolved.relative_path) {
Ok(target_id) => {
self.program.resolved_imports.push(ResolvedImport {
from: module_id.clone(),
@@ -231,7 +315,7 @@ impl Loader {
}),
}
}
None => {
Err(error) => {
self.diagnostics.push(FileDiagnostic {
module_id: module_id.clone(),
path: path.clone(),
@@ -239,7 +323,7 @@ impl Loader {
diagnostic: Diagnostic::error(
codes::MODULE_UNRESOLVED,
Some(span),
format!("cannot resolve imported module `{target}`"),
format!("cannot resolve imported module `{target}`: {error}"),
)
.with_hint(
"use `@dependency` for marqdo.yaml dependencies or `./file.mq.md` for project files",
@@ -267,7 +351,6 @@ impl Loader {
analysis,
},
);
// Move the analysed module into the program.
if let Some(file) = self.files.get(&module_id) {
self.program
.modules
@@ -275,22 +358,4 @@ impl Loader {
}
Ok(module_id)
}
fn resolve_import(&self, importing: &Path, target: &str) -> Option<PathBuf> {
if let Some(alias) = target.strip_prefix('@') {
let manifest = self.manifest.as_ref()?;
let dependency = manifest.dependencies.get(alias)?;
let path = dependency.path.as_ref()?;
let dependency_path = self.root.join(path);
let candidate = if dependency_path.is_dir() {
dependency_path.join("index.mq.md")
} else {
dependency_path
};
return candidate.canonicalize().ok();
}
let base = importing.parent()?;
let candidate = base.join(target);
candidate.canonicalize().ok()
}
}
+1 -1
View File
@@ -66,7 +66,7 @@ pub fn diagnostic_json(item: &FileDiagnostic) -> Json {
.unwrap_or((Json::Null, Json::Null));
json!({
"file": item.path.display().to_string(),
"module": item.module_id.0,
"module": item.module_id.to_string(),
"line": line,
"column": column,
"severity": severity_name(diagnostic),