fix(package): unify package file-set hashing and materialization (B-05)
Add content_profile_version 1 with one selection shared by hash_tree and materialization: exclude .git/.marqdo, include target, reject symlinks and special files, propagate walk errors, enforce file/size/depth budgets, and use a domain-separated length-prefixed encoding (fixes the old NUL-delimiter ambiguity). Materialization hashes exactly the bytes it writes and verifies the expected digest before publishing. Record the decision in ADR-0016 and update the package-format hash contract.
This commit is contained in:
1 parent
74e34c668b
commit
cd90f38ce1
4 files changed
+557
-87
No files matched your search
@@ -0,0 +1,366 @@
|
||||
//! Deterministic package file-set selection, hashing and materialization.
|
||||
//!
|
||||
//! `content_profile_version` 1 defines one selection used by both hashing and
|
||||
//! copying. Hashing and copying must never disagree: a digest that does not
|
||||
//! describe the materialized environment is not a reproducibility gate.
|
||||
//!
|
||||
//! Normative design: `docs/adr/0016-lock-v3-graph-and-package-file-set.md` §4.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::ffi::OsStr;
|
||||
use std::io::Read;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
/// Version of the file-set selection and digest encoding.
|
||||
pub const CONTENT_PROFILE_VERSION: u32 = 1;
|
||||
/// Maximum number of files in one package.
|
||||
pub const MAX_FILES: usize = 10_000;
|
||||
/// Maximum size of one file.
|
||||
pub const MAX_FILE_BYTES: u64 = 64 * 1024 * 1024;
|
||||
/// Maximum total size of one package.
|
||||
pub const MAX_TOTAL_BYTES: u64 = 512 * 1024 * 1024;
|
||||
/// Maximum directory depth below the package root.
|
||||
pub const MAX_DEPTH: usize = 64;
|
||||
|
||||
/// One regular file selected from a package tree.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct FileSetEntry {
|
||||
pub(crate) relative: String,
|
||||
pub(crate) path: PathBuf,
|
||||
}
|
||||
|
||||
fn excluded_component(name: &OsStr) -> bool {
|
||||
name == OsStr::new(".git") || name == OsStr::new(".marqdo")
|
||||
}
|
||||
|
||||
fn normalize_relative(relative: &Path) -> Result<String, String> {
|
||||
let mut parts = Vec::new();
|
||||
for component in relative.components() {
|
||||
match component {
|
||||
Component::Normal(part) => {
|
||||
let part = part
|
||||
.to_str()
|
||||
.ok_or_else(|| {
|
||||
format!("package path `{}` is not valid UTF-8", relative.display())
|
||||
})?
|
||||
.to_string();
|
||||
if part.is_empty() {
|
||||
return Err("package path contains an empty component".to_string());
|
||||
}
|
||||
parts.push(part);
|
||||
}
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"package path `{}` is absolute or contains `.`/`..`",
|
||||
relative.display()
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
if parts.is_empty() {
|
||||
return Err("package path is empty".to_string());
|
||||
}
|
||||
Ok(parts.join("/"))
|
||||
}
|
||||
|
||||
/// Select the regular files of a package deterministically.
|
||||
///
|
||||
/// Excludes any path component `.git` or `.marqdo`; includes `target/`.
|
||||
/// Symlinks and special files are hard errors, never skipped or followed.
|
||||
pub(crate) fn collect_file_set(root: &Path) -> Result<Vec<FileSetEntry>, String> {
|
||||
let metadata = std::fs::symlink_metadata(root)
|
||||
.map_err(|error| format!("cannot inspect package `{}`: {error}", root.display()))?;
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(format!(
|
||||
"package root `{}` is a symlink; refusing to hash or materialize it",
|
||||
root.display()
|
||||
));
|
||||
}
|
||||
if !metadata.is_dir() {
|
||||
return Err(format!(
|
||||
"package source `{}` is not a directory",
|
||||
root.display()
|
||||
));
|
||||
}
|
||||
|
||||
let walker = WalkDir::new(root)
|
||||
.follow_links(false)
|
||||
.sort_by_file_name()
|
||||
.into_iter()
|
||||
.filter_entry(|entry| {
|
||||
entry.depth() == 0
|
||||
|| !(entry.file_type().is_dir() && excluded_component(entry.file_name()))
|
||||
});
|
||||
|
||||
let mut files = Vec::new();
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut total: u64 = 0;
|
||||
for entry in walker {
|
||||
let entry = entry.map_err(|error| {
|
||||
let path = error
|
||||
.path()
|
||||
.map(|path| path.display().to_string())
|
||||
.unwrap_or_else(|| root.display().to_string());
|
||||
format!("cannot walk package `{path}`: {error}")
|
||||
})?;
|
||||
if entry.depth() == 0 {
|
||||
continue;
|
||||
}
|
||||
if entry.depth() > MAX_DEPTH {
|
||||
return Err(format!(
|
||||
"package `{}` exceeds the maximum directory depth of {MAX_DEPTH}",
|
||||
root.display()
|
||||
));
|
||||
}
|
||||
let file_type = entry.file_type();
|
||||
if file_type.is_symlink() {
|
||||
return Err(format!(
|
||||
"package contains a symlink `{}`; refusing to hash or materialize it",
|
||||
entry.path().display()
|
||||
));
|
||||
}
|
||||
if file_type.is_dir() {
|
||||
continue;
|
||||
}
|
||||
if !file_type.is_file() {
|
||||
return Err(format!(
|
||||
"package contains special file `{}`; only regular files are package content",
|
||||
entry.path().display()
|
||||
));
|
||||
}
|
||||
let relative = entry
|
||||
.path()
|
||||
.strip_prefix(root)
|
||||
.map_err(|_| format!("package path `{}` escaped its root", entry.path().display()))?;
|
||||
let relative = normalize_relative(relative)?;
|
||||
if !seen.insert(relative.clone()) {
|
||||
return Err(format!(
|
||||
"package `{}` contains duplicate normalized path `{relative}`",
|
||||
root.display()
|
||||
));
|
||||
}
|
||||
let file_metadata = entry.metadata().map_err(|error| {
|
||||
format!(
|
||||
"cannot inspect package file `{}`: {error}",
|
||||
entry.path().display()
|
||||
)
|
||||
})?;
|
||||
let len = file_metadata.len();
|
||||
if len > MAX_FILE_BYTES {
|
||||
return Err(format!(
|
||||
"package file `{}` is {len} bytes; maximum is {MAX_FILE_BYTES}",
|
||||
entry.path().display()
|
||||
));
|
||||
}
|
||||
total = total
|
||||
.checked_add(len)
|
||||
.ok_or_else(|| "package size overflowed".to_string())?;
|
||||
if total > MAX_TOTAL_BYTES {
|
||||
return Err(format!(
|
||||
"package `{}` exceeds the maximum total size of {MAX_TOTAL_BYTES} bytes",
|
||||
root.display()
|
||||
));
|
||||
}
|
||||
if files.len() >= MAX_FILES {
|
||||
return Err(format!(
|
||||
"package `{}` exceeds the maximum file count of {MAX_FILES}",
|
||||
root.display()
|
||||
));
|
||||
}
|
||||
files.push(FileSetEntry {
|
||||
relative,
|
||||
path: entry.path().to_path_buf(),
|
||||
});
|
||||
}
|
||||
files.sort_by(|a, b| a.relative.as_bytes().cmp(b.relative.as_bytes()));
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn read_file(path: &Path) -> Result<Vec<u8>, String> {
|
||||
let mut file = std::fs::File::open(path)
|
||||
.map_err(|error| format!("cannot read `{}`: {error}", path.display()))?;
|
||||
let mut bytes = Vec::new();
|
||||
file.read_to_end(&mut bytes)
|
||||
.map_err(|error| format!("cannot read `{}`: {error}", path.display()))?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn update_digest(hasher: &mut Sha256, relative: &str, bytes: &[u8]) {
|
||||
hasher.update((relative.len() as u64).to_le_bytes());
|
||||
hasher.update(relative.as_bytes());
|
||||
hasher.update((bytes.len() as u64).to_le_bytes());
|
||||
hasher.update(bytes);
|
||||
}
|
||||
|
||||
/// Hash a package directory with the profile-v1 file-set encoding.
|
||||
pub(crate) fn hash_tree(root: &Path) -> Result<String, String> {
|
||||
let files = collect_file_set(root)?;
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"marqdo.package.file-set\0");
|
||||
hasher.update(CONTENT_PROFILE_VERSION.to_le_bytes());
|
||||
hasher.update((files.len() as u64).to_le_bytes());
|
||||
for file in &files {
|
||||
let bytes = read_file(&file.path)?;
|
||||
update_digest(&mut hasher, &file.relative, &bytes);
|
||||
}
|
||||
Ok(hex::encode(hasher.finalize()))
|
||||
}
|
||||
|
||||
/// Materialize a package and verify that the written bytes match `expected`.
|
||||
///
|
||||
/// The source is read once: the same bytes are hashed and written, so a
|
||||
/// source mutation during materialization is detected before publication.
|
||||
/// Callers materialize into staging and publish only on success.
|
||||
pub(crate) fn materialize_file_set(
|
||||
source: &Path,
|
||||
target: &Path,
|
||||
expected_hex: &str,
|
||||
) -> Result<(), String> {
|
||||
let files = collect_file_set(source)?;
|
||||
if target.exists() {
|
||||
std::fs::remove_dir_all(target)
|
||||
.map_err(|error| format!("cannot reset `{}`: {error}", target.display()))?;
|
||||
}
|
||||
std::fs::create_dir_all(target)
|
||||
.map_err(|error| format!("cannot create `{}`: {error}", target.display()))?;
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"marqdo.package.file-set\0");
|
||||
hasher.update(CONTENT_PROFILE_VERSION.to_le_bytes());
|
||||
hasher.update((files.len() as u64).to_le_bytes());
|
||||
for file in &files {
|
||||
let bytes = read_file(&file.path)?;
|
||||
update_digest(&mut hasher, &file.relative, &bytes);
|
||||
let destination = target.join(&file.relative);
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.map_err(|error| format!("cannot create `{}`: {error}", parent.display()))?;
|
||||
}
|
||||
std::fs::write(&destination, &bytes)
|
||||
.map_err(|error| format!("cannot write `{}`: {error}", destination.display()))?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let _ = std::fs::set_permissions(&destination, std::fs::Permissions::from_mode(0o644));
|
||||
}
|
||||
}
|
||||
|
||||
let actual = hex::encode(hasher.finalize());
|
||||
if actual != expected_hex {
|
||||
return Err(format!(
|
||||
"package `{}` changed while materializing (expected digest {expected_hex}, got {actual}); run `marqdo lock`",
|
||||
source.display()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn write(path: &Path, bytes: &[u8]) {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
std::fs::write(path, bytes).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excludes_git_and_marqdo_directories() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
write(&root.join("index.mq.md"), b"## main\n");
|
||||
write(&root.join(".git/HEAD"), b"ref: main\n");
|
||||
write(&root.join(".marqdo/env/env.json"), b"{}");
|
||||
write(&root.join("nested/.marqdo/state"), b"x");
|
||||
let first = hash_tree(root).unwrap();
|
||||
|
||||
write(&root.join(".git/HEAD"), b"ref: other\n");
|
||||
write(&root.join(".marqdo/env/env.json"), b"{\"changed\":true}");
|
||||
assert_eq!(hash_tree(root).unwrap(), first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_directory_is_package_content() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
write(&root.join("index.mq.md"), b"x");
|
||||
let first = hash_tree(root).unwrap();
|
||||
write(&root.join("target/generated"), b"y");
|
||||
assert_ne!(hash_tree(root).unwrap(), first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_symlinks() {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
write(&root.join("index.mq.md"), b"x");
|
||||
std::os::unix::fs::symlink("index.mq.md", root.join("link.mq.md")).unwrap();
|
||||
let error = hash_tree(root).unwrap_err();
|
||||
assert!(error.contains("symlink"), "{error}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_special_files() {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
write(&root.join("index.mq.md"), b"x");
|
||||
let status = std::process::Command::new("mkfifo")
|
||||
.arg(root.join("pipe"))
|
||||
.status();
|
||||
if status.map(|status| status.success()).unwrap_or(false) {
|
||||
let error = hash_tree(root).unwrap_err();
|
||||
assert!(error.contains("special file"), "{error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn digest_encoding_is_unambiguous() {
|
||||
// The v1 delimiter-only encoding produced the same stream for both
|
||||
// trees: `a\0x\0y\0z\0`. Length prefixes must distinguish them.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let first = dir.path().join("first");
|
||||
let second = dir.path().join("second");
|
||||
write(&first.join("a"), b"x\0y");
|
||||
write(&first.join("z"), b"");
|
||||
write(&second.join("a"), b"x");
|
||||
write(&second.join("y"), b"z");
|
||||
assert_ne!(hash_tree(&first).unwrap(), hash_tree(&second).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialization_uses_the_same_selection_and_digest() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let source = dir.path().join("source");
|
||||
let target = dir.path().join("target");
|
||||
write(&source.join("index.mq.md"), b"## main\n");
|
||||
write(&source.join("data/value.txt"), b"42");
|
||||
write(&source.join(".git/HEAD"), b"ignored");
|
||||
let digest = hash_tree(&source).unwrap();
|
||||
materialize_file_set(&source, &target, &digest).unwrap();
|
||||
assert!(target.join("index.mq.md").is_file());
|
||||
assert!(target.join("data/value.txt").is_file());
|
||||
assert!(!target.join(".git").exists());
|
||||
assert_eq!(hash_tree(&target).unwrap(), digest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialization_rejects_a_wrong_expected_digest() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let source = dir.path().join("source");
|
||||
let target = dir.path().join("target");
|
||||
write(&source.join("index.mq.md"), b"x");
|
||||
let error = materialize_file_set(&source, &target, "0000").unwrap_err();
|
||||
assert!(error.contains("changed while materializing"), "{error}");
|
||||
}
|
||||
}
|
||||
@@ -5,12 +5,16 @@
|
||||
//! bindings. This crate only parses and serialises these files; it never runs
|
||||
//! project code and never executes install scripts.
|
||||
|
||||
mod fileset;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub use fileset::CONTENT_PROFILE_VERSION;
|
||||
|
||||
/// Canonical instance id used by a standalone document without a manifest.
|
||||
pub const DEFAULT_INSTANCE: &str = "root:default";
|
||||
|
||||
@@ -286,39 +290,11 @@ fn normalize_relative(path: &Path) -> Result<String, String> {
|
||||
}
|
||||
|
||||
/// Deterministic content digest of a package tree (relative path + bytes).
|
||||
///
|
||||
/// Selection and encoding are defined by `content_profile_version` 1 in
|
||||
/// [`fileset`]; hashing and materialization share that selection.
|
||||
pub fn hash_tree(root: &Path) -> Result<String, String> {
|
||||
if !root.exists() {
|
||||
return Err(format!(
|
||||
"dependency path `{}` does not exist",
|
||||
root.display()
|
||||
));
|
||||
}
|
||||
let mut files = Vec::new();
|
||||
for entry in walkdir::WalkDir::new(root)
|
||||
.sort_by_file_name()
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
{
|
||||
if entry.file_type().is_file() {
|
||||
let relative = entry
|
||||
.path()
|
||||
.strip_prefix(root)
|
||||
.map(|path| path.to_string_lossy().replace('\\', "/"))
|
||||
.unwrap_or_default();
|
||||
files.push((relative, entry.path().to_path_buf()));
|
||||
}
|
||||
}
|
||||
files.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
let mut hasher = Sha256::new();
|
||||
for (relative, path) in files {
|
||||
hasher.update(relative.as_bytes());
|
||||
hasher.update([0u8]);
|
||||
let bytes = std::fs::read(&path)
|
||||
.map_err(|error| format!("cannot read `{}`: {error}", path.display()))?;
|
||||
hasher.update(&bytes);
|
||||
hasher.update([0u8]);
|
||||
}
|
||||
Ok(hex::encode(hasher.finalize()))
|
||||
fileset::hash_tree(root)
|
||||
}
|
||||
|
||||
/// Sanitize a logical instance id into one filesystem component.
|
||||
@@ -698,7 +674,13 @@ impl PackageResolver {
|
||||
let mut records = Vec::new();
|
||||
for (alias, package) in &packages {
|
||||
let directory = staging.join(env_dir_name(&package.instance_id));
|
||||
copy_tree(&package.module_root, &directory)?;
|
||||
let expected = package.content_digest.as_deref().ok_or_else(|| {
|
||||
format!(
|
||||
"package `{}` has no content digest; refusing to materialize",
|
||||
package.instance_id
|
||||
)
|
||||
})?;
|
||||
fileset::materialize_file_set(&package.module_root, &directory, expected)?;
|
||||
records.push(serde_json::json!({
|
||||
"alias": alias,
|
||||
"instance_id": package.instance_id,
|
||||
@@ -824,49 +806,6 @@ fn safe_component(value: &str) -> String {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn copy_tree(source: &Path, target: &Path) -> Result<(), String> {
|
||||
if !source.is_dir() {
|
||||
return Err(format!(
|
||||
"package source `{}` is not a directory",
|
||||
source.display()
|
||||
));
|
||||
}
|
||||
for entry in walkdir::WalkDir::new(source).follow_links(false) {
|
||||
let entry = entry.map_err(|error| format!("cannot walk package: {error}"))?;
|
||||
let relative = entry
|
||||
.path()
|
||||
.strip_prefix(source)
|
||||
.map_err(|_| "package path escaped its root".to_string())?;
|
||||
if relative.as_os_str() == ".git" || relative.starts_with(".git/") {
|
||||
continue;
|
||||
}
|
||||
let destination = target.join(relative);
|
||||
if entry.file_type().is_symlink() {
|
||||
return Err(format!(
|
||||
"package contains a symlink `{}`; refusing to materialize it",
|
||||
entry.path().display()
|
||||
));
|
||||
}
|
||||
if entry.file_type().is_dir() {
|
||||
std::fs::create_dir_all(&destination)
|
||||
.map_err(|error| format!("cannot create `{}`: {error}", destination.display()))?;
|
||||
} else if entry.file_type().is_file() {
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.map_err(|error| format!("cannot create `{}`: {error}", parent.display()))?;
|
||||
}
|
||||
std::fs::copy(entry.path(), &destination).map_err(|error| {
|
||||
format!(
|
||||
"cannot copy `{}` to `{}`: {error}",
|
||||
entry.path().display(),
|
||||
destination.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One pinned dependency entry in `marqdo.lock`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct LockedPackage {
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
# ADR-0016: Lock v3 package graph and unified package file set
|
||||
|
||||
Status: **Accepted**
|
||||
Date: 2026-11-20
|
||||
Supersedes: ADR-0008 §4 (flat direct-dependency lock), `spec/package-format.md`
|
||||
lock_version 2 and its `hash_tree` description
|
||||
|
||||
## Context
|
||||
|
||||
The M0.6 lockfile records the root manifest's direct dependencies as a flat
|
||||
list. The resolver, however, reads each dependency's own manifest and can load
|
||||
transitive `@alias` imports. "Executable transitive dependency" therefore did
|
||||
not imply "locked transitive dependency" (B-02). Two further defects made the
|
||||
flat format unsafe as a reproducibility contract:
|
||||
|
||||
* a missing lock entry was accepted (`check_lock` returned `Ok(())`), and the
|
||||
lookup key was the bare alias, so the same alias in two packages could
|
||||
resolve to the wrong instance (B-03);
|
||||
* `sync` materialized a freshly computed graph while leaving the on-disk lock
|
||||
untouched, allowing manifest/lock/env divergence (B-04).
|
||||
|
||||
Package content was also hashed and copied with different file selections:
|
||||
`hash_tree` swallowed walk errors, included `.git`/`.marqdo`, and silently
|
||||
skipped symlinks, while `copy_tree` excluded `.git`, rejected symlinks and
|
||||
created empty directories. The hash encoding was ambiguous and unversioned
|
||||
(B-05). A digest that does not describe the executed environment cannot be a
|
||||
reproducibility gate.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Lock v3 is a package-instance graph
|
||||
|
||||
`marqdo.lock` records the full closure reachable from the root manifest:
|
||||
|
||||
```yaml
|
||||
lock_version: 3
|
||||
content_profile_version: 1
|
||||
root:
|
||||
package: root:project@0.1.0
|
||||
manifest_digest: sha256:<64 hex>
|
||||
packages:
|
||||
- id: dep:a@0.1.0#sha256:<64 hex>
|
||||
name: a
|
||||
version: 0.1.0
|
||||
source: {kind: path, path: deps/a}
|
||||
content_digest: sha256:<64 hex>
|
||||
manifest_digest: sha256:<64 hex>
|
||||
dependencies: {b: dep:b@...}
|
||||
root_dependencies: {a: dep:a@...}
|
||||
```
|
||||
|
||||
* `id`, not the alias, identifies a package instance. Path, Git and multi-version
|
||||
instances have distinct ids; ids embed the full content digest.
|
||||
* Each package record has its own alias→instance dependency edges. Root edges
|
||||
live in `root_dependencies`. The same alias in different packages may point
|
||||
at different instances.
|
||||
* `source` is structured (`path` is project-relative and normalized; `git` is
|
||||
URL plus full commit hash). Absolute or escaping paths are rejected.
|
||||
* `content_profile_version` gates digest comparability. An unknown profile is
|
||||
a hard error, never a comparison against an incompatible hash.
|
||||
* Readers accept only lock v3. v1/v2 fail with an instruction to run
|
||||
`marqdo lock`; a flat lock cannot be upgraded without resolving, so silent
|
||||
migration would re-open B-03. `lock` is the only writer and produces v3.
|
||||
|
||||
`LockFile::validate` enforces: unique package ids; sorted ids; every edge target
|
||||
exists; no unreachable package; digests match `sha256:<64 lowercase hex>`;
|
||||
`root.package` and `root.manifest_digest` are present.
|
||||
|
||||
### 2. Locking is a closed operation; resolution is fail-closed
|
||||
|
||||
* `marqdo lock` builds the full closure and writes the graph atomically
|
||||
(temporary file, fsync, rename).
|
||||
* `run`, `call`, `check` and `inspect` never fetch, never write the lock and
|
||||
require a lock whenever the project declares dependencies.
|
||||
* Every locked edge is checked by `(parent instance, alias)`. A missing lock,
|
||||
missing edge, missing package record, content-digest mismatch, source
|
||||
mismatch or root-manifest mismatch is an error with a stable message telling
|
||||
the user to run `marqdo lock`. There is no `Ok(())` fallback.
|
||||
* A resolved directory (source, cache or env copy) is trusted only after its
|
||||
file-set digest matches the lock.
|
||||
* Git revisions must be full lowercase 40- or 64-character commit hashes. After
|
||||
checkout, `git rev-parse HEAD` must equal the requested revision. Cache
|
||||
entries carry a marker with URL, revision, profile and digest, and are
|
||||
reverified before every use; a cache directory's existence is never proof of
|
||||
correctness.
|
||||
|
||||
### 3. `sync` is strict and never rewrites the lock
|
||||
|
||||
`sync` verifies the lock against the current manifest and source contents, then
|
||||
materializes it. Drift fails with "lockfile is out of date or dependency
|
||||
contents changed; run `marqdo lock`". Fresh pinned Git revisions may be fetched
|
||||
by plain `sync`; `--locked` additionally forbids fetching. In both modes the
|
||||
lock bytes are unchanged. This removes the two-file transaction and the
|
||||
manifest/lock/env divergence in B-04.
|
||||
|
||||
Materialization covers every package in the lock closure, writes
|
||||
`env.json` schema 2 with the lock digest and content profile, and publishes via
|
||||
staging + atomic rename with recovery of the previous environment on failure.
|
||||
|
||||
### 4. One file-set profile for hashing and materialization
|
||||
|
||||
`content_profile_version: 1` defines exactly one selection used by both
|
||||
`hash_tree` and materialization:
|
||||
|
||||
* include regular files; exclude any path component `.git` or `.marqdo`;
|
||||
include `target/` (not declared generated by this profile);
|
||||
* symlinks and special files are errors, not skipped or followed;
|
||||
* relative paths must be UTF-8, `/`-separated, without absolute or `..`
|
||||
components; duplicate normalized paths are errors;
|
||||
* walk errors, unreadable files and budget overruns abort the operation; no
|
||||
`filter_map(Result::ok)`;
|
||||
* budgets: 10 000 files, 64 MiB per file, 512 MiB total, depth 64;
|
||||
* hash encoding is domain-separated, profile-versioned, count-prefixed and
|
||||
length-prefixed (`rel_len`, `rel`, `content_len`, `content`), eliminating the
|
||||
old delimiter ambiguity;
|
||||
* materialization reads each file once, hashes exactly the bytes it writes and
|
||||
compares the final digest with the expected digest before publishing.
|
||||
|
||||
File modes are not part of the digest; materialized files use `0644` and
|
||||
directories `0755` on Unix. Empty directories are not package content.
|
||||
|
||||
### 5. Identity strings
|
||||
|
||||
Dependency instance ids embed the full content digest:
|
||||
`dep:<name>@<version>#sha256:<64 hex>`. `env_dir_name` is injective
|
||||
(`pkg-<sha256(instance id)>`). The current short-prefix id is replaced; this is
|
||||
a deliberate 0.x break.
|
||||
|
||||
## Consequences
|
||||
|
||||
* Existing v2 locks and example locks must be regenerated. Because hashes now
|
||||
cover a different file set and encoding, all content digests change.
|
||||
* Path dependencies must live inside the project root; `../shared` is rejected
|
||||
and must become a pinned Git dependency or move into the project.
|
||||
* Root manifest edits (including capabilities/plugins) invalidate the lock and
|
||||
require `marqdo lock`; this is fail-closed by design.
|
||||
* `add`/`remove` no longer rewrite lock entries. A stale lock is rejected with
|
||||
the standard relock message.
|
||||
* A same-user attacker who can rewrite a source during hashing is only detected
|
||||
(final digest mismatch), not prevented. Signatures and publisher
|
||||
authentication remain out of scope.
|
||||
* Strict OS isolation is unaffected; this ADR changes reproducibility and
|
||||
integrity, not the sandbox claim.
|
||||
|
||||
## Tests
|
||||
|
||||
* file-set: excludes `.git`/`.marqdo`, rejects symlinks/special files, reports
|
||||
walk errors, enforces budgets, unambiguous encoding with embedded NUL bytes,
|
||||
hash and materialization select the same files, digest mismatch leaves no
|
||||
published environment.
|
||||
* lock graph: A→B→C closure, multi-version, diamond dedup, cycle rejection,
|
||||
duplicate id with different content, byte-identical output across project
|
||||
copies, v1/v2 rejection, missing edge/record and drift failures.
|
||||
* sync: strict without `--locked`, never rewrites the lock, full closure in
|
||||
`env.json`, failed publish restores the previous environment.
|
||||
* CLI: run/check require a lock when dependencies exist; lock errors exit 1.
|
||||
+20
-11
@@ -120,19 +120,28 @@ order-insensitive.
|
||||
|
||||
### 3.2 Tree hash
|
||||
|
||||
For a path or Git dependency, `lock` hashes the resolved package tree
|
||||
(`marqdo_package::hash_tree`):
|
||||
For a path or Git dependency, `lock` hashes the resolved package tree with
|
||||
`content_profile_version: 1` (`crates/marqdo-package/src/fileset.rs`, ADR-0016):
|
||||
|
||||
1. walk the dependency directory with `walkdir` (`sort_by_file_name`), keeping
|
||||
regular files only (symlinks are not followed and are skipped);
|
||||
2. sort files by their root-relative path (normalized to `/`);
|
||||
3. feed into SHA-256, for each file in order:
|
||||
`relative_path bytes`, a `0x00` byte, `file bytes`, a `0x00` byte;
|
||||
4. hex-encode the digest.
|
||||
1. walk the dependency directory with `walkdir` (`sort_by_file_name`,
|
||||
`follow_links(false)`), propagating every walk error;
|
||||
2. exclude any path component `.git` or `.marqdo`; include `target/`;
|
||||
3. accept regular files only. Symlinks and special files are hard errors, not
|
||||
skipped or followed; paths must be UTF-8, `/`-separated, relative and
|
||||
duplicate-free;
|
||||
4. sort files by root-relative path (UTF-8 bytes); enforce budgets of 10 000
|
||||
files, 64 MiB per file, 512 MiB total and depth 64;
|
||||
5. hash with domain separation and length prefixes:
|
||||
`b"marqdo.package.file-set\0"`, `profile_version: u32le`,
|
||||
`entry_count: u64le`, then for each file
|
||||
`rel_len: u64le`, `rel_bytes`, `content_len: u64le`, `content_bytes`;
|
||||
6. hex-encode the digest.
|
||||
|
||||
This is a deterministic content fingerprint, not a Merkle tree and not a
|
||||
cryptographic commitment to file modes, symlinks or empty directories. It
|
||||
detects content changes and additions/removals of regular files.
|
||||
Materialization uses the same selection and reads each file once, hashing the
|
||||
exact bytes it writes; the published environment is accepted only when the
|
||||
final digest equals the lock. This is a deterministic content fingerprint, not
|
||||
a Merkle tree and not a commitment to file modes or empty directories. Modes
|
||||
are normalized on materialization (`0644` files, `0755` directories on Unix).
|
||||
|
||||
### 3.3 Lock generation rules
|
||||
|
||||
|
||||
Reference in new issue
Block a user