fix(ci): anchor AST goldens to LF bytes and test CRLF explicitly
The Windows CI failure was a checkout byte-identity problem, not a Markdown semantics change: CRLF goldens compared with LF parser output produced '0 line(s) differ'. - add .gitattributes: text files are LF, native products are binary, and the intentional CRLF fixture stays a byte fixture - report the first differing byte and whether only line endings differ - fail early when a golden contains CR instead of the opaque mismatch - assert exactly one CRLF fixture and that its golden is canonical LF - add a markdown test proving CRLF/LF have equal structure while spans remain byte offsets into the original source
This commit is contained in:
1 parent
e966b7661f
commit
c7f3a8422c
4 files changed
+244
-3
No files matched your search
@@ -0,0 +1,20 @@
|
||||
# Canonical byte identity for the repository.
|
||||
#
|
||||
# The frozen Markdown profile and its AST goldens are compared byte-for-byte.
|
||||
# Normalize every text file to LF on checkout so a Windows clone does not turn
|
||||
# golden JSON into CRLF (which previously produced a confusing "0 line(s)
|
||||
# differ" failure). Line-ending semantics are tested explicitly with the
|
||||
# dedicated CRLF fixture below instead of relying on the checkout platform.
|
||||
* text=auto eol=lf
|
||||
|
||||
# Intentional byte-level CRLF fixture (see spec/markdown-profile.md §3).
|
||||
# It must keep its original CRLF bytes on every platform.
|
||||
tests/conformance/ast/146-crlf-line-endings.mq.md -text -eol
|
||||
|
||||
# Native build products are not text.
|
||||
*.so binary
|
||||
*.dylib binary
|
||||
*.dll binary
|
||||
*.a binary
|
||||
*.lib binary
|
||||
*.exe binary
|
||||
@@ -7,3 +7,10 @@
|
||||
# Built plugin artifacts
|
||||
examples/plugin-demo/solver.so
|
||||
examples/plugin-multi-export/solver.so
|
||||
|
||||
# Local cargo home used on hosts with a read-only $HOME
|
||||
.cargo-home/
|
||||
|
||||
# Generated plugin fixture manifests/artifacts
|
||||
examples/plugin-demo/build/
|
||||
examples/plugin-multi-export/build/
|
||||
@@ -71,7 +71,46 @@ fn write_golden(path: &Path, text: &str) {
|
||||
.unwrap_or_else(|error| panic!("cannot write golden {}: {error}", path.display()));
|
||||
}
|
||||
|
||||
/// Report a compact, line-based diff summary for one fixture.
|
||||
/// First byte offset at which two inputs differ, if any.
|
||||
fn first_byte_difference(expected: &[u8], actual: &[u8]) -> Option<usize> {
|
||||
expected
|
||||
.iter()
|
||||
.zip(actual.iter())
|
||||
.position(|(left, right)| left != right)
|
||||
.or_else(|| (expected.len() != actual.len()).then_some(expected.len().min(actual.len())))
|
||||
}
|
||||
|
||||
/// Render a short escaped byte window around `offset`.
|
||||
fn byte_window(bytes: &[u8], offset: usize) -> String {
|
||||
let start = offset.saturating_sub(8);
|
||||
let end = (offset + 8).min(bytes.len());
|
||||
bytes[start..end]
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
/// Whether two byte strings are identical after normalizing CRLF to LF.
|
||||
fn only_line_endings_differ(expected: &[u8], actual: &[u8]) -> bool {
|
||||
fn normalize(bytes: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(bytes.len());
|
||||
let mut index = 0;
|
||||
while index < bytes.len() {
|
||||
if bytes[index] == b'\r' && bytes.get(index + 1) == Some(&b'\n') {
|
||||
out.push(b'\n');
|
||||
index += 2;
|
||||
} else {
|
||||
out.push(bytes[index]);
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
expected != actual && normalize(expected) == normalize(actual)
|
||||
}
|
||||
|
||||
/// Report a compact, byte-aware diff summary for one fixture.
|
||||
fn report_mismatch(fixture: &Path, expected: &str, actual: &str) {
|
||||
let expected_lines: Vec<&str> = expected.lines().collect();
|
||||
let actual_lines: Vec<&str> = actual.lines().collect();
|
||||
@@ -90,12 +129,36 @@ fn report_mismatch(fixture: &Path, expected: &str, actual: &str) {
|
||||
let end = (first + 8).min(expected_lines.len().max(actual_lines.len()));
|
||||
let mut message = format!(
|
||||
"AST golden mismatch for `{}`.\n\
|
||||
{} line(s) differ (first at line {}).\n\
|
||||
--- expected (golden) ---\n",
|
||||
expected {} byte(s), actual {} byte(s).\n\
|
||||
{} line(s) differ (first at line {}).\n",
|
||||
fixture.display(),
|
||||
expected.len(),
|
||||
actual.len(),
|
||||
differences,
|
||||
first + 1
|
||||
);
|
||||
match first_byte_difference(expected.as_bytes(), actual.as_bytes()) {
|
||||
Some(offset) => {
|
||||
message.push_str(&format!(
|
||||
"first byte difference at offset {offset}.\n\
|
||||
expected bytes: {}\n\
|
||||
actual bytes: {}\n",
|
||||
byte_window(expected.as_bytes(), offset),
|
||||
byte_window(actual.as_bytes(), offset)
|
||||
));
|
||||
if only_line_endings_differ(expected.as_bytes(), actual.as_bytes()) {
|
||||
message.push_str(
|
||||
"the only difference is line endings (CRLF vs LF); \
|
||||
this is a checkout/fixture byte-identity problem, not a \
|
||||
Markdown semantics change. Ensure .gitattributes keeps \
|
||||
goldens at LF and keep intentional CRLF inputs as byte \
|
||||
fixtures.\n",
|
||||
);
|
||||
}
|
||||
}
|
||||
None => message.push_str("the two strings are byte-identical.\n"),
|
||||
}
|
||||
message.push_str("--- expected (golden) ---\n");
|
||||
for index in start..end {
|
||||
message.push_str(&format!(
|
||||
"{:>4} | {}\n",
|
||||
@@ -213,3 +276,106 @@ fn ast_goldens_match_the_frozen_profile() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Goldens are canonical JSON: UTF-8, LF line endings and a trailing newline.
|
||||
/// A CRLF checkout used to fail with "0 line(s) differ", which hid the real
|
||||
/// cause. This test fails early with a direct byte-level diagnosis instead.
|
||||
#[test]
|
||||
fn ast_goldens_use_canonical_lf_bytes() {
|
||||
let dir = ast_dir();
|
||||
let mut checked = 0usize;
|
||||
for entry in fs::read_dir(&dir).expect("ast fixture directory is readable") {
|
||||
let path = entry.expect("directory entry").path();
|
||||
if !path
|
||||
.extension()
|
||||
.is_some_and(|extension| extension == "json")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let bytes = fs::read(&path).expect("golden is readable");
|
||||
assert!(
|
||||
std::str::from_utf8(&bytes).is_ok(),
|
||||
"golden {} is not valid UTF-8",
|
||||
path.display()
|
||||
);
|
||||
assert!(
|
||||
!bytes.contains(&b'\r'),
|
||||
"golden {} contains CR bytes; canonical JSON goldens must use LF only",
|
||||
path.display()
|
||||
);
|
||||
assert!(
|
||||
bytes.ends_with(b"\n"),
|
||||
"golden {} must end with exactly one trailing LF",
|
||||
path.display()
|
||||
);
|
||||
checked += 1;
|
||||
}
|
||||
assert!(
|
||||
checked >= 100,
|
||||
"expected the full AST golden corpus, saw {checked}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The profile's line-ending semantics are tested with an explicit byte
|
||||
/// fixture, not by relying on the checkout platform. Exactly one fixture keeps
|
||||
/// CRLF bytes; see `.gitattributes`.
|
||||
#[test]
|
||||
fn crlf_fixture_is_preserved_as_bytes() {
|
||||
let fixture = ast_dir().join("146-crlf-line-endings.mq.md");
|
||||
let bytes = fs::read(&fixture).expect("CRLF fixture is readable");
|
||||
assert!(
|
||||
bytes.windows(2).any(|window| window == b"\r\n"),
|
||||
"the CRLF fixture lost its CRLF bytes (checkout normalization?)"
|
||||
);
|
||||
assert!(
|
||||
bytes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.all(|(index, byte)| *byte != b'\r' || bytes.get(index + 1) == Some(&b'\n')),
|
||||
"CRLF fixture contains a lone CR byte"
|
||||
);
|
||||
assert!(
|
||||
!bytes
|
||||
.split(|byte| *byte == b'\n')
|
||||
.any(|line| !line.ends_with(b"\r") && !line.is_empty()),
|
||||
"CRLF fixture mixes bare LF line endings with CRLF"
|
||||
);
|
||||
|
||||
let golden = ast_dir().join("146-crlf-line-endings.ast.json");
|
||||
let golden_bytes = fs::read(&golden).expect("CRLF golden is readable");
|
||||
assert!(
|
||||
!golden_bytes.contains(&b'\r'),
|
||||
"the CRLF fixture's golden is canonical LF JSON, not CRLF text"
|
||||
);
|
||||
|
||||
let mut crlf_fixtures = 0usize;
|
||||
for entry in fs::read_dir(ast_dir()).expect("ast fixture directory is readable") {
|
||||
let path = entry.expect("directory entry").path();
|
||||
if path.extension().is_some_and(|extension| extension == "md") {
|
||||
let bytes = fs::read(&path).expect("fixture is readable");
|
||||
if bytes.contains(&b'\r') {
|
||||
assert_eq!(
|
||||
path.file_name(),
|
||||
fixture.file_name(),
|
||||
"unexpected CRLF fixture {}; add it deliberately and document it",
|
||||
path.display()
|
||||
);
|
||||
crlf_fixtures += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
crlf_fixtures, 1,
|
||||
"exactly one CRLF byte fixture is expected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_diff_helpers_distinguish_line_endings_from_semantics() {
|
||||
assert_eq!(first_byte_difference(b"abc", b"abc"), None);
|
||||
assert_eq!(first_byte_difference(b"abc", b"abd"), Some(2));
|
||||
assert_eq!(first_byte_difference(b"ab", b"abc"), Some(2));
|
||||
assert!(only_line_endings_differ(b"a\nb\n", b"a\r\nb\r\n"));
|
||||
assert!(!only_line_endings_differ(b"a\nb\n", b"a\nc\n"));
|
||||
assert!(!only_line_endings_differ(b"a\r\nb\n", b"a\r\nb\n"));
|
||||
}
|
||||
@@ -370,6 +370,54 @@ mod tests {
|
||||
assert_eq!(&source[heading.span.start..heading.span.end], "# T");
|
||||
}
|
||||
|
||||
/// CRLF is a line-ending spelling, not a different document: the
|
||||
/// structural AST is identical to the LF form while every span remains a
|
||||
/// byte offset into the original CRLF source (so it never includes the CR
|
||||
/// of a line terminator for content nodes).
|
||||
#[test]
|
||||
fn crlf_and_lf_sources_keep_structure_and_byte_spans() {
|
||||
fn structure(node: &MdNode) -> MdNode {
|
||||
MdNode {
|
||||
kind: node.kind.clone(),
|
||||
span: Span::new(0, 0),
|
||||
literal: node.literal.clone(),
|
||||
children: node.children.iter().map(structure).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn check_text_spans(node: &MdNode, source: &str) {
|
||||
if node.kind == MdKind::Text {
|
||||
let literal = node.literal.as_deref().expect("text nodes carry a literal");
|
||||
assert_eq!(
|
||||
&source[node.span.start..node.span.end],
|
||||
literal,
|
||||
"text span must slice exactly its literal"
|
||||
);
|
||||
}
|
||||
for child in &node.children {
|
||||
check_text_spans(child, source);
|
||||
}
|
||||
}
|
||||
|
||||
let lf = "# T\n\nHello *world*\n\n- a\n- b\n";
|
||||
let crlf = "# T\r\n\r\nHello *world*\r\n\r\n- a\r\n- b\r\n";
|
||||
let lf_doc = parse(lf);
|
||||
let crlf_doc = parse(crlf);
|
||||
assert_eq!(structure(&lf_doc.root), structure(&crlf_doc.root));
|
||||
|
||||
let heading = &crlf_doc.root.children[0];
|
||||
assert_eq!(&crlf[heading.span.start..heading.span.end], "# T");
|
||||
check_text_spans(&crlf_doc.root, crlf);
|
||||
|
||||
// The CRLF input is longer, so absolute offsets shift; the derived
|
||||
// line/column of the same content stays the same.
|
||||
assert!(crlf_doc.root.span.end > lf_doc.root.span.end);
|
||||
assert_eq!(
|
||||
crlf_doc.line_col(heading.span.start),
|
||||
lf_doc.line_col(lf_doc.root.children[0].span.start)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn math_nodes_are_native_and_inert() {
|
||||
let doc = parse("inline $a+b$ math\n\n$$\nx+y\n$$\n");
|
||||
|
||||
Reference in new issue
Block a user