fix(ci): anchor AST goldens to LF bytes and test CRLF explicitly
The Windows CI failure was a checkout byte-identity problem, not a Markdown semantics change: CRLF goldens compared with LF parser output produced '0 line(s) differ'. - add .gitattributes: text files are LF, native products are binary, and the intentional CRLF fixture stays a byte fixture - report the first differing byte and whether only line endings differ - fail early when a golden contains CR instead of the opaque mismatch - assert exactly one CRLF fixture and that its golden is canonical LF - add a markdown test proving CRLF/LF have equal structure while spans remain byte offsets into the original source
This commit is contained in:
1 parent
e966b7661f
commit
c7f3a8422c
4 files changed
+244
-3
No files matched your search
@@ -0,0 +1,20 @@
|
|||||||
|
# Canonical byte identity for the repository.
|
||||||
|
#
|
||||||
|
# The frozen Markdown profile and its AST goldens are compared byte-for-byte.
|
||||||
|
# Normalize every text file to LF on checkout so a Windows clone does not turn
|
||||||
|
# golden JSON into CRLF (which previously produced a confusing "0 line(s)
|
||||||
|
# differ" failure). Line-ending semantics are tested explicitly with the
|
||||||
|
# dedicated CRLF fixture below instead of relying on the checkout platform.
|
||||||
|
* text=auto eol=lf
|
||||||
|
|
||||||
|
# Intentional byte-level CRLF fixture (see spec/markdown-profile.md §3).
|
||||||
|
# It must keep its original CRLF bytes on every platform.
|
||||||
|
tests/conformance/ast/146-crlf-line-endings.mq.md -text -eol
|
||||||
|
|
||||||
|
# Native build products are not text.
|
||||||
|
*.so binary
|
||||||
|
*.dylib binary
|
||||||
|
*.dll binary
|
||||||
|
*.a binary
|
||||||
|
*.lib binary
|
||||||
|
*.exe binary
|
||||||
@@ -7,3 +7,10 @@
|
|||||||
# Built plugin artifacts
|
# Built plugin artifacts
|
||||||
examples/plugin-demo/solver.so
|
examples/plugin-demo/solver.so
|
||||||
examples/plugin-multi-export/solver.so
|
examples/plugin-multi-export/solver.so
|
||||||
|
|
||||||
|
# Local cargo home used on hosts with a read-only $HOME
|
||||||
|
.cargo-home/
|
||||||
|
|
||||||
|
# Generated plugin fixture manifests/artifacts
|
||||||
|
examples/plugin-demo/build/
|
||||||
|
examples/plugin-multi-export/build/
|
||||||
@@ -71,7 +71,46 @@ fn write_golden(path: &Path, text: &str) {
|
|||||||
.unwrap_or_else(|error| panic!("cannot write golden {}: {error}", path.display()));
|
.unwrap_or_else(|error| panic!("cannot write golden {}: {error}", path.display()));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Report a compact, line-based diff summary for one fixture.
|
/// First byte offset at which two inputs differ, if any.
|
||||||
|
fn first_byte_difference(expected: &[u8], actual: &[u8]) -> Option<usize> {
|
||||||
|
expected
|
||||||
|
.iter()
|
||||||
|
.zip(actual.iter())
|
||||||
|
.position(|(left, right)| left != right)
|
||||||
|
.or_else(|| (expected.len() != actual.len()).then_some(expected.len().min(actual.len())))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Render a short escaped byte window around `offset`.
|
||||||
|
fn byte_window(bytes: &[u8], offset: usize) -> String {
|
||||||
|
let start = offset.saturating_sub(8);
|
||||||
|
let end = (offset + 8).min(bytes.len());
|
||||||
|
bytes[start..end]
|
||||||
|
.iter()
|
||||||
|
.map(|byte| format!("{byte:02x}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether two byte strings are identical after normalizing CRLF to LF.
|
||||||
|
fn only_line_endings_differ(expected: &[u8], actual: &[u8]) -> bool {
|
||||||
|
fn normalize(bytes: &[u8]) -> Vec<u8> {
|
||||||
|
let mut out = Vec::with_capacity(bytes.len());
|
||||||
|
let mut index = 0;
|
||||||
|
while index < bytes.len() {
|
||||||
|
if bytes[index] == b'\r' && bytes.get(index + 1) == Some(&b'\n') {
|
||||||
|
out.push(b'\n');
|
||||||
|
index += 2;
|
||||||
|
} else {
|
||||||
|
out.push(bytes[index]);
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
expected != actual && normalize(expected) == normalize(actual)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Report a compact, byte-aware diff summary for one fixture.
|
||||||
fn report_mismatch(fixture: &Path, expected: &str, actual: &str) {
|
fn report_mismatch(fixture: &Path, expected: &str, actual: &str) {
|
||||||
let expected_lines: Vec<&str> = expected.lines().collect();
|
let expected_lines: Vec<&str> = expected.lines().collect();
|
||||||
let actual_lines: Vec<&str> = actual.lines().collect();
|
let actual_lines: Vec<&str> = actual.lines().collect();
|
||||||
@@ -90,12 +129,36 @@ fn report_mismatch(fixture: &Path, expected: &str, actual: &str) {
|
|||||||
let end = (first + 8).min(expected_lines.len().max(actual_lines.len()));
|
let end = (first + 8).min(expected_lines.len().max(actual_lines.len()));
|
||||||
let mut message = format!(
|
let mut message = format!(
|
||||||
"AST golden mismatch for `{}`.\n\
|
"AST golden mismatch for `{}`.\n\
|
||||||
{} line(s) differ (first at line {}).\n\
|
expected {} byte(s), actual {} byte(s).\n\
|
||||||
--- expected (golden) ---\n",
|
{} line(s) differ (first at line {}).\n",
|
||||||
fixture.display(),
|
fixture.display(),
|
||||||
|
expected.len(),
|
||||||
|
actual.len(),
|
||||||
differences,
|
differences,
|
||||||
first + 1
|
first + 1
|
||||||
);
|
);
|
||||||
|
match first_byte_difference(expected.as_bytes(), actual.as_bytes()) {
|
||||||
|
Some(offset) => {
|
||||||
|
message.push_str(&format!(
|
||||||
|
"first byte difference at offset {offset}.\n\
|
||||||
|
expected bytes: {}\n\
|
||||||
|
actual bytes: {}\n",
|
||||||
|
byte_window(expected.as_bytes(), offset),
|
||||||
|
byte_window(actual.as_bytes(), offset)
|
||||||
|
));
|
||||||
|
if only_line_endings_differ(expected.as_bytes(), actual.as_bytes()) {
|
||||||
|
message.push_str(
|
||||||
|
"the only difference is line endings (CRLF vs LF); \
|
||||||
|
this is a checkout/fixture byte-identity problem, not a \
|
||||||
|
Markdown semantics change. Ensure .gitattributes keeps \
|
||||||
|
goldens at LF and keep intentional CRLF inputs as byte \
|
||||||
|
fixtures.\n",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => message.push_str("the two strings are byte-identical.\n"),
|
||||||
|
}
|
||||||
|
message.push_str("--- expected (golden) ---\n");
|
||||||
for index in start..end {
|
for index in start..end {
|
||||||
message.push_str(&format!(
|
message.push_str(&format!(
|
||||||
"{:>4} | {}\n",
|
"{:>4} | {}\n",
|
||||||
@@ -213,3 +276,106 @@ fn ast_goldens_match_the_frozen_profile() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Goldens are canonical JSON: UTF-8, LF line endings and a trailing newline.
|
||||||
|
/// A CRLF checkout used to fail with "0 line(s) differ", which hid the real
|
||||||
|
/// cause. This test fails early with a direct byte-level diagnosis instead.
|
||||||
|
#[test]
|
||||||
|
fn ast_goldens_use_canonical_lf_bytes() {
|
||||||
|
let dir = ast_dir();
|
||||||
|
let mut checked = 0usize;
|
||||||
|
for entry in fs::read_dir(&dir).expect("ast fixture directory is readable") {
|
||||||
|
let path = entry.expect("directory entry").path();
|
||||||
|
if !path
|
||||||
|
.extension()
|
||||||
|
.is_some_and(|extension| extension == "json")
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bytes = fs::read(&path).expect("golden is readable");
|
||||||
|
assert!(
|
||||||
|
std::str::from_utf8(&bytes).is_ok(),
|
||||||
|
"golden {} is not valid UTF-8",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!bytes.contains(&b'\r'),
|
||||||
|
"golden {} contains CR bytes; canonical JSON goldens must use LF only",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
bytes.ends_with(b"\n"),
|
||||||
|
"golden {} must end with exactly one trailing LF",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
checked += 1;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
checked >= 100,
|
||||||
|
"expected the full AST golden corpus, saw {checked}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The profile's line-ending semantics are tested with an explicit byte
|
||||||
|
/// fixture, not by relying on the checkout platform. Exactly one fixture keeps
|
||||||
|
/// CRLF bytes; see `.gitattributes`.
|
||||||
|
#[test]
|
||||||
|
fn crlf_fixture_is_preserved_as_bytes() {
|
||||||
|
let fixture = ast_dir().join("146-crlf-line-endings.mq.md");
|
||||||
|
let bytes = fs::read(&fixture).expect("CRLF fixture is readable");
|
||||||
|
assert!(
|
||||||
|
bytes.windows(2).any(|window| window == b"\r\n"),
|
||||||
|
"the CRLF fixture lost its CRLF bytes (checkout normalization?)"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
bytes
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.all(|(index, byte)| *byte != b'\r' || bytes.get(index + 1) == Some(&b'\n')),
|
||||||
|
"CRLF fixture contains a lone CR byte"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!bytes
|
||||||
|
.split(|byte| *byte == b'\n')
|
||||||
|
.any(|line| !line.ends_with(b"\r") && !line.is_empty()),
|
||||||
|
"CRLF fixture mixes bare LF line endings with CRLF"
|
||||||
|
);
|
||||||
|
|
||||||
|
let golden = ast_dir().join("146-crlf-line-endings.ast.json");
|
||||||
|
let golden_bytes = fs::read(&golden).expect("CRLF golden is readable");
|
||||||
|
assert!(
|
||||||
|
!golden_bytes.contains(&b'\r'),
|
||||||
|
"the CRLF fixture's golden is canonical LF JSON, not CRLF text"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut crlf_fixtures = 0usize;
|
||||||
|
for entry in fs::read_dir(ast_dir()).expect("ast fixture directory is readable") {
|
||||||
|
let path = entry.expect("directory entry").path();
|
||||||
|
if path.extension().is_some_and(|extension| extension == "md") {
|
||||||
|
let bytes = fs::read(&path).expect("fixture is readable");
|
||||||
|
if bytes.contains(&b'\r') {
|
||||||
|
assert_eq!(
|
||||||
|
path.file_name(),
|
||||||
|
fixture.file_name(),
|
||||||
|
"unexpected CRLF fixture {}; add it deliberately and document it",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
crlf_fixtures += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
crlf_fixtures, 1,
|
||||||
|
"exactly one CRLF byte fixture is expected"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn byte_diff_helpers_distinguish_line_endings_from_semantics() {
|
||||||
|
assert_eq!(first_byte_difference(b"abc", b"abc"), None);
|
||||||
|
assert_eq!(first_byte_difference(b"abc", b"abd"), Some(2));
|
||||||
|
assert_eq!(first_byte_difference(b"ab", b"abc"), Some(2));
|
||||||
|
assert!(only_line_endings_differ(b"a\nb\n", b"a\r\nb\r\n"));
|
||||||
|
assert!(!only_line_endings_differ(b"a\nb\n", b"a\nc\n"));
|
||||||
|
assert!(!only_line_endings_differ(b"a\r\nb\n", b"a\r\nb\n"));
|
||||||
|
}
|
||||||
@@ -370,6 +370,54 @@ mod tests {
|
|||||||
assert_eq!(&source[heading.span.start..heading.span.end], "# T");
|
assert_eq!(&source[heading.span.start..heading.span.end], "# T");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// CRLF is a line-ending spelling, not a different document: the
|
||||||
|
/// structural AST is identical to the LF form while every span remains a
|
||||||
|
/// byte offset into the original CRLF source (so it never includes the CR
|
||||||
|
/// of a line terminator for content nodes).
|
||||||
|
#[test]
|
||||||
|
fn crlf_and_lf_sources_keep_structure_and_byte_spans() {
|
||||||
|
fn structure(node: &MdNode) -> MdNode {
|
||||||
|
MdNode {
|
||||||
|
kind: node.kind.clone(),
|
||||||
|
span: Span::new(0, 0),
|
||||||
|
literal: node.literal.clone(),
|
||||||
|
children: node.children.iter().map(structure).collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_text_spans(node: &MdNode, source: &str) {
|
||||||
|
if node.kind == MdKind::Text {
|
||||||
|
let literal = node.literal.as_deref().expect("text nodes carry a literal");
|
||||||
|
assert_eq!(
|
||||||
|
&source[node.span.start..node.span.end],
|
||||||
|
literal,
|
||||||
|
"text span must slice exactly its literal"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for child in &node.children {
|
||||||
|
check_text_spans(child, source);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let lf = "# T\n\nHello *world*\n\n- a\n- b\n";
|
||||||
|
let crlf = "# T\r\n\r\nHello *world*\r\n\r\n- a\r\n- b\r\n";
|
||||||
|
let lf_doc = parse(lf);
|
||||||
|
let crlf_doc = parse(crlf);
|
||||||
|
assert_eq!(structure(&lf_doc.root), structure(&crlf_doc.root));
|
||||||
|
|
||||||
|
let heading = &crlf_doc.root.children[0];
|
||||||
|
assert_eq!(&crlf[heading.span.start..heading.span.end], "# T");
|
||||||
|
check_text_spans(&crlf_doc.root, crlf);
|
||||||
|
|
||||||
|
// The CRLF input is longer, so absolute offsets shift; the derived
|
||||||
|
// line/column of the same content stays the same.
|
||||||
|
assert!(crlf_doc.root.span.end > lf_doc.root.span.end);
|
||||||
|
assert_eq!(
|
||||||
|
crlf_doc.line_col(heading.span.start),
|
||||||
|
lf_doc.line_col(lf_doc.root.children[0].span.start)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn math_nodes_are_native_and_inert() {
|
fn math_nodes_are_native_and_inert() {
|
||||||
let doc = parse("inline $a+b$ math\n\n$$\nx+y\n$$\n");
|
let doc = parse("inline $a+b$ math\n\n$$\nx+y\n$$\n");
|
||||||
|
|||||||
Reference in new issue
Block a user