Files
marqdo/src/parse/prose.rs
T
cflmyandCursor 6994acb91d fix(prose): mid-line italic emphasis is not a return
Skip *very important* / *foo.bar* / *code-as-docs* so narrative emphasis cannot end a function; keep whole-line *n* and 返回*n*.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-14 15:25:26 +08:00

259 lines
8.6 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Inline prose scan for Markup v0.3 (叙述中的 `` `名` `` / `**代码**` / `*返回*`).
//!
//! Soft Markdown emphasis that does not look like code/return is skipped (G2 —
//! [code-as-docs-gaps.md](../../../doc/design/code-as-docs-gaps.md)).
use crate::lex::{matching_bold_close_inline, matching_italic_close_inline};
pub use crate::lex::looks_like_bold_code;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProseBit {
/// `` `ident` `` or `` `ident`=default `` (default string is raw RHS text).
Decl {
name: String,
default: Option<String>,
/// Byte offset in the line (for diagnostics).
at: usize,
},
/// Inner text of `**…**` (code segment).
BoldCode { inner: String, at: usize },
/// Inner text of `*…*` (return), when not starting a bold.
ItalicReturn { inner: String, at: usize },
}
/// Scan a narrative (or mixed) line for v0.3 inline bits, left to right.
pub fn scan_prose_line(line: &str) -> Vec<ProseBit> {
let bytes = line.as_bytes();
let mut i = 0usize;
let mut out = Vec::new();
while i < bytes.len() {
let c = bytes[i];
if c == b'`' {
if let Some((bit, next)) = scan_backtick_decl(line, i) {
out.push(bit);
i = next;
continue;
}
// Non-decl span (e.g. `` `OPENAI_*` ``): skip whole ticks so inner `*`
// does not open italic/bold.
if let Some(rel) = line[i + 1..].find('`') {
i = i + 1 + rel + 1;
continue;
}
i += 1;
continue;
}
if c == b'*' && i + 1 < bytes.len() && bytes[i + 1] == b'*' {
let rest = &line[i..];
if let Some(close_rel) = matching_bold_close_inline(rest) {
let inner = rest[2..close_rel].to_string();
let next = i + close_rel + 2;
if looks_like_bold_code(&inner) {
out.push(ProseBit::BoldCode { inner, at: i });
}
i = next;
continue;
}
i += 1;
continue;
}
if c == b'*' {
let rest = &line[i..];
if rest.starts_with("**") {
i += 1;
continue;
}
if let Some(close_rel) = matching_italic_close_inline(rest) {
let inner = rest[1..close_rel].to_string();
let next = i + close_rel + 1;
if looks_like_italic_return(line, i, next, &inner) {
out.push(ProseBit::ItalicReturn { inner, at: i });
}
i = next;
continue;
}
}
i += 1;
}
out
}
/// Whether `*inner*` is a return (vs mid-prose English/CJK emphasis).
///
/// Design §12 #5: mid-line italic is return only for whole-line, `返回*…*`,
/// or an expression-shaped span — not `*very important*` / `*foo.bar*`.
fn looks_like_italic_return(line: &str, at: usize, after: usize, inner: &str) -> bool {
let t = inner.trim();
if t.is_empty()
|| matches!(
t,
"None" | "无" | "空" | "True" | "False" | "真" | "假" | "true" | "false"
)
{
return true;
}
// Whole-line return: only ws outside the italic span (`*n*` / `*总额*`).
let before = line[..at].trim();
let after_s = line.get(after..).unwrap_or("").trim();
if before.is_empty() && after_s.is_empty() {
return true;
}
// Narration "返回*n*" / "return *n*"
if before.ends_with("返回") || before.ends_with("return") || before.ends_with("Return") {
return true;
}
// Mid-prose: expression-shaped only (`*n + 1*` / `*x=1*`); not hyphenated emphasis.
if is_hyphenated_emphasis(t) {
return false;
}
t.starts_with('>')
|| t.contains('=')
|| t.chars()
.any(|c| matches!(c, '+' | '-' | '*' | '/' | '(' | ')' | '[' | ']'))
}
fn is_hyphenated_emphasis(t: &str) -> bool {
!t.chars().any(char::is_whitespace)
&& t.contains('-')
&& !t
.chars()
.any(|c| matches!(c, '+' | '*' | '/' | '(' | ')' | '[' | ']' | '='))
}
fn scan_backtick_decl(line: &str, start: usize) -> Option<(ProseBit, usize)> {
let rest = &line[start + 1..];
let end = rest.find('`')?;
let name = rest[..end].to_string();
if name.is_empty() || !is_prose_decl_name(&name) {
return None;
}
let after_tick = start + 1 + end + 1;
let after = line.get(after_tick..)?;
if let Some(stripped) = after.strip_prefix('=') {
let def_end = stripped
.find(|c: char| {
c.is_whitespace()
|| matches!(c, ',' | '。' | ';' | '、' | ',' | ';' | '*' | '`')
})
.unwrap_or(stripped.len());
let default = stripped[..def_end].to_string();
let next = after_tick + 1 + def_end;
return Some((
ProseBit::Decl {
name,
default: Some(default),
at: start,
},
next,
));
}
Some((
ProseBit::Decl {
name,
default: None,
at: start,
},
after_tick,
))
}
/// Identifiers only: letters / digits / `_` / non-ASCII (CJK). Rejects `` `n+1` ``.
fn is_prose_decl_name(name: &str) -> bool {
let mut chars = name.chars();
let Some(first) = chars.next() else {
return false;
};
if !(first.is_alphabetic() || first == '_' || !first.is_ascii()) {
return false;
}
chars.all(|c| c.is_alphanumeric() || c == '_' || !c.is_ascii())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scan_inc_prose_line() {
let line = "对于输入变量`n`,我们执行操作**n=n+1**接着返回*n*。";
let bits = scan_prose_line(line);
assert_eq!(bits.len(), 3);
match &bits[0] {
ProseBit::Decl { name, default, .. } => {
assert_eq!(name, "n");
assert!(default.is_none());
}
_ => panic!("{:?}", bits[0]),
}
match &bits[1] {
ProseBit::BoldCode { inner, .. } => assert_eq!(inner, "n=n+1"),
_ => panic!("{:?}", bits[1]),
}
match &bits[2] {
ProseBit::ItalicReturn { inner, .. } => assert_eq!(inner, "n"),
_ => panic!("{:?}", bits[2]),
}
}
#[test]
fn scan_default_decl() {
let line = "默认`礼貌`=False。";
let bits = scan_prose_line(line);
assert_eq!(bits.len(), 1);
match &bits[0] {
ProseBit::Decl { name, default, .. } => {
assert_eq!(name, "礼貌");
assert_eq!(default.as_deref(), Some("False"));
}
_ => panic!("{:?}", bits[0]),
}
}
#[test]
fn soft_bold_emphasis_skipped() {
let bits = scan_prose_line("本文件既是**说明**也是**程序**。");
assert!(bits.is_empty(), "{bits:?}");
assert!(!looks_like_bold_code("说明"));
assert!(looks_like_bold_code("n=n+1"));
assert!(looks_like_bold_code("加一 37"));
}
#[test]
fn soft_italic_emphasis_skipped() {
let bits = scan_prose_line("When soft=True, *only* FIND continues.");
assert!(bits.is_empty(), "{bits:?}");
let bits = scan_prose_line("This is *very important* documentation.");
assert!(bits.is_empty(), "multi-word italic is emphasis: {bits:?}");
let bits = scan_prose_line("See *foo.bar* in the text.");
assert!(bits.is_empty(), "dotted italic is emphasis: {bits:?}");
let bits = scan_prose_line("Use *code-as-docs* style.");
assert!(bits.is_empty(), "hyphenated italic is emphasis: {bits:?}");
let bits = scan_prose_line("*总额*");
assert!(
matches!(&bits[..], [ProseBit::ItalicReturn { inner, .. }] if inner == "总额"),
"{bits:?}"
);
let bits = scan_prose_line("接着返回*n*。");
assert!(
matches!(&bits[..], [ProseBit::ItalicReturn { inner, .. }] if inner == "n"),
"{bits:?}"
);
}
#[test]
fn reject_n_plus_one_as_decl() {
let bits = scan_prose_line("输入`n+1`然后继续。");
assert!(bits.is_empty(), "`` `n+1` `` must not Decl: {bits:?}");
}
#[test]
fn skip_glob_backticks_without_italic() {
let bits = scan_prose_line("from `OPENAI_*` / `MARQDO_LLM_*`.");
assert!(
bits.is_empty(),
"glob-like ticks must not yield italic: {bits:?}"
);
}
}