Files
marqdo/src/document_extract.rs
T
cflmyandCursor 03396afd02 feat(workbench): land M00–M06 runtime contracts and read-only view
Add capabilities/selection/usage/DocumentView/locator/RunContext with
honest probes, LLM messages+cancel, sse-live cancel bridge, and CI
workbench_contracts gate; record design evidence for verified slices.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-25 10:55:08 +08:00

300 lines
9.5 KiB
Rust

//! Document extraction protocol (M01): DocumentSource → ExtractedDocument.
//!
//! Format parsers live here / in extensions — language core does not OCR.
//! First slice: plain text and Markdown. PDF/DOCX plug in later via the same shape.
use std::fs;
use std::path::Path;
use sha2::{Digest, Sha256};
use serde_json::{json, Value};
pub const EXTRACTOR_VERSION: &str = "marqdo-document-extract/1";
#[derive(Debug, Clone)]
pub struct DocumentSource {
pub path: std::path::PathBuf,
pub format_hint: Option<String>,
}
#[derive(Debug, Clone)]
pub struct ExtractedSegment {
pub id: String,
pub kind: String,
pub text: String,
pub locator: String,
pub confidence: f64,
}
#[derive(Debug, Clone)]
pub struct ExtractedDocument {
pub source_uri: String,
pub source_digest: String,
pub format: String,
pub extractor_version: String,
pub text: String,
pub segments: Vec<ExtractedSegment>,
pub missing: Vec<String>,
pub diagnostics: Vec<String>,
pub status: String,
}
impl ExtractedDocument {
pub fn to_json(&self) -> Value {
json!({
"schema_version": 1,
"source_uri": self.source_uri,
"source_digest": self.source_digest,
"format": self.format,
"extractor_version": self.extractor_version,
"status": self.status,
"text": self.text,
"segments": self.segments.iter().map(|s| json!({
"id": s.id,
"kind": s.kind,
"text": s.text,
"locator": s.locator,
"confidence": s.confidence,
})).collect::<Vec<_>>(),
"missing": self.missing,
"diagnostics": self.diagnostics,
})
}
}
/// Extract text/Markdown from a file. Uploaded `.mq.md` is treated as untrusted
/// source text and is never executed here.
pub fn extract_document(source: &DocumentSource) -> ExtractedDocument {
let path = &source.path;
let uri = path.to_string_lossy().replace('\\', "/");
let ext = path
.extension()
.and_then(|e| e.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let format = source
.format_hint
.clone()
.unwrap_or_else(|| {
if name.ends_with(".mq.md") || ext == "md" {
"markdown".into()
} else if matches!(ext.as_str(), "txt" | "text") {
"text".into()
} else if matches!(ext.as_str(), "pdf" | "docx") {
ext.clone()
} else {
"unknown".into()
}
});
if matches!(
format.as_str(),
"pdf" | "docx" | "ocr" | "png" | "jpg" | "jpeg" | "webp" | "tiff"
) || matches!(ext.as_str(), "pdf" | "docx" | "png" | "jpg" | "jpeg" | "webp" | "tiff")
{
let kind = if matches!(format.as_str(), "ocr")
|| matches!(ext.as_str(), "png" | "jpg" | "jpeg" | "webp" | "tiff")
{
"ocr/image".to_string()
} else {
format.clone()
};
return ExtractedDocument {
source_uri: uri,
source_digest: String::new(),
format,
extractor_version: EXTRACTOR_VERSION.into(),
text: String::new(),
segments: vec![],
missing: vec!["full_document".into()],
diagnostics: vec![format!(
"format `{kind}` requires an external extractor plugin (not in core); incomplete extraction is explicit failure"
)],
status: "failed".into(),
};
}
let bytes = match fs::read(path) {
Ok(b) => b,
Err(e) => {
return ExtractedDocument {
source_uri: uri,
source_digest: String::new(),
format,
extractor_version: EXTRACTOR_VERSION.into(),
text: String::new(),
segments: vec![],
missing: vec!["full_document".into()],
diagnostics: vec![format!("read error: {e}")],
status: "failed".into(),
};
}
};
if bytes.is_empty() {
return ExtractedDocument {
source_uri: uri,
source_digest: digest_bytes(&bytes),
format,
extractor_version: EXTRACTOR_VERSION.into(),
text: String::new(),
segments: vec![],
missing: vec!["content".into()],
diagnostics: vec!["empty document".into()],
status: "failed".into(),
};
}
// Resource limit: refuse to fully buffer huge uploads as text (M01).
const MAX_EXTRACT_BYTES: usize = 8 << 20;
if bytes.len() > MAX_EXTRACT_BYTES {
let head = &bytes[..MAX_EXTRACT_BYTES];
let lossy = String::from_utf8_lossy(head).into_owned();
let mut doc = extract_markdown_or_text(&uri, &digest_bytes(&bytes), &format, &lossy);
doc.missing.push("tail_beyond_limit".into());
doc.diagnostics.push(format!(
"document truncated at {MAX_EXTRACT_BYTES} bytes (got {}); extraction is partial",
bytes.len()
));
doc.status = "partial".into();
return doc;
}
let text = match String::from_utf8(bytes.clone()) {
Ok(t) => t,
Err(e) => {
let lossy = String::from_utf8_lossy(&bytes).into_owned();
let mut doc = extract_markdown_or_text(&uri, &digest_bytes(&bytes), &format, &lossy);
doc.diagnostics
.push(format!("invalid utf-8 ({e}); decoded lossy"));
doc.status = if doc.text.is_empty() {
"failed".into()
} else {
"partial".into()
};
return doc;
}
};
extract_markdown_or_text(&uri, &digest_bytes(&bytes), &format, &text)
}
fn extract_markdown_or_text(
uri: &str,
digest: &str,
format: &str,
text: &str,
) -> ExtractedDocument {
let mut segments = Vec::new();
let mut n = 0u32;
let mut byte_off = 0usize;
for (i, line) in text.lines().enumerate() {
let line_no = i + 1;
let line_bytes = line.len();
// M05: stable fragment locator with 1-based line + UTF-8 byte offset.
let loc = |kind: &str| format!("{kind}:L{line_no}:B{byte_off}");
if let Some(rest) = line.trim().strip_prefix('#') {
n += 1;
let heading = rest.trim_start_matches('#').trim();
segments.push(ExtractedSegment {
id: format!("seg-{n}"),
kind: "heading".into(),
text: heading.to_string(),
locator: loc("heading"),
confidence: 1.0,
});
} else if line.trim().starts_with('|') {
n += 1;
segments.push(ExtractedSegment {
id: format!("seg-{n}"),
kind: "table_row".into(),
text: line.to_string(),
locator: loc("table_row"),
confidence: 0.9,
});
} else if !line.trim().is_empty() {
n += 1;
segments.push(ExtractedSegment {
id: format!("seg-{n}"),
kind: "paragraph".into(),
text: line.to_string(),
locator: loc("paragraph"),
confidence: 1.0,
});
}
// +1 for '\n' when present in original (lines() strips it).
byte_off += line_bytes + 1;
}
ExtractedDocument {
source_uri: uri.to_string(),
source_digest: digest.to_string(),
format: format.to_string(),
extractor_version: EXTRACTOR_VERSION.into(),
text: text.to_string(),
segments,
missing: vec![],
diagnostics: vec![],
status: "ok".into(),
}
}
fn digest_bytes(bytes: &[u8]) -> String {
let mut h = Sha256::new();
h.update(bytes);
format!("sha256:{:x}", h.finalize())
}
/// Convenience for tests / CLI: extract if path exists.
pub fn extract_path(path: &Path) -> ExtractedDocument {
extract_document(&DocumentSource {
path: path.to_path_buf(),
format_hint: None,
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[test]
fn extracts_markdown_headings() {
let dir = std::env::temp_dir().join(format!("mq-extract-{}", std::process::id()));
let _ = fs::create_dir_all(&dir);
let path = dir.join("note.md");
let mut f = fs::File::create(&path).unwrap();
writeln!(f, "# rate\n\nhello").unwrap();
let doc = extract_path(&path);
assert_eq!(doc.status, "ok");
assert!(doc.segments.iter().any(|s| s.kind == "heading" && s.text == "rate"));
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn pdf_reports_missing_extractor() {
let doc = extract_document(&DocumentSource {
path: Path::new("x.pdf").into(),
format_hint: Some("pdf".into()),
});
assert_eq!(doc.status, "failed");
assert!(!doc.diagnostics.is_empty());
}
#[test]
fn ocr_image_reports_missing_extractor() {
let doc = extract_document(&DocumentSource {
path: Path::new("scan.png").into(),
format_hint: Some("ocr".into()),
});
assert_eq!(doc.status, "failed");
assert!(doc.missing.contains(&"full_document".into()));
assert!(doc.diagnostics.iter().any(|d| d.contains("ocr") || d.contains("extractor")));
}
}