ext/web W4: Postgres, Redis/memory cache, and S3/file storage drivers

URL-scheme backends keep the same # db CRUD author API; add # cache and

Co-authored-by: Cursor <cursoragent@cursor.com>
# storage with offline memory:/file: gold coverage.
This commit is contained in:
cflmy committed 2026-08-28 10:33:05 +08:00
1 parent e7c9c21a49
commit 034f52b7e7
14 files changed
+2504 -17

No files matched your search

+242
View File
@@ -0,0 +1,242 @@
//! `# cache` — Redis or in-process memory (design ext-web-drivers).
use serde_json::{json, Value};
use std::collections::HashMap;
use std::sync::Mutex;
use std::time::{SystemTime, UNIX_EPOCH};
struct Entry {
value: String,
/// Absolute expiry unix secs; `None` = no TTL.
exp: Option<u64>,
}
struct MemStore {
data: HashMap<String, Entry>,
}
enum Backend {
Memory(MemStore),
/// Redis URL — connections opened per call (simple; enough for W4).
RedisUrl(String),
}
static HANDLES: Mutex<Option<HashMap<String, Backend>>> = Mutex::new(None);
fn now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
fn with_map<T>(f: impl FnOnce(&mut HashMap<String, Backend>) -> T) -> T {
let mut g = HANDLES.lock().unwrap_or_else(|e| e.into_inner());
let m = g.get_or_insert_with(HashMap::new);
f(m)
}
fn is_memory(url: &str) -> bool {
let u = url.trim().to_ascii_lowercase();
u.is_empty() || u == "memory:" || u == "memory://" || u.starts_with("memory:")
}
fn is_redis(url: &str) -> bool {
let u = url.trim().to_ascii_lowercase();
u.starts_with("redis://") || u.starts_with("rediss://")
}
/// Open (or reuse) a cache handle. Returns `{ _type, url }`.
pub fn open(url: &str) -> Result<Value, String> {
let url = if url.trim().is_empty() {
"memory:"
} else {
url.trim()
};
if !is_memory(url) && !is_redis(url) {
return Err(format!(
"cache url must be `memory:` or `redis://…`, got `{url}`"
));
}
with_map(|m| {
if !m.contains_key(url) {
let backend = if is_memory(url) {
Backend::Memory(MemStore {
data: HashMap::new(),
})
} else {
Backend::RedisUrl(url.to_string())
};
m.insert(url.to_string(), backend);
}
});
Ok(json!({ "_type": "cache", "url": url }))
}
fn prune_mem(store: &mut MemStore) {
let t = now();
store.data.retain(|_, e| e.exp.map(|x| x > t).unwrap_or(true));
}
fn with_backend<T>(url: &str, f: impl FnOnce(&mut Backend) -> Result<T, String>) -> Result<T, String> {
with_map(|m| {
let b = m
.get_mut(url)
.ok_or_else(|| format!("cache `{url}` not opened — call cache.new first"))?;
f(b)
})
}
pub fn get(url: &str, key: &str) -> Result<Value, String> {
with_backend(url, |b| match b {
Backend::Memory(store) => {
prune_mem(store);
match store.data.get(key) {
Some(e) => Ok(json!({ "ok": true, "value": e.value })),
None => Ok(json!({ "ok": false })),
}
}
Backend::RedisUrl(u) => redis_get(u, key),
})
}
pub fn set(url: &str, key: &str, value: &str, ttl_sec: Option<u64>) -> Result<Value, String> {
with_backend(url, |b| match b {
Backend::Memory(store) => {
prune_mem(store);
let exp = ttl_sec.filter(|t| *t > 0).map(|t| now() + t);
store.data.insert(
key.to_string(),
Entry {
value: value.to_string(),
exp,
},
);
Ok(json!({ "ok": true }))
}
Backend::RedisUrl(u) => redis_set(u, key, value, ttl_sec),
})
}
pub fn del(url: &str, key: &str) -> Result<Value, String> {
with_backend(url, |b| match b {
Backend::Memory(store) => {
let ok = store.data.remove(key).is_some();
Ok(json!({ "ok": ok }))
}
Backend::RedisUrl(u) => redis_del(u, key),
})
}
pub fn exists(url: &str, key: &str) -> Result<Value, String> {
with_backend(url, |b| match b {
Backend::Memory(store) => {
prune_mem(store);
Ok(json!({ "ok": store.data.contains_key(key) }))
}
Backend::RedisUrl(u) => redis_exists(u, key),
})
}
pub fn ttl(url: &str, key: &str) -> Result<Value, String> {
with_backend(url, |b| match b {
Backend::Memory(store) => {
prune_mem(store);
match store.data.get(key) {
Some(e) => {
let left = match e.exp {
Some(x) => x.saturating_sub(now()) as i64,
None => -1,
};
Ok(json!({ "ok": true, "ttl": left }))
}
None => Ok(json!({ "ok": false, "ttl": -2 })),
}
}
Backend::RedisUrl(u) => redis_ttl(u, key),
})
}
fn redis_get(url: &str, key: &str) -> Result<Value, String> {
let client = redis::Client::open(url).map_err(|e| e.to_string())?;
let mut con = client.get_connection().map_err(|e| e.to_string())?;
let v: Option<String> = redis::cmd("GET")
.arg(key)
.query(&mut con)
.map_err(|e| e.to_string())?;
match v {
Some(s) => Ok(json!({ "ok": true, "value": s })),
None => Ok(json!({ "ok": false })),
}
}
fn redis_set(url: &str, key: &str, value: &str, ttl_sec: Option<u64>) -> Result<Value, String> {
let client = redis::Client::open(url).map_err(|e| e.to_string())?;
let mut con = client.get_connection().map_err(|e| e.to_string())?;
match ttl_sec.filter(|t| *t > 0) {
Some(t) => {
redis::cmd("SETEX")
.arg(key)
.arg(t)
.arg(value)
.query::<()>(&mut con)
.map_err(|e| e.to_string())?;
}
None => {
redis::cmd("SET")
.arg(key)
.arg(value)
.query::<()>(&mut con)
.map_err(|e| e.to_string())?;
}
}
Ok(json!({ "ok": true }))
}
fn redis_del(url: &str, key: &str) -> Result<Value, String> {
let client = redis::Client::open(url).map_err(|e| e.to_string())?;
let mut con = client.get_connection().map_err(|e| e.to_string())?;
let n: i64 = redis::cmd("DEL")
.arg(key)
.query(&mut con)
.map_err(|e| e.to_string())?;
Ok(json!({ "ok": n > 0 }))
}
fn redis_exists(url: &str, key: &str) -> Result<Value, String> {
let client = redis::Client::open(url).map_err(|e| e.to_string())?;
let mut con = client.get_connection().map_err(|e| e.to_string())?;
let n: i64 = redis::cmd("EXISTS")
.arg(key)
.query(&mut con)
.map_err(|e| e.to_string())?;
Ok(json!({ "ok": n > 0 }))
}
fn redis_ttl(url: &str, key: &str) -> Result<Value, String> {
let client = redis::Client::open(url).map_err(|e| e.to_string())?;
let mut con = client.get_connection().map_err(|e| e.to_string())?;
let t: i64 = redis::cmd("TTL")
.arg(key)
.query(&mut con)
.map_err(|e| e.to_string())?;
Ok(json!({ "ok": t >= -1, "ttl": t }))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn memory_roundtrip() {
let url = "memory:test-cache";
open(url).unwrap();
set(url, "a", "1", Some(60)).unwrap();
let g = get(url, "a").unwrap();
assert_eq!(g["ok"], true);
assert_eq!(g["value"], "1");
assert_eq!(exists(url, "a").unwrap()["ok"], true);
del(url, "a").unwrap();
assert_eq!(get(url, "a").unwrap()["ok"], false);
}
}
+53
View File
@@ -172,6 +172,9 @@ fn from_sql(v: ValueRef<'_>) -> Value {
/// `fields` = list of {name,type,nullable?} or columnar schema table already normalized.
pub fn init(url: &str, table: &str, fields: &Value) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::init(url, table, fields);
}
let table = ident(table)?;
let cols = crate::table::as_fields(fields);
let arr = cols.as_array().ok_or("fields must be a list")?;
@@ -224,6 +227,9 @@ pub fn init(url: &str, table: &str, fields: &Value) -> Result<Value, String> {
}
pub fn insert(url: &str, table: &str, rows: &Value, txn_id: Option<&str>) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::insert(url, table, rows, txn_id);
}
let table = ident(table)?;
let rows = crate::table::as_rows(rows);
let arr = rows.as_array().ok_or("rows must be a list")?;
@@ -489,6 +495,9 @@ pub fn select_order(
offset: Option<i64>,
txn_id: Option<&str>,
) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::select_order(url, table, limit, where_v, order, offset, txn_id);
}
let table = ident(table)?;
let (exprs, mut vals) = parse_where(where_v)?;
let conn = conn_for(url, txn_id)?;
@@ -584,6 +593,10 @@ fn count_rows(
}
pub fn get(url: &str, table: &str, id: &str, txn_id: Option<&str>) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::get(url, table, id, txn_id);
}
let table = ident(table)?;
let conn = conn_for(url, txn_id)?;
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
@@ -612,6 +625,10 @@ pub fn update(
row: &Value,
txn_id: Option<&str>,
) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::update(url, table, id, row, txn_id);
}
let table = ident(table)?;
let obj = row.as_object().ok_or("row must be a map")?;
let mut sets = Vec::new();
@@ -641,6 +658,10 @@ pub fn update(
}
pub fn delete(url: &str, table: &str, id: &str, txn_id: Option<&str>) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::delete(url, table, id, txn_id);
}
let table = ident(table)?;
let conn = conn_for(url, txn_id)?;
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
@@ -652,6 +673,10 @@ pub fn delete(url: &str, table: &str, id: &str, txn_id: Option<&str>) -> Result<
}
pub fn exec(url: &str, sql: &str, args: Option<&Value>, txn_id: Option<&str>) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::exec(url, sql, args, txn_id);
}
let conn = conn_for(url, txn_id)?;
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
let vals: Vec<rusqlite::types::Value> = match args {
@@ -667,6 +692,10 @@ pub fn exec(url: &str, sql: &str, args: Option<&Value>, txn_id: Option<&str>) ->
/// Run a `SELECT …` and return the result set (count / join / group / subquery).
/// Bare SQL — the result carries `{ rows, count }`.
pub fn query(url: &str, sql: &str, args: Option<&Value>, txn_id: Option<&str>) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::query(url, sql, args, txn_id);
}
let conn = conn_for(url, txn_id)?;
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
let vals: Vec<rusqlite::types::Value> = match args {
@@ -679,6 +708,10 @@ pub fn query(url: &str, sql: &str, args: Option<&Value>, txn_id: Option<&str>) -
/// Count rows matching a `where` filter (aggregation helper).
pub fn count(url: &str, table: &str, where_v: Option<&Value>, txn_id: Option<&str>) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::count(url, table, where_v, txn_id);
}
let table = ident(table)?;
let (exprs, vals) = parse_where(where_v)?;
let conn = conn_for(url, txn_id)?;
@@ -694,6 +727,10 @@ pub fn count(url: &str, table: &str, where_v: Option<&Value>, txn_id: Option<&st
/// Begin a transaction. Borrows the pooled connection exclusively and records
/// it under a fresh txn id; returns `{ txn, url }` for `commit` / `rollback`.
pub fn begin(url: &str) -> Result<Value, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::begin(url);
}
let conn = pooled(url)?;
{
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
@@ -711,6 +748,10 @@ pub fn begin(url: &str) -> Result<Value, String> {
/// Commit a transaction and return its connection to the pool.
pub fn commit(txn_id: &str) -> Result<Value, String> {
if txn_id.starts_with("pg-txn-") {
return crate::db_pg::commit(txn_id);
}
let (_, conn) = take_txn(txn_id)?;
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
c.execute_batch("COMMIT").map_err(|e| e.to_string())?;
@@ -719,6 +760,10 @@ pub fn commit(txn_id: &str) -> Result<Value, String> {
/// Roll back a transaction and return its connection to the pool.
pub fn rollback(txn_id: &str) -> Result<Value, String> {
if txn_id.starts_with("pg-txn-") {
return crate::db_pg::rollback(txn_id);
}
let (_, conn) = take_txn(txn_id)?;
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
c.execute_batch("ROLLBACK").map_err(|e| e.to_string())?;
@@ -735,6 +780,10 @@ fn take_txn(txn_id: &str) -> Result<(String, Arc<Mutex<Connection>>), String> {
}
pub fn list_tables(url: &str) -> Result<Vec<String>, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::list_tables(url);
}
let conn = pooled(url)?;
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
let mut stmt = c
@@ -760,6 +809,10 @@ pub struct ColumnInfo {
/// `PRAGMA table_info` → column metadata for admin forms.
pub fn table_info(url: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
if crate::db_pg::is_postgres(url) {
return crate::db_pg::table_info(url, table);
}
let table = ident(table)?;
let conn = pooled(url)?;
let c = conn.lock().unwrap_or_else(|e| e.into_inner());
+534
View File
@@ -0,0 +1,534 @@
//! Postgres backend for `# db` (same author-facing API as SQLite).
use crate::driver::{db_kind, rewrite_placeholders_pg, DbKind};
use postgres::types::ToSql;
use postgres::{Client, NoTls, Row};
use serde_json::{json, Map, Value};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
static POOL: Mutex<Option<HashMap<String, Arc<Mutex<Client>>>>> = Mutex::new(None);
static TXNS: Mutex<Option<HashMap<String, (String, Arc<Mutex<Client>>)>>> = Mutex::new(None);
pub fn is_postgres(url: &str) -> bool {
matches!(db_kind(url), DbKind::Postgres)
}
fn pooled(url: &str) -> Result<Arc<Mutex<Client>>, String> {
let mut guard = POOL.lock().unwrap_or_else(|e| e.into_inner());
let pool = guard.get_or_insert_with(HashMap::new);
if let Some(c) = pool.get(url) {
return Ok(c.clone());
}
let client = Client::connect(url, NoTls).map_err(|e| format!("postgres connect: {e}"))?;
let arc = Arc::new(Mutex::new(client));
pool.insert(url.to_string(), arc.clone());
Ok(arc)
}
fn conn_for(url: &str, txn_id: Option<&str>) -> Result<Arc<Mutex<Client>>, String> {
if let Some(tid) = txn_id {
let guard = TXNS.lock().unwrap_or_else(|e| e.into_inner());
match guard.as_ref().and_then(|m| m.get(tid)) {
Some((_, c)) => Ok(c.clone()),
None => Err(format!("unknown transaction `{tid}`")),
}
} else {
pooled(url)
}
}
fn next_txn_id() -> String {
use std::time::{SystemTime, UNIX_EPOCH};
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.subsec_nanos())
.unwrap_or(0);
format!("pg-txn-{}-{nanos}", std::process::id())
}
fn ident(s: &str) -> Result<&str, String> {
let s = s.trim();
if s.is_empty()
|| !s
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_')
{
return Err(format!("bad identifier `{s}`"));
}
Ok(s)
}
fn cell_str(v: &Value) -> String {
match v {
Value::Null => String::new(),
Value::String(s) => s.clone(),
Value::Bool(b) => b.to_string(),
Value::Number(n) => n.to_string(),
other => other.to_string(),
}
}
fn pg_type(t: &str) -> &'static str {
match t.trim().to_ascii_lowercase().as_str() {
"int" | "integer" => "INTEGER",
"real" | "float" | "double" => "DOUBLE PRECISION",
"blob" | "bytea" => "BYTEA",
"bool" | "boolean" => "BOOLEAN",
_ => "TEXT",
}
}
fn to_pg_box(v: &Value) -> Box<dyn ToSql + Sync + Send> {
match v {
Value::Null => Box::new(None::<String>),
Value::Bool(b) => Box::new(*b),
Value::Number(n) => {
if let Some(i) = n.as_i64() {
Box::new(i)
} else if let Some(f) = n.as_f64() {
Box::new(f)
} else {
Box::new(n.to_string())
}
}
Value::String(s) => Box::new(s.clone()),
other => Box::new(other.to_string()),
}
}
fn row_to_json(row: &Row) -> Value {
let mut m = Map::new();
for (i, col) in row.columns().iter().enumerate() {
let name = col.name().to_string();
let v: Value = if let Ok(x) = row.try_get::<_, Option<i64>>(i) {
x.map(|n| json!(n)).unwrap_or(Value::Null)
} else if let Ok(x) = row.try_get::<_, Option<f64>>(i) {
x.map(|n| json!(n)).unwrap_or(Value::Null)
} else if let Ok(x) = row.try_get::<_, Option<bool>>(i) {
x.map(|b| json!(b)).unwrap_or(Value::Null)
} else if let Ok(x) = row.try_get::<_, Option<String>>(i) {
x.map(Value::String).unwrap_or(Value::Null)
} else {
Value::Null
};
m.insert(name, v);
}
Value::Object(m)
}
fn exec_sql(client: &mut Client, sql: &str, params: &[Value]) -> Result<u64, String> {
let sql = rewrite_placeholders_pg(sql);
let boxes: Vec<Box<dyn ToSql + Sync + Send>> = params.iter().map(to_pg_box).collect();
let refs: Vec<&(dyn ToSql + Sync)> = boxes
.iter()
.map(|b| b.as_ref() as &(dyn ToSql + Sync))
.collect();
client
.execute(sql.as_str(), refs.as_slice())
.map_err(|e| e.to_string())
}
fn query_sql(client: &mut Client, sql: &str, params: &[Value]) -> Result<Vec<Value>, String> {
let sql = rewrite_placeholders_pg(sql);
let boxes: Vec<Box<dyn ToSql + Sync + Send>> = params.iter().map(to_pg_box).collect();
let refs: Vec<&(dyn ToSql + Sync)> = boxes
.iter()
.map(|b| b.as_ref() as &(dyn ToSql + Sync))
.collect();
let rows = client
.query(sql.as_str(), refs.as_slice())
.map_err(|e| e.to_string())?;
Ok(rows.iter().map(row_to_json).collect())
}
/// Equality-map / filter-table → `(AND exprs with ?, vals)`.
fn parse_where_simple(where_v: Option<&Value>) -> Result<(Vec<String>, Vec<Value>), String> {
let Some(w) = where_v else {
return Ok((vec![], vec![]));
};
let w = crate::table::as_rows(w);
let mut exprs = Vec::new();
let mut vals = Vec::new();
match w {
Value::Array(arr) => {
for item in arr {
let Value::Object(m) = item else {
continue;
};
let field = m
.get("field")
.or_else(|| m.get("字段"))
.or_else(|| m.get("列"))
.map(cell_str)
.unwrap_or_default();
if !field.is_empty() {
let col = ident(&field)?;
let op = m
.get("op")
.or_else(|| m.get("操作"))
.map(cell_str)
.unwrap_or_else(|| "=".into());
let op = match op.trim().to_ascii_lowercase().as_str() {
"" | "=" | "eq" | "等于" => "=",
"!=" | "<>" | "ne" | "不等于" => "!=",
">" | "gt" | "大于" => ">",
">=" | "gte" => ">=",
"<" | "lt" | "小于" => "<",
"<=" | "lte" => "<=",
"like" | "包含" => "LIKE",
other => return Err(format!("unsupported where op `{other}`")),
};
let val = m
.get("value")
.or_else(|| m.get("值"))
.cloned()
.unwrap_or(Value::Null);
exprs.push(format!("\"{col}\" {op} ?"));
vals.push(val);
} else {
for (k, v) in &m {
if k == "@" || k == "行" || k == "row" || k.starts_with('_') {
continue;
}
let col = ident(k)?;
exprs.push(format!("\"{col}\" = ?"));
vals.push(v.clone());
}
}
}
}
_ => {}
}
Ok((exprs, vals))
}
pub fn init(url: &str, table: &str, fields: &Value) -> Result<Value, String> {
let table = ident(table)?;
let cols = crate::table::as_fields(fields);
let arr = cols.as_array().ok_or("fields must be a list")?;
if arr.is_empty() {
return Err("empty fields".into());
}
let mut parts = Vec::new();
let mut has_pk = false;
for c in arr {
let name = c
.get("name")
.or_else(|| c.get("字段"))
.map(cell_str)
.unwrap_or_default();
let name = ident(&name)?;
let ty = c
.get("type")
.or_else(|| c.get("类型"))
.map(cell_str)
.unwrap_or_else(|| "text".into());
let nullable = c
.get("nullable")
.or_else(|| c.get("可空"))
.and_then(|v| match v {
Value::Bool(b) => Some(*b),
Value::String(s) => {
Some(matches!(s.as_str(), "true" | "True" | "1" | "是" | "可"))
}
_ => None,
})
.unwrap_or(true);
let part = if name == "id" && !has_pk && pg_type(&ty) == "INTEGER" {
has_pk = true;
format!("\"{name}\" SERIAL PRIMARY KEY")
} else if name == "id" && !has_pk {
has_pk = true;
format!("\"{name}\" {} PRIMARY KEY", pg_type(&ty))
} else {
let mut col = format!("\"{name}\" {}", pg_type(&ty));
if !nullable {
col.push_str(" NOT NULL");
}
col
};
parts.push(part);
}
let sql = format!(
"CREATE TABLE IF NOT EXISTS \"{table}\" ({})",
parts.join(", ")
);
let conn = pooled(url)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
c.batch_execute(&sql).map_err(|e| e.to_string())?;
Ok(json!({ "_type": "db_table", "name": table, "url": url }))
}
pub fn insert(url: &str, table: &str, rows: &Value, txn_id: Option<&str>) -> Result<Value, String> {
let table = ident(table)?;
let rows = crate::table::as_rows(rows);
let arr = rows.as_array().ok_or("rows must be a list")?;
let conn = conn_for(url, txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let mut n = 0i64;
for row in arr {
let obj = row.as_object().ok_or("row must be a map")?;
let mut cols = Vec::new();
let mut ph = Vec::new();
let mut vals = Vec::new();
for (k, v) in obj {
if k == "@" || k == "行" || k == "row" {
continue;
}
let _ = ident(k)?;
cols.push(format!("\"{k}\""));
ph.push("?");
vals.push(v.clone());
}
if cols.is_empty() {
continue;
}
let sql = format!(
"INSERT INTO \"{table}\" ({}) VALUES ({})",
cols.join(", "),
ph.join(", ")
);
exec_sql(&mut c, &sql, &vals)?;
n += 1;
}
Ok(json!({ "ok": true, "inserted": n }))
}
pub fn select_order(
url: &str,
table: &str,
limit: i64,
where_v: Option<&Value>,
order: Option<&str>,
offset: Option<i64>,
txn_id: Option<&str>,
) -> Result<Value, String> {
let table = ident(table)?;
let (exprs, mut vals) = parse_where_simple(where_v)?;
let mut sql = format!("SELECT * FROM \"{table}\"");
let where_sql = if exprs.is_empty() {
String::new()
} else {
format!(" WHERE {}", exprs.join(" AND "))
};
sql.push_str(&where_sql);
if let Some(order) = order.map(str::trim).filter(|s| !s.is_empty()) {
let mut order_exprs = Vec::new();
for part in order.split(',').map(str::trim).filter(|p| !p.is_empty()) {
let (col, dir) = match part.strip_prefix('-') {
Some(c) => (c, " DESC"),
None => (part, ""),
};
order_exprs.push(format!("\"{}\"{}", ident(col)?, dir));
}
if !order_exprs.is_empty() {
sql.push_str(" ORDER BY ");
sql.push_str(&order_exprs.join(", "));
}
}
sql.push_str(" LIMIT ?");
vals.push(json!(limit));
if let Some(off) = offset {
if off > 0 {
sql.push_str(" OFFSET ?");
vals.push(json!(off));
}
}
let conn = conn_for(url, txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let rows = query_sql(&mut c, &sql, &vals)?;
if offset.is_some() {
let mut count_sql = format!("SELECT COUNT(*) AS count FROM \"{table}\"");
count_sql.push_str(&where_sql);
let count_vals: Vec<Value> = vals
.iter()
.take(exprs.len())
.cloned()
.collect();
let total = query_sql(&mut c, &count_sql, &count_vals)?
.first()
.and_then(|r| r.get("count"))
.and_then(|v| v.as_i64())
.unwrap_or(rows.len() as i64);
Ok(json!({ "rows": rows, "total": total }))
} else {
Ok(json!({ "rows": rows }))
}
}
pub fn get(url: &str, table: &str, id: &str, txn_id: Option<&str>) -> Result<Value, String> {
let table = ident(table)?;
let sql = format!("SELECT * FROM \"{table}\" WHERE \"id\" = ? LIMIT 1");
let conn = conn_for(url, txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let rows = query_sql(&mut c, &sql, &[json!(id)])?;
Ok(rows.into_iter().next().unwrap_or(Value::Null))
}
pub fn update(
url: &str,
table: &str,
id: &str,
row: &Value,
txn_id: Option<&str>,
) -> Result<Value, String> {
let table = ident(table)?;
let obj = row.as_object().ok_or("row must be a map")?;
let mut sets = Vec::new();
let mut vals = Vec::new();
for (k, v) in obj {
if k == "id" || k == "@" || k == "行" || k == "row" {
continue;
}
let _ = ident(k)?;
sets.push(format!("\"{k}\" = ?"));
vals.push(v.clone());
}
if sets.is_empty() {
return Ok(json!({ "ok": true, "updated": 0 }));
}
vals.push(json!(id));
let sql = format!(
"UPDATE \"{table}\" SET {} WHERE \"id\" = ?",
sets.join(", ")
);
let conn = conn_for(url, txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let n = exec_sql(&mut c, &sql, &vals)?;
Ok(json!({ "ok": true, "updated": n }))
}
pub fn delete(url: &str, table: &str, id: &str, txn_id: Option<&str>) -> Result<Value, String> {
let table = ident(table)?;
let sql = format!("DELETE FROM \"{table}\" WHERE \"id\" = ?");
let conn = conn_for(url, txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let n = exec_sql(&mut c, &sql, &[json!(id)])?;
Ok(json!({ "ok": true, "deleted": n }))
}
pub fn exec(url: &str, sql: &str, args: Option<&Value>, txn_id: Option<&str>) -> Result<Value, String> {
let vals: Vec<Value> = match args {
Some(Value::Array(a)) => a.clone(),
_ => Vec::new(),
};
let conn = conn_for(url, txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let n = exec_sql(&mut c, sql, &vals)?;
Ok(json!({ "ok": true, "changes": n }))
}
pub fn query(url: &str, sql: &str, args: Option<&Value>, txn_id: Option<&str>) -> Result<Value, String> {
let vals: Vec<Value> = match args {
Some(Value::Array(a)) => a.clone(),
_ => Vec::new(),
};
let conn = conn_for(url, txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let rows = query_sql(&mut c, sql, &vals)?;
Ok(json!({ "rows": rows, "count": rows.len() }))
}
pub fn count(
url: &str,
table: &str,
where_v: Option<&Value>,
txn_id: Option<&str>,
) -> Result<Value, String> {
let table = ident(table)?;
let (exprs, vals) = parse_where_simple(where_v)?;
let mut sql = format!("SELECT COUNT(*) AS count FROM \"{table}\"");
if !exprs.is_empty() {
sql.push_str(&format!(" WHERE {}", exprs.join(" AND ")));
}
let conn = conn_for(url, txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let rows = query_sql(&mut c, &sql, &vals)?;
let n = rows
.first()
.and_then(|r| r.get("count"))
.and_then(|v| v.as_i64())
.unwrap_or(0);
Ok(json!({ "count": n }))
}
pub fn begin(url: &str) -> Result<Value, String> {
let conn = pooled(url)?;
{
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
c.batch_execute("BEGIN").map_err(|e| e.to_string())?;
}
let id = next_txn_id();
{
let mut guard = TXNS.lock().unwrap_or_else(|e| e.into_inner());
guard
.get_or_insert_with(HashMap::new)
.insert(id.clone(), (url.to_string(), conn));
}
Ok(json!({ "_type": "txn", "txn": id, "url": url, "事务": id, "地址": url }))
}
pub fn commit(txn_id: &str) -> Result<Value, String> {
let (_, conn) = take_txn(txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
c.batch_execute("COMMIT").map_err(|e| e.to_string())?;
Ok(json!({ "ok": true }))
}
pub fn rollback(txn_id: &str) -> Result<Value, String> {
let (_, conn) = take_txn(txn_id)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
c.batch_execute("ROLLBACK").map_err(|e| e.to_string())?;
Ok(json!({ "ok": true }))
}
fn take_txn(txn_id: &str) -> Result<(String, Arc<Mutex<Client>>), String> {
let mut guard = TXNS.lock().unwrap_or_else(|e| e.into_inner());
guard
.as_mut()
.and_then(|m| m.remove(txn_id))
.ok_or_else(|| format!("unknown transaction `{txn_id}`"))
}
pub fn list_tables(url: &str) -> Result<Vec<String>, String> {
let conn = pooled(url)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let rows = query_sql(
&mut c,
"SELECT tablename AS name FROM pg_catalog.pg_tables WHERE schemaname = 'public' ORDER BY tablename",
&[],
)?;
Ok(rows
.iter()
.filter_map(|r| r.get("name").and_then(|v| v.as_str()).map(|s| s.to_string()))
.collect())
}
pub fn table_info(url: &str, table: &str) -> Result<Vec<crate::db::ColumnInfo>, String> {
let table = ident(table)?;
let conn = pooled(url)?;
let mut c = conn.lock().unwrap_or_else(|e| e.into_inner());
let rows = query_sql(
&mut c,
"SELECT column_name AS name, data_type AS sql_type, \
CASE WHEN is_nullable = 'NO' THEN true ELSE false END AS notnull \
FROM information_schema.columns \
WHERE table_schema = 'public' AND table_name = ? \
ORDER BY ordinal_position",
&[json!(table)],
)?;
Ok(rows
.iter()
.map(|r| crate::db::ColumnInfo {
name: r.get("name").and_then(|v| v.as_str()).unwrap_or("").into(),
sql_type: r
.get("sql_type")
.and_then(|v| v.as_str())
.unwrap_or("text")
.into(),
notnull: r.get("notnull").and_then(|v| v.as_bool()).unwrap_or(false),
pk: r.get("name").and_then(|v| v.as_str()) == Some("id"),
})
.collect())
}
+54
View File
@@ -0,0 +1,54 @@
//! Driver URL schemes for `# db` / `# cache` / `# storage` (design ext-web-drivers).
/// Which SQL backend a `# db` URL selects.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DbKind {
Sqlite,
Postgres,
}
pub fn db_kind(url: &str) -> DbKind {
let u = url.trim().to_ascii_lowercase();
if u.starts_with("postgres://") || u.starts_with("postgresql://") {
DbKind::Postgres
} else {
DbKind::Sqlite
}
}
/// Rewrite SQL written with `?` placeholders into Postgres `$1,$2,…` form.
pub fn rewrite_placeholders_pg(sql: &str) -> String {
let mut out = String::with_capacity(sql.len() + 8);
let mut n = 0usize;
let mut chars = sql.chars().peekable();
while let Some(c) = chars.next() {
if c == '?' {
n += 1;
out.push('$');
out.push_str(&n.to_string());
} else {
out.push(c);
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn kind_from_url() {
assert_eq!(db_kind("sqlite:a.db"), DbKind::Sqlite);
assert_eq!(db_kind("postgres://u:p@h/db"), DbKind::Postgres);
assert_eq!(db_kind("postgresql://localhost/x"), DbKind::Postgres);
}
#[test]
fn rewrite_qmark() {
assert_eq!(
rewrite_placeholders_pg("SELECT * FROM t WHERE a = ? AND b IN (?, ?)"),
"SELECT * FROM t WHERE a = $1 AND b IN ($2, $3)"
);
}
}
+99 -2
View File
@@ -2,8 +2,11 @@
mod compose;
mod db;
mod db_pg;
mod driver;
mod form;
mod http;
mod cache;
mod markdown;
mod middleware;
mod password;
@@ -11,6 +14,7 @@ mod rate_limit;
mod render;
mod rss;
mod session;
mod storage;
mod table;
mod ws;
@@ -158,8 +162,15 @@ fn entry_dir() -> PathBuf {
})
}
/// `sqlite:` URL with a relative path → absolute against `entry_dir()`.
/// Resolve a DB URL for the plugin.
///
/// - `postgres://` / `postgresql://` — passed through unchanged.
/// - `sqlite:` / bare relative path — made absolute against `entry_dir()`.
fn resolve_db_url(url: &str) -> String {
let lower = url.to_ascii_lowercase();
if lower.starts_with("postgres://") || lower.starts_with("postgresql://") {
return url.to_string();
}
let stripped = url
.strip_prefix("sqlite:")
.or_else(|| url.strip_prefix("SQLITE:"))
@@ -452,7 +463,78 @@ web_ffi!(web_render, |args: &Value| {
web_ffi!(web_db_new, |args: &Value| {
let url = arg_str_opt(args, "url").unwrap_or("sqlite:site.db");
Ok(json!({ "url": url }))
Ok(json!({ "url": resolve_db_url(url), "_type": "db" }))
});
web_ffi!(web_cache_new, |args: &Value| {
let url = arg_str_opt(args, "url").unwrap_or("memory:");
cache::open(url)
});
web_ffi!(web_cache_get, |args: &Value| {
let url = arg_str(args, "url")?;
let key = arg_str(args, "key")?;
cache::get(url, key)
});
web_ffi!(web_cache_set, |args: &Value| {
let url = arg_str(args, "url")?;
let key = arg_str(args, "key")?;
let value = arg_text(args, "value")?;
let ttl = args
.get("ttl")
.and_then(|v| v.as_u64().or_else(|| v.as_i64().map(|i| i as u64)));
cache::set(url, key, &value, ttl)
});
web_ffi!(web_cache_del, |args: &Value| {
let url = arg_str(args, "url")?;
let key = arg_str(args, "key")?;
cache::del(url, key)
});
web_ffi!(web_cache_exists, |args: &Value| {
let url = arg_str(args, "url")?;
let key = arg_str(args, "key")?;
cache::exists(url, key)
});
web_ffi!(web_cache_ttl, |args: &Value| {
let url = arg_str(args, "url")?;
let key = arg_str(args, "key")?;
cache::ttl(url, key)
});
web_ffi!(web_storage_new, |args: &Value| {
let url = arg_str(args, "url").unwrap_or("file:data/blobs");
storage::open(url)
});
web_ffi!(web_storage_put, |args: &Value| {
let url = arg_str(args, "url")?;
let key = arg_str(args, "key")?;
let body = arg_str_opt(args, "body");
let path = arg_str_opt(args, "path");
let content_type = arg_str_opt(args, "content_type");
storage::put(url, key, body, path, content_type)
});
web_ffi!(web_storage_get, |args: &Value| {
let url = arg_str(args, "url")?;
let key = arg_str(args, "key")?;
storage::get(url, key)
});
web_ffi!(web_storage_delete, |args: &Value| {
let url = arg_str(args, "url")?;
let key = arg_str(args, "key")?;
storage::delete(url, key)
});
web_ffi!(web_storage_list, |args: &Value| {
let url = arg_str(args, "url")?;
let prefix = arg_str_opt(args, "prefix");
storage::list(url, prefix)
});
web_ffi!(web_db_init, |args: &Value| {
@@ -1249,6 +1331,21 @@ pub unsafe extern "C" fn marqdo_plugin_init(host: *const MarqdoHostApi) -> c_int
),
("web_render", "page", web_render as PluginFn),
("web_db_new", "url", web_db_new as PluginFn),
("web_cache_new", "url", web_cache_new as PluginFn),
("web_cache_get", "url,key", web_cache_get as PluginFn),
("web_cache_set", "url,key,value,ttl", web_cache_set as PluginFn),
("web_cache_del", "url,key", web_cache_del as PluginFn),
("web_cache_exists", "url,key", web_cache_exists as PluginFn),
("web_cache_ttl", "url,key", web_cache_ttl as PluginFn),
("web_storage_new", "url", web_storage_new as PluginFn),
(
"web_storage_put",
"url,key,body,path,content_type",
web_storage_put as PluginFn,
),
("web_storage_get", "url,key", web_storage_get as PluginFn),
("web_storage_delete", "url,key", web_storage_delete as PluginFn),
("web_storage_list", "url,prefix", web_storage_list as PluginFn),
("web_db_init", "url,name,fields", web_db_init as PluginFn),
("web_db_insert", "url,table,rows,txn", web_db_insert as PluginFn),
(
+607
View File
@@ -0,0 +1,607 @@
//! `# storage` — local `file:` blobs; `s3://` reserved for live S3/MinIO (SigV4 later).
use serde_json::{json, Value};
use std::fs;
use std::path::{Path, PathBuf};
fn is_file(url: &str) -> bool {
let u = url.trim().to_ascii_lowercase();
u.starts_with("file:") || u.starts_with("file://")
}
fn is_s3(url: &str) -> bool {
url.trim().to_ascii_lowercase().starts_with("s3://")
}
fn file_root(url: &str) -> Result<PathBuf, String> {
let raw = url
.trim()
.strip_prefix("file://")
.or_else(|| url.trim().strip_prefix("file:"))
.unwrap_or(url.trim());
let p = PathBuf::from(raw);
if !p.as_os_str().is_empty() {
fs::create_dir_all(&p).map_err(|e| format!("storage mkdir: {e}"))?;
}
Ok(p)
}
fn safe_key(key: &str) -> Result<String, String> {
let k = key.trim().trim_start_matches('/');
if k.is_empty() {
return Err("storage key is empty".into());
}
if k.contains("..") {
return Err("storage key must not contain `..`".into());
}
Ok(k.replace('\\', "/"))
}
fn path_for(root: &Path, key: &str) -> Result<PathBuf, String> {
let k = safe_key(key)?;
Ok(root.join(k))
}
/// Open storage. For `file:` ensures the directory exists.
pub fn open(url: &str) -> Result<Value, String> {
let url = url.trim();
if is_file(url) {
let _ = file_root(url)?;
Ok(json!({ "_type": "storage", "url": url, "backend": "file" }))
} else if is_s3(url) {
// Validate shape early so author typos fail at open, not at put.
let rest = url
.strip_prefix("s3://")
.ok_or_else(|| "bad s3 url".to_string())?;
let bucket = rest.split('?').next().unwrap_or("");
if bucket.is_empty() {
return Err("s3 url missing bucket (s3://bucket?endpoint=…)".into());
}
Ok(json!({ "_type": "storage", "url": url, "backend": "s3" }))
} else {
Err(format!(
"storage url must be `file:…` or `s3://bucket?…`, got `{url}`"
))
}
}
fn load_bytes(body: Option<&str>, path: Option<&str>) -> Result<Vec<u8>, String> {
if let Some(p) = path.filter(|s| !s.is_empty()) {
return fs::read(p).map_err(|e| format!("read path `{p}`: {e}"));
}
if let Some(b) = body {
return Ok(b.as_bytes().to_vec());
}
Err("storage.put needs `body` or `path`".into())
}
pub fn put(
url: &str,
key: &str,
body: Option<&str>,
path: Option<&str>,
content_type: Option<&str>,
) -> Result<Value, String> {
let bytes = load_bytes(body, path)?;
let ct = content_type.unwrap_or("application/octet-stream");
if is_file(url) {
let root = file_root(url)?;
let dest = path_for(&root, key)?;
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent).map_err(|e| e.to_string())?;
}
fs::write(&dest, &bytes).map_err(|e| e.to_string())?;
let _ = fs::write(ctype_path(&dest), ct);
Ok(json!({ "ok": true, "key": key, "size": bytes.len() }))
} else if is_s3(url) {
s3_put_http(url, key, &bytes, ct)
} else {
Err("unknown storage backend".into())
}
}
fn ctype_path(dest: &Path) -> PathBuf {
PathBuf::from(format!("{}.ctype", dest.display()))
}
pub fn get(url: &str, key: &str) -> Result<Value, String> {
if is_file(url) {
let root = file_root(url)?;
let dest = path_for(&root, key)?;
if !dest.is_file() {
return Ok(json!({ "ok": false }));
}
let bytes = fs::read(&dest).map_err(|e| e.to_string())?;
let ct = fs::read_to_string(ctype_path(&dest))
.unwrap_or_else(|_| "application/octet-stream".into());
match String::from_utf8(bytes.clone()) {
Ok(s) => Ok(json!({
"ok": true,
"key": key,
"body": s,
"content_type": ct,
"size": s.len(),
})),
Err(_) => Ok(json!({
"ok": true,
"key": key,
"path": dest.to_string_lossy(),
"content_type": ct,
"size": bytes.len(),
"binary": true,
})),
}
} else if is_s3(url) {
s3_get_http(url, key)
} else {
Err("unknown storage backend".into())
}
}
pub fn delete(url: &str, key: &str) -> Result<Value, String> {
if is_file(url) {
let root = file_root(url)?;
let dest = path_for(&root, key)?;
let ok = if dest.is_file() {
fs::remove_file(&dest).map_err(|e| e.to_string())?;
let _ = fs::remove_file(ctype_path(&dest));
true
} else {
false
};
Ok(json!({ "ok": ok }))
} else if is_s3(url) {
s3_delete_http(url, key)
} else {
Err("unknown storage backend".into())
}
}
pub fn list(url: &str, prefix: Option<&str>) -> Result<Value, String> {
let prefix = prefix.unwrap_or("").trim_start_matches('/');
if is_file(url) {
let root = file_root(url)?;
let mut keys = Vec::new();
walk_keys(&root, &root, prefix, &mut keys)?;
keys.sort();
Ok(json!({ "ok": true, "keys": keys, "count": keys.len() }))
} else if is_s3(url) {
s3_list_http(url, prefix)
} else {
Err("unknown storage backend".into())
}
}
fn walk_keys(root: &Path, dir: &Path, prefix: &str, out: &mut Vec<String>) -> Result<(), String> {
let entries = match fs::read_dir(dir) {
Ok(e) => e,
Err(_) => return Ok(()),
};
for ent in entries {
let ent = ent.map_err(|e| e.to_string())?;
let path = ent.path();
if path
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.ends_with(".ctype"))
{
continue;
}
if path.is_dir() {
walk_keys(root, &path, prefix, out)?;
} else if path.is_file() {
let rel = path
.strip_prefix(root)
.map_err(|e| e.to_string())?
.to_string_lossy()
.replace('\\', "/");
if prefix.is_empty() || rel.starts_with(prefix) {
out.push(rel);
}
}
}
Ok(())
}
/// Live S3/MinIO via AWS SigV4. Requires `endpoint` query for path-style (MinIO).
fn s3_put_http(url: &str, key: &str, body: &[u8], content_type: &str) -> Result<Value, String> {
let cfg = S3Conf::parse(url)?;
let target = cfg.object_url(key)?;
let status = cfg.request("PUT", &target, body, content_type)?;
if (200..300).contains(&status) {
Ok(json!({ "ok": true, "key": key, "size": body.len() }))
} else {
Err(format!("s3 put HTTP {status}"))
}
}
fn s3_get_http(url: &str, key: &str) -> Result<Value, String> {
let cfg = S3Conf::parse(url)?;
let target = cfg.object_url(key)?;
let (status, ct, bytes) = cfg.request_bytes("GET", &target)?;
if status == 404 {
return Ok(json!({ "ok": false }));
}
if !(200..300).contains(&status) {
return Err(format!("s3 get HTTP {status}"));
}
match String::from_utf8(bytes.clone()) {
Ok(s) => Ok(json!({
"ok": true, "key": key, "body": s, "content_type": ct, "size": s.len()
})),
Err(_) => Ok(json!({
"ok": true, "key": key, "content_type": ct, "size": bytes.len(), "binary": true
})),
}
}
fn s3_delete_http(url: &str, key: &str) -> Result<Value, String> {
let cfg = S3Conf::parse(url)?;
let target = cfg.object_url(key)?;
let status = cfg.request("DELETE", &target, &[], "")?;
if (200..300).contains(&status) || status == 204 || status == 404 {
Ok(json!({ "ok": status != 404 }))
} else {
Err(format!("s3 delete HTTP {status}"))
}
}
fn s3_list_http(url: &str, prefix: &str) -> Result<Value, String> {
let cfg = S3Conf::parse(url)?;
let list_url = cfg.list_url(prefix);
let (status, _ct, bytes) = cfg.request_bytes("GET", &list_url)?;
let text = String::from_utf8_lossy(&bytes).into_owned();
if !(200..300).contains(&status) {
return Err(format!("s3 list HTTP {status}: {text}"));
}
let mut keys = Vec::new();
for part in text.split("<Key>").skip(1) {
if let Some(end) = part.find("</Key>") {
keys.push(part[..end].to_string());
}
}
Ok(json!({ "ok": true, "keys": keys, "count": keys.len() }))
}
struct S3Conf {
bucket: String,
endpoint: String,
region: String,
access_key: String,
secret_key: String,
path_style: bool,
}
impl S3Conf {
fn parse(url: &str) -> Result<Self, String> {
let rest = url
.trim()
.strip_prefix("s3://")
.ok_or_else(|| "not an s3 url".to_string())?;
let (bucket, query) = match rest.split_once('?') {
Some((b, q)) => (b.to_string(), q),
None => (rest.to_string(), ""),
};
if bucket.is_empty() {
return Err("s3 url missing bucket".into());
}
let mut endpoint = String::new();
let mut region = "us-east-1".to_string();
let mut access_key = std::env::var("AWS_ACCESS_KEY_ID")
.or_else(|_| std::env::var("S3_ACCESS_KEY"))
.unwrap_or_default();
let mut secret_key = std::env::var("AWS_SECRET_ACCESS_KEY")
.or_else(|_| std::env::var("S3_SECRET_KEY"))
.unwrap_or_default();
let mut path_style = true;
for pair in query.split('&').filter(|p| !p.is_empty()) {
let (k, v) = pair.split_once('=').unwrap_or((pair, ""));
let v = pct_decode(v);
match k {
"endpoint" => endpoint = v,
"region" => region = v,
"access_key" | "accessKey" => access_key = v,
"secret_key" | "secretKey" => secret_key = v,
"path_style" => {
path_style = matches!(v.as_str(), "1" | "true" | "True" | "yes")
}
_ => {}
}
}
if endpoint.is_empty() {
return Err(
"s3 url needs `endpoint=` for MinIO/path-style (e.g. s3://bucket?endpoint=http://127.0.0.1:9000)"
.into(),
);
}
if access_key.is_empty() || secret_key.is_empty() {
return Err(
"s3 credentials missing (access_key/secret_key query or AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY)"
.into(),
);
}
Ok(Self {
bucket,
endpoint: endpoint.trim_end_matches('/').to_string(),
region,
access_key,
secret_key,
path_style,
})
}
fn object_url(&self, key: &str) -> Result<String, String> {
let k = safe_key(key)?;
if self.path_style {
Ok(format!("{}/{}/{}", self.endpoint, self.bucket, k))
} else {
Err("virtual-hosted s3 not implemented in this wave — set path_style=true".into())
}
}
fn list_url(&self, prefix: &str) -> String {
format!(
"{}/{}?list-type=2&prefix={}",
self.endpoint,
self.bucket,
uri_encode(prefix)
)
}
fn request(
&self,
method: &str,
url: &str,
body: &[u8],
content_type: &str,
) -> Result<u16, String> {
let (status, _, _) = self.request_inner(method, url, body, content_type)?;
Ok(status)
}
fn request_bytes(&self, method: &str, url: &str) -> Result<(u16, String, Vec<u8>), String> {
self.request_inner(method, url, &[], "")
}
fn request_inner(
&self,
method: &str,
url: &str,
body: &[u8],
content_type: &str,
) -> Result<(u16, String, Vec<u8>), String> {
use hmac::{Hmac, Mac};
use sha2::{Digest, Sha256};
type HmacSha256 = Hmac<Sha256>;
let (scheme_host, path_query) = split_url(url)?;
let (path, query) = match path_query.split_once('?') {
Some((p, q)) => (p, q),
None => (path_query.as_str(), ""),
};
let host = scheme_host
.split("://")
.nth(1)
.ok_or_else(|| "bad url host".to_string())?;
let amz_date = amz_timestamp();
let date_stamp = &amz_date[..8];
let payload_hash = hex::encode(Sha256::digest(body));
let mut hdrs: Vec<(String, String)> = vec![
("host".into(), host.into()),
("x-amz-content-sha256".into(), payload_hash.clone()),
("x-amz-date".into(), amz_date.clone()),
];
if method == "PUT" && !content_type.is_empty() {
hdrs.push(("content-type".into(), content_type.into()));
}
hdrs.sort_by(|a, b| a.0.cmp(&b.0));
let signed_headers = hdrs.iter().map(|(k, _)| k.as_str()).collect::<Vec<_>>().join(";");
let canonical_headers = hdrs
.iter()
.map(|(k, v)| format!("{k}:{}\n", v.trim()))
.collect::<String>();
let canonical_query = canonical_query_string(query);
let canonical_request = format!(
"{method}\n{path}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{payload_hash}"
);
let scope = format!("{date_stamp}/{}/s3/aws4_request", self.region);
let string_to_sign = format!(
"AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}",
hex::encode(Sha256::digest(canonical_request.as_bytes()))
);
let mut mac =
HmacSha256::new_from_slice(format!("AWS4{}", self.secret_key).as_bytes()).unwrap();
mac.update(date_stamp.as_bytes());
let k_date = mac.finalize().into_bytes();
let mut mac = HmacSha256::new_from_slice(&k_date).unwrap();
mac.update(self.region.as_bytes());
let k_region = mac.finalize().into_bytes();
let mut mac = HmacSha256::new_from_slice(&k_region).unwrap();
mac.update(b"s3");
let k_service = mac.finalize().into_bytes();
let mut mac = HmacSha256::new_from_slice(&k_service).unwrap();
mac.update(b"aws4_request");
let k_signing = mac.finalize().into_bytes();
let mut mac = HmacSha256::new_from_slice(&k_signing).unwrap();
mac.update(string_to_sign.as_bytes());
let signature = hex::encode(mac.finalize().into_bytes());
let auth = format!(
"AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={signed_headers}, Signature={signature}",
self.access_key
);
let agent = ureq::AgentBuilder::new()
.timeout(std::time::Duration::from_secs(30))
.build();
let mut req = agent.request(method, url);
req = req
.set("Authorization", &auth)
.set("x-amz-content-sha256", &payload_hash)
.set("x-amz-date", &amz_date);
if method == "PUT" && !content_type.is_empty() {
req = req.set("Content-Type", content_type);
}
let result = if method == "PUT" {
req.send_bytes(body)
} else {
req.call()
};
match result {
Ok(r) => {
let status = r.status();
let ct = r
.header("content-type")
.unwrap_or("application/octet-stream")
.to_string();
let mut buf = Vec::new();
let _ = std::io::Read::read_to_end(&mut r.into_reader(), &mut buf);
Ok((status, ct, buf))
}
Err(ureq::Error::Status(code, r)) => {
let ct = r
.header("content-type")
.unwrap_or("application/octet-stream")
.to_string();
let mut buf = Vec::new();
let _ = std::io::Read::read_to_end(&mut r.into_reader(), &mut buf);
Ok((code, ct, buf))
}
Err(e) => Err(e.to_string()),
}
}
}
fn split_url(url: &str) -> Result<(String, String), String> {
let without = url
.strip_prefix("https://")
.or_else(|| url.strip_prefix("http://"))
.ok_or_else(|| "url must be http(s)".to_string())?;
let scheme = if url.starts_with("https://") {
"https"
} else {
"http"
};
let (hostport, rest) = match without.find('/') {
Some(i) => (&without[..i], &without[i..]),
None => (without, "/"),
};
Ok((format!("{scheme}://{hostport}"), rest.to_string()))
}
fn amz_timestamp() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
// UTC YYYYMMDDTHHMMSSZ via a tiny formatter (no chrono dep).
let days = secs / 86400;
let tod = secs % 86400;
let hour = tod / 3600;
let min = (tod % 3600) / 60;
let sec = tod % 60;
let (y, m, d) = civil_from_days(days as i64);
format!("{y:04}{m:02}{d:02}T{hour:02}{min:02}{sec:02}Z")
}
/// Algorithm from Howard Hinnant's date algorithms (days since 1970-01-01 → Y-M-D).
fn civil_from_days(z: i64) -> (i32, u32, u32) {
let z = z + 719468;
let era = if z >= 0 { z } else { z - 146096 } / 146097;
let doe = (z - era * 146097) as u64;
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
let y = (yoe as i64) + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = if m <= 2 { y + 1 } else { y };
(y as i32, m as u32, d as u32)
}
fn uri_encode(s: &str) -> String {
let mut out = String::new();
for b in s.bytes() {
match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
out.push(b as char)
}
_ => out.push_str(&format!("%{b:02X}")),
}
}
out
}
fn canonical_query_string(query: &str) -> String {
if query.is_empty() {
return String::new();
}
let mut pairs: Vec<(String, String)> = query
.split('&')
.filter(|p| !p.is_empty())
.map(|p| {
let (k, v) = p.split_once('=').unwrap_or((p, ""));
(pct_decode(k), pct_decode(v))
})
.collect();
pairs.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
pairs
.iter()
.map(|(k, v)| format!("{}={}", uri_encode(k), uri_encode(v)))
.collect::<Vec<_>>()
.join("&")
}
fn pct_decode(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let b = s.as_bytes();
let mut i = 0;
while i < b.len() {
if b[i] == b'%' && i + 2 < b.len() {
if let Ok(v) = u8::from_str_radix(std::str::from_utf8(&b[i + 1..i + 3]).unwrap_or(""), 16)
{
out.push(v as char);
i += 3;
continue;
}
}
out.push(b[i] as char);
i += 1;
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
fn file_put_get_list_delete() {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let dir = std::env::temp_dir().join(format!("marqdo-storage-{nanos}"));
let url = format!("file:{}", dir.display());
open(&url).unwrap();
put(&url, "a/hello.txt", Some("hi"), None, Some("text/plain")).unwrap();
let g = get(&url, "a/hello.txt").unwrap();
assert_eq!(g["ok"], true);
assert_eq!(g["body"], "hi");
let ls = list(&url, Some("a/")).unwrap();
assert_eq!(ls["count"], 1);
delete(&url, "a/hello.txt").unwrap();
assert_eq!(get(&url, "a/hello.txt").unwrap()["ok"], false);
let _ = fs::remove_dir_all(dir);
}
#[test]
fn s3_open_validates_bucket() {
let r = open("s3://mybucket?endpoint=http://127.0.0.1:9000");
assert!(r.is_ok());
assert!(open("s3://").is_err());
}
}