254 lines
11 KiB
Rust
254 lines
11 KiB
Rust
//! First-change snapshots per tag, shared across views of one virtual namespace.
|
|
//! The SQLite index and deduplicated blobs live outside that namespace.
|
|
mod layer;
|
|
mod restore;
|
|
#[cfg(test)]
|
|
mod tests;
|
|
use crate::{Layer, Open, Result, observe::under};
|
|
use rusqlite::{Connection, OptionalExtension, params};
|
|
use serde::{Deserialize, Serialize};
|
|
use sha2::{Digest, Sha256};
|
|
use std::{
|
|
collections::HashSet,
|
|
io,
|
|
path::{Path, PathBuf},
|
|
sync::{Arc, Mutex},
|
|
time::{SystemTime, UNIX_EPOCH},
|
|
};
|
|
|
|
const MAX_FILE: u64 = 64 * 1024 * 1024;
|
|
const MAX_BYTES: u64 = 4 * 1024 * 1024 * 1024;
|
|
const MAX_AGE: u64 = 7 * 24 * 60 * 60;
|
|
fn err(e: impl std::fmt::Display) -> io::Error {
|
|
io::Error::other(e.to_string())
|
|
}
|
|
fn now() -> u64 {
|
|
SystemTime::now()
|
|
.duration_since(UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_secs()
|
|
}
|
|
|
|
/// A tool call, or the originating process and its inherited conversation chain.
|
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
pub struct Tag {
|
|
pub id: String,
|
|
pub pid: u32,
|
|
pub program: String,
|
|
pub conversations: Vec<String>,
|
|
}
|
|
impl Tag {
|
|
pub fn process(pid: u32, program: String, conversations: Vec<String>) -> Result<Self> {
|
|
let mut random = [0; 16];
|
|
getrandom::fill(&mut random).map_err(err)?;
|
|
Ok(Self {
|
|
id: format!("process:{:032x}", u128::from_le_bytes(random)),
|
|
pid,
|
|
program,
|
|
conversations,
|
|
})
|
|
}
|
|
}
|
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
pub struct Entry {
|
|
pub entry: i64,
|
|
pub tag: String,
|
|
pub pid: u32,
|
|
pub program: String,
|
|
pub conversations: Vec<String>,
|
|
pub created: u64,
|
|
pub paths: usize,
|
|
pub changed_paths: Vec<String>,
|
|
}
|
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
pub struct Snapshot {
|
|
pub entry: i64,
|
|
pub tag: String,
|
|
pub path: String,
|
|
/// absent, directory, file, or not_undoable.
|
|
pub kind: String,
|
|
pub hash: Option<String>,
|
|
pub size: u64,
|
|
}
|
|
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
|
pub struct Undo {
|
|
pub tag: Option<String>,
|
|
pub since: Option<i64>,
|
|
#[serde(default)]
|
|
pub force: bool,
|
|
}
|
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
pub struct Outcome {
|
|
pub tag: Option<String>,
|
|
pub conflicts: Vec<String>,
|
|
pub skipped: Vec<String>,
|
|
pub restored: Vec<String>,
|
|
}
|
|
struct Storage {
|
|
db: Connection,
|
|
directory: PathBuf,
|
|
}
|
|
/// Shared journal. Reads of the world do not access its database or mutation lock.
|
|
pub struct Journal {
|
|
root: Arc<dyn Layer>,
|
|
storage: Arc<Mutex<Storage>>,
|
|
seen: Mutex<HashSet<(String, String)>>,
|
|
gate: tokio::sync::Mutex<()>,
|
|
paths: Mutex<Vec<std::sync::Weak<Mutex<Option<String>>>>>,
|
|
}
|
|
impl Journal {
|
|
/// Open the host-owned store outside an asynchronous worker.
|
|
pub fn open(directory: &Path, root: Arc<dyn Layer>) -> Result<Arc<Self>> {
|
|
std::fs::create_dir_all(directory.join("blobs"))?;
|
|
let db = Connection::open(directory.join("index.sqlite")).map_err(err)?;
|
|
db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;
|
|
CREATE TABLE IF NOT EXISTS tags(entry INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT UNIQUE NOT NULL, pid INTEGER NOT NULL, program TEXT NOT NULL, conversations TEXT NOT NULL, created INTEGER NOT NULL);
|
|
CREATE TABLE IF NOT EXISTS snapshots(entry INTEGER PRIMARY KEY AUTOINCREMENT, tag TEXT NOT NULL REFERENCES tags(id) ON DELETE CASCADE, path TEXT NOT NULL, kind TEXT NOT NULL, hash TEXT, size INTEGER NOT NULL, UNIQUE(tag,path));
|
|
CREATE INDEX IF NOT EXISTS snapshots_path ON snapshots(path);").map_err(err)?;
|
|
let seen = {
|
|
let mut statement = db.prepare("SELECT tag,path FROM snapshots").map_err(err)?;
|
|
statement
|
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))
|
|
.map_err(err)?
|
|
.collect::<rusqlite::Result<_>>()
|
|
.map_err(err)?
|
|
};
|
|
Ok(Arc::new(Self {
|
|
root,
|
|
storage: Arc::new(Mutex::new(Storage {
|
|
db,
|
|
directory: directory.into(),
|
|
})),
|
|
seen: Mutex::new(seen),
|
|
gate: tokio::sync::Mutex::new(()),
|
|
paths: Mutex::new(Vec::new()),
|
|
}))
|
|
}
|
|
pub fn layer(self: &Arc<Self>, tag: Tag) -> Arc<dyn Layer> {
|
|
Arc::new(layer::View {
|
|
journal: self.clone(),
|
|
tag,
|
|
})
|
|
}
|
|
async fn db<T: Send + 'static>(
|
|
&self,
|
|
f: impl FnOnce(&mut Storage) -> Result<T> + Send + 'static,
|
|
) -> Result<T> {
|
|
let storage = self.storage.clone();
|
|
tokio::task::spawn_blocking(move || f(&mut storage.lock().unwrap()))
|
|
.await
|
|
.map_err(err)?
|
|
}
|
|
fn excluded(path: &str) -> bool {
|
|
["/proc", "/run", "/tmp"]
|
|
.iter()
|
|
.any(|base| under(path, base))
|
|
}
|
|
/// Caller holds the mutation gate until the actual mutation completes.
|
|
async fn capture(&self, tag: &Tag, path: &str) -> Result<()> {
|
|
if Self::excluded(path)
|
|
|| self
|
|
.seen
|
|
.lock()
|
|
.unwrap()
|
|
.contains(&(tag.id.clone(), path.into()))
|
|
{
|
|
return Ok(());
|
|
}
|
|
let (kind, bytes, size) = match self.root.stat(path, false).await {
|
|
Err(e) if e.kind() == io::ErrorKind::NotFound => ("absent", None, 0),
|
|
Err(e) => return Err(e),
|
|
Ok(stat) if stat.directory => ("directory", None, 0),
|
|
Ok(stat) if stat.symlink || stat.size > MAX_FILE => ("not_undoable", None, stat.size),
|
|
Ok(stat) => {
|
|
let file = self.root.open(path, Open::default()).await?;
|
|
let mut bytes = Vec::new();
|
|
while (bytes.len() as u64) < stat.size {
|
|
let chunk = file
|
|
.read(bytes.len() as u64, (stat.size as usize) - bytes.len())
|
|
.await?;
|
|
if chunk.is_empty() {
|
|
return Err(io::ErrorKind::UnexpectedEof.into());
|
|
}
|
|
bytes.extend(chunk);
|
|
}
|
|
("file", Some(bytes), stat.size)
|
|
}
|
|
};
|
|
let (kind, bytes) = if bytes.as_ref().is_some_and(|b| b.len() as u64 > MAX_FILE) {
|
|
("not_undoable", None)
|
|
} else {
|
|
(kind, bytes)
|
|
};
|
|
let tag = tag.clone();
|
|
let key = (tag.id.clone(), path.to_owned());
|
|
let path = path.to_owned();
|
|
self.db(move |s| {
|
|
let hash = if let Some(bytes) = bytes {
|
|
let hash = format!("{:x}", Sha256::digest(&bytes));
|
|
let blob = s.directory.join("blobs").join(&hash);
|
|
if !blob.exists() { std::fs::write(blob, bytes)?; }
|
|
Some(hash)
|
|
} else { None };
|
|
let transaction = s.db.transaction().map_err(err)?;
|
|
transaction.execute("INSERT OR IGNORE INTO tags(id,pid,program,conversations,created) VALUES (?1,?2,?3,?4,?5)", params![tag.id, tag.pid, tag.program, serde_json::to_string(&tag.conversations).map_err(err)?, now() as i64]).map_err(err)?;
|
|
transaction.execute("INSERT OR IGNORE INTO snapshots(tag,path,kind,hash,size) VALUES (?1,?2,?3,?4,?5)", params![tag.id,path,kind,hash,size as i64]).map_err(err)?;
|
|
transaction.commit().map_err(err)?;
|
|
Ok(())
|
|
}).await?;
|
|
self.seen.lock().unwrap().insert(key);
|
|
Ok(())
|
|
}
|
|
/// Reserve caller metadata before a process takes its first snapshot.
|
|
pub async fn register(&self, tag: Tag) -> Result<()> {
|
|
self.db(move |s| {
|
|
s.db.execute("INSERT OR IGNORE INTO tags(id,pid,program,conversations,created) VALUES (?1,?2,?3,?4,?5)", params![tag.id, tag.pid, tag.program, serde_json::to_string(&tag.conversations).map_err(err)?, now() as i64]).map_err(err)?;
|
|
Ok(())
|
|
}).await
|
|
}
|
|
pub async fn list(&self, conversation: Option<String>) -> Result<Vec<Entry>> {
|
|
self.db(move |s| {
|
|
let mut statement = s.db.prepare("SELECT t.entry,t.id,t.pid,t.program,t.conversations,t.created,COUNT(s.entry),json_group_array(s.path) FROM tags t JOIN snapshots s ON s.tag=t.id GROUP BY t.id ORDER BY t.entry DESC").map_err(err)?;
|
|
let entries = statement.query_map([], |r| Ok(Entry { entry:r.get(0)?, tag:r.get(1)?, pid:r.get(2)?, program:r.get(3)?, conversations:serde_json::from_str(&r.get::<_,String>(4)?).unwrap_or_default(), created:r.get::<_, i64>(5)? as u64, paths:r.get::<_, i64>(6)? as usize, changed_paths:serde_json::from_str(&r.get::<_,String>(7)?).map_err(|e| rusqlite::Error::FromSqlConversionFailure(7, rusqlite::types::Type::Text, Box::new(e)))? })).map_err(err)?.collect::<rusqlite::Result<Vec<_>>>().map_err(err)?;
|
|
Ok(entries.into_iter().filter(|e| conversation.as_ref().is_none_or(|c| e.conversations.contains(c))).collect())
|
|
}).await
|
|
}
|
|
pub async fn show(&self, tag: String) -> Result<Vec<Snapshot>> {
|
|
self.db(move |s| snapshots(&s.db, Some(&tag))).await
|
|
}
|
|
/// Drop old tags until both retention limits hold. Content accounting deduplicates hashes.
|
|
pub async fn retain(&self) -> Result<()> {
|
|
let _gate = self.gate.lock().await;
|
|
let remaining = self.db(|s| {
|
|
s.db.execute("DELETE FROM tags WHERE created < ?1", [now().saturating_sub(MAX_AGE) as i64]).map_err(err)?;
|
|
loop {
|
|
let size: u64 = s.db.query_row("SELECT COALESCE(SUM(size),0) FROM (SELECT hash,MAX(size) size FROM snapshots WHERE hash IS NOT NULL GROUP BY hash)", [], |r| Ok(r.get::<_, i64>(0)? as u64)).map_err(err)?;
|
|
if size <= MAX_BYTES { break; }
|
|
s.db.execute("DELETE FROM tags WHERE entry=(SELECT MIN(entry) FROM tags)", []).map_err(err)?;
|
|
}
|
|
let hashes = { let mut q = s.db.prepare("SELECT DISTINCT hash FROM snapshots WHERE hash IS NOT NULL").map_err(err)?; q.query_map([], |r| r.get::<_,String>(0)).map_err(err)?.collect::<rusqlite::Result<HashSet<_>>>().map_err(err)? };
|
|
for file in std::fs::read_dir(s.directory.join("blobs"))? { let file=file?; if !hashes.contains(&file.file_name().to_string_lossy().into_owned()) { std::fs::remove_file(file.path())?; } }
|
|
let mut q=s.db.prepare("SELECT tag,path FROM snapshots").map_err(err)?;
|
|
q.query_map([], |r| Ok((r.get(0)?,r.get(1)?))).map_err(err)?.collect::<rusqlite::Result<HashSet<_>>>().map_err(err)
|
|
}).await?;
|
|
*self.seen.lock().unwrap() = remaining;
|
|
Ok(())
|
|
}
|
|
}
|
|
fn snapshots(db: &Connection, tag: Option<&str>) -> Result<Vec<Snapshot>> {
|
|
let mut q = db.prepare("SELECT entry,tag,path,kind,hash,size FROM snapshots WHERE ?1 IS NULL OR tag=?1 ORDER BY entry").map_err(err)?;
|
|
q.query_map([tag], |r| {
|
|
Ok(Snapshot {
|
|
entry: r.get(0)?,
|
|
tag: r.get(1)?,
|
|
path: r.get(2)?,
|
|
kind: r.get(3)?,
|
|
hash: r.get(4)?,
|
|
size: r.get::<_, i64>(5)? as u64,
|
|
})
|
|
})
|
|
.map_err(err)?
|
|
.collect::<rusqlite::Result<_>>()
|
|
.map_err(err)
|
|
}
|