//! Audio and video blobs kept on disk, so the proxy can answer `Range` //! requests: PDSes answer only whole blobs, and a player seeks. A blob is //! kept as `/blobs/`, beside `.type`, the content //! type its PDS gave it. A blob is downloaded once however many ask for it //! (`fill`, `filling`): written to a scratch file by a task of its own, //! hashed as it goes, and kept only if its bytes are what its CID names, so //! a PDS cannot put anything else in the cache. Every request for it reads //! the scratch file as it grows (`Filling::read`), so a player starts on //! the first bytes, not the last; its last byte is held back until the //! whole is known to be the CID's, so no read of it completes otherwise. //! The cache holds at most its budget (`HUD_BLOB_CACHE_BYTES`); after each //! write, the blobs least recently used (by modification time, which a //! read updates) go until it fits. use atproto::Cid; use futures::{Stream, StreamExt}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::fs; use std::io; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::io::AsyncWriteExt; use tokio::sync::watch; /// How much of a blob is read from the disk, and sent, at a time. pub const READ_CHUNK: usize = 64 * 1024; /// The blob cache: its directory, its budget in bytes, and the blobs being /// downloaded into it. pub struct BlobCache { inner: Arc, } struct Inner { dir: PathBuf, max_bytes: u64, /// Held while evicting, so two writes do not evict at once. evicting: tokio::sync::Mutex<()>, /// The downloads under way, by ` `: a blob's bytes are known /// to be its CID's only once all here, so a download is shared only by /// those asking the same PDS for it. filling: Mutex>>, } /// A kept blob, open to read: its file, length and content type. pub struct Cached { pub file: tokio::fs::File, pub len: u64, pub mime: String, } /// Why a blob was not kept. #[derive(Debug)] pub enum PutError { /// It is longer than the most allowed. TooLarge, /// Its bytes are not what its CID names. Mismatch, /// Its download failed. Download(String), /// The disk refused it. Disk(io::Error), } impl std::fmt::Display for PutError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { PutError::TooLarge => f.write_str("the blob is too large"), PutError::Mismatch => f.write_str("the blob's bytes do not match its CID"), PutError::Download(why) => write!(f, "its download failed: {why}"), PutError::Disk(e) => write!(f, "the cache could not keep it: {e}"), } } } impl From for PutError { fn from(e: io::Error) -> PutError { PutError::Disk(e) } } /// A blob being downloaded: its type, the length its PDS said it has, and /// its scratch file, read by every request for it as it grows. pub struct Filling { pub mime: String, pub said: Option, /// The scratch file, open to read: still the blob's once it is kept, /// or evicted, under whatever name. file: fs::File, progress: watch::Sender, } /// How far a download is: the bytes written, and once it is done, the /// blob's length or why it was not kept. struct Progress { written: u64, done: Option>>, } /// What a filling's sender is: kept by the filling itself, so never gone /// while anyone reads it. const HELD: &str = "a filling holds its own sender"; impl Filling { /// Its length once it is kept, or why it was not. pub async fn kept(&self) -> Result> { let mut progress = self.progress.subscribe(); let done = progress.wait_for(|p| p.done.is_some()).await.expect(HELD); done.done.clone().expect("waited for") } /// `count` of its bytes from `start`, each as it is written; the last /// of the blob only once it is kept. The stream ends in an error where /// the blob is not kept, or is shorter than asked. pub fn read( self: &Arc, start: u64, count: u64, ) -> impl Stream> + Send + 'static { let end = start + count; futures::stream::try_unfold(start, { let me = self.clone(); move |at| { let me = me.clone(); async move { if at >= end { return Ok(None); } let upto = me.readable(at).await?.min(end).min(at + READ_CHUNK as u64); let read = tokio::task::spawn_blocking(move || { let mut buf = vec![0; usize::try_from(upto - at).expect("a chunk")]; me.file.read_exact_at(&mut buf, at).map(|()| buf) }); let bytes = read.await.map_err(io::Error::other)??; Ok(Some((bytes::Bytes::from(bytes), upto))) } } }) } /// How far it can be read from `at`, once it can be: what is written /// but its last byte, or all of it once kept. async fn readable(&self, at: u64) -> io::Result { let mut progress = self.progress.subscribe(); let p = progress .wait_for(|p| p.done.is_some() || p.written > at + 1) .await .expect(HELD); match &p.done { None => Ok(p.written - 1), Some(Ok(len)) if *len > at => Ok(*len), Some(Ok(_)) => Err(io::Error::other("the blob is shorter than it said")), Some(Err(e)) => Err(io::Error::other(e.to_string())), } } } /// A file name unlike any other: hidden, so eviction skips it. fn scratch_name(cid: &Cid) -> String { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) .map_or(0, |d| d.as_nanos()); format!(".{cid}.{}.{nanos:x}.tmp", std::process::id()) } impl BlobCache { pub fn new(dir: PathBuf, max_bytes: u64) -> BlobCache { BlobCache { inner: Arc::new(Inner { dir, max_bytes, evicting: tokio::sync::Mutex::new(()), filling: Mutex::new(HashMap::new()), }), } } /// A kept blob, if there is one, marked as just used. pub async fn get(&self, cid: &Cid) -> Option { let mime = tokio::fs::read_to_string(self.inner.type_path(cid)) .await .ok()?; let file = tokio::fs::File::open(self.inner.path(cid)).await.ok()?; let len = file.metadata().await.ok()?.len(); let std = file.into_std().await; // A read refreshes its place in the eviction order; failing that // costs only an earlier eviction. let _ = std.set_modified(SystemTime::now()); Some(Cached { file: tokio::fs::File::from_std(std), len, mime: mime.trim().to_string(), }) } /// The download of `did`'s blob `cid` under way, if there is one. pub fn filling(&self, did: &str, cid: &Cid) -> Option> { let filling = self .inner .filling .lock() .unwrap_or_else(PoisonError::into_inner); filling.get(&format!("{did} {cid}")).cloned() } /// Download `did`'s blob `cid` from `body`, at most `max` bytes, into /// the cache, by a task of its own that goes on however many read it: /// kept if its bytes hash to `cid`. Ask `filling` first: this starts /// another download. pub async fn fill( &self, did: &str, cid: &Cid, mime: &str, said: Option, body: S, max: u64, ) -> io::Result> where S: Stream> + Send + Unpin + 'static, B: AsRef<[u8]> + Send + 'static, E: std::fmt::Display + Send + 'static, { tokio::fs::create_dir_all(&self.inner.dir).await?; let scratch = self.inner.dir.join(scratch_name(cid)); let writer = tokio::fs::File::create(&scratch).await?; let file = tokio::fs::File::open(&scratch).await?.into_std().await; let filling = Arc::new(Filling { mime: mime.to_string(), said, file, progress: watch::Sender::new(Progress { written: 0, done: None, }), }); let key = format!("{did} {cid}"); let mut under_way = self .inner .filling .lock() .unwrap_or_else(PoisonError::into_inner); under_way.insert(key.clone(), filling.clone()); drop(under_way); let download = Download { inner: self.inner.clone(), key, cid: *cid, scratch, filling: filling.clone(), }; crate::telemetry::spawn(download.run(writer, body, max)); Ok(filling) } /// Remove the least recently used blobs until the cache fits its budget. #[cfg(test)] async fn evict(&self) { self.inner.evict().await; } } /// A download into the cache, written by a task of its own. struct Download { inner: Arc, key: String, cid: Cid, scratch: PathBuf, filling: Arc, } impl Download { /// Write `body` to the scratch file, telling its readers of each /// chunk, and keep it if it is the CID's; then say how it ended. async fn run(self, file: tokio::fs::File, body: S, max: u64) where S: Stream> + Unpin, B: AsRef<[u8]>, E: std::fmt::Display, { let done = self.write(file, body, max).await; if done.is_ok() { self.inner.evict().await; } else { let _ = tokio::fs::remove_file(&self.scratch).await; } let mut under_way = self .inner .filling .lock() .unwrap_or_else(PoisonError::into_inner); under_way.remove(&self.key); drop(under_way); let done = done.map_err(Arc::new); self.filling.progress.send_modify(|p| p.done = Some(done)); } async fn write( &self, mut file: tokio::fs::File, mut body: S, max: u64, ) -> Result where S: Stream> + Unpin, B: AsRef<[u8]>, E: std::fmt::Display, { let mut hash = Sha256::new(); let mut len = 0u64; while let Some(chunk) = body.next().await { let chunk = chunk.map_err(|e| PutError::Download(e.to_string()))?; let chunk = chunk.as_ref(); len += chunk.len() as u64; if len > max { return Err(PutError::TooLarge); } hash.update(chunk); // Flushed, so a reader finds what it is told is written. file.write_all(chunk).await?; file.flush().await?; self.filling.progress.send_modify(|p| p.written = len); } file.sync_all().await?; let digest: [u8; 32] = hash.finalize().into(); if self.cid.codec() != atproto::cid::RAW || &digest != self.cid.digest() { return Err(PutError::Mismatch); } self.inner .keep(&self.cid, &self.filling.mime, &self.scratch) .await?; Ok(len) } } impl Inner { fn path(&self, cid: &Cid) -> PathBuf { self.dir.join(cid.to_string()) } fn type_path(&self, cid: &Cid) -> PathBuf { self.dir.join(format!("{cid}.type")) } /// Keep a downloaded blob under its CID: its type first, so a kept /// blob always has one. async fn keep(&self, cid: &Cid, mime: &str, scratch: &Path) -> io::Result<()> { let type_scratch = self.dir.join(format!("{}.type", scratch_name(cid))); let kept = async { tokio::fs::write(&type_scratch, mime).await?; tokio::fs::rename(&type_scratch, self.type_path(cid)).await?; tokio::fs::rename(scratch, self.path(cid)).await } .await; if kept.is_err() { let _ = tokio::fs::remove_file(&type_scratch).await; } kept } /// Remove the least recently used blobs until the cache fits its budget. async fn evict(&self) { let _held = self.evicting.lock().await; let dir = self.dir.clone(); let max = self.max_bytes; let done = tokio::task::spawn_blocking(move || evict_dir(&dir, max)).await; if let Ok(Err(e)) = done { tracing::warn!(error = %e, "could not evict blobs from the cache"); } } } /// Remove `dir`'s least recently modified blobs, with their types, until /// what is left is at most `max` bytes. Scratch files (hidden) and types /// are not counted. fn evict_dir(dir: &std::path::Path, max: u64) -> io::Result<()> { let mut blobs: Vec<(SystemTime, u64, PathBuf)> = Vec::new(); for entry in fs::read_dir(dir)? { let entry = entry?; let name = entry.file_name(); let name = name.to_string_lossy(); if name.starts_with('.') || name.contains('.') { continue; } let meta = entry.metadata()?; if meta.is_file() { blobs.push((meta.modified()?, meta.len(), entry.path())); } } let mut total: u64 = blobs.iter().map(|(_, len, _)| len).sum(); blobs.sort(); for (_, len, path) in blobs { if total <= max { break; } fs::remove_file(&path)?; let _ = fs::remove_file(path.with_extension("type")); total -= len; } Ok(()) } #[cfg(test)] mod tests { use super::*; use std::time::Duration; const DID: &str = "did:plc:alice"; fn dir(name: &str) -> PathBuf { let d = std::env::temp_dir().join(format!("hud-blobcache-{name}-{}", std::process::id())); let _ = fs::remove_dir_all(&d); d } fn body(bytes: &[u8]) -> impl Stream, String>> + Send + Unpin + 'static { futures::stream::iter(bytes.chunks(3).map(|c| Ok(c.to_vec())).collect::>()) } /// `bytes` downloaded as `cid`'s blob, its length unsaid: its length /// once kept, or why it was not. async fn keep( cache: &BlobCache, cid: &Cid, bytes: &[u8], max: u64, ) -> Result> { let filling = cache .fill(DID, cid, "audio/wav", None, body(bytes), max) .await .unwrap(); filling.kept().await } /// What a read gives, or the error it ends in. async fn all(read: impl Stream>) -> io::Result> { let mut read = std::pin::pin!(read); let mut out = Vec::new(); while let Some(chunk) = read.next().await { out.extend_from_slice(&chunk?); } Ok(out) } async fn read(c: Cached) -> Vec { use tokio::io::AsyncReadExt; let mut out = Vec::new(); let mut f = c.file; f.read_to_end(&mut out).await.unwrap(); out } #[tokio::test] async fn keeps_only_what_hashes_to_its_cid() { let d = dir("verify"); let cache = BlobCache::new(d.clone(), 1 << 20); let cid = Cid::of_raw(b"the real bytes"); let refused = keep(&cache, &cid, b"other bytes", 1 << 20).await; assert!(matches!(*refused.unwrap_err(), PutError::Mismatch)); assert!(cache.get(&cid).await.is_none()); assert_eq!(fs::read_dir(&d).unwrap().count(), 0, "nothing left behind"); // A record's CID (DAG-CBOR) over the same bytes is no blob's. let cbor = Cid::of_cbor(b"the real bytes"); let refused = keep(&cache, &cbor, b"the real bytes", 1 << 20).await; assert!(matches!(*refused.unwrap_err(), PutError::Mismatch)); let refused = keep(&cache, &cid, b"the real bytes", 4).await; assert!(matches!(*refused.unwrap_err(), PutError::TooLarge)); assert_eq!( keep(&cache, &cid, b"the real bytes", 1 << 20) .await .unwrap(), 14 ); let got = cache.get(&cid).await.unwrap(); assert_eq!((got.len, got.mime.as_str()), (14, "audio/wav")); assert_eq!(read(got).await, b"the real bytes"); let _ = fs::remove_dir_all(&d); } #[tokio::test] async fn readers_follow_the_download() { let d = dir("follow"); let cache = BlobCache::new(d.clone(), 1 << 20); let bytes: Vec = (0..200u8).collect(); let cid = Cid::of_raw(&bytes); let (tx, rx) = futures::channel::mpsc::unbounded::, String>>(); let filling = cache .fill(DID, &cid, "video/mp4", Some(200), rx, 1 << 20) .await .unwrap(); // Under way for those asking the same PDS, and no one else. assert!(Arc::ptr_eq(&cache.filling(DID, &cid).unwrap(), &filling)); assert!(cache.filling("did:plc:bob", &cid).is_none()); // Its first bytes are read before the rest has come. tx.unbounded_send(Ok(bytes[..50].to_vec())).unwrap(); assert_eq!(all(filling.read(10, 20)).await.unwrap(), bytes[10..30]); let whole = tokio::spawn(all(filling.read(0, 200))); tx.unbounded_send(Ok(bytes[50..].to_vec())).unwrap(); // All of it here, but not yet known to be the CID's: its last byte waits. let last = tokio::time::timeout(Duration::from_millis(100), all(filling.read(199, 1))).await; assert!(last.is_err(), "the last byte waited"); assert!(!whole.is_finished()); drop(tx); assert_eq!(whole.await.unwrap().unwrap(), bytes); assert_eq!(filling.kept().await.unwrap(), 200); assert_eq!(all(filling.read(199, 1)).await.unwrap(), [199]); // Kept, and no longer under way. assert!(cache.filling(DID, &cid).is_none()); assert_eq!(cache.get(&cid).await.unwrap().len, 200); let _ = fs::remove_dir_all(&d); } #[tokio::test] async fn reads_of_a_blob_not_kept_end_in_why() { let d = dir("unkept"); let cache = BlobCache::new(d.clone(), 1 << 20); // Bytes not its CID's: a read ends in the reason. let cid = Cid::of_raw(b"the real bytes"); let wrong = cache .fill( DID, &cid, "video/mp4", Some(11), body(b"other bytes"), 1 << 20, ) .await .unwrap(); let e = all(wrong.read(0, 11)).await.unwrap_err(); assert_eq!(e.to_string(), "the blob's bytes do not match its CID"); // Kept, but shorter than its PDS said: a read past it ends in an error. let short = cache .fill( DID, &cid, "video/mp4", Some(100), body(b"the real bytes"), 1 << 20, ) .await .unwrap(); let e = all(short.read(0, 100)).await.unwrap_err(); assert_eq!(e.to_string(), "the blob is shorter than it said"); assert!(all(short.read(20, 5)).await.is_err()); let _ = fs::remove_dir_all(&d); } #[tokio::test] async fn evicts_least_recently_used() { let d = dir("evict"); let cache = BlobCache::new(d.clone(), 25); let blobs: Vec<(Cid, Vec)> = (0..3u8) .map(|i| { let bytes = vec![i; 10]; (Cid::of_raw(&bytes), bytes) }) .collect(); keep(&cache, &blobs[0].0, &blobs[0].1, 100).await.unwrap(); tokio::time::sleep(Duration::from_millis(20)).await; keep(&cache, &blobs[1].0, &blobs[1].1, 100).await.unwrap(); tokio::time::sleep(Duration::from_millis(20)).await; // Reading the first makes the second the least recently used. assert!(cache.get(&blobs[0].0).await.is_some()); tokio::time::sleep(Duration::from_millis(20)).await; keep(&cache, &blobs[2].0, &blobs[2].1, 100).await.unwrap(); assert!(cache.get(&blobs[0].0).await.is_some()); assert!(cache.get(&blobs[1].0).await.is_none()); assert!(!d.join(format!("{}.type", blobs[1].0)).exists()); assert!(cache.get(&blobs[2].0).await.is_some()); let _ = fs::remove_dir_all(&d); } #[tokio::test] async fn failures_say_why() { let d = dir("fail"); let cid = Cid::of_raw(b"the real bytes"); // A download that breaks off keeps nothing. let cache = BlobCache::new(d.clone(), 1 << 20); let broken = futures::stream::iter(vec![Ok(b"the ".to_vec()), Err("reset".to_string())]); let filling = cache .fill(DID, &cid, "audio/wav", None, broken, 1 << 20) .await .unwrap(); let e = filling.kept().await.unwrap_err(); assert_eq!(e.to_string(), "its download failed: reset"); assert_eq!(fs::read_dir(&d).unwrap().count(), 0, "nothing left behind"); assert_eq!(PutError::TooLarge.to_string(), "the blob is too large"); // A type that cannot be kept keeps neither it nor the blob. fs::create_dir_all(d.join(format!("{cid}.type")).join("in-the-way")).unwrap(); let e = keep(&cache, &cid, b"the real bytes", 1 << 20) .await .unwrap_err(); assert!(matches!(*e, PutError::Disk(_)), "{e}"); assert!(e.to_string().starts_with("the cache could not keep it: ")); assert!(cache.get(&cid).await.is_none()); assert_eq!( fs::read_dir(&d).unwrap().count(), 1, "the scratch files went" ); // A cache whose directory cannot be made. let file = d.join("a-file"); fs::write(&file, "").unwrap(); let nowhere = BlobCache::new(file.join("blobs"), 1 << 20); let made = nowhere .fill( DID, &cid, "audio/wav", None, body(b"the real bytes"), 1 << 20, ) .await; assert!(made.is_err()); let _ = fs::remove_dir_all(&d); } #[tokio::test] async fn eviction_leaves_what_is_no_blob() { let d = dir("stray"); let cache = BlobCache::new(d.clone(), 5); fs::create_dir_all(d.join("stray")).unwrap(); let bytes = vec![7u8; 10]; let cid = Cid::of_raw(&bytes); let filling = cache .fill(DID, &cid, "audio/a", None, body(&bytes), 100) .await .unwrap(); // Over its budget, the blob went at once; the directory stays, and // what was downloaded is still read whole. assert_eq!(filling.kept().await.unwrap(), 10); assert!(cache.get(&cid).await.is_none()); assert!(d.join("stray").is_dir()); assert_eq!(all(filling.read(0, 10)).await.unwrap(), bytes); // A cache whose directory went away has nothing to evict, and // eviction makes none. let _ = fs::remove_dir_all(&d); cache.evict().await; assert!(!d.exists()); } }