//! Blobs: `uploadBlob`, `getBlob` and `listBlobs`, and the blob refs in //! records, as `spec/BLOBS.md` says, within the quota and the time an //! unreferenced upload is held that `spec/LIMITS.md` sets. //! //! Each account keeps its own blobs. A blob's bytes are the file //! `repos//blobs/`, and the account's log has an entry for each //! upload it stored or uploaded again, with its type, size and time, and //! one for each upload it stopped holding for want of a reference in time //! (`blobGone`). Which records reference a blob is not logged apart: the //! records say, so the commits that write them carry it, and replaying the //! log counts the references again. A space record's refs hold a blob as a //! public record's do (`space_repo` counts them), but the public `getBlob` //! and `listBlobs` serve and list only blobs public records reference. //! //! An upload is written to a temporary file in that directory and renamed //! into place before its entry is logged, and answered once both are handed //! to the operating system: an entry always has its file, and an answered //! upload survives the server being killed. Neither is synced then. The //! commit that first references a blob has its file and directory synced //! before the commit's frame is (`sync_before`, `Wal::sync_first`), in the //! group commit, so a referenced blob survives a power cut wherever its //! commit does; an upload no record references yet may not //! (`spec/DURABILITY.md`), and recovery drops one whose file did not. //! A write that removes a blob's last //! reference stops holding it at once, under the account's lock, but removes //! its file only once the write is on disk. If the server stops between the //! two, it starts again with the file held by no entry, and removes it, as it //! does a temporary file an upload left. An upload no record references is //! no longer held once `PDS_BLOB_TTL_SECS` have passed since it was last //! uploaded: refused as a ref and counted against no quota at once, and //! logged as gone, its file removed, by `sweep`. //! //! With blobs in a bucket (`blobstore`), a blob's bytes are an object there //! and no file: an upload is read whole into memory and put before it is //! answered, and logged as above once the store has kept it; `getBlob` //! reads the object as it sends it. The log says which objects the store //! may keep (`kept`): `blobPut` before an upload puts one, `blobDeleted` //! once one is deleted. Those of blobs no longer held are deleted by the //! deleter (`blobstore`), which a write that stops holding a blob, an //! upload, and `sweep` wake, where they would remove its file. A move of //! the blobs into the bucket (`migrate_blobs`) logs `blobPut` for each blob //! held before it puts their objects, and a move out of it //! `blobObjectsDeleted` once it has deleted every object the store may keep. use crate::{ Account, Call, Handler, Resp, Service, Shared, Shown, Store, XrpcError, err, internal, invalid, opt, read, required, write, }; use axum::body::{Body, Bytes, HttpBody}; use axum::extract::{Extension, State as AxState}; use axum::http::{HeaderMap, HeaderValue, Method, header}; use axum::response::Response; use http_body::{Frame, SizeHint}; use pds_repo::data::{V, cid_of, cid_string, parse_cid, valid_cid_string}; use pds_repo::{Change, Op, OpResult, put_record}; use serde_json::{Map, Value as Json, json}; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; use std::ops::Bound::{Excluded, Unbounded}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::{Arc, LazyLock, OnceLock}; use std::task::{Context, Poll}; /// The largest body `uploadBlob` takes unless `PDS_BLOB_UPLOAD_LIMIT` sets /// another (`Config::blob_upload_limit`): the reference's default. pub(crate) const DEFAULT_UPLOAD_LIMIT: u64 = 5 * 1024 * 1024; const UPLOAD: &str = "com.atproto.repo.uploadBlob"; /// The entry of an object the bucket may keep from now on. const PUT: &str = "blobPut"; /// The entry of an object the bucket answered the delete of. const DELETED: &str = "blobDeleted"; /// The entry of every object the bucket may keep deleted, by a move of the /// blobs out of it. const OBJECTS_DELETED: &str = "blobObjectsDeleted"; /// The kinds of entry an account's log has of its blobs: an upload, one no /// longer held, a takedown or its lifting (`spec/LIFECYCLE.md`), and with /// blobs in a bucket, an object it may keep and one deleted (`blobstore`), /// and every one deleted (`migrate_blobs`). pub(crate) const ENTRIES: &[&str] = &[ "blob", "blobGone", "takedownBlob", PUT, DELETED, OBJECTS_DELETED, ]; /// How often uploads held past their time are logged as gone, and their files removed. const SWEEP_EVERY: std::time::Duration = std::time::Duration::from_secs(30); pub(crate) fn handler(nsid: &str) -> Option { Some(match nsid { "com.atproto.sync.getBlob" => Handler::Query(get_blob), "com.atproto.sync.listBlobs" => Handler::Query(list_blobs), _ => return None, }) } /// Where an account's blobs are kept. pub(crate) fn blob_dir(dir: &Path, did: &str) -> PathBuf { dir.join("repos").join(did).join("blobs") } /// A blob an account holds, as it was first stored. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct Held { pub mime: String, pub size: u64, /// When it was last uploaded, in seconds since the epoch. pub at: u64, } /// A record with blob refs: the rev of the commit that last wrote it, and the /// CIDs it references, each once, in order. type RecordRefs = (String, Vec); /// The bytes a record with blob refs, at `path`, is counted as in the cache. fn record_cost(path: &str, (rev, cids): &RecordRefs) -> usize { path.len() + rev.len() + cids.iter().map(|c| c.len() + 24).sum::() + 96 } /// What a commit changed of an account's blobs, as they were before it. #[derive(Default)] struct Undo { records: Vec<(String, Option)>, refs: Vec<(String, Option)>, held: Vec<(String, Option)>, bytes: u64, doomed: usize, /// The takedowns of the blobs it stopped holding. takedowns: Vec<(String, String)>, } /// An account's blobs, kept with its repository and changed under its lock. #[derive(Default)] pub(crate) struct Blobs { /// The blobs it holds, by CID. held: BTreeMap, /// How many public records reference each CID, for those some record does. refs: HashMap, /// How many records of the account's space repos reference each CID, /// for those some such record does (`space_repo`): these hold a blob as /// public records do, but the public methods neither serve nor list it. space_refs: HashMap, /// Each record with blob refs, by path. records: HashMap, /// The bytes `records` is counted as in the cache. record_bytes: usize, /// The blobs held that no record references, by when they were last uploaded. unreferenced: BTreeSet<(u64, String)>, /// The bytes of the blobs held. bytes: u64, /// Uploads moving their files into place, or putting their objects, by /// CID: those files stay, and those objects are deleted by none. placing: HashMap, /// The files of uploads not yet synced, by CID: synced before the frame /// of the commit that first references them. unsynced: HashMap, /// Blobs the last write stopped holding, whose files go once it is on disk. doomed: Vec, /// The last commit's changes, until it is in the log. undo: Option, /// The time of the write under way, and how long an unreferenced upload /// is held, both in seconds. now: u64, ttl: u64, /// The blobs held taken down, by CID, each with its reference: a blob no /// longer held is no longer taken down. takedowns: BTreeMap, /// With blobs in a bucket, the CIDs whose objects it may keep: each blob /// held, and those of the others not known to be deleted, its strays. /// Empty with blobs in the data directory, but while a move of them /// into the bucket, or out of it, is unfinished (`migrate_blobs`). kept: BTreeSet, } /// A blob ref in a record. pub(crate) struct BlobRef { cid: Vec, mime: String, size: i64, } /// The blob ref a map is: exactly `$type` blob, a link `ref`, a `mimeType` /// with a `/`, and a `size` of 0 or more, or -1. A `ref` that is a map of /// the one key `$link`, a CID string, is that link, as JSON writes both /// (`spec/BLOBS.md`, "Blob refs in records"). fn blob_ref(m: &[(String, V)]) -> Option { if m.len() != 4 { return None; } let get = |k: &str| m.iter().find(|(n, _)| n == k).map(|(_, v)| v); let cid = match get("ref")? { V::Link(cid) => cid.clone(), V::Map(r) => match r.as_slice() { [(k, V::Str(cid))] if k == "$link" => parse_cid(cid)?, _ => return None, }, _ => return None, }; match (get("$type")?, get("mimeType")?, get("size")?) { (V::Str(t), V::Str(mime), V::Int(size)) if t == "blob" && mime.contains('/') && *size >= -1 => { Some(BlobRef { cid, mime: mime.clone(), size: *size, }) } _ => None, } } /// Every blob ref in a value, at any depth. pub(crate) fn refs_in(v: &V, out: &mut Vec) { match v { V::Map(m) => match blob_ref(m) { Some(r) => out.push(r), None => m.iter().for_each(|(_, x)| refs_in(x, out)), }, V::Arr(a) => a.iter().for_each(|x| refs_in(x, out)), _ => {} } } /// The CIDs a record's blob refs name, each once, in order. pub(crate) fn cids_in(v: &V) -> Vec { let mut refs = Vec::new(); refs_in(v, &mut refs); let set: BTreeSet = refs.iter().map(|r| cid_string(&r.cid)).collect(); set.into_iter().collect() } /// Whether a value holds a blob ref of the legacy shape: exactly a CID /// string `cid` and a non-empty `mimeType`. fn legacy_in(v: &V) -> bool { match v { V::Map(m) => { let get = |k: &str| m.iter().find(|(n, _)| n == k).map(|(_, v)| v); let legacy = m.len() == 2 && matches!(get("cid"), Some(V::Str(c)) if valid_cid_string(c)) && matches!(get("mimeType"), Some(V::Str(t)) if !t.is_empty()); legacy || m.iter().any(|(_, x)| legacy_in(x)) } V::Arr(a) => a.iter().any(legacy_in), _ => false, } } /// A record may not hold a legacy blob ref: checked right after its `$type`. pub(crate) fn refuse_legacy(v: &V) -> Result<(), XrpcError> { if legacy_in(v) { invalid("Legacy blob refs are not supported") } else { Ok(()) } } /// The log entry of an upload the account stored, or uploaded again. fn upload_entry(cid: &str, h: &Held) -> Json { json!({ "t": "blob", "cid": cid, "mimeType": h.mime, "size": h.size, "at": h.at }) } /// The log entry of an upload no longer held, as no record referenced it in time. fn gone_entry(cid: &str) -> Json { json!({ "t": "blobGone", "cid": cid }) } /// The log entry of an object the bucket may keep from now on, before it is put. fn put_entry(cid: &str) -> Json { json!({ "t": PUT, "cid": cid }) } /// The log entry of an object whose delete the bucket answered. pub(crate) fn deleted_entry(cid: &str) -> Json { json!({ "t": DELETED, "cid": cid }) } /// What an upload does to the blobs held. #[derive(Clone, Copy, PartialEq, Eq, Debug)] enum Upload { /// It is held, and referenced: nothing. Kept, /// It is held, unreferenced: its time starts again. Again, /// It is held, unreferenced, past its time: it is stored afresh. Afresh, /// It is not held: it is stored. New, } impl Blobs { /// Set the time of the write under way, and how long uploads are held. pub(crate) fn time(&mut self, now: u64, ttl: u64) { (self.now, self.ttl) = (now, ttl); } /// Whether nothing of it is in flight: no upload moving into place, no /// file doomed and not yet removed. pub(crate) fn settled(&self) -> bool { self.placing.is_empty() && self.doomed.is_empty() } /// About the bytes it holds in memory: each a count kept as it changes. pub(crate) fn cost(&self) -> u64 { let undo = self .undo .as_ref() .map_or(0, |u| u.records.len() + u.refs.len() + u.held.len()); (self.held.len() * 160 + self.refs.len() * 96 + self.space_refs.len() * 96 + self.takedowns.len() * 128 + self.kept.len() * 96 + self.unreferenced.len() * 112 + self.unsynced.len() * 256 + self.placing.len() * 96 + self.doomed.len() * 96 + undo * 160 + self.record_bytes) as u64 } /// The CIDs whose objects a bucket may keep that it does not hold, and /// no upload is putting: those to delete. pub(crate) fn strays(&self) -> impl Iterator { self.kept .iter() .filter(|c| !self.held.contains_key(*c) && !self.placing.contains_key(*c)) } pub(crate) fn is_stray(&self, cid: &str) -> bool { self.kept.contains(cid) && !self.held.contains_key(cid) && !self.placing.contains_key(cid) } /// Whether a bucket may keep `cid`'s object. pub(crate) fn keeps(&self, cid: &str) -> bool { self.kept.contains(cid) } /// The CIDs whose objects a bucket may keep: those of an account deleted. pub(crate) fn kept_list(&self) -> Vec { self.kept.iter().cloned().collect() } /// An object deleted, its entry logged. pub(crate) fn forget_object(&mut self, cid: &str) { self.kept.remove(cid); } /// The blobs it holds at `now`, with uploads held for `ttl`, each with /// its size, in order of CID. pub(crate) fn holding(&self, now: u64, ttl: u64) -> Vec<(String, u64)> { self.held .iter() .filter(|(cid, h)| self.referenced(cid) || h.at.saturating_add(ttl) > now) .map(|(cid, h)| (cid.clone(), h.size)) .collect() } /// The CIDs of every object a bucket may keep, and of every blob held: /// those a move out of the bucket deletes. pub(crate) fn objects(&self) -> Vec { let mut all: BTreeSet<&String> = self.kept.iter().collect(); all.extend(self.held.keys()); all.into_iter().cloned().collect() } /// Log, in `wal`, that a bucket may keep the objects of `cids`, those no /// entry names yet, before a move puts them: the frame to wait on, if /// one was appended. pub(crate) fn note_kept( &mut self, wal: &pds_store::Wal, cids: &[String], ) -> io::Result> { let new: Vec<&String> = cids.iter().filter(|c| !self.kept.contains(*c)).collect(); if new.is_empty() { return Ok(None); } let n = wal.append(&new.iter().map(|c| put_entry(c)).collect::>())?; self.kept.extend(new.into_iter().cloned()); Ok(Some(n)) } /// Log, in `wal`, that the strays `cids` are deleted, as the deleter /// does: the frame to wait on, if one was appended. pub(crate) fn strays_deleted( &mut self, wal: &pds_store::Wal, cids: &[String], ) -> io::Result> { if cids.is_empty() { return Ok(None); } let n = wal.append(&cids.iter().map(|c| deleted_entry(c)).collect::>())?; for c in cids { self.kept.remove(c); } Ok(Some(n)) } /// Log, in `wal`, that a move out of the bucket deleted every object it /// may keep: the frame to wait on, if one was appended. pub(crate) fn objects_deleted(&mut self, wal: &pds_store::Wal) -> io::Result> { if self.kept.is_empty() { return Ok(None); } let n = wal.append(&[json!({ "t": OBJECTS_DELETED })])?; self.kept.clear(); Ok(Some(n)) } /// Whether an upload is on its way, or a stray waits to be deleted. pub(crate) fn objects_unsettled(&self) -> bool { !self.placing.is_empty() || self.strays().next().is_some() } /// Whether the write under way stops holding a blob. pub(crate) fn dooms(&self) -> bool { !self.doomed.is_empty() } /// Whether a change to its space repos may stop holding a blob: whether /// any of their records references one. pub(crate) fn space_held(&self) -> bool { !self.space_refs.is_empty() } /// Whether some record references `cid`, public or in a space repo. fn referenced(&self, cid: &str) -> bool { self.refs.contains_key(cid) || self.space_refs.contains_key(cid) } /// Have the files of the uploads not yet synced among `cids` synced /// before `wal` next is: a commit referencing them is about to be logged /// in it. pub(crate) fn sync_before( &mut self, wal: &pds_store::Wal, cids: impl IntoIterator, ) { let paths: Vec = cids .into_iter() .filter_map(|c| self.unsynced.remove(&c)) .collect(); if !paths.is_empty() { wal.sync_first(paths); } } /// Whether a blob held is unreferenced and past its time. fn expired(&self, cid: &str) -> bool { !self.referenced(cid) && self .held .get(cid) .is_some_and(|h| h.at.saturating_add(self.ttl) <= self.now) } /// The unreferenced uploads past their time. fn past_time(&self) -> impl Iterator { self.unreferenced .iter() .take_while(|(at, _)| at.saturating_add(self.ttl) <= self.now) .map(|(_, cid)| cid) } /// The bytes held that count against the quota: those of the blobs held /// but uploads past their time. fn counted(&self) -> u64 { let past: u64 = self.past_time().map(|cid| self.held[cid].size).sum(); self.bytes - past } /// A blob that a record references: what `getBlob` serves and `listBlobs` lists. fn served(&self, cid: &str) -> Option<&Held> { self.held.get(cid).filter(|_| self.refs.contains_key(cid)) } /// Whether it holds `cid`: stored, and if unreferenced, within its time. fn holds(&self, cid: &str) -> bool { self.held.contains_key(cid) && !self.expired(cid) } /// A blob it holds: its takedown's reference, if it is down. pub(crate) fn held_takedown(&self, cid: &str) -> Option> { self.holds(cid) .then(|| self.takedowns.get(cid).map(String::as_str)) } fn taken_down(&self, cid: &str) -> bool { self.holds(cid) && self.takedowns.contains_key(cid) } /// The entry that takes down a blob it holds with reference `r`, or /// lifts that with none: none if that changes nothing. pub(crate) fn takedown_entry(&self, cid: &str, r: Option<&str>) -> Option { let changes = match r { Some(_) => self.holds(cid), None => self.takedowns.contains_key(cid), }; changes.then(|| json!({ "t": "takedownBlob", "cid": cid, "ref": r })) } /// How many distinct blobs its records reference, and how many it holds, /// at `now` with uploads held for `ttl`. pub(crate) fn counts(&self, now: u64, ttl: u64) -> (usize, usize) { let past = self .unreferenced .iter() .take_while(|(at, _)| at.saturating_add(ttl) <= now) .count(); (self.refs.len(), self.held.len() - past) } /// The blobs its records reference that it does not hold, in order of /// CID, those after `cursor` (as strings), at most `limit`: each with /// the path of a record referencing it, the first in order of path /// (`spec/MIGRATION.md`). pub(crate) fn missing(&self, cursor: Option<&str>, limit: usize) -> Vec<(String, String)> { let mut cids: Vec<&String> = self .refs .keys() .filter(|c| !self.held.contains_key(*c) && cursor.is_none_or(|k| c.as_str() > k)) .collect(); cids.sort_unstable(); cids.truncate(limit); let mut paths: BTreeMap<&str, &str> = BTreeMap::new(); for (path, (_, refs)) in &self.records { for cid in refs { if cids.binary_search(&cid).is_ok() { let p = paths.entry(cid.as_str()).or_insert(path.as_str()); if path.as_str() < *p { *p = path.as_str(); } } } } cids.into_iter() .filter_map(|c| Some((c.clone(), paths.get(c.as_str())?.to_string()))) .collect() } /// Check a write's blob refs against the blobs as they were before it: /// a public write's, or a space write's (`space_repo`). pub(crate) fn check(&self, refs: &[BlobRef]) -> Result<(), XrpcError> { for r in refs { let cid = cid_string(&r.cid); let Some(h) = self .held .get(&cid) .filter(|_| !self.expired(&cid) && !self.takedowns.contains_key(&cid)) else { return err(400, "BlobNotFound", format!("Could not find blob: {cid}")); }; // One some record, public or in a space, references already is taken as it is. if self.referenced(&cid) { continue; } if h.mime != r.mime { return err( 400, "InvalidMimeType", format!( "Referenced Mimetype does not match stored blob. Expected: {}, Got: {}", h.mime, r.mime ), ); } if u64::try_from(r.size).ok() != Some(h.size) { return err( 400, "InvalidSize", format!( "Referenced size does not match stored blob. Expected: {}, Got: {}", h.size, r.size ), ); } } Ok(()) } /// Hold `cid`, or stop holding it, keeping the index of those /// unreferenced and the bytes held: one no longer held is no longer /// taken down. fn set_held(&mut self, cid: &str, h: Option) { if let Some(old) = self.held.remove(cid) { self.unreferenced.remove(&(old.at, cid.to_string())); self.bytes -= old.size; } if h.is_none() { self.takedowns.remove(cid); } if let Some(h) = h { if !self.referenced(cid) { self.unreferenced.insert((h.at, cid.to_string())); } self.bytes += h.size; self.held.insert(cid.to_string(), h); } } /// Count the references a commit changed: each record it wrote or /// deleted references what it holds now. A blob no record references any /// more is no longer held, and its file is doomed. What it changed is /// kept, to `undo` should the commit not reach the log. pub(crate) fn committed(&mut self, c: &Change) { let mut undo = Undo { bytes: self.bytes, doomed: self.doomed.len(), ..Default::default() }; let mut delta: BTreeMap = BTreeMap::new(); let written: BTreeMap<&String, Vec> = c .puts .iter() .map(|(p, j)| { ( p, put_record(j).map(|r| cids_in(&r.value)).unwrap_or_default(), ) }) .collect(); for path in c.puts.iter().map(|(p, _)| p).chain(&c.dels) { let now = written.get(path).cloned().unwrap_or_default(); let before = self.records.remove(path); if let Some(b) = &before { self.record_bytes -= record_cost(path, b); } undo.records.push((path.clone(), before.clone())); let before = before.map(|(_, cids)| cids).unwrap_or_default(); for cid in before.iter().filter(|x| !now.contains(x)) { *delta.entry(cid.clone()).or_default() -= 1; } for cid in now.iter().filter(|x| !before.contains(x)) { *delta.entry(cid.clone()).or_default() += 1; } if !now.is_empty() { let refs = (c.rev.clone(), now); self.record_bytes += record_cost(path, &refs); self.records.insert(path.clone(), refs); } } for (cid, d) in delta { let had = self.refs.get(&cid).copied(); undo.refs.push((cid.clone(), had)); let held = self.held.get(&cid).cloned(); undo.held.push((cid.clone(), held.clone())); let n = i64::from(had.unwrap_or(0)) + d; if n > 0 { self.refs.insert(cid.clone(), n as u32); if let Some(h) = held { self.unreferenced.remove(&(h.at, cid)); } continue; } self.refs.remove(&cid); if d < 0 && held.is_some() && !self.space_refs.contains_key(&cid) { if let Some(t) = self.takedowns.get(&cid) { undo.takedowns.push((cid.clone(), t.clone())); } self.set_held(&cid, None); self.doomed.push(cid); } } self.undo = Some(undo); } /// Count the references a space repo's commit changed, given each CID's /// change in the number of its records referencing it: a blob no record /// references any more, public or in a space, is no longer held, and its /// file is doomed. Made only once the commit is in the log. pub(crate) fn space_counted(&mut self, delta: BTreeMap) { for (cid, d) in delta { let n = i64::from(self.space_refs.get(&cid).copied().unwrap_or(0)) + d; if n > 0 { self.space_refs.insert(cid.clone(), n as u32); if let Some(h) = self.held.get(&cid) { self.unreferenced.remove(&(h.at, cid)); } continue; } self.space_refs.remove(&cid); if d < 0 && self.held.contains_key(&cid) && !self.refs.contains_key(&cid) { self.set_held(&cid, None); self.doomed.push(cid); } } } /// A blob a space repo's records reference, as its `getBlob` serves it: /// `400 InvalidRequest` if the account does not hold its bytes. pub(crate) fn serve_space(&self, s: &Service, did: &str, cid: &str) -> Result { let Some(h) = self.held.get(cid) else { return invalid(format!("Blob not held: {cid}")); }; serve(s, did, cid, h) } /// Put the blobs back as they were before the last commit. pub(crate) fn undo(&mut self) { let Some(u) = self.undo.take() else { return }; for (path, before) in u.records { if let Some(now) = self.records.remove(&path) { self.record_bytes -= record_cost(&path, &now); } if let Some(r) = before { self.record_bytes += record_cost(&path, &r); self.records.insert(path, r); } } for (cid, n) in u.refs { match n { Some(n) => self.refs.insert(cid, n), None => self.refs.remove(&cid), }; } for (cid, h) in u.held { self.held.remove(&cid); self.unreferenced.retain(|(_, c)| *c != cid); if let Some(h) = h { if !self.referenced(&cid) { self.unreferenced.insert((h.at, cid.clone())); } self.held.insert(cid, h); } } self.bytes = u.bytes; self.doomed.truncate(u.doomed); self.takedowns.extend(u.takedowns); } /// An upload of `cid` no longer moving into place, or putting its object. fn unplace(&mut self, cid: &str) { match self.placing.get_mut(cid) { Some(n) if *n > 1 => *n -= 1, _ => { self.placing.remove(cid); } } } /// What uploading `cid` would do. fn upload_of(&self, cid: &str) -> Upload { match self.held.get(cid) { None => Upload::New, Some(_) if self.referenced(cid) => Upload::Kept, Some(_) if self.expired(cid) => Upload::Afresh, Some(_) => Upload::Again, } } /// Make an upload's change, logged: stored as `h`, keeping the type it /// was first stored with if it is held still. fn uploaded(&mut self, cid: &str, how: Upload, h: Held) { match how { Upload::Kept => {} Upload::Again => { let mime = self.held[cid].mime.clone(); self.set_held(cid, Some(Held { mime, ..h })); } Upload::Afresh | Upload::New => { // One taken down and past its time is no longer held, nor down. self.takedowns.remove(cid); self.set_held(cid, Some(h)); } } } /// Replay an entry of the kinds in `ENTRIES`. pub(crate) fn replay(&mut self, e: &Json) -> Result<(), String> { if e["t"] == OBJECTS_DELETED { self.kept.clear(); return Ok(()); } let Some(cid) = e["cid"].as_str() else { return Err(format!("a blob entry with no cid: {e}")); }; if e["t"] == "blobGone" { if !self.held.contains_key(cid) || self.referenced(cid) { return Err(format!( "an upload gone that was not held, or is referenced: {e}" )); } self.set_held(cid, None); return Ok(()); } if e["t"] == PUT { self.kept.insert(cid.to_string()); return Ok(()); } if e["t"] == DELETED { if self.held.contains_key(cid) || !self.kept.remove(cid) { return Err(format!( "an object deleted of a blob held, or of none kept: {e}" )); } return Ok(()); } if e["t"] == "takedownBlob" { match &e["ref"] { Json::String(r) if self.held.contains_key(cid) => { self.takedowns.insert(cid.to_string(), r.clone()) } Json::Null => self.takedowns.remove(cid), _ => { return Err(format!( "a takedown of a blob not held, or with a ref not a string: {e}" )); } }; return Ok(()); } let (Some(mime), Some(size), Some(at)) = (e["mimeType"].as_str(), e["size"].as_u64(), e["at"].as_u64()) else { return Err(format!("a blob with no mimeType, size or at: {e}")); }; let how = if self.held.contains_key(cid) { Upload::Again } else { Upload::New }; self.uploaded( cid, how, Held { mime: mime.to_string(), size, at, }, ); Ok(()) } /// Once the log is replayed: every blob held has its file, whole, and /// every other file in `dir` goes, as no entry holds it, but an upload /// of this process's on its way in (`temp_prefix`). An upload no /// record references was never synced: one whose file a power cut took /// is held no more, and the others are synced by the commit that first /// references them, as before. pub(crate) fn recover(&mut self, dir: &Path) -> Result<(), String> { self.doomed.clear(); self.undo = None; let mut lost = Vec::new(); for (cid, h) in &self.held { let p = dir.join(cid); let whole = std::fs::metadata(&p).is_ok_and(|m| m.len() == h.size); match (whole, self.referenced(cid)) { (_, true) if !whole => { return Err(format!( "{}: the file of a blob the log holds is missing or not whole", p.display() )); } (_, true) => {} (true, false) => { self.unsynced.insert(cid.clone(), p); } (false, false) => lost.push(cid.clone()), } } for cid in lost { self.set_held(&cid, None); } let Ok(entries) = std::fs::read_dir(dir) else { return Ok(()); }; for e in entries { let e = e.map_err(|e| format!("{}: {e}", dir.display()))?; if !e .file_name() .to_str() .is_some_and(|n| self.held.contains_key(n) || n.starts_with(temp_prefix())) { std::fs::remove_file(e.path()) .map_err(|why| format!("{}: {why}", e.path().display()))?; } } Ok(()) } /// Once the log is replayed, with blobs in a bucket: nothing in flight. /// Its strays are deleted by the deleter (`blobstore`). pub(crate) fn recover_kept(&mut self) { self.doomed.clear(); self.undo = None; } /// The blobs the last write stopped holding. pub(crate) fn take_doomed(&mut self) -> Vec { self.undo = None; std::mem::take(&mut self.doomed) } /// What a checkpoint keeps of the objects a bucket may keep: none with /// blobs in the data directory, and with a bucket, those not held, the /// blobs held's being kept too. pub(crate) fn objects_list(&self) -> Option> { (!self.kept.is_empty()).then(|| { self.kept .iter() .filter(|c| !self.held.contains_key(*c)) .map(String::as_str) .collect() }) } /// The blobs it holds, in ascending order of CID, each with its /// takedown's reference while it is down: what a checkpoint keeps. pub(crate) fn held_list(&self) -> impl Iterator)> { self.held .iter() .map(|(cid, h)| (cid.as_str(), h, self.takedowns.get(cid).map(String::as_str))) } /// Each record with blob refs: its path, the rev that last wrote it, and /// the CIDs it references: what a checkpoint keeps of them. pub(crate) fn record_list(&self) -> Vec<(&str, &str, &[String])> { let mut out: Vec<(&str, &str, &[String])> = self .records .iter() .map(|(p, (r, c))| (p.as_str(), r.as_str(), c.as_slice())) .collect(); out.sort_unstable(); out } /// How many records of the account's space repos reference each CID, /// for those some such record does, in ascending order of CID: what a /// checkpoint keeps of them, as the records are not read to count them. pub(crate) fn space_ref_list(&self) -> Vec<(&str, u32)> { let mut out: Vec<(&str, u32)> = self .space_refs .iter() .map(|(cid, n)| (cid.as_str(), *n)) .collect(); out.sort_unstable(); out } /// The blobs a checkpoint keeps, as replaying the log would leave them: /// those held, each with its takedown, the records with blob refs, each /// with the rev that last wrote it and what it references, and with /// blobs in a bucket, the objects it may keep of those not held /// (`objects_list`), and how many records of the account's space repos /// reference each CID (`space_ref_list`). pub(crate) fn restore( records: Vec<(String, String, Vec)>, held: Vec<(String, Held, Option)>, objects: Option>, space_refs: Vec<(String, u32)>, ) -> Result { let mut b = Blobs { space_refs: space_refs.into_iter().collect(), ..Default::default() }; for (path, rev, cids) in records { if cids.is_empty() { return Err(format!("record {path} kept with no blob refs")); } for cid in &cids { *b.refs.entry(cid.clone()).or_default() += 1; } let refs = (rev, cids); b.record_bytes += record_cost(&path, &refs); b.records.insert(path, refs); } for (cid, h, takedown) in held { if !b.referenced(&cid) { b.unreferenced.insert((h.at, cid.clone())); } if let Some(t) = takedown { b.takedowns.insert(cid.clone(), t); } if objects.is_some() { b.kept.insert(cid.clone()); } b.bytes += h.size; b.held.insert(cid, h); } for cid in objects.into_iter().flatten() { if b.held.contains_key(&cid) { return Err(format!("object {cid} kept apart from its blob, held")); } b.kept.insert(cid); } Ok(b) } /// What a restart must carry over, as one comparable value. #[cfg(test)] pub(crate) fn snapshot(&self) -> String { let refs: BTreeMap<_, _> = self.refs.iter().collect(); let space_refs: BTreeMap<_, _> = self.space_refs.iter().collect(); let records: BTreeMap<_, _> = self.records.iter().collect(); format!( "{:?}", ( &self.held, refs, space_refs, records, &self.unreferenced, self.bytes, &self.takedowns, &self.kept, ) ) } } /// Apply a write's ops to the account's repository, its blob refs checked /// last, against the blobs as they were before it, and their references counted. pub(crate) fn apply( st: &mut Store, ops: Vec, swap_commit: Option<&str>, ) -> Result, XrpcError> { let mut refs = Vec::new(); for op in &ops { if let Op::Create { value, .. } | Op::Update { value, .. } = op { refs_in(value, &mut refs); } } let Store { repo, blobs, disk, ctx, wal, .. } = st; let results = repo.apply_checked(&disk.source(&ctx.blocks), ops, swap_commit, || { blobs.check(&refs) })?; if let Some(c) = &repo.change { blobs.sync_before(wal, refs.iter().map(|r| cid_string(&r.cid))); blobs.committed(c); } Ok(results) } /// Make the account's repository the tree an import brings, by `ops`, as /// one commit: its blob refs are not checked, each blob it does not hold /// becoming missing until it is uploaded (`spec/MIGRATION.md`), and their /// references are counted as a write's. pub(crate) fn import(st: &mut Store, ops: Vec) -> Result<(), XrpcError> { let mut refs = Vec::new(); for op in &ops { if let Op::Create { value, .. } | Op::Update { value, .. } = op { refs_in(value, &mut refs); } } let Store { repo, blobs, disk, ctx, wal, .. } = st; repo.import(&disk.source(&ctx.blocks), ops, || Ok::<(), XrpcError>(()))?; if let Some(c) = &repo.change { blobs.sync_before(wal, refs.iter().map(|r| cid_string(&r.cid))); blobs.committed(c); } Ok(()) } /// Remove the files of blobs a write stopped holding, once it is on disk: /// those not held again, by an upload since, or moving into place. With /// blobs in a bucket, have the deleter delete their objects (`blobstore`). pub(crate) fn forget(svc: &Service, account: &Account, doomed: Vec) { if doomed.is_empty() { return; } if let Some(bucket) = &svc.ctx.bucket { bucket.wake(&account.did); return; } let dir = blob_dir(&svc.dir, &account.did); let st = read(&account.store); for cid in doomed .iter() .filter(|c| !st.blobs.held.contains_key(*c) && !st.blobs.placing.contains_key(*c)) { let _ = std::fs::remove_file(dir.join(cid)); } } /// Log as gone every upload held past its time with no record referencing /// it, and remove its file, for every account whose store is held: one not /// held has its swept when next loaded and written. With blobs in a bucket, /// the account is marked for a start to load first (`blobstore`). pub(crate) fn sweep(svc: &Service) { let accounts: Vec> = read(&svc.directory) .by_did .values() .filter(|a| a.store.is_loaded()) .cloned() .collect(); let (now, ttl) = (svc.limits.clock.now(), svc.config.blob_ttl); for a in accounts { let (wal, n, gone) = { let mut st = write(&a.store); let st = &mut *st; st.blobs.time(now, ttl); let gone: Vec = st.blobs.past_time().cloned().collect(); if gone.is_empty() { continue; } let entries: Vec = gone.iter().map(|c| gone_entry(c)).collect(); let marked = match &st.ctx.bucket { Some(_) => st.ctx.eager.mark(&a.did), None => Ok(()), }; match marked.and_then(|()| st.wal.append(&entries)) { Ok(n) => { for cid in &gone { st.blobs.set_held(cid, None); } (st.wal.clone(), n, gone) } Err(e) => { crate::ops::failed_for( crate::ops::Level::Error, "logging uploads gone failed", &a.did, e, ); continue; } } }; if let Err(e) = wal.wait(n) { crate::ops::failed_for( crate::ops::Level::Error, "syncing uploads gone failed", &a.did, e, ); continue; } forget(svc, &a, gone); } } /// Sweep now, and every `SWEEP_EVERY` while the server runs. pub(crate) fn sweep_every(svc: &Shared) { let weak = Arc::downgrade(svc); tokio::spawn(async move { loop { let Some(svc) = weak.upgrade() else { return }; let _ = tokio::task::spawn_blocking(move || sweep(&svc)).await; tokio::time::sleep(SWEEP_EVERY).await; } }); } // ---- uploadBlob ------------------------------------------------------------ /// `uploadBlob`, which takes its body as it comes rather than as JSON. pub(crate) async fn upload( AxState(svc): AxState, method: Method, headers: HeaderMap, peer: Option>, mut body: Body, ) -> Response { let authorization = crate::authorization(&headers); let bearer = crate::sessions::Bearer::of(authorization.as_deref()); let client = crate::limits::client( &svc.config.trusted_proxies, &headers, peer.map(|Extension(p)| p.0), ); let mut call = Call { nsid: UPLOAD, method: &method, params: &[], body: &[], bearer, headers: &headers, client, charged: OnceLock::new(), http: None, }; let grant = match crate::scopes::look(&svc, &call, &headers).await { Ok(g) => g, Err(r) => { drain(&mut body, DRAIN_LIMIT).await; return *r; } }; if let Some(g) = &grant { call.bearer = crate::sessions::Bearer::OAuth(g); } let answer = upload_blob(svc.clone(), &call, &headers, &mut body).await; // A client still sending its body when it is refused, as one too large // is, may take the connection closing for a failure and never read the // answer: read the rest first, up to a point, storing none of it. if answer.is_err() { drain(&mut body, DRAIN_LIMIT).await; } crate::scopes::answered( &svc, grant.as_ref(), crate::respond(answer, call.charged.get()), ) } /// How much of a refused upload's body is read and thrown away before it is /// answered; past it, the connection closes as the answer goes. const DRAIN_LIMIT: u64 = 16 * 1024 * 1024; /// The next frame of a body, if it has another. async fn next_frame(body: &mut Body) -> Option, axum::Error>> { std::future::poll_fn(|cx| Pin::new(&mut *body).poll_frame(cx)).await } /// Read and throw away the rest of a body, up to `limit` bytes. async fn drain(body: &mut Body, mut limit: u64) { while let Some(Ok(frame)) = next_frame(body).await { let n = frame.data_ref().map_or(0, |d| d.len() as u64); if n > limit { return; } limit -= n; } } /// How the names of this process's uploads on their way in begin, in an /// account's blob directory: a store loaded while one comes in, its /// account's first request or one since it was let go, removes the files a /// process before left there (`recover`, `log::open_repo`), not this one's. pub(crate) fn temp_prefix() -> &'static str { static PREFIX: LazyLock = LazyLock::new(|| format!(".tmp-{}-", hex::encode(rand::random::<[u8; 6]>()))); &PREFIX } /// An upload's bytes on their way in, removed unless they are moved into place. struct Temp { path: PathBuf, file: File, placed: bool, } impl Temp { fn create(dir: &Path) -> io::Result { for made in pds_store::make_dirs(dir)? { if let Some(parent) = made.parent() { pds_store::sync_dir(parent)?; } } let path = dir.join(format!( "{}{}", temp_prefix(), hex::encode(rand::random::<[u8; 12]>()) )); let file = pds_store::open_private( &path, OpenOptions::new().create_new(true).read(true).write(true), )?; Ok(Temp { path, file, placed: false, }) } /// Move the bytes to `to`, unsynced: the commit that first references /// them syncs them. Bytes of the same CID already there stay, as they /// may be synced and these not. fn place(mut self, to: &Path) -> io::Result<()> { if to.exists() { return Ok(()); } std::fs::rename(&self.path, to)?; self.placed = true; Ok(()) } } impl Drop for Temp { fn drop(&mut self) { if !self.placed { let _ = std::fs::remove_file(&self.path); } } } /// Make `bytes` `cid`'s file in `dir`, in place of any there, synced, as a /// move into the data directory writes it (`migrate_blobs`): the directory /// is synced once each of its files is. pub(crate) fn write_synced(dir: &Path, cid: &str, bytes: &[u8]) -> io::Result<()> { let mut t = Temp::create(dir)?; t.file.write_all(bytes)?; pds_store::sync_file(&t.file)?; std::fs::rename(&t.path, dir.join(cid))?; t.placed = true; Ok(()) } /// The type a body's first bytes show, of those `spec/BLOBS.md` lists. fn sniff(head: &[u8], body: &mut (impl Read + Seek)) -> Option<&'static str> { let at = |from: usize, want: &[u8]| head.get(from..from + want.len()) == Some(want); if at(0, b"\x89PNG\r\n\x1a\n") { png_image(body).then_some("image/png") } else if at(0, b"\xff\xd8\xff") && head.get(3).is_some_and(|b| *b != 0xf7) { Some("image/jpeg") } else if at(0, b"GIF89a") { Some("image/gif") } else if at(0, b"RIFF") && at(8, b"WEBP") { Some("image/webp") } else if at(0, b"%PDF-") { Some("application/pdf") } else if at(4, b"ftyp") && at(8, b"isom") { Some("video/mp4") } else { None } } /// Whether a body after the PNG signature is a PNG image, not an animated /// one: an `IHDR` chunk of 13 bytes first, then chunks up to an `IDAT`, with /// no `acTL` before it. Nothing after the first `IDAT` is read: neither its /// length nor `IEND` (the harness's PNG has an `IDAT` 4 bytes shorter than /// it says, and the reference takes it), nor any CRC. fn png_image(body: &mut (impl Read + Seek)) -> bool { let mut f = io::BufReader::new(body); if f.seek(SeekFrom::Start(8)).is_err() { return false; } let mut first = true; loop { let mut h = [0u8; 8]; if f.read_exact(&mut h).is_err() { return false; } let len = u32::from_be_bytes([h[0], h[1], h[2], h[3]]); let kind = &h[4..]; if first && (kind != b"IHDR" || len != 13) { return false; } first = false; match kind { b"acTL" => return false, b"IDAT" => return true, _ => {} } if f.seek_relative(i64::from(len) + 4).is_err() { return false; } } } /// `uploadBlob` checks its token, its account's state, its `Content-Type`, /// what an OAuth scope needs of that media type, then its size, pays its /// rate limits and checks the free space (`limits::guard`), then the /// account's quota, and stores the body under its CID: in a file, or with /// blobs in a bucket, read whole and put there. An OAuth token's call has /// `Bearer::OAuth` already. async fn upload_blob( svc: Shared, call: &Call<'_>, headers: &HeaderMap, body: &mut Body, ) -> Result { crate::method_of(call)?; let did = svc.auth(call.bearer).did()?; crate::lifecycle::gate(&svc, UPLOAD, &did)?; let Some(account) = svc.account(&did) else { return internal(); }; // The media type is the header up to its first `;`, as sent. let given = headers .get(header::CONTENT_TYPE) .map(|v| String::from_utf8_lossy(v.as_bytes()).into_owned()) .unwrap_or_default(); let media = given.split(';').next().unwrap_or_default().to_string(); if media.is_empty() { return invalid("Request encoding (Content-Type) required but not provided"); } // What the scope needs reads each byte of the header as its Latin-1 // character, as the refusal names it (`spec/OAUTH-SCOPES.md`). let raw = headers .get(header::CONTENT_TYPE) .map(|v| v.as_bytes()) .unwrap_or_default(); let latin1: String = raw .split(|&b| b == b';') .next() .unwrap_or_default() .iter() .map(|&b| char::from(b)) .collect(); crate::scopes::may(call.bearer, &crate::scopes::Need::Blob(&latin1))?; let too_large = || err(413, "PayloadTooLarge", "request entity too large"); let limit = svc.config.blob_upload_limit; let length = headers .get(header::CONTENT_LENGTH) .and_then(|v| v.to_str().ok()?.trim().parse::().ok()); if length.is_some_and(|n| n > limit) { return too_large(); } if let Some(bucket) = svc.ctx.bucket.clone() { let bytes = read_whole(body, length, limit).await?; crate::limits::guard(&svc, call, Some(&did), None)?; let cid = cid_string(&cid_of(0x55, &bytes)); let head = &bytes[..bytes.len().min(16)]; let mime = sniff(head, &mut io::Cursor::new(&bytes[..])).map_or(media, str::to_string); let size = bytes.len() as u64; let answer = json!({ "blob": { "$type": "blob", "ref": { "$link": cid }, "mimeType": mime, "size": size } }); let blob = Held { mime, size, at: svc.limits.clock.now(), }; // Once its bytes are in, it is stored even if the caller goes away, // so that no upload is left putting its object. crate::ops::spawn(store_object( svc.clone(), account, bucket, cid, bytes.into(), blob, )) .await .unwrap_or_else(|_| internal())?; return Ok(Resp::Json(answer)); } let dir = blob_dir(&svc.dir, &did); let Ok(mut tmp) = Temp::create(&dir) else { return internal(); }; let (mut hash, mut size, mut head) = (Sha256::new(), 0u64, Vec::with_capacity(16)); while let Some(frame) = next_frame(body).await { let Ok(frame) = frame else { return invalid("The body could not be read"); }; let Ok(data) = frame.into_data() else { continue; }; size += data.len() as u64; if size > limit { return too_large(); } hash.update(&data); head.extend_from_slice(&data[..data.len().min(16 - head.len())]); if tmp.file.write_all(&data).is_err() { return internal(); } } crate::limits::guard(&svc, call, Some(&did), None)?; let mut cid = vec![1, 0x55, 0x12, 0x20]; cid.extend_from_slice(&hash.finalize()); let cid = cid_string(&cid); let mime = sniff(&head, &mut tmp.file).map_or(media, str::to_string); let answer = json!({ "blob": { "$type": "blob", "ref": { "$link": cid }, "mimeType": mime, "size": size } }); // Once its bytes are in, it is stored even if the caller goes away, so // that no upload is left moving into place. let blob = Held { mime, size, at: svc.limits.clock.now(), }; tokio::spawn(store(svc.clone(), account, dir, tmp, cid, blob)) .await .unwrap_or_else(|_| internal())?; Ok(Resp::Json(answer)) } /// An upload's body, whole, in memory: at most `limit` bytes. async fn read_whole( body: &mut Body, length: Option, limit: u64, ) -> Result, XrpcError> { let mut bytes = Vec::with_capacity(length.unwrap_or(0) as usize); while let Some(frame) = next_frame(body).await { let Ok(frame) = frame else { return invalid("The body could not be read"); }; let Ok(data) = frame.into_data() else { continue; }; if (bytes.len() + data.len()) as u64 > limit { return err(413, "PayloadTooLarge", "request entity too large"); } bytes.extend_from_slice(&data); } Ok(bytes) } /// Store an upload's bytes as its CID's, within the account's quota, and log /// it, unless a record references it already; return once that is handed to /// the operating system, unsynced. Its file is put in place first, held by /// none until it is logged. async fn store( svc: Shared, account: Arc, dir: PathBuf, tmp: Temp, cid: String, blob: Held, ) -> Result<(), XrpcError> { *write(&account.store) .blobs .placing .entry(cid.clone()) .or_default() += 1; let to = dir.join(&cid); let placed = tokio::task::spawn_blocking(move || tmp.place(&to)).await; let (logged, doomed) = { let mut st = write(&account.store); let st = &mut *st; st.blobs.unplace(&cid); st.blobs.time(blob.at, svc.config.blob_ttl); let how = st.blobs.upload_of(&cid); let logged = if !matches!(placed, Ok(Ok(()))) { internal() } else { refusal(&svc, &account, &st.blobs, &cid, how, blob.size) .and_then(|()| log_upload(st, &account.did, &cid, how, blob)) }; if logged.is_ok() && how != Upload::Kept && !st.blobs.referenced(&cid) { st.blobs.unsynced.insert(cid.clone(), dir.join(&cid)); } (logged, logged_or_doomed(&st.blobs, &cid)) }; // An upload not logged leaves no file a blob held does not have. forget(&svc, &account, doomed); logged } /// Store an upload's bytes as its CID's object in the bucket, and log it as /// `store` does, once the store has kept it, in the object's turn. It is /// refused before it is put as it would be after, and its object noted in /// the log first if no entry names it yet, so that one the server stops /// during is deleted. One the store does not keep holds nothing more: its /// object, if the store kept it after all, is a stray, deleted. async fn store_object( svc: Shared, account: Arc, bucket: Arc, cid: String, bytes: Bytes, blob: Held, ) -> Result<(), XrpcError> { let did = &account.did; let ttl = svc.config.blob_ttl; let turn = bucket.turn(did, &cid).await; { let mut st = write(&account.store); let st = &mut *st; st.blobs.time(blob.at, ttl); let how = st.blobs.upload_of(&cid); refusal(&svc, &account, &st.blobs, &cid, how, blob.size)?; if !st.blobs.kept.contains(&cid) { let noted = st .ctx .eager .mark(did) .and_then(|()| st.wal.append(&[put_entry(&cid)])); if let Err(e) = noted { crate::ops::failed_for( crate::ops::Level::Error, "logging an upload failed", did, e, ); return internal(); } st.blobs.kept.insert(cid.clone()); } *st.blobs.placing.entry(cid.clone()).or_default() += 1; } let put = bucket.put(did, &cid, bytes).await; let logged = { let mut st = write(&account.store); let st = &mut *st; st.blobs.unplace(&cid); st.blobs.time(blob.at, ttl); let how = st.blobs.upload_of(&cid); put.and_then(|()| refusal(&svc, &account, &st.blobs, &cid, how, blob.size)) .and_then(|()| log_upload(st, did, &cid, how, blob)) }; drop(turn); // An object of a blob not held to delete, or a mark that may go. bucket.wake(did); logged } /// Whether an upload doing `how` is refused, its bytes in: its account /// deleted, the blob taken down, or the account's quota passed. fn refusal( svc: &Service, account: &Account, b: &Blobs, cid: &str, how: Upload, size: u64, ) -> Result<(), XrpcError> { crate::lifecycle::not_deleted(account)?; if b.taken_down(cid) { return invalid("Blob has been takendown, cannot re-upload"); } if matches!(how, Upload::New | Upload::Afresh) && b.counted() + size > svc.config.blob_quota { return err( 400, "BlobQuotaExceeded", "The account's blobs would be past its quota", ); } Ok(()) } /// Log an upload doing `how`, unless a record references it already, and /// make its change, keeping the type a blob held was first stored with. fn log_upload( st: &mut Store, did: &str, cid: &str, how: Upload, blob: Held, ) -> Result<(), XrpcError> { let entries = match how { Upload::Kept => return Ok(()), Upload::Again => vec![upload_entry( cid, &Held { mime: st.blobs.held[cid].mime.clone(), ..blob.clone() }, )], Upload::Afresh => vec![gone_entry(cid), upload_entry(cid, &blob)], Upload::New => vec![upload_entry(cid, &blob)], }; match st.wal.append(&entries) { Ok(_) => { st.blobs.uploaded(cid, how, blob); Ok(()) } Err(e) => { crate::ops::failed_for(crate::ops::Level::Error, "logging an upload failed", did, e); internal() } } } /// The CID, if the account does not hold it, as a file to remove. fn logged_or_doomed(blobs: &Blobs, cid: &str) -> Vec { if blobs.held.contains_key(cid) { vec![] } else { vec![cid.to_string()] } } // ---- getBlob and listBlobs --------------------------------------------------- fn repo_not_found(did: &str) -> Result { err( 400, "RepoNotFound", format!("Could not find repo for DID: {did}"), ) } /// A blob `getBlob` serves: where its bytes are, and what its headers say. pub(crate) struct Served { source: Source, mime: String, size: u64, cid: String, } /// Where a blob's bytes come from. enum Source { /// Its file, opened. File(File), /// Its object in the bucket, of the account by DID, not yet asked for: /// `read_object` asks. Object(String), /// Its object as it comes, its first bytes in. Read(crate::blobstore::Chunks), } /// What `getBlob` answers of a blob held: its file, or its object, to read. fn serve(s: &Service, did: &str, cid: &str, h: &Held) -> Result { let source = match &s.ctx.bucket { Some(_) => Source::Object(did.to_string()), None => match File::open(blob_dir(&s.dir, did).join(cid)) { Ok(file) => Source::File(file), Err(_) => return internal(), }, }; Ok(Resp::Blob(Served { source, mime: h.mime.clone(), size: h.size, cid: cid.to_string(), })) } /// A blob in the bucket an answer serves, asked of the store: `500` if it /// fails before the object's first bytes (`spec/BLOBS.md`). Any other /// answer, as it is. pub(crate) async fn read_object( s: &Service, out: Result, ) -> Result { match out { Ok(Resp::Blob(Served { source: Source::Object(did), mime, size, cid, })) => { let Some(bucket) = &s.ctx.bucket else { return internal(); }; match bucket.get(&did, &cid, size).await { Some(chunks) => Ok(Resp::Blob(Served { source: Source::Read(chunks), mime, size, cid, })), None => internal(), } } out => out, } } /// `getBlob` takes no token, but refuses one the server would not take, as /// the lifecycle's gate checks it (`lifecycle::gate_query`). fn get_blob( s: &Service, _: &Call, shown: &mut Shown, p: &Map, ) -> Result { let (did, cid) = (required(p, "did")?, required(p, "cid")?); let Some(a) = s.account(did) else { return repo_not_found(did); }; let st = read(&a.store); shown.0.push((st.wal.clone(), st.wal.written())); // One taken down is as one not held, to the account and the admin too. let Some(h) = st.blobs.served(cid).filter(|_| !st.blobs.taken_down(cid)) else { return invalid("Blob not found"); }; crate::ops::serving_blob(&a.did, cid); serve(s, &a.did, cid, h) } /// A blob's bytes as `getBlob` answers them, as they are read: no more than /// its size, and an error if they end before it. struct BlobBody { chunks: tokio::sync::mpsc::Receiver>, left: u64, } impl HttpBody for BlobBody { type Data = Bytes; type Error = io::Error; fn poll_frame( self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll, io::Error>>> { let this = self.get_mut(); if this.left == 0 { return Poll::Ready(None); } this.chunks.poll_recv(cx).map(|chunk| match chunk { Some(Ok(mut b)) => { b.truncate(this.left.min(b.len() as u64) as usize); this.left -= b.len() as u64; Some(Ok(Frame::data(b))) } Some(Err(e)) => Some(Err(e)), None => Some(Err(io::Error::new( io::ErrorKind::UnexpectedEof, "the blob's bytes ended early", ))), }) } fn is_end_stream(&self) -> bool { self.left == 0 } fn size_hint(&self) -> SizeHint { SizeHint::with_exact(self.left) } } /// `getBlob`'s answer: the blob's bytes, streamed, with its headers. pub(crate) fn response(b: Served) -> Response { let Served { source, mime, size, cid, } = b; let chunks = match source { Source::File(mut file) => { let (tx, chunks) = tokio::sync::mpsc::channel(4); tokio::task::spawn_blocking(move || { let mut left = size; while left > 0 { let mut buf = vec![0u8; left.min(64 * 1024) as usize]; let read = file.read_exact(&mut buf).map(|()| Bytes::from(buf)); let failed = read.is_err(); if tx.blocking_send(read).is_err() || failed { return; } left -= left.min(64 * 1024); } }); chunks } Source::Read(chunks) => chunks, // Never so: an object is read before its answer is made (`read_object`). Source::Object(_) => return crate::respond(internal(), None), }; let mut res = Response::new(Body::new(BlobBody { chunks, left: size })); let h = res.headers_mut(); h.insert( header::CONTENT_TYPE, HeaderValue::from_bytes(mime.as_bytes()) .unwrap_or(HeaderValue::from_static("application/octet-stream")), ); h.insert(header::CONTENT_LENGTH, HeaderValue::from(size)); h.insert( header::X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff"), ); if let Ok(v) = HeaderValue::from_str(&format!("attachment; filename=\"{cid}\"")) { h.insert(header::CONTENT_DISPOSITION, v); } h.insert( header::CONTENT_SECURITY_POLICY, HeaderValue::from_static("default-src 'none'; sandbox"), ); res } /// `listBlobs`: the CIDs records reference, in order, after the cursor; with /// `since`, only those of records written after that rev. Its /// `Authorization` is the gate's, as `getBlob`'s is. fn list_blobs( s: &Service, _: &Call, shown: &mut Shown, p: &Map, ) -> Result { let did = required(p, "did")?; let limit = p.get("limit").and_then(Json::as_u64).unwrap_or(500) as usize; let cursor = opt(p, "cursor").unwrap_or(""); let Some(a) = s.account(did) else { return repo_not_found(did); }; let st = read(&a.store); shown.0.push((st.wal.clone(), st.wal.written())); let b = &st.blobs; let cids: Vec<&str> = match opt(p, "since") { None => b .held .range::((Excluded(cursor), Unbounded)) .map(|(c, _)| c.as_str()) .filter(|c| b.refs.contains_key(*c)) .take(limit) .collect(), Some(since) => { let since: BTreeSet<&str> = b .records .values() .filter(|(rev, _)| rev.as_str() > since) .flat_map(|(_, cids)| cids.iter().map(String::as_str)) .filter(|c| *c > cursor && b.served(c).is_some()) .collect(); since.into_iter().take(limit).collect() } }; let mut out = json!({ "cids": cids }); if let Some(last) = cids.last() { out["cursor"] = json!(last); } Ok(Resp::Json(out)) } /// Upload `body` as a request would, with its `Content-Length` unless /// `chunked`; answers the JSON, or the error's status and name. #[cfg(test)] pub(crate) async fn test_upload( s: &Shared, token: Option<&str>, content_type: Option<&str>, body: Vec, chunked: bool, ) -> Result { /// A body in pieces, of no length known beforehand. struct Chunks(Vec); impl HttpBody for Chunks { type Data = Bytes; type Error = io::Error; fn poll_frame( self: Pin<&mut Self>, _: &mut Context<'_>, ) -> Poll, io::Error>>> { let this = self.get_mut(); Poll::Ready((!this.0.is_empty()).then(|| Ok(Frame::data(this.0.remove(0))))) } } let mut headers = HeaderMap::new(); if let Some(t) = token { headers.insert( header::AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {t}")).unwrap(), ); } if let Some(c) = content_type { headers.insert(header::CONTENT_TYPE, HeaderValue::from_str(c).unwrap()); } let mut body = if chunked { Body::new(Chunks( body.chunks(100_000).map(Bytes::copy_from_slice).collect(), )) } else { headers.insert(header::CONTENT_LENGTH, HeaderValue::from(body.len())); Body::from(body) }; let authorization = crate::authorization(&headers); let bearer = crate::sessions::Bearer::of(authorization.as_deref()); let call = Call { nsid: UPLOAD, method: &Method::POST, params: &[], body: &[], bearer, headers: &headers, client: "127.0.0.1".into(), charged: OnceLock::new(), http: None, }; match upload_blob(s.clone(), &call, &headers, &mut body).await { Ok(Resp::Json(j)) => Ok(j), Ok(_) => Err((0, "not JSON".into())), Err(e) => Err((e.status, e.name.to_string())), } } #[cfg(test)] mod tests { use super::*; use crate::testing::*; use pds_repo::data::cid_of; use std::sync::atomic::Ordering; /// A blob ref, as a record holds one. fn bref(cid: &str, mime: &str, size: i64) -> Json { json!({ "$type": "blob", "ref": { "$link": cid }, "mimeType": mime, "size": size }) } fn raw_cid(body: &[u8]) -> String { cid_string(&cid_of(0x55, body)) } /// A body that sniffs as no type. fn plain(n: usize) -> Vec { let mut b = b"spec-pds blob ".to_vec(); b.extend((0..n).map(|i| (i * 7 + n) as u8)); b } /// `harness/checks/blobs.mjs`'s PNG, a 1x1 image, then `tail`. fn harness_png(tail: &[u8]) -> Vec { let mut b = hex::decode(concat!( "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c636000", "0002000154a24f5d0000000049454e44ae426082" )) .unwrap(); b.extend(tail); b } fn png(extra: &[&[u8; 4]]) -> Vec { let chunk = |kind: &[u8], data: &[u8]| -> Vec { let mut c = (data.len() as u32).to_be_bytes().to_vec(); c.extend(kind); c.extend(data); c.extend([0; 4]); c }; let mut b = b"\x89PNG\r\n\x1a\n".to_vec(); b.extend(chunk(b"IHDR", &[0; 13])); for k in extra { b.extend(chunk(*k, &[1, 2, 3])); } b.extend(chunk(b"IDAT", &[9; 20])); b.extend(chunk(b"IEND", &[])); b } async fn put( s: &Shared, token: &str, did: &str, rkey: &str, record: Json, ) -> Result { call(s, "com.atproto.repo.putRecord", Some(token), json!({ "repo": did, "collection": "com.example.blob", "rkey": rkey, "record": record })).await } async fn delete(s: &Shared, token: &str, did: &str, rkey: &str) -> Result { call( s, "com.atproto.repo.deleteRecord", Some(token), json!({ "repo": did, "collection": "com.example.blob", "rkey": rkey }), ) .await } /// listBlobs, paged to the end with `limit`. async fn listed(s: &Shared, did: &str, since: Option<&str>, limit: u32) -> Vec { let mut out = Vec::new(); let mut cursor: Option = None; loop { let mut p = vec![ ("did".to_string(), did.to_string()), ("limit".into(), limit.to_string()), ]; p.extend(since.map(|v| ("since".to_string(), v.to_string()))); p.extend(cursor.clone().map(|c| ("cursor".to_string(), c))); let got = call_with(s, "com.atproto.sync.listBlobs", None, &p, json!({})) .await .unwrap(); let page: Vec = got["cids"] .as_array() .unwrap() .iter() .map(|c| c.as_str().unwrap().to_string()) .collect(); assert!(page.len() <= limit as usize); assert_eq!(got["cursor"].as_str(), page.last().map(String::as_str)); if page.is_empty() { return out; } cursor = page.last().cloned(); out.extend(page); } } /// getBlob through the router's handler: status, headers and bytes. async fn get( s: &Shared, did: &str, cid: &str, token: Option<&str>, ) -> (u16, HeaderMap, Vec) { let mut headers = HeaderMap::new(); if let Some(t) = token { headers.insert( header::AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {t}")).unwrap(), ); } let query = url::form_urlencoded::Serializer::new(String::new()) .append_pair("did", did) .append_pair("cid", cid) .finish(); let uri = format!("/xrpc/com.atproto.sync.getBlob?{query}") .parse() .unwrap(); let res = crate::handle( AxState(s.clone()), Method::GET, uri, headers, None, Body::empty(), ) .await; let (parts, body) = res.into_parts(); let bytes = axum::body::to_bytes(body, usize::MAX) .await .unwrap() .to_vec(); (parts.status.as_u16(), parts.headers, bytes) } /// The error name of a getBlob that fails. async fn get_error(s: &Shared, did: &str, cid: &str, token: Option<&str>) -> String { let (status, _, body) = get(s, did, cid, token).await; assert_ne!(status, 200); serde_json::from_slice::(&body).unwrap()["error"] .as_str() .unwrap() .to_string() } fn files(dir: &Path, did: &str) -> Vec { let mut names: Vec = std::fs::read_dir(blob_dir(dir, did)) .map(|d| { d.map(|e| e.unwrap().file_name().into_string().unwrap()) .collect() }) .unwrap_or_default(); names.sort(); names } #[tokio::test] async fn upload_checks_sniffs_and_limits() { let plc = plc().await; let dir = scratch("blob-upload"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let t = Some(token.as_str()); // The token, then the Content-Type, then the size. let big = vec![0u8; DEFAULT_UPLOAD_LIMIT as usize + 1]; assert_eq!( test_upload(&s, None, None, big.clone(), false).await, Err((401, "AuthMissing".into())) ); assert_eq!( test_upload(&s, Some("forged"), None, big.clone(), false).await, Err((400, "InvalidToken".into())) ); for ct in [None, Some(""), Some("; charset=utf-8")] { assert_eq!( test_upload(&s, t, ct, big.clone(), false).await, Err((400, "InvalidRequest".into())), "{ct:?}" ); } for chunked in [false, true] { assert_eq!( test_upload(&s, t, Some("text/plain"), big.clone(), chunked).await, Err((413, "PayloadTooLarge".into())) ); } let mut headers = HeaderMap::new(); headers.insert( header::AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {token}")).unwrap(), ); let wrong = Call { nsid: UPLOAD, method: &Method::GET, params: &[], body: &[], bearer: crate::sessions::Bearer::Token(&token), headers: &headers, client: String::new(), charged: OnceLock::new(), http: None, }; assert_eq!( upload_blob(s.clone(), &wrong, &headers, &mut Body::empty()) .await .err() .map(|e| e.name), Some("InvalidRequest") ); // A refused upload leaves nothing behind. assert_eq!(files(&dir, &did), Vec::::new()); // Exactly the limit is taken, chunked or not, and so is nothing. for chunked in [false, true] { let exact = vec![7u8; DEFAULT_UPLOAD_LIMIT as usize]; let got = test_upload( &s, t, Some("application/octet-stream"), exact.clone(), chunked, ) .await .unwrap(); assert_eq!( got, json!({ "blob": bref(&raw_cid(&exact), "application/octet-stream", DEFAULT_UPLOAD_LIMIT as i64) }) ); } let got = test_upload(&s, t, Some("image/png"), vec![], false) .await .unwrap(); assert_eq!(got["blob"], bref(&raw_cid(&[]), "image/png", 0)); // The type the bytes show, or else the media type as sent. let mut jpeg = b"\xff\xd8\xff\xe0".to_vec(); jpeg.extend(plain(3)); let mut not_jpeg = b"\xff\xd8\xff\xf7".to_vec(); not_jpeg.extend(plain(3)); let mut webp = b"RIFF\x01\x02\x03\x04WEBP".to_vec(); webp.extend(plain(2)); let mut mp4 = b"\0\0\0\x18ftypisom".to_vec(); mp4.extend(plain(4)); let cases: Vec<(Vec, &str, &str)> = vec![ // The harness's PNG, with a tail of its own after `IEND`. (harness_png(b"spec-pds blob 1 "), "image/jpeg", "image/png"), (harness_png(b"\x00\x01"), "TEXT/Plain", "image/png"), (harness_png(b""), "text/plain ;q=1", "image/png"), (png(&[]), "text/plain", "image/png"), (png(&[b"tEXt"]), "text/plain", "image/png"), (png(&[b"acTL"]), "text/plain", "text/plain"), (png(&[])[..30].to_vec(), "text/plain", "text/plain"), (jpeg, "text/plain", "image/jpeg"), (not_jpeg, "text/plain", "text/plain"), ( [b"GIF89a".as_slice(), &plain(5)].concat(), "text/plain", "image/gif", ), (webp, "text/plain", "image/webp"), ( [b"%PDF-1.7".as_slice(), &plain(5)].concat(), "image/png", "application/pdf", ), (mp4, "text/plain", "video/mp4"), (plain(10), "TEXT/Plain", "TEXT/Plain"), (plain(11), "text/plain; charset=utf-8", "text/plain"), (plain(12), "text/plain ;q=1", "text/plain "), (plain(13), "a/b/c", "a/b/c"), (plain(14), "*/*", "*/*"), ]; for (body, ct, want) in cases { for chunked in [false, true] { let got = test_upload(&s, t, Some(ct), body.clone(), chunked) .await .unwrap(); assert_eq!( got["blob"], bref(&raw_cid(&body), want, body.len() as i64), "{ct} {want}" ); } } // Again with another type: answered with it, but stored with the first. let b = plain(20); let cid = raw_cid(&b); test_upload(&s, t, Some("text/plain"), b.clone(), false) .await .unwrap(); assert_eq!( test_upload(&s, t, Some("image/x"), b.clone(), true) .await .unwrap()["blob"]["mimeType"], json!("image/x") ); assert_eq!( put( &s, &token, &did, "x", json!({ "f": bref(&cid, "image/x", 34) }) ) .await, Err("InvalidMimeType".into()) ); put( &s, &token, &did, "x", json!({ "f": bref(&cid, "text/plain", 34) }), ) .await .unwrap(); let (status, headers, bytes) = get(&s, &did, &cid, None).await; assert_eq!( ( status, headers[header::CONTENT_TYPE].to_str().unwrap(), bytes ), (200, "text/plain", b) ); // Only whole blobs are left: no temporary file. assert!(files(&dir, &did).iter().all(|f| !f.starts_with('.'))); let got = call(&s, "com.atproto.server.describeServer", None, json!({})) .await .unwrap(); assert_eq!(got["blobUploadLimit"], json!(5242880)); let _ = std::fs::remove_dir_all(&dir); } /// Refs are checked last, against the blobs before the write; the legacy /// shape right after the record's `$type`. #[tokio::test] async fn refs_are_checked_in_order() { let plc = plc().await; let dir = scratch("blob-refs"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let (bob, bob_token) = create(&s, "bob").await; let (x, y, theirs) = (plain(1), plain(2), plain(3)); for b in [&x, &y] { test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false) .await .unwrap(); } test_upload( &s, Some(&bob_token), Some("text/plain"), theirs.clone(), false, ) .await .unwrap(); let (xc, yc) = (raw_cid(&x), raw_cid(&y)); let (size, ysize) = (x.len() as i64, y.len() as i64); let rev = |s: &Service| repo_of(s, &did, |r| r.rev.clone()); let before = rev(&s); let record_cid = repo_of(&s, &did, |r| cid_string(&r.head)); let refused = [ (bref(&raw_cid(b"never"), "text/plain", 5), "BlobNotFound"), (bref(&raw_cid(&theirs), "text/plain", size), "BlobNotFound"), (bref(&record_cid, "text/plain", size), "BlobNotFound"), (bref(&xc, "image/png", size), "InvalidMimeType"), (bref(&xc, "image/png", 1), "InvalidMimeType"), (bref(&xc, "text/plain", size + 1), "InvalidSize"), (bref(&xc, "text/plain", -1), "InvalidSize"), ( json!({ "cid": xc, "mimeType": "text/plain" }), "InvalidRequest", ), ]; for (r, want) in &refused { assert_eq!( put(&s, &token, &did, "a", json!({ "deep": [{ "f": r }] })).await, Err(want.to_string()), "{r}" ); let create = json!({ "repo": did, "collection": "com.example.blob", "record": { "f": r } }); assert_eq!( call(&s, "com.atproto.repo.createRecord", Some(&token), create).await, Err(want.to_string()), "{r}" ); } assert_eq!(rev(&s), before, "a refused write makes no commit"); // Everything else a write checks comes first. put(&s, &token, &did, "k", json!({ "n": 1 })).await.unwrap(); let bad = bref(&raw_cid(b"never"), "text/plain", 5); let body = json!({ "repo": did, "collection": "com.example.blob", "rkey": "k", "record": { "f": bad }, "swapRecord": record_cid }); assert_eq!( call(&s, "com.atproto.repo.putRecord", Some(&token), body).await, Err("InvalidSwap".into()) ); let body = json!({ "repo": did, "collection": "com.example.blob", "rkey": "k", "record": { "f": bad } }); assert_eq!( call(&s, "com.atproto.repo.createRecord", Some(&token), body).await, Err("InternalServerError".into()) ); let body = json!({ "repo": did, "collection": "com.example.blob", "rkey": "k", "record": { "$type": "a.b.c", "f": bad } }); assert_eq!( call(&s, "com.atproto.repo.putRecord", Some(&token), body).await, Err("InvalidRequest".into()) ); let body = json!({ "repo": bob, "collection": "com.example.blob", "rkey": "k", "record": { "f": bad } }); assert_eq!( call(&s, "com.atproto.repo.putRecord", Some(&token), body).await, Err("AuthenticationRequired".into()) ); // The legacy shape before a swap. let legacy = json!({ "cid": xc, "mimeType": "text/plain" }); let body = json!({ "repo": did, "collection": "com.example.blob", "rkey": "k", "record": { "f": legacy }, "swapRecord": record_cid }); assert_eq!( call(&s, "com.atproto.repo.putRecord", Some(&token), body).await, Err("InvalidRequest".into()) ); // Maps that are no blob ref are kept as they are, and reference nothing. let kept = [ json!({ "cid": "not a cid", "mimeType": "text/plain" }), json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "text/plain", "size": size, "x": 1 }), json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "text/plain", "size": -2 }), json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "textplain", "size": size }), json!({ "$type": "blob", "ref": xc, "mimeType": "text/plain", "size": size }), json!({ "$type": "blob", "ref": { "$link": xc, "x": 1 }, "mimeType": "text/plain", "size": size }), json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "text/plain" }), json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "", "size": "1" }), json!({ "$type": "Blob", "ref": { "$link": xc }, "mimeType": "text/plain", "size": size }), ]; for (i, k) in kept.iter().enumerate() { put(&s, &token, &did, &format!("kept{i}"), json!({ "f": k })) .await .unwrap(); let p = [ ("repo".to_string(), did.clone()), ("collection".into(), "com.example.blob".into()), ("rkey".into(), format!("kept{i}")), ]; let got = call_with(&s, "com.atproto.repo.getRecord", None, &p, json!({})) .await .unwrap(); assert_eq!(&got["value"]["f"], k); } assert_eq!(listed(&s, &did, None, 10).await, Vec::::new()); // In one write, a ref that matches a new blob lets no other through. let writes = json!({ "repo": did, "writes": [ { "$type": "com.atproto.repo.applyWrites#create", "collection": "com.example.blob", "rkey": "a", "value": { "f": bref(&yc, "text/plain", ysize) } }, { "$type": "com.atproto.repo.applyWrites#create", "collection": "com.example.blob", "rkey": "b", "value": { "f": bref(&yc, "image/png", ysize) } }, ] }); assert_eq!( call(&s, "com.atproto.repo.applyWrites", Some(&token), writes).await, Err("InvalidMimeType".into()) ); let r = json!({ "one": bref(&yc, "text/plain", ysize), "two": bref(&yc, "text/plain", -1) }); assert_eq!( put(&s, &token, &did, "a", r).await, Err("InvalidSize".into()) ); // Once referenced, a blob takes refs of any type and size. put( &s, &token, &did, "a", json!({ "f": bref(&yc, "text/plain", ysize) }), ) .await .unwrap(); put( &s, &token, &did, "b", json!({ "f": bref(&yc, "image/png", -1) }), ) .await .unwrap(); let (_, headers, bytes) = get(&s, &did, &yc, None).await; assert_eq!( (headers[header::CONTENT_TYPE].to_str().unwrap(), bytes), ("text/plain", y) ); let _ = std::fs::remove_dir_all(&dir); } /// An upload is answered unsynced; the commit that first references it /// syncs its file before the commit's frame. A power cut is modelled as /// losing every file not synced: the referenced blob survives with its /// commit, the unreferenced upload is lost, and the server starts again, /// holding the one and not the other. Under `together`, a commit whose /// blob cannot be synced, with another account's write waited for after /// it, which syncs the journal: the commit was never written, so the /// power cut that loses its blob leaves nothing that references it. #[tokio::test] async fn a_referenced_blob_survives_where_its_commit_does() { for sync in SYNCS { syncing(sync); a_referenced_blob_survives_once(sync).await; } } async fn a_referenced_blob_survives_once(sync: crate::Syncing) { let plc = plc().await; let dir = scratch("blob-power-cut"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let (kept, lost) = (plain(41), plain(42)); let (kc, lc) = (raw_cid(&kept), raw_cid(&lost)); for b in [&kept, &lost] { test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false) .await .unwrap(); } let account = s.account(&did).unwrap(); { let st = read(&account.store); assert!( st.blobs.unsynced.contains_key(&kc) && st.blobs.unsynced.contains_key(&lc), "uploads are answered unsynced" ); } put( &s, &token, &did, "a", json!({ "f": bref(&kc, "text/plain", kept.len() as i64) }), ) .await .unwrap(); let wal = { let st = read(&account.store); assert!( !st.blobs.unsynced.contains_key(&kc), "the commit referencing it took it to sync" ); assert!( st.blobs.unsynced.contains_key(&lc), "an unreferenced upload stays unsynced" ); st.wal.clone() }; // The commit was answered: its frame is on disk, so the blob's file is. assert!(wal.is_durable(wal.written())); drop((wal, account)); let mut never_synced = vec![blob_dir(&dir, &did).join(&lc)]; if sync == crate::Syncing::Together { let (bob, bob_token) = create(&s, "bob").await; let racy = plain(43); let rc = raw_cid(&racy); test_upload(&s, Some(&token), Some("text/plain"), racy.clone(), false) .await .unwrap(); // In the file's place a link to itself: it cannot be opened to // be synced, by any user. let file = blob_dir(&dir, &did).join(&rc); let aside = file.with_extension("aside"); std::fs::rename(&file, &aside).unwrap(); std::os::unix::fs::symlink(&file, &file).unwrap(); let r = json!({ "f": bref(&rc, "text/plain", racy.len() as i64) }); let refused = put(&s, &token, &did, "r", r).await; assert_eq!(refused, Err("InternalServerError".into())); put(&s, &bob_token, &bob, "x", json!({ "n": 1 })) .await .unwrap(); std::fs::remove_file(&file).unwrap(); std::fs::rename(&aside, &file).unwrap(); never_synced.push(file); } drop(s); // The power cut: the files never synced are gone. for f in &never_synced { std::fs::remove_file(f).unwrap(); } let s = open(&dir, &plc); if sync == crate::Syncing::Together { let p = [ ("repo".to_string(), did.clone()), ("collection".into(), "com.example.blob".into()), ("rkey".into(), "r".into()), ]; assert_eq!( call_with(&s, "com.atproto.repo.getRecord", None, &p, json!({})).await, Err("RecordNotFound".into()) ); } let (status, _, bytes) = get(&s, &did, &kc, None).await; assert_eq!((status, bytes), (200, kept.clone())); let r = json!({ "f": bref(&lc, "text/plain", lost.len() as i64) }); assert_eq!( put(&s, &token, &did, "b", r).await, Err("BlobNotFound".into()) ); // Uploaded again, it is held, and its next reference syncs it. test_upload(&s, Some(&token), Some("text/plain"), lost.clone(), false) .await .unwrap(); put( &s, &token, &did, "b", json!({ "f": bref(&lc, "text/plain", lost.len() as i64) }), ) .await .unwrap(); assert!( !read(&s.account(&did).unwrap().store) .blobs .unsynced .contains_key(&lc) ); let _ = std::fs::remove_dir_all(&dir); } /// A blob is held while some record references it, counted across /// creates, updates, deletes and applyWrites. #[tokio::test] async fn references_are_counted() { let plc = plc().await; let dir = scratch("blob-count"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let blobs: Vec> = (0..4).map(|i| plain(30 + i)).collect(); let cids: Vec = blobs.iter().map(|b| raw_cid(b)).collect(); for b in &blobs { test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false) .await .unwrap(); } let r = |i: usize| bref(&cids[i], "text/plain", blobs[i].len() as i64); // Held but not referenced: neither served nor listed. assert_eq!(get_error(&s, &did, &cids[0], None).await, "InvalidRequest"); assert_eq!(listed(&s, &did, None, 3).await, Vec::::new()); // Two records reference blob 0, one of them twice. put(&s, &token, &did, "a", json!({ "f": r(0), "g": [r(0)] })) .await .unwrap(); put(&s, &token, &did, "b", json!({ "f": r(0) })) .await .unwrap(); assert_eq!(get(&s, &did, &cids[0], None).await.0, 200); delete(&s, &token, &did, "a").await.unwrap(); assert_eq!(get(&s, &did, &cids[0], None).await.0, 200); // Replaced by a record referencing another: blob 0 goes. put(&s, &token, &did, "b", json!({ "f": r(1) })) .await .unwrap(); assert_eq!(get_error(&s, &did, &cids[0], None).await, "InvalidRequest"); assert_eq!( put(&s, &token, &did, "c", json!({ "f": r(0) })).await, Err("BlobNotFound".into()) ); assert!(!files(&dir, &did).contains(&cids[0])); assert_eq!(listed(&s, &did, None, 3).await, vec![cids[1].clone()]); // Uploaded again, with another type, it is stored afresh. test_upload(&s, Some(&token), Some("image/x"), blobs[0].clone(), false) .await .unwrap(); put( &s, &token, &did, "c", json!({ "f": bref(&cids[0], "image/x", blobs[0].len() as i64) }), ) .await .unwrap(); assert_eq!( get(&s, &did, &cids[0], None).await.1[header::CONTENT_TYPE], "image/x" ); // putRecord keeping the ref in the same record keeps the blob. put(&s, &token, &did, "b", json!({ "f": r(1), "n": 2 })) .await .unwrap(); assert_eq!(get(&s, &did, &cids[1], None).await.0, 200); // applyWrites deleting one record and referencing its blob from another keeps it. let w = |t: &str, rkey: &str, value: Option| { let mut w = json!({ "$type": format!("com.atproto.repo.applyWrites#{t}"), "collection": "com.example.blob", "rkey": rkey }); if let Some(v) = value { w["value"] = v; } w }; let batch = |writes: Vec| json!({ "repo": did, "writes": writes }); call( &s, "com.atproto.repo.applyWrites", Some(&token), batch(vec![ w("delete", "b", None), w("create", "d", Some(json!({ "f": r(1) }))), ]), ) .await .unwrap(); assert_eq!(get(&s, &did, &cids[1], None).await.0, 200); // References are counted as the commit leaves the records: a batch // whose create references blob 2 and whose update of the same record // then does not leaves no record referencing it, before or after, so // it stays held, unreferenced. call( &s, "com.atproto.repo.applyWrites", Some(&token), batch(vec![ w("create", "e", Some(json!({ "f": r(2) }))), w("update", "e", Some(json!({ "f": r(3) }))), ]), ) .await .unwrap(); assert_eq!(get_error(&s, &did, &cids[2], None).await, "InvalidRequest"); assert_eq!(get(&s, &did, &cids[3], None).await.0, 200); let mut want = vec![cids[0].clone(), cids[1].clone(), cids[3].clone()]; want.sort(); assert_eq!(listed(&s, &did, None, 2).await, want); // Deleting the rest drops them, files too, once the writes are on disk. call( &s, "com.atproto.repo.applyWrites", Some(&token), batch(vec![ w("delete", "c", None), w("delete", "d", None), w("delete", "e", None), ]), ) .await .unwrap(); assert_eq!(listed(&s, &did, None, 2).await, Vec::::new()); assert_eq!(files(&dir, &did), vec![cids[2].clone()]); let _ = std::fs::remove_dir_all(&dir); } /// Writes at once, each referencing the same blob from its own record, /// count every reference. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn concurrent_writes_count_right() { let plc = plc().await; let dir = scratch("blob-concurrent"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let b = plain(40); let cid = raw_cid(&b); test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false) .await .unwrap(); let r = bref(&cid, "text/plain", b.len() as i64); let spawn_all = |f: &dyn Fn(usize) -> tokio::task::JoinHandle>| { (0..16).map(f).collect::>() }; for h in spawn_all(&|i| { let (s, token, did, r) = (s.clone(), token.clone(), did.clone(), r.clone()); tokio::spawn( async move { put(&s, &token, &did, &format!("r{i}"), json!({ "f": r })).await }, ) }) { h.await.unwrap().unwrap(); } assert_eq!(read(&s.account(&did).unwrap().store).blobs.refs[&cid], 16); // Uploads of it meanwhile change nothing. let deletes = spawn_all(&|i| { let (s, token, did) = (s.clone(), token.clone(), did.clone()); tokio::spawn(async move { delete(&s, &token, &did, &format!("r{i}")).await }) }); let uploads: Vec<_> = (0..4) .map(|_| { let (s, token, b) = (s.clone(), token.clone(), b.clone()); tokio::spawn(async move { test_upload(&s, Some(&token), Some("image/y"), b, true).await }) }) .collect(); for h in deletes { h.await.unwrap().unwrap(); } for h in uploads { h.await.unwrap().unwrap(); } assert!( !read(&s.account(&did).unwrap().store) .blobs .refs .contains_key(&cid) ); assert_eq!(get_error(&s, &did, &cid, None).await, "InvalidRequest"); // Whether an upload came after the last delete or before, the files // are what is held: the blob's if held, nothing else. let held = read(&s.account(&did).unwrap().store) .blobs .held .contains_key(&cid); assert_eq!( files(&dir, &did), if held { vec![cid.clone()] } else { vec![] } ); let _ = std::fs::remove_dir_all(&dir); } /// Over HTTP, a client still sending a body that is too large, with its /// `Content-Length` or chunked, or refused for its token, reads the answer. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn refusals_reach_a_client_still_sending() { let plc = plc().await; let dir = scratch("blob-http"); let s = open(&dir, &plc); let (_, token) = create(&s, "alice").await; let app = axum::Router::new() .route( "/xrpc/com.atproto.repo.uploadBlob", axum::routing::any(upload), ) .with_state(s.clone()); let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) .await .unwrap(); let url = format!( "http://{}/xrpc/com.atproto.repo.uploadBlob", listener.local_addr().unwrap() ); tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); let answers = tokio::task::spawn_blocking(move || { let http = reqwest::blocking::Client::builder() .no_proxy() .build() .unwrap(); let send = |token: &str, body: reqwest::blocking::Body| { let res = http .post(&url) .bearer_auth(token) .header("content-type", "text/plain") .body(body) .send() .unwrap(); let status = res.status().as_u16(); ( status, serde_json::from_slice::(&res.bytes().unwrap()).unwrap()["error"].clone(), ) }; let big = || vec![b'a'; DEFAULT_UPLOAD_LIMIT as usize + 1]; vec![ send(&token, big().into()), send(&token, reqwest::blocking::Body::new(io::Cursor::new(big()))), send("forged", big().into()), send(&token, plain(1).into()), ] }) .await .unwrap(); let too_large = (413, json!("PayloadTooLarge")); assert_eq!( answers, vec![ too_large.clone(), too_large, (400, json!("InvalidToken")), (200, Json::Null) ] ); let _ = std::fs::remove_dir_all(&dir); } #[tokio::test] async fn get_blob_answers_and_headers() { let plc = plc().await; let dir = scratch("blob-get"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let (_, other) = create(&s, "bob").await; let b = vec![5u8; 300_000]; let cid = raw_cid(&b); test_upload( &s, Some(&token), Some("application/x-thing"), b.clone(), true, ) .await .unwrap(); put( &s, &token, &did, "a", json!({ "f": bref(&cid, "application/x-thing", b.len() as i64) }), ) .await .unwrap(); // Any account's token, or none, is the same. for t in [None, Some(token.as_str()), Some(other.as_str())] { let (status, h, bytes) = get(&s, &did, &cid, t).await; assert_eq!(status, 200); assert_eq!(bytes, b); assert_eq!(h[header::CONTENT_TYPE], "application/x-thing"); assert_eq!(h[header::CONTENT_LENGTH], "300000"); assert_eq!(h[header::X_CONTENT_TYPE_OPTIONS], "nosniff"); assert_eq!( h[header::CONTENT_DISPOSITION].to_str().unwrap(), format!("attachment; filename=\"{cid}\"") ); assert_eq!( h[header::CONTENT_SECURITY_POLICY], "default-src 'none'; sandbox" ); } // Parameters first, then the token, then the account, then the blob. assert_eq!( get_error(&s, "not a did", &cid, Some("forged")).await, "InvalidRequest" ); assert_eq!( get_error(&s, &did, "not a cid", Some("forged")).await, "InvalidRequest" ); assert_eq!( get_error(&s, "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", &cid, Some("forged")).await, "InvalidToken" ); assert_eq!( get_error(&s, "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", &cid, None).await, "RepoNotFound" ); assert_eq!( get_error(&s, &did, &raw_cid(b"never"), None).await, "InvalidRequest" ); let record = cid_string(&records_of(&s, &did)[0].1.cid); assert_eq!(get_error(&s, &did, &record, None).await, "InvalidRequest"); let bob = s.account("bob.test").unwrap().did.clone(); assert_eq!(get_error(&s, &bob, &cid, None).await, "InvalidRequest"); let _ = std::fs::remove_dir_all(&dir); } #[tokio::test] async fn list_blobs_pages_and_since() { let plc = plc().await; let dir = scratch("blob-list"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let blobs: Vec> = (0..7).map(|i| plain(50 + i)).collect(); let cids: Vec = blobs.iter().map(|b| raw_cid(b)).collect(); for b in &blobs { test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false) .await .unwrap(); } let r = |i: usize| bref(&cids[i], "text/plain", blobs[i].len() as i64); let rev = || repo_of(&s, &did, |r| r.rev.clone()); let start = rev(); put(&s, &token, &did, "a", json!({ "f": [r(0), r(1)] })) .await .unwrap(); let after_a = rev(); put(&s, &token, &did, "b", json!({ "f": r(2) })) .await .unwrap(); put(&s, &token, &did, "c", json!({ "f": r(3) })) .await .unwrap(); let after_c = rev(); // An update lists every blob the record references now. put(&s, &token, &did, "a", json!({ "f": [r(0), r(1), r(4)] })) .await .unwrap(); put(&s, &token, &did, "d", json!({ "f": r(5) })) .await .unwrap(); delete(&s, &token, &did, "d").await.unwrap(); let sorted = |mut v: Vec| { v.sort(); v }; let all = sorted(cids[..5].to_vec()); for limit in [1, 2, 3, 500] { assert_eq!(listed(&s, &did, None, limit).await, all, "{limit}"); } assert_eq!(listed(&s, &did, Some(&start), 2).await, all); assert_eq!(listed(&s, &did, Some(&after_a), 2).await, all); assert_eq!( listed(&s, &did, Some(&after_c), 2).await, sorted(vec![cids[0].clone(), cids[1].clone(), cids[4].clone()]) ); assert_eq!( listed(&s, &did, Some(&rev()), 2).await, Vec::::new() ); assert_eq!( listed(&s, &did, Some("jzzzzzzzzzzzz"), 2).await, Vec::::new() ); assert_eq!(listed(&s, &did, Some("2222222222222"), 2).await, all); // A cursor need not be a CID; an empty one is none. let page = |cursor: &str| { let p = vec![ ("did".to_string(), did.clone()), ("cursor".to_string(), cursor.to_string()), ]; let s = s.clone(); async move { call_with(&s, "com.atproto.sync.listBlobs", None, &p, json!({})).await } }; assert_eq!( page("").await.unwrap(), json!({ "cids": all, "cursor": all.last().unwrap() }) ); assert_eq!(page("bafkrei").await.unwrap()["cids"], json!(all)); assert_eq!(page("c").await.unwrap(), json!({ "cids": [] })); assert_eq!(page(&all[2]).await.unwrap()["cids"], json!(all[3..])); let bad = |p: Vec<(&str, &str)>| { let p: Vec<(String, String)> = p .into_iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); let s = s.clone(); async move { call_with(&s, "com.atproto.sync.listBlobs", None, &p, json!({})).await } }; assert_eq!( bad(vec![("did", &did), ("since", "notatid")]).await, Err("InvalidRequest".into()) ); assert_eq!( bad(vec![("did", &did), ("limit", "0")]).await, Err("InvalidRequest".into()) ); assert_eq!( bad(vec![("did", &did), ("limit", "1001")]).await, Err("InvalidRequest".into()) ); assert_eq!( bad(vec![("did", "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa")]).await, Err("RepoNotFound".into()) ); let p = [("did".to_string(), did.clone())]; assert_eq!( call_with( &s, "com.atproto.sync.listBlobs", Some("forged"), &p, json!({}) ) .await, Err("InvalidToken".into()) ); assert!( call_with( &s, "com.atproto.sync.listBlobs", Some(&token), &p, json!({}) ) .await .is_ok() ); let _ = std::fs::remove_dir_all(&dir); } /// A restart keeps every upload, referenced or not, with its first /// type; a blob dropped before it stays gone; stray files go. #[tokio::test] async fn restart_keeps_blobs() { let plc = plc().await; let dir = scratch("blob-restart"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let (kept, loose, dropped) = (plain(60), plain(61), plain(62)); for b in [&kept, &loose, &dropped] { test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false) .await .unwrap(); } test_upload(&s, Some(&token), Some("image/other"), loose.clone(), false) .await .unwrap(); let r = |b: &[u8]| bref(&raw_cid(b), "text/plain", b.len() as i64); put(&s, &token, &did, "k", json!({ "f": r(&kept) })) .await .unwrap(); put(&s, &token, &did, "d", json!({ "f": r(&dropped) })) .await .unwrap(); delete(&s, &token, &did, "d").await.unwrap(); let listed_before = listed(&s, &did, None, 10).await; drop(s); std::fs::write(blob_dir(&dir, &did).join(".tmp-left"), b"half").unwrap(); std::fs::write(blob_dir(&dir, &did).join(raw_cid(&dropped)), &dropped).unwrap(); let s = open(&dir, &plc); assert_eq!( files(&dir, &did), sorted_pair(raw_cid(&kept), raw_cid(&loose)) ); assert_eq!(listed(&s, &did, None, 10).await, listed_before); assert_eq!(get(&s, &did, &raw_cid(&kept), None).await.2, kept); assert_eq!( put( &s, &token, &did, "l", json!({ "f": bref(&raw_cid(&loose), "image/other", loose.len() as i64) }) ) .await, Err("InvalidMimeType".into()) ); put(&s, &token, &did, "l", json!({ "f": r(&loose) })) .await .unwrap(); assert_eq!( put(&s, &token, &did, "d", json!({ "f": r(&dropped) })).await, Err("BlobNotFound".into()) ); drop(s); // A blob the log holds whose file is gone is not served: the server does not start. std::fs::remove_file(blob_dir(&dir, &did).join(raw_cid(&loose))).unwrap(); assert!( try_open(&dir, &plc) .err() .unwrap() .contains("missing or not whole") ); let _ = std::fs::remove_dir_all(&dir); } fn sorted_pair(a: String, b: String) -> Vec { if a < b { vec![a, b] } else { vec![b, a] } } /// The quota: an upload past it is refused and stores nothing; bytes /// held already cost nothing again; a blob no longer held frees its bytes. #[tokio::test] async fn the_quota() { let plc = plc().await; let dir = scratch("blob-quota"); let s = open_with(&dir, &plc, |c| c.blob_quota = 100); let (did, token) = create(&s, "alice").await; let t = Some(token.as_str()); let (a, b) = (plain(46), plain(36)); assert_eq!((a.len(), b.len()), (60, 50)); test_upload(&s, t, Some("text/plain"), a.clone(), false) .await .unwrap(); test_upload(&s, t, Some("text/plain"), a.clone(), true) .await .unwrap(); assert_eq!( test_upload(&s, t, Some("text/plain"), b.clone(), false).await, Err((400, "BlobQuotaExceeded".into())) ); assert_eq!(files(&dir, &did), vec![raw_cid(&a)]); // Referenced, then the record deleted: no longer held, its bytes free. put( &s, &token, &did, "r", json!({ "f": bref(&raw_cid(&a), "text/plain", 60) }), ) .await .unwrap(); delete(&s, &token, &did, "r").await.unwrap(); test_upload(&s, t, Some("text/plain"), b.clone(), false) .await .unwrap(); assert_eq!(files(&dir, &did), vec![raw_cid(&b)]); let _ = std::fs::remove_dir_all(&dir); } /// An upload no record references is held for its time from when it was /// last uploaded: after it, a ref to it is `BlobNotFound`, it counts /// against no quota, its file goes when the server sweeps, and after a /// restart it is gone too. One referenced in time is kept. #[tokio::test] async fn unreferenced_uploads_expire() { let plc = plc().await; let dir = scratch("blob-ttl"); let s = open_with(&dir, &plc, |c| (c.blob_ttl, c.blob_quota) = (10, 150)); // The clock stands still but for the skew, so each upload's time is exact. s.limits.clock.freeze(); let (did, token) = create(&s, "alice").await; let t = Some(token.as_str()); let (kept, lost, late) = (plain(1), plain(2), plain(80)); for b in [&kept, &lost] { test_upload(&s, t, Some("text/plain"), b.clone(), false) .await .unwrap(); } let r = |b: &[u8], mime: &str| json!({ "f": bref(&raw_cid(b), mime, b.len() as i64) }); put(&s, &token, &did, "kept", r(&kept, "text/plain")) .await .unwrap(); // Uploaded again, the time starts again. s.limits.clock.skew.store(6, Ordering::SeqCst); test_upload(&s, t, Some("image/whatever"), lost.clone(), false) .await .unwrap(); s.limits.clock.skew.store(12, Ordering::SeqCst); put(&s, &token, &did, "lost", r(&lost, "text/plain")) .await .unwrap(); delete(&s, &token, &did, "lost").await.unwrap(); test_upload(&s, t, Some("text/plain"), lost.clone(), false) .await .unwrap(); test_upload(&s, t, Some("text/plain"), late.clone(), false) .await .unwrap(); s.limits.clock.skew.store(30, Ordering::SeqCst); assert_eq!( put(&s, &token, &did, "lost", r(&lost, "text/plain")).await, Err("BlobNotFound".into()) ); // Its bytes count against no quota, though its file is there until a sweep. let big = plain(120); test_upload(&s, t, Some("text/plain"), big.clone(), false) .await .unwrap(); assert!(files(&dir, &did).contains(&raw_cid(&late))); sweep(&s); let mut want = vec![raw_cid(&kept), raw_cid(&big)]; want.sort(); assert_eq!(files(&dir, &did), want); assert_eq!(get(&s, &did, &raw_cid(&kept), None).await.0, 200); // Stored afresh, with its new type. s.limits.clock.skew.store(60, Ordering::SeqCst); test_upload(&s, t, Some("text/other"), lost.clone(), false) .await .unwrap(); put(&s, &token, &did, "again", r(&lost, "text/other")) .await .unwrap(); let blobs = |s: &Service| read(&s.account(&did).unwrap().store).blobs.snapshot(); let before = blobs(&s); drop(s); // After a restart: as it was, and an upload that outlived its time while // the server was down is gone once it starts. let s = open_with(&dir, &plc, |c| (c.blob_ttl, c.blob_quota) = (10, 150)); assert_eq!(blobs(&s), before); s.limits.clock.skew.store(100, Ordering::SeqCst); assert_eq!( put(&s, &token, &did, "big", r(&big, "text/plain")).await, Err("BlobNotFound".into()) ); sweep(&s); let mut want = vec![raw_cid(&kept), raw_cid(&lost)]; want.sort(); assert_eq!(files(&dir, &did), want); drop(s); let s = open(&dir, &plc); assert_eq!( put(&s, &token, &did, "big", r(&big, "text/plain")).await, Err("BlobNotFound".into()) ); let _ = std::fs::remove_dir_all(&dir); } }