chadtmiller.com/monolith
An atproto PDS
0
0
//! Blobs: `uploadBlob`, `getBlob` and `listBlobs`, and the blob refs in
//! records, as `spec/BLOBS.md` says, within the quota and the time an
//! unreferenced upload is held that `spec/LIMITS.md` sets.
//!
//! Each account keeps its own blobs. A blob's bytes are the file
//! `repos/<did>/blobs/<cid>`, and the account's log has an entry for each
//! upload it stored or uploaded again, with its type, size and time, and
//! one for each upload it stopped holding for want of a reference in time
//! (`blobGone`). Which records reference a blob is not logged apart: the
//! records say, so the commits that write them carry it, and replaying the
//! log counts the references again. A space record's refs hold a blob as a
//! public record's do (`space_repo` counts them), but the public `getBlob`
//! and `listBlobs` serve and list only blobs public records reference.
//!
//! An upload is written to a temporary file in that directory and renamed
//! into place before its entry is logged, and answered once both are handed
//! to the operating system: an entry always has its file, and an answered
//! upload survives the server being killed. Neither is synced then. The
//! commit that first references a blob has its file and directory synced
//! before the commit's frame is (`sync_before`, `Wal::sync_first`), in the
//! group commit, so a referenced blob survives a power cut wherever its
//! commit does; an upload no record references yet may not
//! (`spec/DURABILITY.md`), and recovery drops one whose file did not.
//! A write that removes a blob's last
//! reference stops holding it at once, under the account's lock, but removes
//! its file only once the write is on disk. If the server stops between the
//! two, it starts again with the file held by no entry, and removes it, as it
//! does a temporary file an upload left. An upload no record references is
//! no longer held once `PDS_BLOB_TTL_SECS` have passed since it was last
//! uploaded: refused as a ref and counted against no quota at once, and
//! logged as gone, its file removed, by `sweep`.
//!
//! With blobs in a bucket (`blobstore`), a blob's bytes are an object there
//! and no file: an upload is read whole into memory and put before it is
//! answered, and logged as above once the store has kept it; `getBlob`
//! reads the object as it sends it. The log says which objects the store
//! may keep (`kept`): `blobPut` before an upload puts one, `blobDeleted`
//! once one is deleted. Those of blobs no longer held are deleted by the
//! deleter (`blobstore`), which a write that stops holding a blob, an
//! upload, and `sweep` wake, where they would remove its file. A move of
//! the blobs into the bucket (`migrate_blobs`) logs `blobPut` for each blob
//! held before it puts their objects, and a move out of it
//! `blobObjectsDeleted` once it has deleted every object the store may keep.
use crate::{
Account, Call, Handler, Resp, Service, Shared, Shown, Store, XrpcError, err, internal, invalid,
opt, read, required, write,
};
use axum::body::{Body, Bytes, HttpBody};
use axum::extract::{Extension, State as AxState};
use axum::http::{HeaderMap, HeaderValue, Method, header};
use axum::response::Response;
use http_body::{Frame, SizeHint};
use pds_repo::data::{V, cid_of, cid_string, parse_cid, valid_cid_string};
use pds_repo::{Change, Op, OpResult, put_record};
use serde_json::{Map, Value as Json, json};
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::ops::Bound::{Excluded, Unbounded};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::{Arc, LazyLock, OnceLock};
use std::task::{Context, Poll};
/// The largest body `uploadBlob` takes unless `PDS_BLOB_UPLOAD_LIMIT` sets
/// another (`Config::blob_upload_limit`): the reference's default.
pub(crate) const DEFAULT_UPLOAD_LIMIT: u64 = 5 * 1024 * 1024;
const UPLOAD: &str = "com.atproto.repo.uploadBlob";
/// The entry of an object the bucket may keep from now on.
const PUT: &str = "blobPut";
/// The entry of an object the bucket answered the delete of.
const DELETED: &str = "blobDeleted";
/// The entry of every object the bucket may keep deleted, by a move of the
/// blobs out of it.
const OBJECTS_DELETED: &str = "blobObjectsDeleted";
/// The kinds of entry an account's log has of its blobs: an upload, one no
/// longer held, a takedown or its lifting (`spec/LIFECYCLE.md`), and with
/// blobs in a bucket, an object it may keep and one deleted (`blobstore`),
/// and every one deleted (`migrate_blobs`).
pub(crate) const ENTRIES: &[&str] = &[
"blob",
"blobGone",
"takedownBlob",
PUT,
DELETED,
OBJECTS_DELETED,
];
/// How often uploads held past their time are logged as gone, and their files removed.
const SWEEP_EVERY: std::time::Duration = std::time::Duration::from_secs(30);
pub(crate) fn handler(nsid: &str) -> Option<Handler> {
Some(match nsid {
"com.atproto.sync.getBlob" => Handler::Query(get_blob),
"com.atproto.sync.listBlobs" => Handler::Query(list_blobs),
_ => return None,
})
}
/// Where an account's blobs are kept.
pub(crate) fn blob_dir(dir: &Path, did: &str) -> PathBuf {
dir.join("repos").join(did).join("blobs")
}
/// A blob an account holds, as it was first stored.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Held {
pub mime: String,
pub size: u64,
/// When it was last uploaded, in seconds since the epoch.
pub at: u64,
}
/// A record with blob refs: the rev of the commit that last wrote it, and the
/// CIDs it references, each once, in order.
type RecordRefs = (String, Vec<String>);
/// The bytes a record with blob refs, at `path`, is counted as in the cache.
fn record_cost(path: &str, (rev, cids): &RecordRefs) -> usize {
path.len() + rev.len() + cids.iter().map(|c| c.len() + 24).sum::<usize>() + 96
}
/// What a commit changed of an account's blobs, as they were before it.
#[derive(Default)]
struct Undo {
records: Vec<(String, Option<RecordRefs>)>,
refs: Vec<(String, Option<u32>)>,
held: Vec<(String, Option<Held>)>,
bytes: u64,
doomed: usize,
/// The takedowns of the blobs it stopped holding.
takedowns: Vec<(String, String)>,
}
/// An account's blobs, kept with its repository and changed under its lock.
#[derive(Default)]
pub(crate) struct Blobs {
/// The blobs it holds, by CID.
held: BTreeMap<String, Held>,
/// How many public records reference each CID, for those some record does.
refs: HashMap<String, u32>,
/// How many records of the account's space repos reference each CID,
/// for those some such record does (`space_repo`): these hold a blob as
/// public records do, but the public methods neither serve nor list it.
space_refs: HashMap<String, u32>,
/// Each record with blob refs, by path.
records: HashMap<String, RecordRefs>,
/// The bytes `records` is counted as in the cache.
record_bytes: usize,
/// The blobs held that no record references, by when they were last uploaded.
unreferenced: BTreeSet<(u64, String)>,
/// The bytes of the blobs held.
bytes: u64,
/// Uploads moving their files into place, or putting their objects, by
/// CID: those files stay, and those objects are deleted by none.
placing: HashMap<String, u32>,
/// The files of uploads not yet synced, by CID: synced before the frame
/// of the commit that first references them.
unsynced: HashMap<String, PathBuf>,
/// Blobs the last write stopped holding, whose files go once it is on disk.
doomed: Vec<String>,
/// The last commit's changes, until it is in the log.
undo: Option<Undo>,
/// The time of the write under way, and how long an unreferenced upload
/// is held, both in seconds.
now: u64,
ttl: u64,
/// The blobs held taken down, by CID, each with its reference: a blob no
/// longer held is no longer taken down.
takedowns: BTreeMap<String, String>,
/// With blobs in a bucket, the CIDs whose objects it may keep: each blob
/// held, and those of the others not known to be deleted, its strays.
/// Empty with blobs in the data directory, but while a move of them
/// into the bucket, or out of it, is unfinished (`migrate_blobs`).
kept: BTreeSet<String>,
}
/// A blob ref in a record.
pub(crate) struct BlobRef {
cid: Vec<u8>,
mime: String,
size: i64,
}
/// The blob ref a map is: exactly `$type` blob, a link `ref`, a `mimeType`
/// with a `/`, and a `size` of 0 or more, or -1. A `ref` that is a map of
/// the one key `$link`, a CID string, is that link, as JSON writes both
/// (`spec/BLOBS.md`, "Blob refs in records").
fn blob_ref(m: &[(String, V)]) -> Option<BlobRef> {
if m.len() != 4 {
return None;
}
let get = |k: &str| m.iter().find(|(n, _)| n == k).map(|(_, v)| v);
let cid = match get("ref")? {
V::Link(cid) => cid.clone(),
V::Map(r) => match r.as_slice() {
[(k, V::Str(cid))] if k == "$link" => parse_cid(cid)?,
_ => return None,
},
_ => return None,
};
match (get("$type")?, get("mimeType")?, get("size")?) {
(V::Str(t), V::Str(mime), V::Int(size))
if t == "blob" && mime.contains('/') && *size >= -1 =>
{
Some(BlobRef {
cid,
mime: mime.clone(),
size: *size,
})
}
_ => None,
}
}
/// Every blob ref in a value, at any depth.
pub(crate) fn refs_in(v: &V, out: &mut Vec<BlobRef>) {
match v {
V::Map(m) => match blob_ref(m) {
Some(r) => out.push(r),
None => m.iter().for_each(|(_, x)| refs_in(x, out)),
},
V::Arr(a) => a.iter().for_each(|x| refs_in(x, out)),
_ => {}
}
}
/// The CIDs a record's blob refs name, each once, in order.
pub(crate) fn cids_in(v: &V) -> Vec<String> {
let mut refs = Vec::new();
refs_in(v, &mut refs);
let set: BTreeSet<String> = refs.iter().map(|r| cid_string(&r.cid)).collect();
set.into_iter().collect()
}
/// Whether a value holds a blob ref of the legacy shape: exactly a CID
/// string `cid` and a non-empty `mimeType`.
fn legacy_in(v: &V) -> bool {
match v {
V::Map(m) => {
let get = |k: &str| m.iter().find(|(n, _)| n == k).map(|(_, v)| v);
let legacy = m.len() == 2
&& matches!(get("cid"), Some(V::Str(c)) if valid_cid_string(c))
&& matches!(get("mimeType"), Some(V::Str(t)) if !t.is_empty());
legacy || m.iter().any(|(_, x)| legacy_in(x))
}
V::Arr(a) => a.iter().any(legacy_in),
_ => false,
}
}
/// A record may not hold a legacy blob ref: checked right after its `$type`.
pub(crate) fn refuse_legacy(v: &V) -> Result<(), XrpcError> {
if legacy_in(v) {
invalid("Legacy blob refs are not supported")
} else {
Ok(())
}
}
/// The log entry of an upload the account stored, or uploaded again.
fn upload_entry(cid: &str, h: &Held) -> Json {
json!({ "t": "blob", "cid": cid, "mimeType": h.mime, "size": h.size, "at": h.at })
}
/// The log entry of an upload no longer held, as no record referenced it in time.
fn gone_entry(cid: &str) -> Json {
json!({ "t": "blobGone", "cid": cid })
}
/// The log entry of an object the bucket may keep from now on, before it is put.
fn put_entry(cid: &str) -> Json {
json!({ "t": PUT, "cid": cid })
}
/// The log entry of an object whose delete the bucket answered.
pub(crate) fn deleted_entry(cid: &str) -> Json {
json!({ "t": DELETED, "cid": cid })
}
/// What an upload does to the blobs held.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum Upload {
/// It is held, and referenced: nothing.
Kept,
/// It is held, unreferenced: its time starts again.
Again,
/// It is held, unreferenced, past its time: it is stored afresh.
Afresh,
/// It is not held: it is stored.
New,
}
impl Blobs {
/// Set the time of the write under way, and how long uploads are held.
pub(crate) fn time(&mut self, now: u64, ttl: u64) {
(self.now, self.ttl) = (now, ttl);
}
/// Whether nothing of it is in flight: no upload moving into place, no
/// file doomed and not yet removed.
pub(crate) fn settled(&self) -> bool {
self.placing.is_empty() && self.doomed.is_empty()
}
/// About the bytes it holds in memory: each a count kept as it changes.
pub(crate) fn cost(&self) -> u64 {
let undo = self
.undo
.as_ref()
.map_or(0, |u| u.records.len() + u.refs.len() + u.held.len());
(self.held.len() * 160
+ self.refs.len() * 96
+ self.space_refs.len() * 96
+ self.takedowns.len() * 128
+ self.kept.len() * 96
+ self.unreferenced.len() * 112
+ self.unsynced.len() * 256
+ self.placing.len() * 96
+ self.doomed.len() * 96
+ undo * 160
+ self.record_bytes) as u64
}
/// The CIDs whose objects a bucket may keep that it does not hold, and
/// no upload is putting: those to delete.
pub(crate) fn strays(&self) -> impl Iterator<Item = &String> {
self.kept
.iter()
.filter(|c| !self.held.contains_key(*c) && !self.placing.contains_key(*c))
}
pub(crate) fn is_stray(&self, cid: &str) -> bool {
self.kept.contains(cid) && !self.held.contains_key(cid) && !self.placing.contains_key(cid)
}
/// Whether a bucket may keep `cid`'s object.
pub(crate) fn keeps(&self, cid: &str) -> bool {
self.kept.contains(cid)
}
/// The CIDs whose objects a bucket may keep: those of an account deleted.
pub(crate) fn kept_list(&self) -> Vec<String> {
self.kept.iter().cloned().collect()
}
/// An object deleted, its entry logged.
pub(crate) fn forget_object(&mut self, cid: &str) {
self.kept.remove(cid);
}
/// The blobs it holds at `now`, with uploads held for `ttl`, each with
/// its size, in order of CID.
pub(crate) fn holding(&self, now: u64, ttl: u64) -> Vec<(String, u64)> {
self.held
.iter()
.filter(|(cid, h)| self.referenced(cid) || h.at.saturating_add(ttl) > now)
.map(|(cid, h)| (cid.clone(), h.size))
.collect()
}
/// The CIDs of every object a bucket may keep, and of every blob held:
/// those a move out of the bucket deletes.
pub(crate) fn objects(&self) -> Vec<String> {
let mut all: BTreeSet<&String> = self.kept.iter().collect();
all.extend(self.held.keys());
all.into_iter().cloned().collect()
}
/// Log, in `wal`, that a bucket may keep the objects of `cids`, those no
/// entry names yet, before a move puts them: the frame to wait on, if
/// one was appended.
pub(crate) fn note_kept(
&mut self,
wal: &pds_store::Wal,
cids: &[String],
) -> io::Result<Option<u64>> {
let new: Vec<&String> = cids.iter().filter(|c| !self.kept.contains(*c)).collect();
if new.is_empty() {
return Ok(None);
}
let n = wal.append(&new.iter().map(|c| put_entry(c)).collect::<Vec<_>>())?;
self.kept.extend(new.into_iter().cloned());
Ok(Some(n))
}
/// Log, in `wal`, that the strays `cids` are deleted, as the deleter
/// does: the frame to wait on, if one was appended.
pub(crate) fn strays_deleted(
&mut self,
wal: &pds_store::Wal,
cids: &[String],
) -> io::Result<Option<u64>> {
if cids.is_empty() {
return Ok(None);
}
let n = wal.append(&cids.iter().map(|c| deleted_entry(c)).collect::<Vec<_>>())?;
for c in cids {
self.kept.remove(c);
}
Ok(Some(n))
}
/// Log, in `wal`, that a move out of the bucket deleted every object it
/// may keep: the frame to wait on, if one was appended.
pub(crate) fn objects_deleted(&mut self, wal: &pds_store::Wal) -> io::Result<Option<u64>> {
if self.kept.is_empty() {
return Ok(None);
}
let n = wal.append(&[json!({ "t": OBJECTS_DELETED })])?;
self.kept.clear();
Ok(Some(n))
}
/// Whether an upload is on its way, or a stray waits to be deleted.
pub(crate) fn objects_unsettled(&self) -> bool {
!self.placing.is_empty() || self.strays().next().is_some()
}
/// Whether the write under way stops holding a blob.
pub(crate) fn dooms(&self) -> bool {
!self.doomed.is_empty()
}
/// Whether a change to its space repos may stop holding a blob: whether
/// any of their records references one.
pub(crate) fn space_held(&self) -> bool {
!self.space_refs.is_empty()
}
/// Whether some record references `cid`, public or in a space repo.
fn referenced(&self, cid: &str) -> bool {
self.refs.contains_key(cid) || self.space_refs.contains_key(cid)
}
/// Have the files of the uploads not yet synced among `cids` synced
/// before `wal` next is: a commit referencing them is about to be logged
/// in it.
pub(crate) fn sync_before(
&mut self,
wal: &pds_store::Wal,
cids: impl IntoIterator<Item = String>,
) {
let paths: Vec<PathBuf> = cids
.into_iter()
.filter_map(|c| self.unsynced.remove(&c))
.collect();
if !paths.is_empty() {
wal.sync_first(paths);
}
}
/// Whether a blob held is unreferenced and past its time.
fn expired(&self, cid: &str) -> bool {
!self.referenced(cid)
&& self
.held
.get(cid)
.is_some_and(|h| h.at.saturating_add(self.ttl) <= self.now)
}
/// The unreferenced uploads past their time.
fn past_time(&self) -> impl Iterator<Item = &String> {
self.unreferenced
.iter()
.take_while(|(at, _)| at.saturating_add(self.ttl) <= self.now)
.map(|(_, cid)| cid)
}
/// The bytes held that count against the quota: those of the blobs held
/// but uploads past their time.
fn counted(&self) -> u64 {
let past: u64 = self.past_time().map(|cid| self.held[cid].size).sum();
self.bytes - past
}
/// A blob that a record references: what `getBlob` serves and `listBlobs` lists.
fn served(&self, cid: &str) -> Option<&Held> {
self.held.get(cid).filter(|_| self.refs.contains_key(cid))
}
/// Whether it holds `cid`: stored, and if unreferenced, within its time.
fn holds(&self, cid: &str) -> bool {
self.held.contains_key(cid) && !self.expired(cid)
}
/// A blob it holds: its takedown's reference, if it is down.
pub(crate) fn held_takedown(&self, cid: &str) -> Option<Option<&str>> {
self.holds(cid)
.then(|| self.takedowns.get(cid).map(String::as_str))
}
fn taken_down(&self, cid: &str) -> bool {
self.holds(cid) && self.takedowns.contains_key(cid)
}
/// The entry that takes down a blob it holds with reference `r`, or
/// lifts that with none: none if that changes nothing.
pub(crate) fn takedown_entry(&self, cid: &str, r: Option<&str>) -> Option<Json> {
let changes = match r {
Some(_) => self.holds(cid),
None => self.takedowns.contains_key(cid),
};
changes.then(|| json!({ "t": "takedownBlob", "cid": cid, "ref": r }))
}
/// How many distinct blobs its records reference, and how many it holds,
/// at `now` with uploads held for `ttl`.
pub(crate) fn counts(&self, now: u64, ttl: u64) -> (usize, usize) {
let past = self
.unreferenced
.iter()
.take_while(|(at, _)| at.saturating_add(ttl) <= now)
.count();
(self.refs.len(), self.held.len() - past)
}
/// The blobs its records reference that it does not hold, in order of
/// CID, those after `cursor` (as strings), at most `limit`: each with
/// the path of a record referencing it, the first in order of path
/// (`spec/MIGRATION.md`).
pub(crate) fn missing(&self, cursor: Option<&str>, limit: usize) -> Vec<(String, String)> {
let mut cids: Vec<&String> = self
.refs
.keys()
.filter(|c| !self.held.contains_key(*c) && cursor.is_none_or(|k| c.as_str() > k))
.collect();
cids.sort_unstable();
cids.truncate(limit);
let mut paths: BTreeMap<&str, &str> = BTreeMap::new();
for (path, (_, refs)) in &self.records {
for cid in refs {
if cids.binary_search(&cid).is_ok() {
let p = paths.entry(cid.as_str()).or_insert(path.as_str());
if path.as_str() < *p {
*p = path.as_str();
}
}
}
}
cids.into_iter()
.filter_map(|c| Some((c.clone(), paths.get(c.as_str())?.to_string())))
.collect()
}
/// Check a write's blob refs against the blobs as they were before it:
/// a public write's, or a space write's (`space_repo`).
pub(crate) fn check(&self, refs: &[BlobRef]) -> Result<(), XrpcError> {
for r in refs {
let cid = cid_string(&r.cid);
let Some(h) = self
.held
.get(&cid)
.filter(|_| !self.expired(&cid) && !self.takedowns.contains_key(&cid))
else {
return err(400, "BlobNotFound", format!("Could not find blob: {cid}"));
};
// One some record, public or in a space, references already is taken as it is.
if self.referenced(&cid) {
continue;
}
if h.mime != r.mime {
return err(
400,
"InvalidMimeType",
format!(
"Referenced Mimetype does not match stored blob. Expected: {}, Got: {}",
h.mime, r.mime
),
);
}
if u64::try_from(r.size).ok() != Some(h.size) {
return err(
400,
"InvalidSize",
format!(
"Referenced size does not match stored blob. Expected: {}, Got: {}",
h.size, r.size
),
);
}
}
Ok(())
}
/// Hold `cid`, or stop holding it, keeping the index of those
/// unreferenced and the bytes held: one no longer held is no longer
/// taken down.
fn set_held(&mut self, cid: &str, h: Option<Held>) {
if let Some(old) = self.held.remove(cid) {
self.unreferenced.remove(&(old.at, cid.to_string()));
self.bytes -= old.size;
}
if h.is_none() {
self.takedowns.remove(cid);
}
if let Some(h) = h {
if !self.referenced(cid) {
self.unreferenced.insert((h.at, cid.to_string()));
}
self.bytes += h.size;
self.held.insert(cid.to_string(), h);
}
}
/// Count the references a commit changed: each record it wrote or
/// deleted references what it holds now. A blob no record references any
/// more is no longer held, and its file is doomed. What it changed is
/// kept, to `undo` should the commit not reach the log.
pub(crate) fn committed(&mut self, c: &Change) {
let mut undo = Undo {
bytes: self.bytes,
doomed: self.doomed.len(),
..Default::default()
};
let mut delta: BTreeMap<String, i64> = BTreeMap::new();
let written: BTreeMap<&String, Vec<String>> = c
.puts
.iter()
.map(|(p, j)| {
(
p,
put_record(j).map(|r| cids_in(&r.value)).unwrap_or_default(),
)
})
.collect();
for path in c.puts.iter().map(|(p, _)| p).chain(&c.dels) {
let now = written.get(path).cloned().unwrap_or_default();
let before = self.records.remove(path);
if let Some(b) = &before {
self.record_bytes -= record_cost(path, b);
}
undo.records.push((path.clone(), before.clone()));
let before = before.map(|(_, cids)| cids).unwrap_or_default();
for cid in before.iter().filter(|x| !now.contains(x)) {
*delta.entry(cid.clone()).or_default() -= 1;
}
for cid in now.iter().filter(|x| !before.contains(x)) {
*delta.entry(cid.clone()).or_default() += 1;
}
if !now.is_empty() {
let refs = (c.rev.clone(), now);
self.record_bytes += record_cost(path, &refs);
self.records.insert(path.clone(), refs);
}
}
for (cid, d) in delta {
let had = self.refs.get(&cid).copied();
undo.refs.push((cid.clone(), had));
let held = self.held.get(&cid).cloned();
undo.held.push((cid.clone(), held.clone()));
let n = i64::from(had.unwrap_or(0)) + d;
if n > 0 {
self.refs.insert(cid.clone(), n as u32);
if let Some(h) = held {
self.unreferenced.remove(&(h.at, cid));
}
continue;
}
self.refs.remove(&cid);
if d < 0 && held.is_some() && !self.space_refs.contains_key(&cid) {
if let Some(t) = self.takedowns.get(&cid) {
undo.takedowns.push((cid.clone(), t.clone()));
}
self.set_held(&cid, None);
self.doomed.push(cid);
}
}
self.undo = Some(undo);
}
/// Count the references a space repo's commit changed, given each CID's
/// change in the number of its records referencing it: a blob no record
/// references any more, public or in a space, is no longer held, and its
/// file is doomed. Made only once the commit is in the log.
pub(crate) fn space_counted(&mut self, delta: BTreeMap<String, i64>) {
for (cid, d) in delta {
let n = i64::from(self.space_refs.get(&cid).copied().unwrap_or(0)) + d;
if n > 0 {
self.space_refs.insert(cid.clone(), n as u32);
if let Some(h) = self.held.get(&cid) {
self.unreferenced.remove(&(h.at, cid));
}
continue;
}
self.space_refs.remove(&cid);
if d < 0 && self.held.contains_key(&cid) && !self.refs.contains_key(&cid) {
self.set_held(&cid, None);
self.doomed.push(cid);
}
}
}
/// A blob a space repo's records reference, as its `getBlob` serves it:
/// `400 InvalidRequest` if the account does not hold its bytes.
pub(crate) fn serve_space(&self, s: &Service, did: &str, cid: &str) -> Result<Resp, XrpcError> {
let Some(h) = self.held.get(cid) else {
return invalid(format!("Blob not held: {cid}"));
};
serve(s, did, cid, h)
}
/// Put the blobs back as they were before the last commit.
pub(crate) fn undo(&mut self) {
let Some(u) = self.undo.take() else { return };
for (path, before) in u.records {
if let Some(now) = self.records.remove(&path) {
self.record_bytes -= record_cost(&path, &now);
}
if let Some(r) = before {
self.record_bytes += record_cost(&path, &r);
self.records.insert(path, r);
}
}
for (cid, n) in u.refs {
match n {
Some(n) => self.refs.insert(cid, n),
None => self.refs.remove(&cid),
};
}
for (cid, h) in u.held {
self.held.remove(&cid);
self.unreferenced.retain(|(_, c)| *c != cid);
if let Some(h) = h {
if !self.referenced(&cid) {
self.unreferenced.insert((h.at, cid.clone()));
}
self.held.insert(cid, h);
}
}
self.bytes = u.bytes;
self.doomed.truncate(u.doomed);
self.takedowns.extend(u.takedowns);
}
/// An upload of `cid` no longer moving into place, or putting its object.
fn unplace(&mut self, cid: &str) {
match self.placing.get_mut(cid) {
Some(n) if *n > 1 => *n -= 1,
_ => {
self.placing.remove(cid);
}
}
}
/// What uploading `cid` would do.
fn upload_of(&self, cid: &str) -> Upload {
match self.held.get(cid) {
None => Upload::New,
Some(_) if self.referenced(cid) => Upload::Kept,
Some(_) if self.expired(cid) => Upload::Afresh,
Some(_) => Upload::Again,
}
}
/// Make an upload's change, logged: stored as `h`, keeping the type it
/// was first stored with if it is held still.
fn uploaded(&mut self, cid: &str, how: Upload, h: Held) {
match how {
Upload::Kept => {}
Upload::Again => {
let mime = self.held[cid].mime.clone();
self.set_held(cid, Some(Held { mime, ..h }));
}
Upload::Afresh | Upload::New => {
// One taken down and past its time is no longer held, nor down.
self.takedowns.remove(cid);
self.set_held(cid, Some(h));
}
}
}
/// Replay an entry of the kinds in `ENTRIES`.
pub(crate) fn replay(&mut self, e: &Json) -> Result<(), String> {
if e["t"] == OBJECTS_DELETED {
self.kept.clear();
return Ok(());
}
let Some(cid) = e["cid"].as_str() else {
return Err(format!("a blob entry with no cid: {e}"));
};
if e["t"] == "blobGone" {
if !self.held.contains_key(cid) || self.referenced(cid) {
return Err(format!(
"an upload gone that was not held, or is referenced: {e}"
));
}
self.set_held(cid, None);
return Ok(());
}
if e["t"] == PUT {
self.kept.insert(cid.to_string());
return Ok(());
}
if e["t"] == DELETED {
if self.held.contains_key(cid) || !self.kept.remove(cid) {
return Err(format!(
"an object deleted of a blob held, or of none kept: {e}"
));
}
return Ok(());
}
if e["t"] == "takedownBlob" {
match &e["ref"] {
Json::String(r) if self.held.contains_key(cid) => {
self.takedowns.insert(cid.to_string(), r.clone())
}
Json::Null => self.takedowns.remove(cid),
_ => {
return Err(format!(
"a takedown of a blob not held, or with a ref not a string: {e}"
));
}
};
return Ok(());
}
let (Some(mime), Some(size), Some(at)) =
(e["mimeType"].as_str(), e["size"].as_u64(), e["at"].as_u64())
else {
return Err(format!("a blob with no mimeType, size or at: {e}"));
};
let how = if self.held.contains_key(cid) {
Upload::Again
} else {
Upload::New
};
self.uploaded(
cid,
how,
Held {
mime: mime.to_string(),
size,
at,
},
);
Ok(())
}
/// Once the log is replayed: every blob held has its file, whole, and
/// every other file in `dir` goes, as no entry holds it, but an upload
/// of this process's on its way in (`temp_prefix`). An upload no
/// record references was never synced: one whose file a power cut took
/// is held no more, and the others are synced by the commit that first
/// references them, as before.
pub(crate) fn recover(&mut self, dir: &Path) -> Result<(), String> {
self.doomed.clear();
self.undo = None;
let mut lost = Vec::new();
for (cid, h) in &self.held {
let p = dir.join(cid);
let whole = std::fs::metadata(&p).is_ok_and(|m| m.len() == h.size);
match (whole, self.referenced(cid)) {
(_, true) if !whole => {
return Err(format!(
"{}: the file of a blob the log holds is missing or not whole",
p.display()
));
}
(_, true) => {}
(true, false) => {
self.unsynced.insert(cid.clone(), p);
}
(false, false) => lost.push(cid.clone()),
}
}
for cid in lost {
self.set_held(&cid, None);
}
let Ok(entries) = std::fs::read_dir(dir) else {
return Ok(());
};
for e in entries {
let e = e.map_err(|e| format!("{}: {e}", dir.display()))?;
if !e
.file_name()
.to_str()
.is_some_and(|n| self.held.contains_key(n) || n.starts_with(temp_prefix()))
{
std::fs::remove_file(e.path())
.map_err(|why| format!("{}: {why}", e.path().display()))?;
}
}
Ok(())
}
/// Once the log is replayed, with blobs in a bucket: nothing in flight.
/// Its strays are deleted by the deleter (`blobstore`).
pub(crate) fn recover_kept(&mut self) {
self.doomed.clear();
self.undo = None;
}
/// The blobs the last write stopped holding.
pub(crate) fn take_doomed(&mut self) -> Vec<String> {
self.undo = None;
std::mem::take(&mut self.doomed)
}
/// What a checkpoint keeps of the objects a bucket may keep: none with
/// blobs in the data directory, and with a bucket, those not held, the
/// blobs held's being kept too.
pub(crate) fn objects_list(&self) -> Option<Vec<&str>> {
(!self.kept.is_empty()).then(|| {
self.kept
.iter()
.filter(|c| !self.held.contains_key(*c))
.map(String::as_str)
.collect()
})
}
/// The blobs it holds, in ascending order of CID, each with its
/// takedown's reference while it is down: what a checkpoint keeps.
pub(crate) fn held_list(&self) -> impl Iterator<Item = (&str, &Held, Option<&str>)> {
self.held
.iter()
.map(|(cid, h)| (cid.as_str(), h, self.takedowns.get(cid).map(String::as_str)))
}
/// Each record with blob refs: its path, the rev that last wrote it, and
/// the CIDs it references: what a checkpoint keeps of them.
pub(crate) fn record_list(&self) -> Vec<(&str, &str, &[String])> {
let mut out: Vec<(&str, &str, &[String])> = self
.records
.iter()
.map(|(p, (r, c))| (p.as_str(), r.as_str(), c.as_slice()))
.collect();
out.sort_unstable();
out
}
/// How many records of the account's space repos reference each CID,
/// for those some such record does, in ascending order of CID: what a
/// checkpoint keeps of them, as the records are not read to count them.
pub(crate) fn space_ref_list(&self) -> Vec<(&str, u32)> {
let mut out: Vec<(&str, u32)> = self
.space_refs
.iter()
.map(|(cid, n)| (cid.as_str(), *n))
.collect();
out.sort_unstable();
out
}
/// The blobs a checkpoint keeps, as replaying the log would leave them:
/// those held, each with its takedown, the records with blob refs, each
/// with the rev that last wrote it and what it references, and with
/// blobs in a bucket, the objects it may keep of those not held
/// (`objects_list`), and how many records of the account's space repos
/// reference each CID (`space_ref_list`).
pub(crate) fn restore(
records: Vec<(String, String, Vec<String>)>,
held: Vec<(String, Held, Option<String>)>,
objects: Option<Vec<String>>,
space_refs: Vec<(String, u32)>,
) -> Result<Blobs, String> {
let mut b = Blobs {
space_refs: space_refs.into_iter().collect(),
..Default::default()
};
for (path, rev, cids) in records {
if cids.is_empty() {
return Err(format!("record {path} kept with no blob refs"));
}
for cid in &cids {
*b.refs.entry(cid.clone()).or_default() += 1;
}
let refs = (rev, cids);
b.record_bytes += record_cost(&path, &refs);
b.records.insert(path, refs);
}
for (cid, h, takedown) in held {
if !b.referenced(&cid) {
b.unreferenced.insert((h.at, cid.clone()));
}
if let Some(t) = takedown {
b.takedowns.insert(cid.clone(), t);
}
if objects.is_some() {
b.kept.insert(cid.clone());
}
b.bytes += h.size;
b.held.insert(cid, h);
}
for cid in objects.into_iter().flatten() {
if b.held.contains_key(&cid) {
return Err(format!("object {cid} kept apart from its blob, held"));
}
b.kept.insert(cid);
}
Ok(b)
}
/// What a restart must carry over, as one comparable value.
#[cfg(test)]
pub(crate) fn snapshot(&self) -> String {
let refs: BTreeMap<_, _> = self.refs.iter().collect();
let space_refs: BTreeMap<_, _> = self.space_refs.iter().collect();
let records: BTreeMap<_, _> = self.records.iter().collect();
format!(
"{:?}",
(
&self.held,
refs,
space_refs,
records,
&self.unreferenced,
self.bytes,
&self.takedowns,
&self.kept,
)
)
}
}
/// Apply a write's ops to the account's repository, its blob refs checked
/// last, against the blobs as they were before it, and their references counted.
pub(crate) fn apply(
st: &mut Store,
ops: Vec<Op>,
swap_commit: Option<&str>,
) -> Result<Vec<OpResult>, XrpcError> {
let mut refs = Vec::new();
for op in &ops {
if let Op::Create { value, .. } | Op::Update { value, .. } = op {
refs_in(value, &mut refs);
}
}
let Store {
repo,
blobs,
disk,
ctx,
wal,
..
} = st;
let results = repo.apply_checked(&disk.source(&ctx.blocks), ops, swap_commit, || {
blobs.check(&refs)
})?;
if let Some(c) = &repo.change {
blobs.sync_before(wal, refs.iter().map(|r| cid_string(&r.cid)));
blobs.committed(c);
}
Ok(results)
}
/// Make the account's repository the tree an import brings, by `ops`, as
/// one commit: its blob refs are not checked, each blob it does not hold
/// becoming missing until it is uploaded (`spec/MIGRATION.md`), and their
/// references are counted as a write's.
pub(crate) fn import(st: &mut Store, ops: Vec<Op>) -> Result<(), XrpcError> {
let mut refs = Vec::new();
for op in &ops {
if let Op::Create { value, .. } | Op::Update { value, .. } = op {
refs_in(value, &mut refs);
}
}
let Store {
repo,
blobs,
disk,
ctx,
wal,
..
} = st;
repo.import(&disk.source(&ctx.blocks), ops, || Ok::<(), XrpcError>(()))?;
if let Some(c) = &repo.change {
blobs.sync_before(wal, refs.iter().map(|r| cid_string(&r.cid)));
blobs.committed(c);
}
Ok(())
}
/// Remove the files of blobs a write stopped holding, once it is on disk:
/// those not held again, by an upload since, or moving into place. With
/// blobs in a bucket, have the deleter delete their objects (`blobstore`).
pub(crate) fn forget(svc: &Service, account: &Account, doomed: Vec<String>) {
if doomed.is_empty() {
return;
}
if let Some(bucket) = &svc.ctx.bucket {
bucket.wake(&account.did);
return;
}
let dir = blob_dir(&svc.dir, &account.did);
let st = read(&account.store);
for cid in doomed
.iter()
.filter(|c| !st.blobs.held.contains_key(*c) && !st.blobs.placing.contains_key(*c))
{
let _ = std::fs::remove_file(dir.join(cid));
}
}
/// Log as gone every upload held past its time with no record referencing
/// it, and remove its file, for every account whose store is held: one not
/// held has its swept when next loaded and written. With blobs in a bucket,
/// the account is marked for a start to load first (`blobstore`).
pub(crate) fn sweep(svc: &Service) {
let accounts: Vec<Arc<Account>> = read(&svc.directory)
.by_did
.values()
.filter(|a| a.store.is_loaded())
.cloned()
.collect();
let (now, ttl) = (svc.limits.clock.now(), svc.config.blob_ttl);
for a in accounts {
let (wal, n, gone) = {
let mut st = write(&a.store);
let st = &mut *st;
st.blobs.time(now, ttl);
let gone: Vec<String> = st.blobs.past_time().cloned().collect();
if gone.is_empty() {
continue;
}
let entries: Vec<Json> = gone.iter().map(|c| gone_entry(c)).collect();
let marked = match &st.ctx.bucket {
Some(_) => st.ctx.eager.mark(&a.did),
None => Ok(()),
};
match marked.and_then(|()| st.wal.append(&entries)) {
Ok(n) => {
for cid in &gone {
st.blobs.set_held(cid, None);
}
(st.wal.clone(), n, gone)
}
Err(e) => {
crate::ops::failed_for(
crate::ops::Level::Error,
"logging uploads gone failed",
&a.did,
e,
);
continue;
}
}
};
if let Err(e) = wal.wait(n) {
crate::ops::failed_for(
crate::ops::Level::Error,
"syncing uploads gone failed",
&a.did,
e,
);
continue;
}
forget(svc, &a, gone);
}
}
/// Sweep now, and every `SWEEP_EVERY` while the server runs.
pub(crate) fn sweep_every(svc: &Shared) {
let weak = Arc::downgrade(svc);
tokio::spawn(async move {
loop {
let Some(svc) = weak.upgrade() else { return };
let _ = tokio::task::spawn_blocking(move || sweep(&svc)).await;
tokio::time::sleep(SWEEP_EVERY).await;
}
});
}
// ---- uploadBlob ------------------------------------------------------------
/// `uploadBlob`, which takes its body as it comes rather than as JSON.
pub(crate) async fn upload(
AxState(svc): AxState<Shared>,
method: Method,
headers: HeaderMap,
peer: Option<Extension<crate::limits::Peer>>,
mut body: Body,
) -> Response {
let authorization = crate::authorization(&headers);
let bearer = crate::sessions::Bearer::of(authorization.as_deref());
let client = crate::limits::client(
&svc.config.trusted_proxies,
&headers,
peer.map(|Extension(p)| p.0),
);
let mut call = Call {
nsid: UPLOAD,
method: &method,
params: &[],
body: &[],
bearer,
headers: &headers,
client,
charged: OnceLock::new(),
http: None,
};
let grant = match crate::scopes::look(&svc, &call, &headers).await {
Ok(g) => g,
Err(r) => {
drain(&mut body, DRAIN_LIMIT).await;
return *r;
}
};
if let Some(g) = &grant {
call.bearer = crate::sessions::Bearer::OAuth(g);
}
let answer = upload_blob(svc.clone(), &call, &headers, &mut body).await;
// A client still sending its body when it is refused, as one too large
// is, may take the connection closing for a failure and never read the
// answer: read the rest first, up to a point, storing none of it.
if answer.is_err() {
drain(&mut body, DRAIN_LIMIT).await;
}
crate::scopes::answered(
&svc,
grant.as_ref(),
crate::respond(answer, call.charged.get()),
)
}
/// How much of a refused upload's body is read and thrown away before it is
/// answered; past it, the connection closes as the answer goes.
const DRAIN_LIMIT: u64 = 16 * 1024 * 1024;
/// The next frame of a body, if it has another.
async fn next_frame(body: &mut Body) -> Option<Result<Frame<Bytes>, axum::Error>> {
std::future::poll_fn(|cx| Pin::new(&mut *body).poll_frame(cx)).await
}
/// Read and throw away the rest of a body, up to `limit` bytes.
async fn drain(body: &mut Body, mut limit: u64) {
while let Some(Ok(frame)) = next_frame(body).await {
let n = frame.data_ref().map_or(0, |d| d.len() as u64);
if n > limit {
return;
}
limit -= n;
}
}
/// How the names of this process's uploads on their way in begin, in an
/// account's blob directory: a store loaded while one comes in, its
/// account's first request or one since it was let go, removes the files a
/// process before left there (`recover`, `log::open_repo`), not this one's.
pub(crate) fn temp_prefix() -> &'static str {
static PREFIX: LazyLock<String> =
LazyLock::new(|| format!(".tmp-{}-", hex::encode(rand::random::<[u8; 6]>())));
&PREFIX
}
/// An upload's bytes on their way in, removed unless they are moved into place.
struct Temp {
path: PathBuf,
file: File,
placed: bool,
}
impl Temp {
fn create(dir: &Path) -> io::Result<Temp> {
for made in pds_store::make_dirs(dir)? {
if let Some(parent) = made.parent() {
pds_store::sync_dir(parent)?;
}
}
let path = dir.join(format!(
"{}{}",
temp_prefix(),
hex::encode(rand::random::<[u8; 12]>())
));
let file = pds_store::open_private(
&path,
OpenOptions::new().create_new(true).read(true).write(true),
)?;
Ok(Temp {
path,
file,
placed: false,
})
}
/// Move the bytes to `to`, unsynced: the commit that first references
/// them syncs them. Bytes of the same CID already there stay, as they
/// may be synced and these not.
fn place(mut self, to: &Path) -> io::Result<()> {
if to.exists() {
return Ok(());
}
std::fs::rename(&self.path, to)?;
self.placed = true;
Ok(())
}
}
impl Drop for Temp {
fn drop(&mut self) {
if !self.placed {
let _ = std::fs::remove_file(&self.path);
}
}
}
/// Make `bytes` `cid`'s file in `dir`, in place of any there, synced, as a
/// move into the data directory writes it (`migrate_blobs`): the directory
/// is synced once each of its files is.
pub(crate) fn write_synced(dir: &Path, cid: &str, bytes: &[u8]) -> io::Result<()> {
let mut t = Temp::create(dir)?;
t.file.write_all(bytes)?;
pds_store::sync_file(&t.file)?;
std::fs::rename(&t.path, dir.join(cid))?;
t.placed = true;
Ok(())
}
/// The type a body's first bytes show, of those `spec/BLOBS.md` lists.
fn sniff(head: &[u8], body: &mut (impl Read + Seek)) -> Option<&'static str> {
let at = |from: usize, want: &[u8]| head.get(from..from + want.len()) == Some(want);
if at(0, b"\x89PNG\r\n\x1a\n") {
png_image(body).then_some("image/png")
} else if at(0, b"\xff\xd8\xff") && head.get(3).is_some_and(|b| *b != 0xf7) {
Some("image/jpeg")
} else if at(0, b"GIF89a") {
Some("image/gif")
} else if at(0, b"RIFF") && at(8, b"WEBP") {
Some("image/webp")
} else if at(0, b"%PDF-") {
Some("application/pdf")
} else if at(4, b"ftyp") && at(8, b"isom") {
Some("video/mp4")
} else {
None
}
}
/// Whether a body after the PNG signature is a PNG image, not an animated
/// one: an `IHDR` chunk of 13 bytes first, then chunks up to an `IDAT`, with
/// no `acTL` before it. Nothing after the first `IDAT` is read: neither its
/// length nor `IEND` (the harness's PNG has an `IDAT` 4 bytes shorter than
/// it says, and the reference takes it), nor any CRC.
fn png_image(body: &mut (impl Read + Seek)) -> bool {
let mut f = io::BufReader::new(body);
if f.seek(SeekFrom::Start(8)).is_err() {
return false;
}
let mut first = true;
loop {
let mut h = [0u8; 8];
if f.read_exact(&mut h).is_err() {
return false;
}
let len = u32::from_be_bytes([h[0], h[1], h[2], h[3]]);
let kind = &h[4..];
if first && (kind != b"IHDR" || len != 13) {
return false;
}
first = false;
match kind {
b"acTL" => return false,
b"IDAT" => return true,
_ => {}
}
if f.seek_relative(i64::from(len) + 4).is_err() {
return false;
}
}
}
/// `uploadBlob` checks its token, its account's state, its `Content-Type`,
/// what an OAuth scope needs of that media type, then its size, pays its
/// rate limits and checks the free space (`limits::guard`), then the
/// account's quota, and stores the body under its CID: in a file, or with
/// blobs in a bucket, read whole and put there. An OAuth token's call has
/// `Bearer::OAuth` already.
async fn upload_blob(
svc: Shared,
call: &Call<'_>,
headers: &HeaderMap,
body: &mut Body,
) -> Result<Resp, XrpcError> {
crate::method_of(call)?;
let did = svc.auth(call.bearer).did()?;
crate::lifecycle::gate(&svc, UPLOAD, &did)?;
let Some(account) = svc.account(&did) else {
return internal();
};
// The media type is the header up to its first `;`, as sent.
let given = headers
.get(header::CONTENT_TYPE)
.map(|v| String::from_utf8_lossy(v.as_bytes()).into_owned())
.unwrap_or_default();
let media = given.split(';').next().unwrap_or_default().to_string();
if media.is_empty() {
return invalid("Request encoding (Content-Type) required but not provided");
}
// What the scope needs reads each byte of the header as its Latin-1
// character, as the refusal names it (`spec/OAUTH-SCOPES.md`).
let raw = headers
.get(header::CONTENT_TYPE)
.map(|v| v.as_bytes())
.unwrap_or_default();
let latin1: String = raw
.split(|&b| b == b';')
.next()
.unwrap_or_default()
.iter()
.map(|&b| char::from(b))
.collect();
crate::scopes::may(call.bearer, &crate::scopes::Need::Blob(&latin1))?;
let too_large = || err(413, "PayloadTooLarge", "request entity too large");
let limit = svc.config.blob_upload_limit;
let length = headers
.get(header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok()?.trim().parse::<u64>().ok());
if length.is_some_and(|n| n > limit) {
return too_large();
}
if let Some(bucket) = svc.ctx.bucket.clone() {
let bytes = read_whole(body, length, limit).await?;
crate::limits::guard(&svc, call, Some(&did), None)?;
let cid = cid_string(&cid_of(0x55, &bytes));
let head = &bytes[..bytes.len().min(16)];
let mime = sniff(head, &mut io::Cursor::new(&bytes[..])).map_or(media, str::to_string);
let size = bytes.len() as u64;
let answer = json!({ "blob": { "$type": "blob", "ref": { "$link": cid }, "mimeType": mime, "size": size } });
let blob = Held {
mime,
size,
at: svc.limits.clock.now(),
};
// Once its bytes are in, it is stored even if the caller goes away,
// so that no upload is left putting its object.
crate::ops::spawn(store_object(
svc.clone(),
account,
bucket,
cid,
bytes.into(),
blob,
))
.await
.unwrap_or_else(|_| internal())?;
return Ok(Resp::Json(answer));
}
let dir = blob_dir(&svc.dir, &did);
let Ok(mut tmp) = Temp::create(&dir) else {
return internal();
};
let (mut hash, mut size, mut head) = (Sha256::new(), 0u64, Vec::with_capacity(16));
while let Some(frame) = next_frame(body).await {
let Ok(frame) = frame else {
return invalid("The body could not be read");
};
let Ok(data) = frame.into_data() else {
continue;
};
size += data.len() as u64;
if size > limit {
return too_large();
}
hash.update(&data);
head.extend_from_slice(&data[..data.len().min(16 - head.len())]);
if tmp.file.write_all(&data).is_err() {
return internal();
}
}
crate::limits::guard(&svc, call, Some(&did), None)?;
let mut cid = vec![1, 0x55, 0x12, 0x20];
cid.extend_from_slice(&hash.finalize());
let cid = cid_string(&cid);
let mime = sniff(&head, &mut tmp.file).map_or(media, str::to_string);
let answer = json!({ "blob": { "$type": "blob", "ref": { "$link": cid }, "mimeType": mime, "size": size } });
// Once its bytes are in, it is stored even if the caller goes away, so
// that no upload is left moving into place.
let blob = Held {
mime,
size,
at: svc.limits.clock.now(),
};
tokio::spawn(store(svc.clone(), account, dir, tmp, cid, blob))
.await
.unwrap_or_else(|_| internal())?;
Ok(Resp::Json(answer))
}
/// An upload's body, whole, in memory: at most `limit` bytes.
async fn read_whole(
body: &mut Body,
length: Option<u64>,
limit: u64,
) -> Result<Vec<u8>, XrpcError> {
let mut bytes = Vec::with_capacity(length.unwrap_or(0) as usize);
while let Some(frame) = next_frame(body).await {
let Ok(frame) = frame else {
return invalid("The body could not be read");
};
let Ok(data) = frame.into_data() else {
continue;
};
if (bytes.len() + data.len()) as u64 > limit {
return err(413, "PayloadTooLarge", "request entity too large");
}
bytes.extend_from_slice(&data);
}
Ok(bytes)
}
/// Store an upload's bytes as its CID's, within the account's quota, and log
/// it, unless a record references it already; return once that is handed to
/// the operating system, unsynced. Its file is put in place first, held by
/// none until it is logged.
async fn store(
svc: Shared,
account: Arc<Account>,
dir: PathBuf,
tmp: Temp,
cid: String,
blob: Held,
) -> Result<(), XrpcError> {
*write(&account.store)
.blobs
.placing
.entry(cid.clone())
.or_default() += 1;
let to = dir.join(&cid);
let placed = tokio::task::spawn_blocking(move || tmp.place(&to)).await;
let (logged, doomed) = {
let mut st = write(&account.store);
let st = &mut *st;
st.blobs.unplace(&cid);
st.blobs.time(blob.at, svc.config.blob_ttl);
let how = st.blobs.upload_of(&cid);
let logged = if !matches!(placed, Ok(Ok(()))) {
internal()
} else {
refusal(&svc, &account, &st.blobs, &cid, how, blob.size)
.and_then(|()| log_upload(st, &account.did, &cid, how, blob))
};
if logged.is_ok() && how != Upload::Kept && !st.blobs.referenced(&cid) {
st.blobs.unsynced.insert(cid.clone(), dir.join(&cid));
}
(logged, logged_or_doomed(&st.blobs, &cid))
};
// An upload not logged leaves no file a blob held does not have.
forget(&svc, &account, doomed);
logged
}
/// Store an upload's bytes as its CID's object in the bucket, and log it as
/// `store` does, once the store has kept it, in the object's turn. It is
/// refused before it is put as it would be after, and its object noted in
/// the log first if no entry names it yet, so that one the server stops
/// during is deleted. One the store does not keep holds nothing more: its
/// object, if the store kept it after all, is a stray, deleted.
async fn store_object(
svc: Shared,
account: Arc<Account>,
bucket: Arc<crate::blobstore::Bucket>,
cid: String,
bytes: Bytes,
blob: Held,
) -> Result<(), XrpcError> {
let did = &account.did;
let ttl = svc.config.blob_ttl;
let turn = bucket.turn(did, &cid).await;
{
let mut st = write(&account.store);
let st = &mut *st;
st.blobs.time(blob.at, ttl);
let how = st.blobs.upload_of(&cid);
refusal(&svc, &account, &st.blobs, &cid, how, blob.size)?;
if !st.blobs.kept.contains(&cid) {
let noted = st
.ctx
.eager
.mark(did)
.and_then(|()| st.wal.append(&[put_entry(&cid)]));
if let Err(e) = noted {
crate::ops::failed_for(
crate::ops::Level::Error,
"logging an upload failed",
did,
e,
);
return internal();
}
st.blobs.kept.insert(cid.clone());
}
*st.blobs.placing.entry(cid.clone()).or_default() += 1;
}
let put = bucket.put(did, &cid, bytes).await;
let logged = {
let mut st = write(&account.store);
let st = &mut *st;
st.blobs.unplace(&cid);
st.blobs.time(blob.at, ttl);
let how = st.blobs.upload_of(&cid);
put.and_then(|()| refusal(&svc, &account, &st.blobs, &cid, how, blob.size))
.and_then(|()| log_upload(st, did, &cid, how, blob))
};
drop(turn);
// An object of a blob not held to delete, or a mark that may go.
bucket.wake(did);
logged
}
/// Whether an upload doing `how` is refused, its bytes in: its account
/// deleted, the blob taken down, or the account's quota passed.
fn refusal(
svc: &Service,
account: &Account,
b: &Blobs,
cid: &str,
how: Upload,
size: u64,
) -> Result<(), XrpcError> {
crate::lifecycle::not_deleted(account)?;
if b.taken_down(cid) {
return invalid("Blob has been takendown, cannot re-upload");
}
if matches!(how, Upload::New | Upload::Afresh) && b.counted() + size > svc.config.blob_quota {
return err(
400,
"BlobQuotaExceeded",
"The account's blobs would be past its quota",
);
}
Ok(())
}
/// Log an upload doing `how`, unless a record references it already, and
/// make its change, keeping the type a blob held was first stored with.
fn log_upload(
st: &mut Store,
did: &str,
cid: &str,
how: Upload,
blob: Held,
) -> Result<(), XrpcError> {
let entries = match how {
Upload::Kept => return Ok(()),
Upload::Again => vec![upload_entry(
cid,
&Held {
mime: st.blobs.held[cid].mime.clone(),
..blob.clone()
},
)],
Upload::Afresh => vec![gone_entry(cid), upload_entry(cid, &blob)],
Upload::New => vec![upload_entry(cid, &blob)],
};
match st.wal.append(&entries) {
Ok(_) => {
st.blobs.uploaded(cid, how, blob);
Ok(())
}
Err(e) => {
crate::ops::failed_for(crate::ops::Level::Error, "logging an upload failed", did, e);
internal()
}
}
}
/// The CID, if the account does not hold it, as a file to remove.
fn logged_or_doomed(blobs: &Blobs, cid: &str) -> Vec<String> {
if blobs.held.contains_key(cid) {
vec![]
} else {
vec![cid.to_string()]
}
}
// ---- getBlob and listBlobs ---------------------------------------------------
fn repo_not_found<T>(did: &str) -> Result<T, XrpcError> {
err(
400,
"RepoNotFound",
format!("Could not find repo for DID: {did}"),
)
}
/// A blob `getBlob` serves: where its bytes are, and what its headers say.
pub(crate) struct Served {
source: Source,
mime: String,
size: u64,
cid: String,
}
/// Where a blob's bytes come from.
enum Source {
/// Its file, opened.
File(File),
/// Its object in the bucket, of the account by DID, not yet asked for:
/// `read_object` asks.
Object(String),
/// Its object as it comes, its first bytes in.
Read(crate::blobstore::Chunks),
}
/// What `getBlob` answers of a blob held: its file, or its object, to read.
fn serve(s: &Service, did: &str, cid: &str, h: &Held) -> Result<Resp, XrpcError> {
let source = match &s.ctx.bucket {
Some(_) => Source::Object(did.to_string()),
None => match File::open(blob_dir(&s.dir, did).join(cid)) {
Ok(file) => Source::File(file),
Err(_) => return internal(),
},
};
Ok(Resp::Blob(Served {
source,
mime: h.mime.clone(),
size: h.size,
cid: cid.to_string(),
}))
}
/// A blob in the bucket an answer serves, asked of the store: `500` if it
/// fails before the object's first bytes (`spec/BLOBS.md`). Any other
/// answer, as it is.
pub(crate) async fn read_object(
s: &Service,
out: Result<Resp, XrpcError>,
) -> Result<Resp, XrpcError> {
match out {
Ok(Resp::Blob(Served {
source: Source::Object(did),
mime,
size,
cid,
})) => {
let Some(bucket) = &s.ctx.bucket else {
return internal();
};
match bucket.get(&did, &cid, size).await {
Some(chunks) => Ok(Resp::Blob(Served {
source: Source::Read(chunks),
mime,
size,
cid,
})),
None => internal(),
}
}
out => out,
}
}
/// `getBlob` takes no token, but refuses one the server would not take, as
/// the lifecycle's gate checks it (`lifecycle::gate_query`).
fn get_blob(
s: &Service,
_: &Call,
shown: &mut Shown,
p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
let (did, cid) = (required(p, "did")?, required(p, "cid")?);
let Some(a) = s.account(did) else {
return repo_not_found(did);
};
let st = read(&a.store);
shown.0.push((st.wal.clone(), st.wal.written()));
// One taken down is as one not held, to the account and the admin too.
let Some(h) = st.blobs.served(cid).filter(|_| !st.blobs.taken_down(cid)) else {
return invalid("Blob not found");
};
crate::ops::serving_blob(&a.did, cid);
serve(s, &a.did, cid, h)
}
/// A blob's bytes as `getBlob` answers them, as they are read: no more than
/// its size, and an error if they end before it.
struct BlobBody {
chunks: tokio::sync::mpsc::Receiver<io::Result<Bytes>>,
left: u64,
}
impl HttpBody for BlobBody {
type Data = Bytes;
type Error = io::Error;
fn poll_frame(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Bytes>, io::Error>>> {
let this = self.get_mut();
if this.left == 0 {
return Poll::Ready(None);
}
this.chunks.poll_recv(cx).map(|chunk| match chunk {
Some(Ok(mut b)) => {
b.truncate(this.left.min(b.len() as u64) as usize);
this.left -= b.len() as u64;
Some(Ok(Frame::data(b)))
}
Some(Err(e)) => Some(Err(e)),
None => Some(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"the blob's bytes ended early",
))),
})
}
fn is_end_stream(&self) -> bool {
self.left == 0
}
fn size_hint(&self) -> SizeHint {
SizeHint::with_exact(self.left)
}
}
/// `getBlob`'s answer: the blob's bytes, streamed, with its headers.
pub(crate) fn response(b: Served) -> Response {
let Served {
source,
mime,
size,
cid,
} = b;
let chunks = match source {
Source::File(mut file) => {
let (tx, chunks) = tokio::sync::mpsc::channel(4);
tokio::task::spawn_blocking(move || {
let mut left = size;
while left > 0 {
let mut buf = vec![0u8; left.min(64 * 1024) as usize];
let read = file.read_exact(&mut buf).map(|()| Bytes::from(buf));
let failed = read.is_err();
if tx.blocking_send(read).is_err() || failed {
return;
}
left -= left.min(64 * 1024);
}
});
chunks
}
Source::Read(chunks) => chunks,
// Never so: an object is read before its answer is made (`read_object`).
Source::Object(_) => return crate::respond(internal(), None),
};
let mut res = Response::new(Body::new(BlobBody { chunks, left: size }));
let h = res.headers_mut();
h.insert(
header::CONTENT_TYPE,
HeaderValue::from_bytes(mime.as_bytes())
.unwrap_or(HeaderValue::from_static("application/octet-stream")),
);
h.insert(header::CONTENT_LENGTH, HeaderValue::from(size));
h.insert(
header::X_CONTENT_TYPE_OPTIONS,
HeaderValue::from_static("nosniff"),
);
if let Ok(v) = HeaderValue::from_str(&format!("attachment; filename=\"{cid}\"")) {
h.insert(header::CONTENT_DISPOSITION, v);
}
h.insert(
header::CONTENT_SECURITY_POLICY,
HeaderValue::from_static("default-src 'none'; sandbox"),
);
res
}
/// `listBlobs`: the CIDs records reference, in order, after the cursor; with
/// `since`, only those of records written after that rev. Its
/// `Authorization` is the gate's, as `getBlob`'s is.
fn list_blobs(
s: &Service,
_: &Call,
shown: &mut Shown,
p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
let did = required(p, "did")?;
let limit = p.get("limit").and_then(Json::as_u64).unwrap_or(500) as usize;
let cursor = opt(p, "cursor").unwrap_or("");
let Some(a) = s.account(did) else {
return repo_not_found(did);
};
let st = read(&a.store);
shown.0.push((st.wal.clone(), st.wal.written()));
let b = &st.blobs;
let cids: Vec<&str> = match opt(p, "since") {
None => b
.held
.range::<str, _>((Excluded(cursor), Unbounded))
.map(|(c, _)| c.as_str())
.filter(|c| b.refs.contains_key(*c))
.take(limit)
.collect(),
Some(since) => {
let since: BTreeSet<&str> = b
.records
.values()
.filter(|(rev, _)| rev.as_str() > since)
.flat_map(|(_, cids)| cids.iter().map(String::as_str))
.filter(|c| *c > cursor && b.served(c).is_some())
.collect();
since.into_iter().take(limit).collect()
}
};
let mut out = json!({ "cids": cids });
if let Some(last) = cids.last() {
out["cursor"] = json!(last);
}
Ok(Resp::Json(out))
}
/// Upload `body` as a request would, with its `Content-Length` unless
/// `chunked`; answers the JSON, or the error's status and name.
#[cfg(test)]
pub(crate) async fn test_upload(
s: &Shared,
token: Option<&str>,
content_type: Option<&str>,
body: Vec<u8>,
chunked: bool,
) -> Result<Json, (u16, String)> {
/// A body in pieces, of no length known beforehand.
struct Chunks(Vec<Bytes>);
impl HttpBody for Chunks {
type Data = Bytes;
type Error = io::Error;
fn poll_frame(
self: Pin<&mut Self>,
_: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Bytes>, io::Error>>> {
let this = self.get_mut();
Poll::Ready((!this.0.is_empty()).then(|| Ok(Frame::data(this.0.remove(0)))))
}
}
let mut headers = HeaderMap::new();
if let Some(t) = token {
headers.insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {t}")).unwrap(),
);
}
if let Some(c) = content_type {
headers.insert(header::CONTENT_TYPE, HeaderValue::from_str(c).unwrap());
}
let mut body = if chunked {
Body::new(Chunks(
body.chunks(100_000).map(Bytes::copy_from_slice).collect(),
))
} else {
headers.insert(header::CONTENT_LENGTH, HeaderValue::from(body.len()));
Body::from(body)
};
let authorization = crate::authorization(&headers);
let bearer = crate::sessions::Bearer::of(authorization.as_deref());
let call = Call {
nsid: UPLOAD,
method: &Method::POST,
params: &[],
body: &[],
bearer,
headers: &headers,
client: "127.0.0.1".into(),
charged: OnceLock::new(),
http: None,
};
match upload_blob(s.clone(), &call, &headers, &mut body).await {
Ok(Resp::Json(j)) => Ok(j),
Ok(_) => Err((0, "not JSON".into())),
Err(e) => Err((e.status, e.name.to_string())),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::testing::*;
use pds_repo::data::cid_of;
use std::sync::atomic::Ordering;
/// A blob ref, as a record holds one.
fn bref(cid: &str, mime: &str, size: i64) -> Json {
json!({ "$type": "blob", "ref": { "$link": cid }, "mimeType": mime, "size": size })
}
fn raw_cid(body: &[u8]) -> String {
cid_string(&cid_of(0x55, body))
}
/// A body that sniffs as no type.
fn plain(n: usize) -> Vec<u8> {
let mut b = b"spec-pds blob ".to_vec();
b.extend((0..n).map(|i| (i * 7 + n) as u8));
b
}
/// `harness/checks/blobs.mjs`'s PNG, a 1x1 image, then `tail`.
fn harness_png(tail: &[u8]) -> Vec<u8> {
let mut b = hex::decode(concat!(
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c636000",
"0002000154a24f5d0000000049454e44ae426082"
))
.unwrap();
b.extend(tail);
b
}
fn png(extra: &[&[u8; 4]]) -> Vec<u8> {
let chunk = |kind: &[u8], data: &[u8]| -> Vec<u8> {
let mut c = (data.len() as u32).to_be_bytes().to_vec();
c.extend(kind);
c.extend(data);
c.extend([0; 4]);
c
};
let mut b = b"\x89PNG\r\n\x1a\n".to_vec();
b.extend(chunk(b"IHDR", &[0; 13]));
for k in extra {
b.extend(chunk(*k, &[1, 2, 3]));
}
b.extend(chunk(b"IDAT", &[9; 20]));
b.extend(chunk(b"IEND", &[]));
b
}
async fn put(
s: &Shared,
token: &str,
did: &str,
rkey: &str,
record: Json,
) -> Result<Json, String> {
call(s, "com.atproto.repo.putRecord", Some(token), json!({ "repo": did, "collection": "com.example.blob", "rkey": rkey, "record": record })).await
}
async fn delete(s: &Shared, token: &str, did: &str, rkey: &str) -> Result<Json, String> {
call(
s,
"com.atproto.repo.deleteRecord",
Some(token),
json!({ "repo": did, "collection": "com.example.blob", "rkey": rkey }),
)
.await
}
/// listBlobs, paged to the end with `limit`.
async fn listed(s: &Shared, did: &str, since: Option<&str>, limit: u32) -> Vec<String> {
let mut out = Vec::new();
let mut cursor: Option<String> = None;
loop {
let mut p = vec![
("did".to_string(), did.to_string()),
("limit".into(), limit.to_string()),
];
p.extend(since.map(|v| ("since".to_string(), v.to_string())));
p.extend(cursor.clone().map(|c| ("cursor".to_string(), c)));
let got = call_with(s, "com.atproto.sync.listBlobs", None, &p, json!({}))
.await
.unwrap();
let page: Vec<String> = got["cids"]
.as_array()
.unwrap()
.iter()
.map(|c| c.as_str().unwrap().to_string())
.collect();
assert!(page.len() <= limit as usize);
assert_eq!(got["cursor"].as_str(), page.last().map(String::as_str));
if page.is_empty() {
return out;
}
cursor = page.last().cloned();
out.extend(page);
}
}
/// getBlob through the router's handler: status, headers and bytes.
async fn get(
s: &Shared,
did: &str,
cid: &str,
token: Option<&str>,
) -> (u16, HeaderMap, Vec<u8>) {
let mut headers = HeaderMap::new();
if let Some(t) = token {
headers.insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {t}")).unwrap(),
);
}
let query = url::form_urlencoded::Serializer::new(String::new())
.append_pair("did", did)
.append_pair("cid", cid)
.finish();
let uri = format!("/xrpc/com.atproto.sync.getBlob?{query}")
.parse()
.unwrap();
let res = crate::handle(
AxState(s.clone()),
Method::GET,
uri,
headers,
None,
Body::empty(),
)
.await;
let (parts, body) = res.into_parts();
let bytes = axum::body::to_bytes(body, usize::MAX)
.await
.unwrap()
.to_vec();
(parts.status.as_u16(), parts.headers, bytes)
}
/// The error name of a getBlob that fails.
async fn get_error(s: &Shared, did: &str, cid: &str, token: Option<&str>) -> String {
let (status, _, body) = get(s, did, cid, token).await;
assert_ne!(status, 200);
serde_json::from_slice::<Json>(&body).unwrap()["error"]
.as_str()
.unwrap()
.to_string()
}
fn files(dir: &Path, did: &str) -> Vec<String> {
let mut names: Vec<String> = std::fs::read_dir(blob_dir(dir, did))
.map(|d| {
d.map(|e| e.unwrap().file_name().into_string().unwrap())
.collect()
})
.unwrap_or_default();
names.sort();
names
}
#[tokio::test]
async fn upload_checks_sniffs_and_limits() {
let plc = plc().await;
let dir = scratch("blob-upload");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let t = Some(token.as_str());
// The token, then the Content-Type, then the size.
let big = vec![0u8; DEFAULT_UPLOAD_LIMIT as usize + 1];
assert_eq!(
test_upload(&s, None, None, big.clone(), false).await,
Err((401, "AuthMissing".into()))
);
assert_eq!(
test_upload(&s, Some("forged"), None, big.clone(), false).await,
Err((400, "InvalidToken".into()))
);
for ct in [None, Some(""), Some("; charset=utf-8")] {
assert_eq!(
test_upload(&s, t, ct, big.clone(), false).await,
Err((400, "InvalidRequest".into())),
"{ct:?}"
);
}
for chunked in [false, true] {
assert_eq!(
test_upload(&s, t, Some("text/plain"), big.clone(), chunked).await,
Err((413, "PayloadTooLarge".into()))
);
}
let mut headers = HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {token}")).unwrap(),
);
let wrong = Call {
nsid: UPLOAD,
method: &Method::GET,
params: &[],
body: &[],
bearer: crate::sessions::Bearer::Token(&token),
headers: &headers,
client: String::new(),
charged: OnceLock::new(),
http: None,
};
assert_eq!(
upload_blob(s.clone(), &wrong, &headers, &mut Body::empty())
.await
.err()
.map(|e| e.name),
Some("InvalidRequest")
);
// A refused upload leaves nothing behind.
assert_eq!(files(&dir, &did), Vec::<String>::new());
// Exactly the limit is taken, chunked or not, and so is nothing.
for chunked in [false, true] {
let exact = vec![7u8; DEFAULT_UPLOAD_LIMIT as usize];
let got = test_upload(
&s,
t,
Some("application/octet-stream"),
exact.clone(),
chunked,
)
.await
.unwrap();
assert_eq!(
got,
json!({ "blob": bref(&raw_cid(&exact), "application/octet-stream", DEFAULT_UPLOAD_LIMIT as i64) })
);
}
let got = test_upload(&s, t, Some("image/png"), vec![], false)
.await
.unwrap();
assert_eq!(got["blob"], bref(&raw_cid(&[]), "image/png", 0));
// The type the bytes show, or else the media type as sent.
let mut jpeg = b"\xff\xd8\xff\xe0".to_vec();
jpeg.extend(plain(3));
let mut not_jpeg = b"\xff\xd8\xff\xf7".to_vec();
not_jpeg.extend(plain(3));
let mut webp = b"RIFF\x01\x02\x03\x04WEBP".to_vec();
webp.extend(plain(2));
let mut mp4 = b"\0\0\0\x18ftypisom".to_vec();
mp4.extend(plain(4));
let cases: Vec<(Vec<u8>, &str, &str)> = vec![
// The harness's PNG, with a tail of its own after `IEND`.
(harness_png(b"spec-pds blob 1 "), "image/jpeg", "image/png"),
(harness_png(b"\x00\x01"), "TEXT/Plain", "image/png"),
(harness_png(b""), "text/plain ;q=1", "image/png"),
(png(&[]), "text/plain", "image/png"),
(png(&[b"tEXt"]), "text/plain", "image/png"),
(png(&[b"acTL"]), "text/plain", "text/plain"),
(png(&[])[..30].to_vec(), "text/plain", "text/plain"),
(jpeg, "text/plain", "image/jpeg"),
(not_jpeg, "text/plain", "text/plain"),
(
[b"GIF89a".as_slice(), &plain(5)].concat(),
"text/plain",
"image/gif",
),
(webp, "text/plain", "image/webp"),
(
[b"%PDF-1.7".as_slice(), &plain(5)].concat(),
"image/png",
"application/pdf",
),
(mp4, "text/plain", "video/mp4"),
(plain(10), "TEXT/Plain", "TEXT/Plain"),
(plain(11), "text/plain; charset=utf-8", "text/plain"),
(plain(12), "text/plain ;q=1", "text/plain "),
(plain(13), "a/b/c", "a/b/c"),
(plain(14), "*/*", "*/*"),
];
for (body, ct, want) in cases {
for chunked in [false, true] {
let got = test_upload(&s, t, Some(ct), body.clone(), chunked)
.await
.unwrap();
assert_eq!(
got["blob"],
bref(&raw_cid(&body), want, body.len() as i64),
"{ct} {want}"
);
}
}
// Again with another type: answered with it, but stored with the first.
let b = plain(20);
let cid = raw_cid(&b);
test_upload(&s, t, Some("text/plain"), b.clone(), false)
.await
.unwrap();
assert_eq!(
test_upload(&s, t, Some("image/x"), b.clone(), true)
.await
.unwrap()["blob"]["mimeType"],
json!("image/x")
);
assert_eq!(
put(
&s,
&token,
&did,
"x",
json!({ "f": bref(&cid, "image/x", 34) })
)
.await,
Err("InvalidMimeType".into())
);
put(
&s,
&token,
&did,
"x",
json!({ "f": bref(&cid, "text/plain", 34) }),
)
.await
.unwrap();
let (status, headers, bytes) = get(&s, &did, &cid, None).await;
assert_eq!(
(
status,
headers[header::CONTENT_TYPE].to_str().unwrap(),
bytes
),
(200, "text/plain", b)
);
// Only whole blobs are left: no temporary file.
assert!(files(&dir, &did).iter().all(|f| !f.starts_with('.')));
let got = call(&s, "com.atproto.server.describeServer", None, json!({}))
.await
.unwrap();
assert_eq!(got["blobUploadLimit"], json!(5242880));
let _ = std::fs::remove_dir_all(&dir);
}
/// Refs are checked last, against the blobs before the write; the legacy
/// shape right after the record's `$type`.
#[tokio::test]
async fn refs_are_checked_in_order() {
let plc = plc().await;
let dir = scratch("blob-refs");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let (bob, bob_token) = create(&s, "bob").await;
let (x, y, theirs) = (plain(1), plain(2), plain(3));
for b in [&x, &y] {
test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false)
.await
.unwrap();
}
test_upload(
&s,
Some(&bob_token),
Some("text/plain"),
theirs.clone(),
false,
)
.await
.unwrap();
let (xc, yc) = (raw_cid(&x), raw_cid(&y));
let (size, ysize) = (x.len() as i64, y.len() as i64);
let rev = |s: &Service| repo_of(s, &did, |r| r.rev.clone());
let before = rev(&s);
let record_cid = repo_of(&s, &did, |r| cid_string(&r.head));
let refused = [
(bref(&raw_cid(b"never"), "text/plain", 5), "BlobNotFound"),
(bref(&raw_cid(&theirs), "text/plain", size), "BlobNotFound"),
(bref(&record_cid, "text/plain", size), "BlobNotFound"),
(bref(&xc, "image/png", size), "InvalidMimeType"),
(bref(&xc, "image/png", 1), "InvalidMimeType"),
(bref(&xc, "text/plain", size + 1), "InvalidSize"),
(bref(&xc, "text/plain", -1), "InvalidSize"),
(
json!({ "cid": xc, "mimeType": "text/plain" }),
"InvalidRequest",
),
];
for (r, want) in &refused {
assert_eq!(
put(&s, &token, &did, "a", json!({ "deep": [{ "f": r }] })).await,
Err(want.to_string()),
"{r}"
);
let create =
json!({ "repo": did, "collection": "com.example.blob", "record": { "f": r } });
assert_eq!(
call(&s, "com.atproto.repo.createRecord", Some(&token), create).await,
Err(want.to_string()),
"{r}"
);
}
assert_eq!(rev(&s), before, "a refused write makes no commit");
// Everything else a write checks comes first.
put(&s, &token, &did, "k", json!({ "n": 1 })).await.unwrap();
let bad = bref(&raw_cid(b"never"), "text/plain", 5);
let body = json!({ "repo": did, "collection": "com.example.blob", "rkey": "k", "record": { "f": bad }, "swapRecord": record_cid });
assert_eq!(
call(&s, "com.atproto.repo.putRecord", Some(&token), body).await,
Err("InvalidSwap".into())
);
let body = json!({ "repo": did, "collection": "com.example.blob", "rkey": "k", "record": { "f": bad } });
assert_eq!(
call(&s, "com.atproto.repo.createRecord", Some(&token), body).await,
Err("InternalServerError".into())
);
let body = json!({ "repo": did, "collection": "com.example.blob", "rkey": "k", "record": { "$type": "a.b.c", "f": bad } });
assert_eq!(
call(&s, "com.atproto.repo.putRecord", Some(&token), body).await,
Err("InvalidRequest".into())
);
let body = json!({ "repo": bob, "collection": "com.example.blob", "rkey": "k", "record": { "f": bad } });
assert_eq!(
call(&s, "com.atproto.repo.putRecord", Some(&token), body).await,
Err("AuthenticationRequired".into())
);
// The legacy shape before a swap.
let legacy = json!({ "cid": xc, "mimeType": "text/plain" });
let body = json!({ "repo": did, "collection": "com.example.blob", "rkey": "k", "record": { "f": legacy }, "swapRecord": record_cid });
assert_eq!(
call(&s, "com.atproto.repo.putRecord", Some(&token), body).await,
Err("InvalidRequest".into())
);
// Maps that are no blob ref are kept as they are, and reference nothing.
let kept = [
json!({ "cid": "not a cid", "mimeType": "text/plain" }),
json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "text/plain", "size": size, "x": 1 }),
json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "text/plain", "size": -2 }),
json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "textplain", "size": size }),
json!({ "$type": "blob", "ref": xc, "mimeType": "text/plain", "size": size }),
json!({ "$type": "blob", "ref": { "$link": xc, "x": 1 }, "mimeType": "text/plain", "size": size }),
json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "text/plain" }),
json!({ "$type": "blob", "ref": { "$link": xc }, "mimeType": "", "size": "1" }),
json!({ "$type": "Blob", "ref": { "$link": xc }, "mimeType": "text/plain", "size": size }),
];
for (i, k) in kept.iter().enumerate() {
put(&s, &token, &did, &format!("kept{i}"), json!({ "f": k }))
.await
.unwrap();
let p = [
("repo".to_string(), did.clone()),
("collection".into(), "com.example.blob".into()),
("rkey".into(), format!("kept{i}")),
];
let got = call_with(&s, "com.atproto.repo.getRecord", None, &p, json!({}))
.await
.unwrap();
assert_eq!(&got["value"]["f"], k);
}
assert_eq!(listed(&s, &did, None, 10).await, Vec::<String>::new());
// In one write, a ref that matches a new blob lets no other through.
let writes = json!({ "repo": did, "writes": [
{ "$type": "com.atproto.repo.applyWrites#create", "collection": "com.example.blob", "rkey": "a", "value": { "f": bref(&yc, "text/plain", ysize) } },
{ "$type": "com.atproto.repo.applyWrites#create", "collection": "com.example.blob", "rkey": "b", "value": { "f": bref(&yc, "image/png", ysize) } },
] });
assert_eq!(
call(&s, "com.atproto.repo.applyWrites", Some(&token), writes).await,
Err("InvalidMimeType".into())
);
let r =
json!({ "one": bref(&yc, "text/plain", ysize), "two": bref(&yc, "text/plain", -1) });
assert_eq!(
put(&s, &token, &did, "a", r).await,
Err("InvalidSize".into())
);
// Once referenced, a blob takes refs of any type and size.
put(
&s,
&token,
&did,
"a",
json!({ "f": bref(&yc, "text/plain", ysize) }),
)
.await
.unwrap();
put(
&s,
&token,
&did,
"b",
json!({ "f": bref(&yc, "image/png", -1) }),
)
.await
.unwrap();
let (_, headers, bytes) = get(&s, &did, &yc, None).await;
assert_eq!(
(headers[header::CONTENT_TYPE].to_str().unwrap(), bytes),
("text/plain", y)
);
let _ = std::fs::remove_dir_all(&dir);
}
/// An upload is answered unsynced; the commit that first references it
/// syncs its file before the commit's frame. A power cut is modelled as
/// losing every file not synced: the referenced blob survives with its
/// commit, the unreferenced upload is lost, and the server starts again,
/// holding the one and not the other. Under `together`, a commit whose
/// blob cannot be synced, with another account's write waited for after
/// it, which syncs the journal: the commit was never written, so the
/// power cut that loses its blob leaves nothing that references it.
#[tokio::test]
async fn a_referenced_blob_survives_where_its_commit_does() {
for sync in SYNCS {
syncing(sync);
a_referenced_blob_survives_once(sync).await;
}
}
async fn a_referenced_blob_survives_once(sync: crate::Syncing) {
let plc = plc().await;
let dir = scratch("blob-power-cut");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let (kept, lost) = (plain(41), plain(42));
let (kc, lc) = (raw_cid(&kept), raw_cid(&lost));
for b in [&kept, &lost] {
test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false)
.await
.unwrap();
}
let account = s.account(&did).unwrap();
{
let st = read(&account.store);
assert!(
st.blobs.unsynced.contains_key(&kc) && st.blobs.unsynced.contains_key(&lc),
"uploads are answered unsynced"
);
}
put(
&s,
&token,
&did,
"a",
json!({ "f": bref(&kc, "text/plain", kept.len() as i64) }),
)
.await
.unwrap();
let wal = {
let st = read(&account.store);
assert!(
!st.blobs.unsynced.contains_key(&kc),
"the commit referencing it took it to sync"
);
assert!(
st.blobs.unsynced.contains_key(&lc),
"an unreferenced upload stays unsynced"
);
st.wal.clone()
};
// The commit was answered: its frame is on disk, so the blob's file is.
assert!(wal.is_durable(wal.written()));
drop((wal, account));
let mut never_synced = vec![blob_dir(&dir, &did).join(&lc)];
if sync == crate::Syncing::Together {
let (bob, bob_token) = create(&s, "bob").await;
let racy = plain(43);
let rc = raw_cid(&racy);
test_upload(&s, Some(&token), Some("text/plain"), racy.clone(), false)
.await
.unwrap();
// In the file's place a link to itself: it cannot be opened to
// be synced, by any user.
let file = blob_dir(&dir, &did).join(&rc);
let aside = file.with_extension("aside");
std::fs::rename(&file, &aside).unwrap();
std::os::unix::fs::symlink(&file, &file).unwrap();
let r = json!({ "f": bref(&rc, "text/plain", racy.len() as i64) });
let refused = put(&s, &token, &did, "r", r).await;
assert_eq!(refused, Err("InternalServerError".into()));
put(&s, &bob_token, &bob, "x", json!({ "n": 1 }))
.await
.unwrap();
std::fs::remove_file(&file).unwrap();
std::fs::rename(&aside, &file).unwrap();
never_synced.push(file);
}
drop(s);
// The power cut: the files never synced are gone.
for f in &never_synced {
std::fs::remove_file(f).unwrap();
}
let s = open(&dir, &plc);
if sync == crate::Syncing::Together {
let p = [
("repo".to_string(), did.clone()),
("collection".into(), "com.example.blob".into()),
("rkey".into(), "r".into()),
];
assert_eq!(
call_with(&s, "com.atproto.repo.getRecord", None, &p, json!({})).await,
Err("RecordNotFound".into())
);
}
let (status, _, bytes) = get(&s, &did, &kc, None).await;
assert_eq!((status, bytes), (200, kept.clone()));
let r = json!({ "f": bref(&lc, "text/plain", lost.len() as i64) });
assert_eq!(
put(&s, &token, &did, "b", r).await,
Err("BlobNotFound".into())
);
// Uploaded again, it is held, and its next reference syncs it.
test_upload(&s, Some(&token), Some("text/plain"), lost.clone(), false)
.await
.unwrap();
put(
&s,
&token,
&did,
"b",
json!({ "f": bref(&lc, "text/plain", lost.len() as i64) }),
)
.await
.unwrap();
assert!(
!read(&s.account(&did).unwrap().store)
.blobs
.unsynced
.contains_key(&lc)
);
let _ = std::fs::remove_dir_all(&dir);
}
/// A blob is held while some record references it, counted across
/// creates, updates, deletes and applyWrites.
#[tokio::test]
async fn references_are_counted() {
let plc = plc().await;
let dir = scratch("blob-count");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let blobs: Vec<Vec<u8>> = (0..4).map(|i| plain(30 + i)).collect();
let cids: Vec<String> = blobs.iter().map(|b| raw_cid(b)).collect();
for b in &blobs {
test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false)
.await
.unwrap();
}
let r = |i: usize| bref(&cids[i], "text/plain", blobs[i].len() as i64);
// Held but not referenced: neither served nor listed.
assert_eq!(get_error(&s, &did, &cids[0], None).await, "InvalidRequest");
assert_eq!(listed(&s, &did, None, 3).await, Vec::<String>::new());
// Two records reference blob 0, one of them twice.
put(&s, &token, &did, "a", json!({ "f": r(0), "g": [r(0)] }))
.await
.unwrap();
put(&s, &token, &did, "b", json!({ "f": r(0) }))
.await
.unwrap();
assert_eq!(get(&s, &did, &cids[0], None).await.0, 200);
delete(&s, &token, &did, "a").await.unwrap();
assert_eq!(get(&s, &did, &cids[0], None).await.0, 200);
// Replaced by a record referencing another: blob 0 goes.
put(&s, &token, &did, "b", json!({ "f": r(1) }))
.await
.unwrap();
assert_eq!(get_error(&s, &did, &cids[0], None).await, "InvalidRequest");
assert_eq!(
put(&s, &token, &did, "c", json!({ "f": r(0) })).await,
Err("BlobNotFound".into())
);
assert!(!files(&dir, &did).contains(&cids[0]));
assert_eq!(listed(&s, &did, None, 3).await, vec![cids[1].clone()]);
// Uploaded again, with another type, it is stored afresh.
test_upload(&s, Some(&token), Some("image/x"), blobs[0].clone(), false)
.await
.unwrap();
put(
&s,
&token,
&did,
"c",
json!({ "f": bref(&cids[0], "image/x", blobs[0].len() as i64) }),
)
.await
.unwrap();
assert_eq!(
get(&s, &did, &cids[0], None).await.1[header::CONTENT_TYPE],
"image/x"
);
// putRecord keeping the ref in the same record keeps the blob.
put(&s, &token, &did, "b", json!({ "f": r(1), "n": 2 }))
.await
.unwrap();
assert_eq!(get(&s, &did, &cids[1], None).await.0, 200);
// applyWrites deleting one record and referencing its blob from another keeps it.
let w = |t: &str, rkey: &str, value: Option<Json>| {
let mut w = json!({ "$type": format!("com.atproto.repo.applyWrites#{t}"), "collection": "com.example.blob", "rkey": rkey });
if let Some(v) = value {
w["value"] = v;
}
w
};
let batch = |writes: Vec<Json>| json!({ "repo": did, "writes": writes });
call(
&s,
"com.atproto.repo.applyWrites",
Some(&token),
batch(vec![
w("delete", "b", None),
w("create", "d", Some(json!({ "f": r(1) }))),
]),
)
.await
.unwrap();
assert_eq!(get(&s, &did, &cids[1], None).await.0, 200);
// References are counted as the commit leaves the records: a batch
// whose create references blob 2 and whose update of the same record
// then does not leaves no record referencing it, before or after, so
// it stays held, unreferenced.
call(
&s,
"com.atproto.repo.applyWrites",
Some(&token),
batch(vec![
w("create", "e", Some(json!({ "f": r(2) }))),
w("update", "e", Some(json!({ "f": r(3) }))),
]),
)
.await
.unwrap();
assert_eq!(get_error(&s, &did, &cids[2], None).await, "InvalidRequest");
assert_eq!(get(&s, &did, &cids[3], None).await.0, 200);
let mut want = vec![cids[0].clone(), cids[1].clone(), cids[3].clone()];
want.sort();
assert_eq!(listed(&s, &did, None, 2).await, want);
// Deleting the rest drops them, files too, once the writes are on disk.
call(
&s,
"com.atproto.repo.applyWrites",
Some(&token),
batch(vec![
w("delete", "c", None),
w("delete", "d", None),
w("delete", "e", None),
]),
)
.await
.unwrap();
assert_eq!(listed(&s, &did, None, 2).await, Vec::<String>::new());
assert_eq!(files(&dir, &did), vec![cids[2].clone()]);
let _ = std::fs::remove_dir_all(&dir);
}
/// Writes at once, each referencing the same blob from its own record,
/// count every reference.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn concurrent_writes_count_right() {
let plc = plc().await;
let dir = scratch("blob-concurrent");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let b = plain(40);
let cid = raw_cid(&b);
test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false)
.await
.unwrap();
let r = bref(&cid, "text/plain", b.len() as i64);
let spawn_all = |f: &dyn Fn(usize) -> tokio::task::JoinHandle<Result<Json, String>>| {
(0..16).map(f).collect::<Vec<_>>()
};
for h in spawn_all(&|i| {
let (s, token, did, r) = (s.clone(), token.clone(), did.clone(), r.clone());
tokio::spawn(
async move { put(&s, &token, &did, &format!("r{i}"), json!({ "f": r })).await },
)
}) {
h.await.unwrap().unwrap();
}
assert_eq!(read(&s.account(&did).unwrap().store).blobs.refs[&cid], 16);
// Uploads of it meanwhile change nothing.
let deletes = spawn_all(&|i| {
let (s, token, did) = (s.clone(), token.clone(), did.clone());
tokio::spawn(async move { delete(&s, &token, &did, &format!("r{i}")).await })
});
let uploads: Vec<_> = (0..4)
.map(|_| {
let (s, token, b) = (s.clone(), token.clone(), b.clone());
tokio::spawn(async move {
test_upload(&s, Some(&token), Some("image/y"), b, true).await
})
})
.collect();
for h in deletes {
h.await.unwrap().unwrap();
}
for h in uploads {
h.await.unwrap().unwrap();
}
assert!(
!read(&s.account(&did).unwrap().store)
.blobs
.refs
.contains_key(&cid)
);
assert_eq!(get_error(&s, &did, &cid, None).await, "InvalidRequest");
// Whether an upload came after the last delete or before, the files
// are what is held: the blob's if held, nothing else.
let held = read(&s.account(&did).unwrap().store)
.blobs
.held
.contains_key(&cid);
assert_eq!(
files(&dir, &did),
if held { vec![cid.clone()] } else { vec![] }
);
let _ = std::fs::remove_dir_all(&dir);
}
/// Over HTTP, a client still sending a body that is too large, with its
/// `Content-Length` or chunked, or refused for its token, reads the answer.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn refusals_reach_a_client_still_sending() {
let plc = plc().await;
let dir = scratch("blob-http");
let s = open(&dir, &plc);
let (_, token) = create(&s, "alice").await;
let app = axum::Router::new()
.route(
"/xrpc/com.atproto.repo.uploadBlob",
axum::routing::any(upload),
)
.with_state(s.clone());
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.unwrap();
let url = format!(
"http://{}/xrpc/com.atproto.repo.uploadBlob",
listener.local_addr().unwrap()
);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let answers = tokio::task::spawn_blocking(move || {
let http = reqwest::blocking::Client::builder()
.no_proxy()
.build()
.unwrap();
let send = |token: &str, body: reqwest::blocking::Body| {
let res = http
.post(&url)
.bearer_auth(token)
.header("content-type", "text/plain")
.body(body)
.send()
.unwrap();
let status = res.status().as_u16();
(
status,
serde_json::from_slice::<Json>(&res.bytes().unwrap()).unwrap()["error"].clone(),
)
};
let big = || vec![b'a'; DEFAULT_UPLOAD_LIMIT as usize + 1];
vec![
send(&token, big().into()),
send(&token, reqwest::blocking::Body::new(io::Cursor::new(big()))),
send("forged", big().into()),
send(&token, plain(1).into()),
]
})
.await
.unwrap();
let too_large = (413, json!("PayloadTooLarge"));
assert_eq!(
answers,
vec![
too_large.clone(),
too_large,
(400, json!("InvalidToken")),
(200, Json::Null)
]
);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test]
async fn get_blob_answers_and_headers() {
let plc = plc().await;
let dir = scratch("blob-get");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let (_, other) = create(&s, "bob").await;
let b = vec![5u8; 300_000];
let cid = raw_cid(&b);
test_upload(
&s,
Some(&token),
Some("application/x-thing"),
b.clone(),
true,
)
.await
.unwrap();
put(
&s,
&token,
&did,
"a",
json!({ "f": bref(&cid, "application/x-thing", b.len() as i64) }),
)
.await
.unwrap();
// Any account's token, or none, is the same.
for t in [None, Some(token.as_str()), Some(other.as_str())] {
let (status, h, bytes) = get(&s, &did, &cid, t).await;
assert_eq!(status, 200);
assert_eq!(bytes, b);
assert_eq!(h[header::CONTENT_TYPE], "application/x-thing");
assert_eq!(h[header::CONTENT_LENGTH], "300000");
assert_eq!(h[header::X_CONTENT_TYPE_OPTIONS], "nosniff");
assert_eq!(
h[header::CONTENT_DISPOSITION].to_str().unwrap(),
format!("attachment; filename=\"{cid}\"")
);
assert_eq!(
h[header::CONTENT_SECURITY_POLICY],
"default-src 'none'; sandbox"
);
}
// Parameters first, then the token, then the account, then the blob.
assert_eq!(
get_error(&s, "not a did", &cid, Some("forged")).await,
"InvalidRequest"
);
assert_eq!(
get_error(&s, &did, "not a cid", Some("forged")).await,
"InvalidRequest"
);
assert_eq!(
get_error(&s, "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", &cid, Some("forged")).await,
"InvalidToken"
);
assert_eq!(
get_error(&s, "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", &cid, None).await,
"RepoNotFound"
);
assert_eq!(
get_error(&s, &did, &raw_cid(b"never"), None).await,
"InvalidRequest"
);
let record = cid_string(&records_of(&s, &did)[0].1.cid);
assert_eq!(get_error(&s, &did, &record, None).await, "InvalidRequest");
let bob = s.account("bob.test").unwrap().did.clone();
assert_eq!(get_error(&s, &bob, &cid, None).await, "InvalidRequest");
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test]
async fn list_blobs_pages_and_since() {
let plc = plc().await;
let dir = scratch("blob-list");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let blobs: Vec<Vec<u8>> = (0..7).map(|i| plain(50 + i)).collect();
let cids: Vec<String> = blobs.iter().map(|b| raw_cid(b)).collect();
for b in &blobs {
test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false)
.await
.unwrap();
}
let r = |i: usize| bref(&cids[i], "text/plain", blobs[i].len() as i64);
let rev = || repo_of(&s, &did, |r| r.rev.clone());
let start = rev();
put(&s, &token, &did, "a", json!({ "f": [r(0), r(1)] }))
.await
.unwrap();
let after_a = rev();
put(&s, &token, &did, "b", json!({ "f": r(2) }))
.await
.unwrap();
put(&s, &token, &did, "c", json!({ "f": r(3) }))
.await
.unwrap();
let after_c = rev();
// An update lists every blob the record references now.
put(&s, &token, &did, "a", json!({ "f": [r(0), r(1), r(4)] }))
.await
.unwrap();
put(&s, &token, &did, "d", json!({ "f": r(5) }))
.await
.unwrap();
delete(&s, &token, &did, "d").await.unwrap();
let sorted = |mut v: Vec<String>| {
v.sort();
v
};
let all = sorted(cids[..5].to_vec());
for limit in [1, 2, 3, 500] {
assert_eq!(listed(&s, &did, None, limit).await, all, "{limit}");
}
assert_eq!(listed(&s, &did, Some(&start), 2).await, all);
assert_eq!(listed(&s, &did, Some(&after_a), 2).await, all);
assert_eq!(
listed(&s, &did, Some(&after_c), 2).await,
sorted(vec![cids[0].clone(), cids[1].clone(), cids[4].clone()])
);
assert_eq!(
listed(&s, &did, Some(&rev()), 2).await,
Vec::<String>::new()
);
assert_eq!(
listed(&s, &did, Some("jzzzzzzzzzzzz"), 2).await,
Vec::<String>::new()
);
assert_eq!(listed(&s, &did, Some("2222222222222"), 2).await, all);
// A cursor need not be a CID; an empty one is none.
let page = |cursor: &str| {
let p = vec![
("did".to_string(), did.clone()),
("cursor".to_string(), cursor.to_string()),
];
let s = s.clone();
async move { call_with(&s, "com.atproto.sync.listBlobs", None, &p, json!({})).await }
};
assert_eq!(
page("").await.unwrap(),
json!({ "cids": all, "cursor": all.last().unwrap() })
);
assert_eq!(page("bafkrei").await.unwrap()["cids"], json!(all));
assert_eq!(page("c").await.unwrap(), json!({ "cids": [] }));
assert_eq!(page(&all[2]).await.unwrap()["cids"], json!(all[3..]));
let bad = |p: Vec<(&str, &str)>| {
let p: Vec<(String, String)> = p
.into_iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
let s = s.clone();
async move { call_with(&s, "com.atproto.sync.listBlobs", None, &p, json!({})).await }
};
assert_eq!(
bad(vec![("did", &did), ("since", "notatid")]).await,
Err("InvalidRequest".into())
);
assert_eq!(
bad(vec![("did", &did), ("limit", "0")]).await,
Err("InvalidRequest".into())
);
assert_eq!(
bad(vec![("did", &did), ("limit", "1001")]).await,
Err("InvalidRequest".into())
);
assert_eq!(
bad(vec![("did", "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa")]).await,
Err("RepoNotFound".into())
);
let p = [("did".to_string(), did.clone())];
assert_eq!(
call_with(
&s,
"com.atproto.sync.listBlobs",
Some("forged"),
&p,
json!({})
)
.await,
Err("InvalidToken".into())
);
assert!(
call_with(
&s,
"com.atproto.sync.listBlobs",
Some(&token),
&p,
json!({})
)
.await
.is_ok()
);
let _ = std::fs::remove_dir_all(&dir);
}
/// A restart keeps every upload, referenced or not, with its first
/// type; a blob dropped before it stays gone; stray files go.
#[tokio::test]
async fn restart_keeps_blobs() {
let plc = plc().await;
let dir = scratch("blob-restart");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let (kept, loose, dropped) = (plain(60), plain(61), plain(62));
for b in [&kept, &loose, &dropped] {
test_upload(&s, Some(&token), Some("text/plain"), b.clone(), false)
.await
.unwrap();
}
test_upload(&s, Some(&token), Some("image/other"), loose.clone(), false)
.await
.unwrap();
let r = |b: &[u8]| bref(&raw_cid(b), "text/plain", b.len() as i64);
put(&s, &token, &did, "k", json!({ "f": r(&kept) }))
.await
.unwrap();
put(&s, &token, &did, "d", json!({ "f": r(&dropped) }))
.await
.unwrap();
delete(&s, &token, &did, "d").await.unwrap();
let listed_before = listed(&s, &did, None, 10).await;
drop(s);
std::fs::write(blob_dir(&dir, &did).join(".tmp-left"), b"half").unwrap();
std::fs::write(blob_dir(&dir, &did).join(raw_cid(&dropped)), &dropped).unwrap();
let s = open(&dir, &plc);
assert_eq!(
files(&dir, &did),
sorted_pair(raw_cid(&kept), raw_cid(&loose))
);
assert_eq!(listed(&s, &did, None, 10).await, listed_before);
assert_eq!(get(&s, &did, &raw_cid(&kept), None).await.2, kept);
assert_eq!(
put(
&s,
&token,
&did,
"l",
json!({ "f": bref(&raw_cid(&loose), "image/other", loose.len() as i64) })
)
.await,
Err("InvalidMimeType".into())
);
put(&s, &token, &did, "l", json!({ "f": r(&loose) }))
.await
.unwrap();
assert_eq!(
put(&s, &token, &did, "d", json!({ "f": r(&dropped) })).await,
Err("BlobNotFound".into())
);
drop(s);
// A blob the log holds whose file is gone is not served: the server does not start.
std::fs::remove_file(blob_dir(&dir, &did).join(raw_cid(&loose))).unwrap();
assert!(
try_open(&dir, &plc)
.err()
.unwrap()
.contains("missing or not whole")
);
let _ = std::fs::remove_dir_all(&dir);
}
fn sorted_pair(a: String, b: String) -> Vec<String> {
if a < b { vec![a, b] } else { vec![b, a] }
}
/// The quota: an upload past it is refused and stores nothing; bytes
/// held already cost nothing again; a blob no longer held frees its bytes.
#[tokio::test]
async fn the_quota() {
let plc = plc().await;
let dir = scratch("blob-quota");
let s = open_with(&dir, &plc, |c| c.blob_quota = 100);
let (did, token) = create(&s, "alice").await;
let t = Some(token.as_str());
let (a, b) = (plain(46), plain(36));
assert_eq!((a.len(), b.len()), (60, 50));
test_upload(&s, t, Some("text/plain"), a.clone(), false)
.await
.unwrap();
test_upload(&s, t, Some("text/plain"), a.clone(), true)
.await
.unwrap();
assert_eq!(
test_upload(&s, t, Some("text/plain"), b.clone(), false).await,
Err((400, "BlobQuotaExceeded".into()))
);
assert_eq!(files(&dir, &did), vec![raw_cid(&a)]);
// Referenced, then the record deleted: no longer held, its bytes free.
put(
&s,
&token,
&did,
"r",
json!({ "f": bref(&raw_cid(&a), "text/plain", 60) }),
)
.await
.unwrap();
delete(&s, &token, &did, "r").await.unwrap();
test_upload(&s, t, Some("text/plain"), b.clone(), false)
.await
.unwrap();
assert_eq!(files(&dir, &did), vec![raw_cid(&b)]);
let _ = std::fs::remove_dir_all(&dir);
}
/// An upload no record references is held for its time from when it was
/// last uploaded: after it, a ref to it is `BlobNotFound`, it counts
/// against no quota, its file goes when the server sweeps, and after a
/// restart it is gone too. One referenced in time is kept.
#[tokio::test]
async fn unreferenced_uploads_expire() {
let plc = plc().await;
let dir = scratch("blob-ttl");
let s = open_with(&dir, &plc, |c| (c.blob_ttl, c.blob_quota) = (10, 150));
// The clock stands still but for the skew, so each upload's time is exact.
s.limits.clock.freeze();
let (did, token) = create(&s, "alice").await;
let t = Some(token.as_str());
let (kept, lost, late) = (plain(1), plain(2), plain(80));
for b in [&kept, &lost] {
test_upload(&s, t, Some("text/plain"), b.clone(), false)
.await
.unwrap();
}
let r = |b: &[u8], mime: &str| json!({ "f": bref(&raw_cid(b), mime, b.len() as i64) });
put(&s, &token, &did, "kept", r(&kept, "text/plain"))
.await
.unwrap();
// Uploaded again, the time starts again.
s.limits.clock.skew.store(6, Ordering::SeqCst);
test_upload(&s, t, Some("image/whatever"), lost.clone(), false)
.await
.unwrap();
s.limits.clock.skew.store(12, Ordering::SeqCst);
put(&s, &token, &did, "lost", r(&lost, "text/plain"))
.await
.unwrap();
delete(&s, &token, &did, "lost").await.unwrap();
test_upload(&s, t, Some("text/plain"), lost.clone(), false)
.await
.unwrap();
test_upload(&s, t, Some("text/plain"), late.clone(), false)
.await
.unwrap();
s.limits.clock.skew.store(30, Ordering::SeqCst);
assert_eq!(
put(&s, &token, &did, "lost", r(&lost, "text/plain")).await,
Err("BlobNotFound".into())
);
// Its bytes count against no quota, though its file is there until a sweep.
let big = plain(120);
test_upload(&s, t, Some("text/plain"), big.clone(), false)
.await
.unwrap();
assert!(files(&dir, &did).contains(&raw_cid(&late)));
sweep(&s);
let mut want = vec![raw_cid(&kept), raw_cid(&big)];
want.sort();
assert_eq!(files(&dir, &did), want);
assert_eq!(get(&s, &did, &raw_cid(&kept), None).await.0, 200);
// Stored afresh, with its new type.
s.limits.clock.skew.store(60, Ordering::SeqCst);
test_upload(&s, t, Some("text/other"), lost.clone(), false)
.await
.unwrap();
put(&s, &token, &did, "again", r(&lost, "text/other"))
.await
.unwrap();
let blobs = |s: &Service| read(&s.account(&did).unwrap().store).blobs.snapshot();
let before = blobs(&s);
drop(s);
// After a restart: as it was, and an upload that outlived its time while
// the server was down is gone once it starts.
let s = open_with(&dir, &plc, |c| (c.blob_ttl, c.blob_quota) = (10, 150));
assert_eq!(blobs(&s), before);
s.limits.clock.skew.store(100, Ordering::SeqCst);
assert_eq!(
put(&s, &token, &did, "big", r(&big, "text/plain")).await,
Err("BlobNotFound".into())
);
sweep(&s);
let mut want = vec![raw_cid(&kept), raw_cid(&lost)];
want.sort();
assert_eq!(files(&dir, &did), want);
drop(s);
let s = open(&dir, &plc);
assert_eq!(
put(&s, &token, &did, "big", r(&big, "text/plain")).await,
Err("BlobNotFound".into())
);
let _ = std::fs::remove_dir_all(&dir);
}
}