chadtmiller.com/monolith
An atproto PDS
0
0
//! `pds migrate-blobs --to bucket|disk` (`spec/BLOBS.md`, "Moving them"):
//! a data directory's blobs moved from where it keeps them, its files or
//! the bucket's objects (`blobstore`), to the other place, while it holds
//! the directory and no server runs on it.
//!
//! The directory is marked as moving its blobs (`blobstore::mark_moving`)
//! before anything else of the move is done, and the mark taken away once
//! it is done: while it is there, `pds serve` refuses the directory. Then:
//!
//! 1. **Copy**, unless the directory keeps its blobs where they go already:
//! every blob each account holds (`Blobs::holding`), referenced or not,
//! taken down or not, read where it is kept, held to its CID, and written
//! to the other place: into the bucket as its object, each account's log
//! saying first, durably, that the bucket may keep them (`blobPut`), so
//! that a move turned back deletes them; into the data directory as its
//! file, synced, and the account's blob directory after. A blob missing,
//! or not its CID's bytes, is noted and the rest copied, and then the
//! move stops, naming each; a request the store fails stops it at once.
//! 2. **Switch**: the directory made to keep its blobs where they go, its
//! mark made or taken away, durably (`blobstore::keep_in`).
//! 3. **Clean**: what is left where they were, deleted. Into the bucket:
//! each account's blob files, and the objects of its strays, logged as
//! the deleter logs them; into the data directory: every object the
//! bucket may keep of each account, and of each blob it holds, then
//! `blobObjectsDeleted` logged. Either way, the objects of deleted
//! accounts the service log owes (`blobstore::Doomed`), logged deleted.
//! 4. **Finish**: the mark taken away.
//!
//! A run after a kill or a stop does each step again, whole: a copy is made
//! again, a delete of what is gone is answered as done. One the other way
//! finds the directory keeping its blobs where they were before the switch,
//! and only cleans; after it, copies them back. The accounts' stores are
//! loaded one at a time, and let go after each step.
use crate::blobs::{blob_dir, write_synced};
use crate::blobstore::{self, Bucket, Place};
use crate::{Account, Config, Service, read};
use pds_repo::data::{cid_of, cid_string};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
/// How many blobs are copied, or objects deleted, at once.
const AT_ONCE: usize = 16;
/// What a move moved: the blobs the accounts hold, and their bytes.
#[derive(Default, Debug, PartialEq, Eq)]
pub(crate) struct Moved {
pub blobs: u64,
pub bytes: u64,
}
/// Move `dir`'s blobs `to` the bucket `config` names, or out of it, the
/// directory held: what the accounts hold, every blob now there; nothing
/// if they are there already, with no move unfinished; or why it stopped,
/// the move unfinished.
pub(crate) async fn run(dir: &Path, config: Config, to: Place) -> Result<Moved, String> {
let kept = blobstore::kept_in(dir)?;
let moving = blobstore::moving(dir)?;
if moving.is_none() && kept == to {
return Ok(Moved::default());
}
if moving != Some(Some(to)) {
blobstore::mark_moving(dir, Some(to))?;
}
let svc =
Service::open(dir, config).map_err(|e| format!("cannot recover {}: {e}", dir.display()))?;
let bucket = svc
.ctx
.bucket
.clone()
.ok_or("PDS_BLOBSTORE_S3_BUCKET is not set")?;
let unfinished = |why: String| {
format!(
"{why}\nthe move of the blobs {} is unfinished: `pds migrate-blobs --to {}` carries on with it, and `pds migrate-blobs --to {}` moves them back",
to.towards(),
to.name(),
to.other().name(),
)
};
if kept != to {
copy(&svc, &bucket, to).await.map_err(unfinished)?;
blobstore::keep_in(dir, to)?;
}
let moved = clean(&svc, &bucket, to).await.map_err(unfinished)?;
blobstore::mark_moving(dir, None)?;
Ok(moved)
}
/// Every account, in the order they were made, each loaded as it is reached.
fn accounts(svc: &Service) -> impl Iterator<Item = Arc<Account>> + '_ {
let dids = read(&svc.directory).dids();
dids.into_iter().filter_map(|did| svc.by_did(&did))
}
/// An account's store, loaded: why not, if it cannot be.
fn loaded(a: &Account) -> Result<crate::cache::StoreWrite<'_>, String> {
a.store
.load()
.map_err(|e| format!("cannot recover {}: {e}", a.did))?;
Ok(crate::write(&a.store))
}
/// Run `job` on each of `items`, `AT_ONCE` at a time, handing each answer
/// to `done` as it comes: the first error, the jobs left stopped.
async fn each<T, R, Fut>(
items: impl IntoIterator<Item = T>,
job: impl Fn(T) -> Fut,
mut done: impl FnMut(R),
) -> Result<(), String>
where
R: Send + 'static,
Fut: Future<Output = Result<R, String>> + Send + 'static,
{
let mut jobs = tokio::task::JoinSet::new();
let mut items = items.into_iter();
loop {
while jobs.len() < AT_ONCE
&& let Some(x) = items.next()
{
jobs.spawn(job(x));
}
let Some(answer) = jobs.join_next().await else {
return Ok(());
};
done(answer.map_err(|e| e.to_string())??);
}
}
/// Copy every blob each account holds from where the directory keeps them
/// `to` the other place: why not, naming each blob whose bytes are missing
/// there or are not its CID's.
async fn copy(svc: &Service, bucket: &Arc<Bucket>, to: Place) -> Result<(), String> {
let (now, ttl) = (svc.limits.clock.now(), svc.config.blob_ttl);
let mut bad: Vec<String> = Vec::new();
for a in accounts(svc) {
let held = {
let mut st = loaded(&a)?;
let st = &mut *st;
let held = st.blobs.holding(now, ttl);
if to == Place::Bucket {
let cids: Vec<String> = held.iter().map(|(cid, _)| cid.clone()).collect();
st.blobs
.note_kept(&st.wal, &cids)
.and_then(|n| n.map_or(Ok(()), |n| st.wal.wait(n)))
.map_err(|e| format!("{}: logging its objects: {e}", a.did))?;
}
held
};
let dir = blob_dir(&svc.dir, &a.did);
each(
held,
|(cid, size)| copy_one(bucket.clone(), dir.clone(), a.did.clone(), cid, size, to),
|b| bad.extend(b),
)
.await?;
if to == Place::Disk && dir.exists() {
pds_store::sync_dir(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
}
a.store.evict();
}
if bad.is_empty() {
return Ok(());
}
let within = match to.other() {
Place::Disk => "in the data directory",
Place::Bucket => "in the bucket",
};
Err(format!(
"blobs missing {within}, or whose bytes there are not their CID's:\n{}",
bad.join("\n")
))
}
/// Copy `did`'s blob `cid`, of `size` bytes, from where the directory keeps
/// it `to` the other place: a note of it if its bytes are missing there or
/// are not its CID's; why not, if it could not be read or written.
async fn copy_one(
bucket: Arc<Bucket>,
dir: PathBuf,
did: String,
cid: String,
size: u64,
to: Place,
) -> Result<Option<String>, String> {
let bad = |what: &str| Ok(Some(format!("{did} {cid}: {what}")));
match to {
Place::Bucket => {
let path = dir.join(&cid);
let file = path.clone();
let bytes = match tokio::task::spawn_blocking(move || std::fs::read(file))
.await
.map_err(|e| e.to_string())?
{
Ok(b) => b,
Err(e) if e.kind() == io::ErrorKind::NotFound => return bad("its file is missing"),
Err(e) => return Err(format!("{}: {e}", path.display())),
};
if !bytes_of(&cid, &bytes) {
return bad("its file's bytes are not its CID's");
}
bucket
.put_object(&did, &cid, bytes.into())
.await
.map_err(|why| format!("putting {did}'s object {cid}: {why}"))?;
}
Place::Disk => {
let fetched = bucket
.fetch(&did, &cid, size)
.await
.map_err(|why| format!("reading {did}'s object {cid}: {why}"))?;
let Some(bytes) = fetched else {
return bad("its object is missing");
};
if !bytes_of(&cid, &bytes) {
return bad("its object's bytes are not its CID's");
}
tokio::task::spawn_blocking(move || {
write_synced(&dir, &cid, &bytes)
.map_err(|e| format!("{}: {e}", dir.join(&cid).display()))
})
.await
.map_err(|e| e.to_string())??;
}
}
Ok(None)
}
/// Whether `bytes` are those of the blob `cid`.
fn bytes_of(cid: &str, bytes: &[u8]) -> bool {
cid_string(&cid_of(0x55, bytes)) == cid
}
/// Delete what is left where the blobs were, now that the directory keeps
/// them `to`, and the objects of deleted accounts the service log owes:
/// what the accounts hold.
async fn clean(svc: &Service, bucket: &Arc<Bucket>, to: Place) -> Result<Moved, String> {
let (now, ttl) = (svc.limits.clock.now(), svc.config.blob_ttl);
let mut moved = Moved::default();
for a in accounts(svc) {
let (held, objects) = {
let st = loaded(&a)?;
let objects = match to {
Place::Bucket => st.blobs.strays().cloned().collect(),
Place::Disk => st.blobs.objects(),
};
(st.blobs.holding(now, ttl), objects)
};
moved.blobs += held.len() as u64;
moved.bytes += held.iter().map(|(_, size)| size).sum::<u64>();
if to == Place::Bucket {
let dir = blob_dir(&svc.dir, &a.did);
if let Err(e) = std::fs::remove_dir_all(&dir)
&& e.kind() != io::ErrorKind::NotFound
{
return Err(format!("{}: {e}", dir.display()));
}
}
delete(bucket, &a.did, &objects).await?;
{
let mut st = loaded(&a)?;
let st = &mut *st;
match to {
Place::Bucket => st.blobs.strays_deleted(&st.wal, &objects),
Place::Disk => st.blobs.objects_deleted(&st.wal),
}
.and_then(|n| n.map_or(Ok(()), |n| st.wal.wait(n)))
.map_err(|e| format!("{}: logging its objects deleted: {e}", a.did))?;
}
a.store.evict();
}
for (did, cids) in bucket.owed() {
delete(bucket, &did, &cids).await?;
svc.log
.append(&[blobstore::deleted_entry(&did, &cids)])
.and_then(|n| svc.log.wait(n))
.map_err(|e| format!("{}: {e}", crate::log::ACCOUNTS_LOG))?;
bucket.forget_doomed(&did, &cids);
}
Ok(moved)
}
/// Delete `did`'s objects `cids`: why not, if the store fails a delete.
async fn delete(bucket: &Arc<Bucket>, did: &str, cids: &[String]) -> Result<(), String> {
each(
cids.iter().cloned(),
|cid| {
let (bucket, did) = (bucket.clone(), did.to_string());
async move {
bucket
.delete(&did, &cid)
.await
.map_err(|why| format!("deleting {did}'s object {cid}: {why}"))
}
},
|()| {},
)
.await
}
#[cfg(test)]
mod tests {
use super::*;
use crate::blobs::test_upload;
use crate::blobstore::testing::*;
use crate::blobstore::{check_dir, kept_in, moving};
use crate::testing::*;
use axum::http::Method;
use serde_json::Value as Json;
/// A server on `dir` with its blobs in itself.
fn open_disk(dir: &Path, plc: &str) -> crate::Shared {
check_dir(dir, false).unwrap();
open(dir, plc)
}
async fn upload(s: &crate::Shared, token: &str, b: &[u8]) {
test_upload(s, Some(token), Some("text/plain"), b.to_vec(), false)
.await
.unwrap();
}
fn has_file(dir: &Path, did: &str, b: &[u8]) -> bool {
blob_dir(dir, did).join(raw_cid(b)).exists()
}
fn key(did: &str, b: &[u8]) -> String {
format!("blocks/{did}/{}", raw_cid(b))
}
fn moved(blobs: &[&[u8]]) -> Moved {
Moved {
blobs: blobs.len() as u64,
bytes: blobs.iter().map(|b| b.len() as u64).sum(),
}
}
/// A move to `to` stopped once it has copied every blob, and, if
/// `switched`, once the directory keeps them there, as a kill leaves it.
async fn stopped(dir: &Path, c: Config, to: Place, switched: bool) {
blobstore::mark_moving(dir, Some(to)).unwrap();
let svc = Service::open(dir, c).unwrap();
let bucket = svc.ctx.bucket.clone().unwrap();
copy(&svc, &bucket, to).await.unwrap();
if switched {
blobstore::keep_in(dir, to).unwrap();
}
}
/// Blobs moved into the bucket and out again: every blob held, its
/// object put and its file gone, then the other way; a server on each
/// serves them, and deletes the objects of blobs it stops holding; a
/// deleted account's objects owed are deleted by the move out; a
/// directory with them where they go already is left as it is.
#[tokio::test]
async fn moves_the_blobs_there_and_back() {
let plc = plc().await;
let fake = Arc::new(Fake::default());
let url = fake_store(fake.clone()).await;
let (dir, sink) = (scratch("move-there"), scratch("move-there-mail"));
let c = || bucket_config(&plc, &sink, &url, true);
let (a, b, x, d) = (body(1), body(2), body(3), body(4));
let s = open_disk(&dir, &plc);
let (alice, token) = create(&s, "alice").await;
let (bob, bob_token) = create(&s, "bob").await;
upload(&s, &token, &a).await;
upload(&s, &token, &b).await;
upload(&s, &bob_token, &x).await;
put(&s, &token, &alice, "a", &a).await.unwrap();
put(&s, &bob_token, &bob, "x", &x).await.unwrap();
drop(s);
assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), Moved::default());
assert_eq!(
run(&dir, c(), Place::Bucket).await.unwrap(),
moved(&[&a, &b, &x])
);
let mut want = vec![key(&alice, &a), key(&alice, &b), key(&bob, &x)];
want.sort();
assert_eq!(fake.keys(), want);
assert!(!blob_dir(&dir, &alice).exists() && !blob_dir(&dir, &bob).exists());
assert_eq!(
(kept_in(&dir).unwrap(), moving(&dir).unwrap()),
(Place::Bucket, None)
);
assert_eq!(
run(&dir, c(), Place::Bucket).await.unwrap(),
Moved::default()
);
assert!(check_dir(&dir, false).is_err());
// Served from the bucket, a blob no longer held has its object deleted.
let s = open_bucket(&dir, &plc, &sink, &url, true);
let bucket = s.ctx.bucket.clone().unwrap();
assert_eq!(get(&s, &alice, &a).await, (200, a.clone()));
assert_eq!(get(&s, &bob, &x).await, (200, x.clone()));
unput(&s, &token, &alice, "a").await;
assert!(blobstore::pass(&s, &bucket).await);
assert!(!fake.has(&key(&alice, &a)));
upload(&s, &token, &d).await;
put(&s, &token, &alice, "d", &d).await.unwrap();
// Bob deleted: his object owed by the service log.
call(
&s,
"com.atproto.server.requestAccountDelete",
Some(&bob_token),
Json::Null,
)
.await
.unwrap();
let mailed = crate::email::testing::last_token(&sink);
call(
&s,
"com.atproto.server.deleteAccount",
None,
serde_json::json!({ "did": bob, "password": "bob-pw", "token": mailed }),
)
.await
.unwrap();
assert!(fake.has(&key(&bob, &x)));
drop((s, bucket));
assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&b, &d]));
assert!(fake.keys().is_empty());
assert!(has_file(&dir, &alice, &b) && has_file(&dir, &alice, &d));
assert_eq!(
(kept_in(&dir).unwrap(), moving(&dir).unwrap()),
(Place::Disk, None)
);
// Nothing owed is left in the logs.
let s = Arc::new(Service::open(&dir, c()).unwrap());
assert!(s.ctx.bucket.clone().unwrap().owed().is_empty());
assert!(store_of(&s, &alice, |st| st.blobs.objects_list().is_none()));
drop(s);
let s = open_disk(&dir, &plc);
assert_eq!(get(&s, &alice, &d).await, (200, d.clone()));
drop(s);
assert!(fake.seen.lock().unwrap().iter().all(|(_, _, ok)| *ok));
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&sink);
}
/// A move stopped before its switch, or after it, carries on to its end
/// with the same `--to`, and goes back with the other: the directory
/// refused meanwhile, and served after from where it keeps its blobs.
#[tokio::test]
async fn a_move_stopped_carries_on_or_goes_back() {
let plc = plc().await;
let fake = Arc::new(Fake::default());
let url = fake_store(fake.clone()).await;
let (dir, sink) = (scratch("move-stopped"), scratch("move-stopped-mail"));
let c = || bucket_config(&plc, &sink, &url, true);
let (a, b) = (body(5), body(6));
let s = open_disk(&dir, &plc);
let (alice, token) = create(&s, "alice").await;
upload(&s, &token, &a).await;
upload(&s, &token, &b).await;
put(&s, &token, &alice, "a", &a).await.unwrap();
drop(s);
let mut both = vec![key(&alice, &a), key(&alice, &b)];
both.sort();
let files = |dir: &Path| (has_file(dir, &alice, &a), has_file(dir, &alice, &b));
// Into the bucket, stopped before the switch, then moved back.
stopped(&dir, c(), Place::Bucket, false).await;
assert_eq!(fake.keys(), both);
assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err());
assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&a, &b]));
assert!(fake.keys().is_empty());
assert_eq!(files(&dir), (true, true));
let s = open_disk(&dir, &plc);
assert_eq!(get(&s, &alice, &a).await, (200, a.clone()));
assert!(store_of(&s, &alice, |st| st.blobs.objects_list().is_none()));
drop(s);
// Into the bucket, stopped after the switch, then carried on.
stopped(&dir, c(), Place::Bucket, true).await;
assert!(check_dir(&dir, true).is_err());
assert_eq!(files(&dir), (true, true));
assert_eq!(
run(&dir, c(), Place::Bucket).await.unwrap(),
moved(&[&a, &b])
);
assert_eq!(files(&dir), (false, false));
assert_eq!(fake.keys(), both);
// Into the data directory, stopped after the switch, then moved back.
stopped(&dir, c(), Place::Disk, true).await;
assert_eq!(files(&dir), (true, true));
assert!(check_dir(&dir, false).is_err());
assert_eq!(
run(&dir, c(), Place::Bucket).await.unwrap(),
moved(&[&a, &b])
);
assert_eq!(files(&dir), (false, false));
assert_eq!(fake.keys(), both);
// Into the data directory, stopped before the switch, its mark's
// bytes lost as a power cut may lose them, and its switch's mark
// written but not renamed: refused still, then moved back, nothing
// of the marks left.
stopped(&dir, c(), Place::Disk, false).await;
assert_eq!(files(&dir), (true, true));
std::fs::write(dir.join("blobstore-moving"), b"").unwrap();
std::fs::write(dir.join("blobstore.tmp"), b"").unwrap();
assert_eq!(moving(&dir).unwrap(), Some(None));
assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err());
assert_eq!(
run(&dir, c(), Place::Bucket).await.unwrap(),
moved(&[&a, &b])
);
assert_eq!(files(&dir), (false, false));
assert!(!dir.join("blobstore.tmp").exists() && !dir.join("blobstore-moving.tmp").exists());
// Served from the bucket, its objects still known to it.
let s = open_bucket(&dir, &plc, &sink, &url, true);
let bucket = s.ctx.bucket.clone().unwrap();
assert_eq!(get(&s, &alice, &a).await, (200, a.clone()));
unput(&s, &token, &alice, "a").await;
assert!(blobstore::pass(&s, &bucket).await);
assert_eq!(fake.keys(), vec![key(&alice, &b)]);
drop((s, bucket));
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&sink);
}
/// A blob whose object is missing, or a request the store fails, stops
/// a move, naming the blob, or saying why; the directory is refused
/// until a move runs to its end.
#[tokio::test]
async fn a_move_stops_at_a_blob_missing_or_a_store_failing() {
let plc = plc().await;
let fake = Arc::new(Fake::default());
let url = fake_store(fake.clone()).await;
let (dir, sink) = (scratch("move-missing"), scratch("move-missing-mail"));
let c = || bucket_config(&plc, &sink, &url, true);
let (a, b) = (body(7), body(8));
let s = open_bucket(&dir, &plc, &sink, &url, true);
let (alice, token) = create(&s, "alice").await;
upload(&s, &token, &a).await;
upload(&s, &token, &b).await;
put(&s, &token, &alice, "a", &a).await.unwrap();
drop(s);
let saved = fake
.objects
.lock()
.unwrap()
.remove(&key(&alice, &a))
.unwrap();
let why = run(&dir, c(), Place::Disk).await.unwrap_err();
assert!(why.contains(&alice) && why.contains(&raw_cid(&a)), "{why}");
assert!(!why.contains(&raw_cid(&b)), "{why}");
assert_eq!(moving(&dir).unwrap(), Some(Some(Place::Disk)));
assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err());
fake.objects.lock().unwrap().insert(key(&alice, &a), saved);
fake.fail(Some((Method::GET, 503)));
let why = run(&dir, c(), Place::Disk).await.unwrap_err();
assert!(why.contains("503"), "{why}");
fake.fail(None);
assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&a, &b]));
let s = open_disk(&dir, &plc);
assert_eq!(get(&s, &alice, &a).await, (200, a.clone()));
drop(s);
assert!(fake.keys().is_empty());
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&sink);
}
}