//! `pds migrate-blobs --to bucket|disk` (`spec/BLOBS.md`, "Moving them"): //! a data directory's blobs moved from where it keeps them, its files or //! the bucket's objects (`blobstore`), to the other place, while it holds //! the directory and no server runs on it. //! //! The directory is marked as moving its blobs (`blobstore::mark_moving`) //! before anything else of the move is done, and the mark taken away once //! it is done: while it is there, `pds serve` refuses the directory. Then: //! //! 1. **Copy**, unless the directory keeps its blobs where they go already: //! every blob each account holds (`Blobs::holding`), referenced or not, //! taken down or not, read where it is kept, held to its CID, and written //! to the other place: into the bucket as its object, each account's log //! saying first, durably, that the bucket may keep them (`blobPut`), so //! that a move turned back deletes them; into the data directory as its //! file, synced, and the account's blob directory after. A blob missing, //! or not its CID's bytes, is noted and the rest copied, and then the //! move stops, naming each; a request the store fails stops it at once. //! 2. **Switch**: the directory made to keep its blobs where they go, its //! mark made or taken away, durably (`blobstore::keep_in`). //! 3. **Clean**: what is left where they were, deleted. Into the bucket: //! each account's blob files, and the objects of its strays, logged as //! the deleter logs them; into the data directory: every object the //! bucket may keep of each account, and of each blob it holds, then //! `blobObjectsDeleted` logged. Either way, the objects of deleted //! accounts the service log owes (`blobstore::Doomed`), logged deleted. //! 4. **Finish**: the mark taken away. //! //! A run after a kill or a stop does each step again, whole: a copy is made //! again, a delete of what is gone is answered as done. One the other way //! finds the directory keeping its blobs where they were before the switch, //! and only cleans; after it, copies them back. The accounts' stores are //! loaded one at a time, and let go after each step. use crate::blobs::{blob_dir, write_synced}; use crate::blobstore::{self, Bucket, Place}; use crate::{Account, Config, Service, read}; use pds_repo::data::{cid_of, cid_string}; use std::io; use std::path::{Path, PathBuf}; use std::sync::Arc; /// How many blobs are copied, or objects deleted, at once. const AT_ONCE: usize = 16; /// What a move moved: the blobs the accounts hold, and their bytes. #[derive(Default, Debug, PartialEq, Eq)] pub(crate) struct Moved { pub blobs: u64, pub bytes: u64, } /// Move `dir`'s blobs `to` the bucket `config` names, or out of it, the /// directory held: what the accounts hold, every blob now there; nothing /// if they are there already, with no move unfinished; or why it stopped, /// the move unfinished. pub(crate) async fn run(dir: &Path, config: Config, to: Place) -> Result { let kept = blobstore::kept_in(dir)?; let moving = blobstore::moving(dir)?; if moving.is_none() && kept == to { return Ok(Moved::default()); } if moving != Some(Some(to)) { blobstore::mark_moving(dir, Some(to))?; } let svc = Service::open(dir, config).map_err(|e| format!("cannot recover {}: {e}", dir.display()))?; let bucket = svc .ctx .bucket .clone() .ok_or("PDS_BLOBSTORE_S3_BUCKET is not set")?; let unfinished = |why: String| { format!( "{why}\nthe move of the blobs {} is unfinished: `pds migrate-blobs --to {}` carries on with it, and `pds migrate-blobs --to {}` moves them back", to.towards(), to.name(), to.other().name(), ) }; if kept != to { copy(&svc, &bucket, to).await.map_err(unfinished)?; blobstore::keep_in(dir, to)?; } let moved = clean(&svc, &bucket, to).await.map_err(unfinished)?; blobstore::mark_moving(dir, None)?; Ok(moved) } /// Every account, in the order they were made, each loaded as it is reached. fn accounts(svc: &Service) -> impl Iterator> + '_ { let dids = read(&svc.directory).dids(); dids.into_iter().filter_map(|did| svc.by_did(&did)) } /// An account's store, loaded: why not, if it cannot be. fn loaded(a: &Account) -> Result, String> { a.store .load() .map_err(|e| format!("cannot recover {}: {e}", a.did))?; Ok(crate::write(&a.store)) } /// Run `job` on each of `items`, `AT_ONCE` at a time, handing each answer /// to `done` as it comes: the first error, the jobs left stopped. async fn each( items: impl IntoIterator, job: impl Fn(T) -> Fut, mut done: impl FnMut(R), ) -> Result<(), String> where R: Send + 'static, Fut: Future> + Send + 'static, { let mut jobs = tokio::task::JoinSet::new(); let mut items = items.into_iter(); loop { while jobs.len() < AT_ONCE && let Some(x) = items.next() { jobs.spawn(job(x)); } let Some(answer) = jobs.join_next().await else { return Ok(()); }; done(answer.map_err(|e| e.to_string())??); } } /// Copy every blob each account holds from where the directory keeps them /// `to` the other place: why not, naming each blob whose bytes are missing /// there or are not its CID's. async fn copy(svc: &Service, bucket: &Arc, to: Place) -> Result<(), String> { let (now, ttl) = (svc.limits.clock.now(), svc.config.blob_ttl); let mut bad: Vec = Vec::new(); for a in accounts(svc) { let held = { let mut st = loaded(&a)?; let st = &mut *st; let held = st.blobs.holding(now, ttl); if to == Place::Bucket { let cids: Vec = held.iter().map(|(cid, _)| cid.clone()).collect(); st.blobs .note_kept(&st.wal, &cids) .and_then(|n| n.map_or(Ok(()), |n| st.wal.wait(n))) .map_err(|e| format!("{}: logging its objects: {e}", a.did))?; } held }; let dir = blob_dir(&svc.dir, &a.did); each( held, |(cid, size)| copy_one(bucket.clone(), dir.clone(), a.did.clone(), cid, size, to), |b| bad.extend(b), ) .await?; if to == Place::Disk && dir.exists() { pds_store::sync_dir(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; } a.store.evict(); } if bad.is_empty() { return Ok(()); } let within = match to.other() { Place::Disk => "in the data directory", Place::Bucket => "in the bucket", }; Err(format!( "blobs missing {within}, or whose bytes there are not their CID's:\n{}", bad.join("\n") )) } /// Copy `did`'s blob `cid`, of `size` bytes, from where the directory keeps /// it `to` the other place: a note of it if its bytes are missing there or /// are not its CID's; why not, if it could not be read or written. async fn copy_one( bucket: Arc, dir: PathBuf, did: String, cid: String, size: u64, to: Place, ) -> Result, String> { let bad = |what: &str| Ok(Some(format!("{did} {cid}: {what}"))); match to { Place::Bucket => { let path = dir.join(&cid); let file = path.clone(); let bytes = match tokio::task::spawn_blocking(move || std::fs::read(file)) .await .map_err(|e| e.to_string())? { Ok(b) => b, Err(e) if e.kind() == io::ErrorKind::NotFound => return bad("its file is missing"), Err(e) => return Err(format!("{}: {e}", path.display())), }; if !bytes_of(&cid, &bytes) { return bad("its file's bytes are not its CID's"); } bucket .put_object(&did, &cid, bytes.into()) .await .map_err(|why| format!("putting {did}'s object {cid}: {why}"))?; } Place::Disk => { let fetched = bucket .fetch(&did, &cid, size) .await .map_err(|why| format!("reading {did}'s object {cid}: {why}"))?; let Some(bytes) = fetched else { return bad("its object is missing"); }; if !bytes_of(&cid, &bytes) { return bad("its object's bytes are not its CID's"); } tokio::task::spawn_blocking(move || { write_synced(&dir, &cid, &bytes) .map_err(|e| format!("{}: {e}", dir.join(&cid).display())) }) .await .map_err(|e| e.to_string())??; } } Ok(None) } /// Whether `bytes` are those of the blob `cid`. fn bytes_of(cid: &str, bytes: &[u8]) -> bool { cid_string(&cid_of(0x55, bytes)) == cid } /// Delete what is left where the blobs were, now that the directory keeps /// them `to`, and the objects of deleted accounts the service log owes: /// what the accounts hold. async fn clean(svc: &Service, bucket: &Arc, to: Place) -> Result { let (now, ttl) = (svc.limits.clock.now(), svc.config.blob_ttl); let mut moved = Moved::default(); for a in accounts(svc) { let (held, objects) = { let st = loaded(&a)?; let objects = match to { Place::Bucket => st.blobs.strays().cloned().collect(), Place::Disk => st.blobs.objects(), }; (st.blobs.holding(now, ttl), objects) }; moved.blobs += held.len() as u64; moved.bytes += held.iter().map(|(_, size)| size).sum::(); if to == Place::Bucket { let dir = blob_dir(&svc.dir, &a.did); if let Err(e) = std::fs::remove_dir_all(&dir) && e.kind() != io::ErrorKind::NotFound { return Err(format!("{}: {e}", dir.display())); } } delete(bucket, &a.did, &objects).await?; { let mut st = loaded(&a)?; let st = &mut *st; match to { Place::Bucket => st.blobs.strays_deleted(&st.wal, &objects), Place::Disk => st.blobs.objects_deleted(&st.wal), } .and_then(|n| n.map_or(Ok(()), |n| st.wal.wait(n))) .map_err(|e| format!("{}: logging its objects deleted: {e}", a.did))?; } a.store.evict(); } for (did, cids) in bucket.owed() { delete(bucket, &did, &cids).await?; svc.log .append(&[blobstore::deleted_entry(&did, &cids)]) .and_then(|n| svc.log.wait(n)) .map_err(|e| format!("{}: {e}", crate::log::ACCOUNTS_LOG))?; bucket.forget_doomed(&did, &cids); } Ok(moved) } /// Delete `did`'s objects `cids`: why not, if the store fails a delete. async fn delete(bucket: &Arc, did: &str, cids: &[String]) -> Result<(), String> { each( cids.iter().cloned(), |cid| { let (bucket, did) = (bucket.clone(), did.to_string()); async move { bucket .delete(&did, &cid) .await .map_err(|why| format!("deleting {did}'s object {cid}: {why}")) } }, |()| {}, ) .await } #[cfg(test)] mod tests { use super::*; use crate::blobs::test_upload; use crate::blobstore::testing::*; use crate::blobstore::{check_dir, kept_in, moving}; use crate::testing::*; use axum::http::Method; use serde_json::Value as Json; /// A server on `dir` with its blobs in itself. fn open_disk(dir: &Path, plc: &str) -> crate::Shared { check_dir(dir, false).unwrap(); open(dir, plc) } async fn upload(s: &crate::Shared, token: &str, b: &[u8]) { test_upload(s, Some(token), Some("text/plain"), b.to_vec(), false) .await .unwrap(); } fn has_file(dir: &Path, did: &str, b: &[u8]) -> bool { blob_dir(dir, did).join(raw_cid(b)).exists() } fn key(did: &str, b: &[u8]) -> String { format!("blocks/{did}/{}", raw_cid(b)) } fn moved(blobs: &[&[u8]]) -> Moved { Moved { blobs: blobs.len() as u64, bytes: blobs.iter().map(|b| b.len() as u64).sum(), } } /// A move to `to` stopped once it has copied every blob, and, if /// `switched`, once the directory keeps them there, as a kill leaves it. async fn stopped(dir: &Path, c: Config, to: Place, switched: bool) { blobstore::mark_moving(dir, Some(to)).unwrap(); let svc = Service::open(dir, c).unwrap(); let bucket = svc.ctx.bucket.clone().unwrap(); copy(&svc, &bucket, to).await.unwrap(); if switched { blobstore::keep_in(dir, to).unwrap(); } } /// Blobs moved into the bucket and out again: every blob held, its /// object put and its file gone, then the other way; a server on each /// serves them, and deletes the objects of blobs it stops holding; a /// deleted account's objects owed are deleted by the move out; a /// directory with them where they go already is left as it is. #[tokio::test] async fn moves_the_blobs_there_and_back() { let plc = plc().await; let fake = Arc::new(Fake::default()); let url = fake_store(fake.clone()).await; let (dir, sink) = (scratch("move-there"), scratch("move-there-mail")); let c = || bucket_config(&plc, &sink, &url, true); let (a, b, x, d) = (body(1), body(2), body(3), body(4)); let s = open_disk(&dir, &plc); let (alice, token) = create(&s, "alice").await; let (bob, bob_token) = create(&s, "bob").await; upload(&s, &token, &a).await; upload(&s, &token, &b).await; upload(&s, &bob_token, &x).await; put(&s, &token, &alice, "a", &a).await.unwrap(); put(&s, &bob_token, &bob, "x", &x).await.unwrap(); drop(s); assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), Moved::default()); assert_eq!( run(&dir, c(), Place::Bucket).await.unwrap(), moved(&[&a, &b, &x]) ); let mut want = vec![key(&alice, &a), key(&alice, &b), key(&bob, &x)]; want.sort(); assert_eq!(fake.keys(), want); assert!(!blob_dir(&dir, &alice).exists() && !blob_dir(&dir, &bob).exists()); assert_eq!( (kept_in(&dir).unwrap(), moving(&dir).unwrap()), (Place::Bucket, None) ); assert_eq!( run(&dir, c(), Place::Bucket).await.unwrap(), Moved::default() ); assert!(check_dir(&dir, false).is_err()); // Served from the bucket, a blob no longer held has its object deleted. let s = open_bucket(&dir, &plc, &sink, &url, true); let bucket = s.ctx.bucket.clone().unwrap(); assert_eq!(get(&s, &alice, &a).await, (200, a.clone())); assert_eq!(get(&s, &bob, &x).await, (200, x.clone())); unput(&s, &token, &alice, "a").await; assert!(blobstore::pass(&s, &bucket).await); assert!(!fake.has(&key(&alice, &a))); upload(&s, &token, &d).await; put(&s, &token, &alice, "d", &d).await.unwrap(); // Bob deleted: his object owed by the service log. call( &s, "com.atproto.server.requestAccountDelete", Some(&bob_token), Json::Null, ) .await .unwrap(); let mailed = crate::email::testing::last_token(&sink); call( &s, "com.atproto.server.deleteAccount", None, serde_json::json!({ "did": bob, "password": "bob-pw", "token": mailed }), ) .await .unwrap(); assert!(fake.has(&key(&bob, &x))); drop((s, bucket)); assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&b, &d])); assert!(fake.keys().is_empty()); assert!(has_file(&dir, &alice, &b) && has_file(&dir, &alice, &d)); assert_eq!( (kept_in(&dir).unwrap(), moving(&dir).unwrap()), (Place::Disk, None) ); // Nothing owed is left in the logs. let s = Arc::new(Service::open(&dir, c()).unwrap()); assert!(s.ctx.bucket.clone().unwrap().owed().is_empty()); assert!(store_of(&s, &alice, |st| st.blobs.objects_list().is_none())); drop(s); let s = open_disk(&dir, &plc); assert_eq!(get(&s, &alice, &d).await, (200, d.clone())); drop(s); assert!(fake.seen.lock().unwrap().iter().all(|(_, _, ok)| *ok)); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&sink); } /// A move stopped before its switch, or after it, carries on to its end /// with the same `--to`, and goes back with the other: the directory /// refused meanwhile, and served after from where it keeps its blobs. #[tokio::test] async fn a_move_stopped_carries_on_or_goes_back() { let plc = plc().await; let fake = Arc::new(Fake::default()); let url = fake_store(fake.clone()).await; let (dir, sink) = (scratch("move-stopped"), scratch("move-stopped-mail")); let c = || bucket_config(&plc, &sink, &url, true); let (a, b) = (body(5), body(6)); let s = open_disk(&dir, &plc); let (alice, token) = create(&s, "alice").await; upload(&s, &token, &a).await; upload(&s, &token, &b).await; put(&s, &token, &alice, "a", &a).await.unwrap(); drop(s); let mut both = vec![key(&alice, &a), key(&alice, &b)]; both.sort(); let files = |dir: &Path| (has_file(dir, &alice, &a), has_file(dir, &alice, &b)); // Into the bucket, stopped before the switch, then moved back. stopped(&dir, c(), Place::Bucket, false).await; assert_eq!(fake.keys(), both); assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err()); assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&a, &b])); assert!(fake.keys().is_empty()); assert_eq!(files(&dir), (true, true)); let s = open_disk(&dir, &plc); assert_eq!(get(&s, &alice, &a).await, (200, a.clone())); assert!(store_of(&s, &alice, |st| st.blobs.objects_list().is_none())); drop(s); // Into the bucket, stopped after the switch, then carried on. stopped(&dir, c(), Place::Bucket, true).await; assert!(check_dir(&dir, true).is_err()); assert_eq!(files(&dir), (true, true)); assert_eq!( run(&dir, c(), Place::Bucket).await.unwrap(), moved(&[&a, &b]) ); assert_eq!(files(&dir), (false, false)); assert_eq!(fake.keys(), both); // Into the data directory, stopped after the switch, then moved back. stopped(&dir, c(), Place::Disk, true).await; assert_eq!(files(&dir), (true, true)); assert!(check_dir(&dir, false).is_err()); assert_eq!( run(&dir, c(), Place::Bucket).await.unwrap(), moved(&[&a, &b]) ); assert_eq!(files(&dir), (false, false)); assert_eq!(fake.keys(), both); // Into the data directory, stopped before the switch, its mark's // bytes lost as a power cut may lose them, and its switch's mark // written but not renamed: refused still, then moved back, nothing // of the marks left. stopped(&dir, c(), Place::Disk, false).await; assert_eq!(files(&dir), (true, true)); std::fs::write(dir.join("blobstore-moving"), b"").unwrap(); std::fs::write(dir.join("blobstore.tmp"), b"").unwrap(); assert_eq!(moving(&dir).unwrap(), Some(None)); assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err()); assert_eq!( run(&dir, c(), Place::Bucket).await.unwrap(), moved(&[&a, &b]) ); assert_eq!(files(&dir), (false, false)); assert!(!dir.join("blobstore.tmp").exists() && !dir.join("blobstore-moving.tmp").exists()); // Served from the bucket, its objects still known to it. let s = open_bucket(&dir, &plc, &sink, &url, true); let bucket = s.ctx.bucket.clone().unwrap(); assert_eq!(get(&s, &alice, &a).await, (200, a.clone())); unput(&s, &token, &alice, "a").await; assert!(blobstore::pass(&s, &bucket).await); assert_eq!(fake.keys(), vec![key(&alice, &b)]); drop((s, bucket)); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&sink); } /// A blob whose object is missing, or a request the store fails, stops /// a move, naming the blob, or saying why; the directory is refused /// until a move runs to its end. #[tokio::test] async fn a_move_stops_at_a_blob_missing_or_a_store_failing() { let plc = plc().await; let fake = Arc::new(Fake::default()); let url = fake_store(fake.clone()).await; let (dir, sink) = (scratch("move-missing"), scratch("move-missing-mail")); let c = || bucket_config(&plc, &sink, &url, true); let (a, b) = (body(7), body(8)); let s = open_bucket(&dir, &plc, &sink, &url, true); let (alice, token) = create(&s, "alice").await; upload(&s, &token, &a).await; upload(&s, &token, &b).await; put(&s, &token, &alice, "a", &a).await.unwrap(); drop(s); let saved = fake .objects .lock() .unwrap() .remove(&key(&alice, &a)) .unwrap(); let why = run(&dir, c(), Place::Disk).await.unwrap_err(); assert!(why.contains(&alice) && why.contains(&raw_cid(&a)), "{why}"); assert!(!why.contains(&raw_cid(&b)), "{why}"); assert_eq!(moving(&dir).unwrap(), Some(Some(Place::Disk))); assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err()); fake.objects.lock().unwrap().insert(key(&alice, &a), saved); fake.fail(Some((Method::GET, 503))); let why = run(&dir, c(), Place::Disk).await.unwrap_err(); assert!(why.contains("503"), "{why}"); fake.fail(None); assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&a, &b])); let s = open_disk(&dir, &plc); assert_eq!(get(&s, &alice, &a).await, (200, a.clone())); drop(s); assert!(fake.keys().is_empty()); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&sink); } }