~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
560 lines · 22.5 KB Raw History
//! `pds migrate-blobs --to bucket|disk` (`spec/BLOBS.md`, "Moving them"):
//! a data directory's blobs moved from where it keeps them, its files or
//! the bucket's objects (`blobstore`), to the other place, while it holds
//! the directory and no server runs on it.
//!
//! The directory is marked as moving its blobs (`blobstore::mark_moving`)
//! before anything else of the move is done, and the mark taken away once
//! it is done: while it is there, `pds serve` refuses the directory. Then:
//!
//! 1. **Copy**, unless the directory keeps its blobs where they go already:
//!    every blob each account holds (`Blobs::holding`), referenced or not,
//!    taken down or not, read where it is kept, held to its CID, and written
//!    to the other place: into the bucket as its object, each account's log
//!    saying first, durably, that the bucket may keep them (`blobPut`), so
//!    that a move turned back deletes them; into the data directory as its
//!    file, synced, and the account's blob directory after. A blob missing,
//!    or not its CID's bytes, is noted and the rest copied, and then the
//!    move stops, naming each; a request the store fails stops it at once.
//! 2. **Switch**: the directory made to keep its blobs where they go, its
//!    mark made or taken away, durably (`blobstore::keep_in`).
//! 3. **Clean**: what is left where they were, deleted. Into the bucket:
//!    each account's blob files, and the objects of its strays, logged as
//!    the deleter logs them; into the data directory: every object the
//!    bucket may keep of each account, and of each blob it holds, then
//!    `blobObjectsDeleted` logged. Either way, the objects of deleted
//!    accounts the service log owes (`blobstore::Doomed`), logged deleted.
//! 4. **Finish**: the mark taken away.
//!
//! A run after a kill or a stop does each step again, whole: a copy is made
//! again, a delete of what is gone is answered as done. One the other way
//! finds the directory keeping its blobs where they were before the switch,
//! and only cleans; after it, copies them back. The accounts' stores are
//! loaded one at a time, and let go after each step.

use crate::blobs::{blob_dir, write_synced};
use crate::blobstore::{self, Bucket, Place};
use crate::{Account, Config, Service, read};
use pds_repo::data::{cid_of, cid_string};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;

/// How many blobs are copied, or objects deleted, at once.
const AT_ONCE: usize = 16;

/// What a move moved: the blobs the accounts hold, and their bytes.
#[derive(Default, Debug, PartialEq, Eq)]
pub(crate) struct Moved {
    pub blobs: u64,
    pub bytes: u64,
}

/// Move `dir`'s blobs `to` the bucket `config` names, or out of it, the
/// directory held: what the accounts hold, every blob now there; nothing
/// if they are there already, with no move unfinished; or why it stopped,
/// the move unfinished.
pub(crate) async fn run(dir: &Path, config: Config, to: Place) -> Result<Moved, String> {
    let kept = blobstore::kept_in(dir)?;
    let moving = blobstore::moving(dir)?;
    if moving.is_none() && kept == to {
        return Ok(Moved::default());
    }
    if moving != Some(Some(to)) {
        blobstore::mark_moving(dir, Some(to))?;
    }
    let svc =
        Service::open(dir, config).map_err(|e| format!("cannot recover {}: {e}", dir.display()))?;
    let bucket = svc
        .ctx
        .bucket
        .clone()
        .ok_or("PDS_BLOBSTORE_S3_BUCKET is not set")?;
    let unfinished = |why: String| {
        format!(
            "{why}\nthe move of the blobs {} is unfinished: `pds migrate-blobs --to {}` carries on with it, and `pds migrate-blobs --to {}` moves them back",
            to.towards(),
            to.name(),
            to.other().name(),
        )
    };
    if kept != to {
        copy(&svc, &bucket, to).await.map_err(unfinished)?;
        blobstore::keep_in(dir, to)?;
    }
    let moved = clean(&svc, &bucket, to).await.map_err(unfinished)?;
    blobstore::mark_moving(dir, None)?;
    Ok(moved)
}

/// Every account, in the order they were made, each loaded as it is reached.
fn accounts(svc: &Service) -> impl Iterator<Item = Arc<Account>> + '_ {
    let dids = read(&svc.directory).dids();
    dids.into_iter().filter_map(|did| svc.by_did(&did))
}

/// An account's store, loaded: why not, if it cannot be.
fn loaded(a: &Account) -> Result<crate::cache::StoreWrite<'_>, String> {
    a.store
        .load()
        .map_err(|e| format!("cannot recover {}: {e}", a.did))?;
    Ok(crate::write(&a.store))
}

/// Run `job` on each of `items`, `AT_ONCE` at a time, handing each answer
/// to `done` as it comes: the first error, the jobs left stopped.
async fn each<T, R, Fut>(
    items: impl IntoIterator<Item = T>,
    job: impl Fn(T) -> Fut,
    mut done: impl FnMut(R),
) -> Result<(), String>
where
    R: Send + 'static,
    Fut: Future<Output = Result<R, String>> + Send + 'static,
{
    let mut jobs = tokio::task::JoinSet::new();
    let mut items = items.into_iter();
    loop {
        while jobs.len() < AT_ONCE
            && let Some(x) = items.next()
        {
            jobs.spawn(job(x));
        }
        let Some(answer) = jobs.join_next().await else {
            return Ok(());
        };
        done(answer.map_err(|e| e.to_string())??);
    }
}

/// Copy every blob each account holds from where the directory keeps them
/// `to` the other place: why not, naming each blob whose bytes are missing
/// there or are not its CID's.
async fn copy(svc: &Service, bucket: &Arc<Bucket>, to: Place) -> Result<(), String> {
    let (now, ttl) = (svc.limits.clock.now(), svc.config.blob_ttl);
    let mut bad: Vec<String> = Vec::new();
    for a in accounts(svc) {
        let held = {
            let mut st = loaded(&a)?;
            let st = &mut *st;
            let held = st.blobs.holding(now, ttl);
            if to == Place::Bucket {
                let cids: Vec<String> = held.iter().map(|(cid, _)| cid.clone()).collect();
                st.blobs
                    .note_kept(&st.wal, &cids)
                    .and_then(|n| n.map_or(Ok(()), |n| st.wal.wait(n)))
                    .map_err(|e| format!("{}: logging its objects: {e}", a.did))?;
            }
            held
        };
        let dir = blob_dir(&svc.dir, &a.did);
        each(
            held,
            |(cid, size)| copy_one(bucket.clone(), dir.clone(), a.did.clone(), cid, size, to),
            |b| bad.extend(b),
        )
        .await?;
        if to == Place::Disk && dir.exists() {
            pds_store::sync_dir(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
        }
        a.store.evict();
    }
    if bad.is_empty() {
        return Ok(());
    }
    let within = match to.other() {
        Place::Disk => "in the data directory",
        Place::Bucket => "in the bucket",
    };
    Err(format!(
        "blobs missing {within}, or whose bytes there are not their CID's:\n{}",
        bad.join("\n")
    ))
}

/// Copy `did`'s blob `cid`, of `size` bytes, from where the directory keeps
/// it `to` the other place: a note of it if its bytes are missing there or
/// are not its CID's; why not, if it could not be read or written.
async fn copy_one(
    bucket: Arc<Bucket>,
    dir: PathBuf,
    did: String,
    cid: String,
    size: u64,
    to: Place,
) -> Result<Option<String>, String> {
    let bad = |what: &str| Ok(Some(format!("{did} {cid}: {what}")));
    match to {
        Place::Bucket => {
            let path = dir.join(&cid);
            let file = path.clone();
            let bytes = match tokio::task::spawn_blocking(move || std::fs::read(file))
                .await
                .map_err(|e| e.to_string())?
            {
                Ok(b) => b,
                Err(e) if e.kind() == io::ErrorKind::NotFound => return bad("its file is missing"),
                Err(e) => return Err(format!("{}: {e}", path.display())),
            };
            if !bytes_of(&cid, &bytes) {
                return bad("its file's bytes are not its CID's");
            }
            bucket
                .put_object(&did, &cid, bytes.into())
                .await
                .map_err(|why| format!("putting {did}'s object {cid}: {why}"))?;
        }
        Place::Disk => {
            let fetched = bucket
                .fetch(&did, &cid, size)
                .await
                .map_err(|why| format!("reading {did}'s object {cid}: {why}"))?;
            let Some(bytes) = fetched else {
                return bad("its object is missing");
            };
            if !bytes_of(&cid, &bytes) {
                return bad("its object's bytes are not its CID's");
            }
            tokio::task::spawn_blocking(move || {
                write_synced(&dir, &cid, &bytes)
                    .map_err(|e| format!("{}: {e}", dir.join(&cid).display()))
            })
            .await
            .map_err(|e| e.to_string())??;
        }
    }
    Ok(None)
}

/// Whether `bytes` are those of the blob `cid`.
fn bytes_of(cid: &str, bytes: &[u8]) -> bool {
    cid_string(&cid_of(0x55, bytes)) == cid
}

/// Delete what is left where the blobs were, now that the directory keeps
/// them `to`, and the objects of deleted accounts the service log owes:
/// what the accounts hold.
async fn clean(svc: &Service, bucket: &Arc<Bucket>, to: Place) -> Result<Moved, String> {
    let (now, ttl) = (svc.limits.clock.now(), svc.config.blob_ttl);
    let mut moved = Moved::default();
    for a in accounts(svc) {
        let (held, objects) = {
            let st = loaded(&a)?;
            let objects = match to {
                Place::Bucket => st.blobs.strays().cloned().collect(),
                Place::Disk => st.blobs.objects(),
            };
            (st.blobs.holding(now, ttl), objects)
        };
        moved.blobs += held.len() as u64;
        moved.bytes += held.iter().map(|(_, size)| size).sum::<u64>();
        if to == Place::Bucket {
            let dir = blob_dir(&svc.dir, &a.did);
            if let Err(e) = std::fs::remove_dir_all(&dir)
                && e.kind() != io::ErrorKind::NotFound
            {
                return Err(format!("{}: {e}", dir.display()));
            }
        }
        delete(bucket, &a.did, &objects).await?;
        {
            let mut st = loaded(&a)?;
            let st = &mut *st;
            match to {
                Place::Bucket => st.blobs.strays_deleted(&st.wal, &objects),
                Place::Disk => st.blobs.objects_deleted(&st.wal),
            }
            .and_then(|n| n.map_or(Ok(()), |n| st.wal.wait(n)))
            .map_err(|e| format!("{}: logging its objects deleted: {e}", a.did))?;
        }
        a.store.evict();
    }
    for (did, cids) in bucket.owed() {
        delete(bucket, &did, &cids).await?;
        svc.log
            .append(&[blobstore::deleted_entry(&did, &cids)])
            .and_then(|n| svc.log.wait(n))
            .map_err(|e| format!("{}: {e}", crate::log::ACCOUNTS_LOG))?;
        bucket.forget_doomed(&did, &cids);
    }
    Ok(moved)
}

/// Delete `did`'s objects `cids`: why not, if the store fails a delete.
async fn delete(bucket: &Arc<Bucket>, did: &str, cids: &[String]) -> Result<(), String> {
    each(
        cids.iter().cloned(),
        |cid| {
            let (bucket, did) = (bucket.clone(), did.to_string());
            async move {
                bucket
                    .delete(&did, &cid)
                    .await
                    .map_err(|why| format!("deleting {did}'s object {cid}: {why}"))
            }
        },
        |()| {},
    )
    .await
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::blobs::test_upload;
    use crate::blobstore::testing::*;
    use crate::blobstore::{check_dir, kept_in, moving};
    use crate::testing::*;
    use axum::http::Method;
    use serde_json::Value as Json;

    /// A server on `dir` with its blobs in itself.
    fn open_disk(dir: &Path, plc: &str) -> crate::Shared {
        check_dir(dir, false).unwrap();
        open(dir, plc)
    }

    async fn upload(s: &crate::Shared, token: &str, b: &[u8]) {
        test_upload(s, Some(token), Some("text/plain"), b.to_vec(), false)
            .await
            .unwrap();
    }

    fn has_file(dir: &Path, did: &str, b: &[u8]) -> bool {
        blob_dir(dir, did).join(raw_cid(b)).exists()
    }

    fn key(did: &str, b: &[u8]) -> String {
        format!("blocks/{did}/{}", raw_cid(b))
    }

    fn moved(blobs: &[&[u8]]) -> Moved {
        Moved {
            blobs: blobs.len() as u64,
            bytes: blobs.iter().map(|b| b.len() as u64).sum(),
        }
    }

    /// A move to `to` stopped once it has copied every blob, and, if
    /// `switched`, once the directory keeps them there, as a kill leaves it.
    async fn stopped(dir: &Path, c: Config, to: Place, switched: bool) {
        blobstore::mark_moving(dir, Some(to)).unwrap();
        let svc = Service::open(dir, c).unwrap();
        let bucket = svc.ctx.bucket.clone().unwrap();
        copy(&svc, &bucket, to).await.unwrap();
        if switched {
            blobstore::keep_in(dir, to).unwrap();
        }
    }

    /// Blobs moved into the bucket and out again: every blob held, its
    /// object put and its file gone, then the other way; a server on each
    /// serves them, and deletes the objects of blobs it stops holding; a
    /// deleted account's objects owed are deleted by the move out; a
    /// directory with them where they go already is left as it is.
    #[tokio::test]
    async fn moves_the_blobs_there_and_back() {
        let plc = plc().await;
        let fake = Arc::new(Fake::default());
        let url = fake_store(fake.clone()).await;
        let (dir, sink) = (scratch("move-there"), scratch("move-there-mail"));
        let c = || bucket_config(&plc, &sink, &url, true);
        let (a, b, x, d) = (body(1), body(2), body(3), body(4));
        let s = open_disk(&dir, &plc);
        let (alice, token) = create(&s, "alice").await;
        let (bob, bob_token) = create(&s, "bob").await;
        upload(&s, &token, &a).await;
        upload(&s, &token, &b).await;
        upload(&s, &bob_token, &x).await;
        put(&s, &token, &alice, "a", &a).await.unwrap();
        put(&s, &bob_token, &bob, "x", &x).await.unwrap();
        drop(s);
        assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), Moved::default());
        assert_eq!(
            run(&dir, c(), Place::Bucket).await.unwrap(),
            moved(&[&a, &b, &x])
        );
        let mut want = vec![key(&alice, &a), key(&alice, &b), key(&bob, &x)];
        want.sort();
        assert_eq!(fake.keys(), want);
        assert!(!blob_dir(&dir, &alice).exists() && !blob_dir(&dir, &bob).exists());
        assert_eq!(
            (kept_in(&dir).unwrap(), moving(&dir).unwrap()),
            (Place::Bucket, None)
        );
        assert_eq!(
            run(&dir, c(), Place::Bucket).await.unwrap(),
            Moved::default()
        );
        assert!(check_dir(&dir, false).is_err());
        // Served from the bucket, a blob no longer held has its object deleted.
        let s = open_bucket(&dir, &plc, &sink, &url, true);
        let bucket = s.ctx.bucket.clone().unwrap();
        assert_eq!(get(&s, &alice, &a).await, (200, a.clone()));
        assert_eq!(get(&s, &bob, &x).await, (200, x.clone()));
        unput(&s, &token, &alice, "a").await;
        assert!(blobstore::pass(&s, &bucket).await);
        assert!(!fake.has(&key(&alice, &a)));
        upload(&s, &token, &d).await;
        put(&s, &token, &alice, "d", &d).await.unwrap();
        // Bob deleted: his object owed by the service log.
        call(
            &s,
            "com.atproto.server.requestAccountDelete",
            Some(&bob_token),
            Json::Null,
        )
        .await
        .unwrap();
        let mailed = crate::email::testing::last_token(&sink);
        call(
            &s,
            "com.atproto.server.deleteAccount",
            None,
            serde_json::json!({ "did": bob, "password": "bob-pw", "token": mailed }),
        )
        .await
        .unwrap();
        assert!(fake.has(&key(&bob, &x)));
        drop((s, bucket));
        assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&b, &d]));
        assert!(fake.keys().is_empty());
        assert!(has_file(&dir, &alice, &b) && has_file(&dir, &alice, &d));
        assert_eq!(
            (kept_in(&dir).unwrap(), moving(&dir).unwrap()),
            (Place::Disk, None)
        );
        // Nothing owed is left in the logs.
        let s = Arc::new(Service::open(&dir, c()).unwrap());
        assert!(s.ctx.bucket.clone().unwrap().owed().is_empty());
        assert!(store_of(&s, &alice, |st| st.blobs.objects_list().is_none()));
        drop(s);
        let s = open_disk(&dir, &plc);
        assert_eq!(get(&s, &alice, &d).await, (200, d.clone()));
        drop(s);
        assert!(fake.seen.lock().unwrap().iter().all(|(_, _, ok)| *ok));
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&sink);
    }

    /// A move stopped before its switch, or after it, carries on to its end
    /// with the same `--to`, and goes back with the other: the directory
    /// refused meanwhile, and served after from where it keeps its blobs.
    #[tokio::test]
    async fn a_move_stopped_carries_on_or_goes_back() {
        let plc = plc().await;
        let fake = Arc::new(Fake::default());
        let url = fake_store(fake.clone()).await;
        let (dir, sink) = (scratch("move-stopped"), scratch("move-stopped-mail"));
        let c = || bucket_config(&plc, &sink, &url, true);
        let (a, b) = (body(5), body(6));
        let s = open_disk(&dir, &plc);
        let (alice, token) = create(&s, "alice").await;
        upload(&s, &token, &a).await;
        upload(&s, &token, &b).await;
        put(&s, &token, &alice, "a", &a).await.unwrap();
        drop(s);
        let mut both = vec![key(&alice, &a), key(&alice, &b)];
        both.sort();
        let files = |dir: &Path| (has_file(dir, &alice, &a), has_file(dir, &alice, &b));
        // Into the bucket, stopped before the switch, then moved back.
        stopped(&dir, c(), Place::Bucket, false).await;
        assert_eq!(fake.keys(), both);
        assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err());
        assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&a, &b]));
        assert!(fake.keys().is_empty());
        assert_eq!(files(&dir), (true, true));
        let s = open_disk(&dir, &plc);
        assert_eq!(get(&s, &alice, &a).await, (200, a.clone()));
        assert!(store_of(&s, &alice, |st| st.blobs.objects_list().is_none()));
        drop(s);
        // Into the bucket, stopped after the switch, then carried on.
        stopped(&dir, c(), Place::Bucket, true).await;
        assert!(check_dir(&dir, true).is_err());
        assert_eq!(files(&dir), (true, true));
        assert_eq!(
            run(&dir, c(), Place::Bucket).await.unwrap(),
            moved(&[&a, &b])
        );
        assert_eq!(files(&dir), (false, false));
        assert_eq!(fake.keys(), both);
        // Into the data directory, stopped after the switch, then moved back.
        stopped(&dir, c(), Place::Disk, true).await;
        assert_eq!(files(&dir), (true, true));
        assert!(check_dir(&dir, false).is_err());
        assert_eq!(
            run(&dir, c(), Place::Bucket).await.unwrap(),
            moved(&[&a, &b])
        );
        assert_eq!(files(&dir), (false, false));
        assert_eq!(fake.keys(), both);
        // Into the data directory, stopped before the switch, its mark's
        // bytes lost as a power cut may lose them, and its switch's mark
        // written but not renamed: refused still, then moved back, nothing
        // of the marks left.
        stopped(&dir, c(), Place::Disk, false).await;
        assert_eq!(files(&dir), (true, true));
        std::fs::write(dir.join("blobstore-moving"), b"").unwrap();
        std::fs::write(dir.join("blobstore.tmp"), b"").unwrap();
        assert_eq!(moving(&dir).unwrap(), Some(None));
        assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err());
        assert_eq!(
            run(&dir, c(), Place::Bucket).await.unwrap(),
            moved(&[&a, &b])
        );
        assert_eq!(files(&dir), (false, false));
        assert!(!dir.join("blobstore.tmp").exists() && !dir.join("blobstore-moving.tmp").exists());
        // Served from the bucket, its objects still known to it.
        let s = open_bucket(&dir, &plc, &sink, &url, true);
        let bucket = s.ctx.bucket.clone().unwrap();
        assert_eq!(get(&s, &alice, &a).await, (200, a.clone()));
        unput(&s, &token, &alice, "a").await;
        assert!(blobstore::pass(&s, &bucket).await);
        assert_eq!(fake.keys(), vec![key(&alice, &b)]);
        drop((s, bucket));
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&sink);
    }

    /// A blob whose object is missing, or a request the store fails, stops
    /// a move, naming the blob, or saying why; the directory is refused
    /// until a move runs to its end.
    #[tokio::test]
    async fn a_move_stops_at_a_blob_missing_or_a_store_failing() {
        let plc = plc().await;
        let fake = Arc::new(Fake::default());
        let url = fake_store(fake.clone()).await;
        let (dir, sink) = (scratch("move-missing"), scratch("move-missing-mail"));
        let c = || bucket_config(&plc, &sink, &url, true);
        let (a, b) = (body(7), body(8));
        let s = open_bucket(&dir, &plc, &sink, &url, true);
        let (alice, token) = create(&s, "alice").await;
        upload(&s, &token, &a).await;
        upload(&s, &token, &b).await;
        put(&s, &token, &alice, "a", &a).await.unwrap();
        drop(s);
        let saved = fake
            .objects
            .lock()
            .unwrap()
            .remove(&key(&alice, &a))
            .unwrap();
        let why = run(&dir, c(), Place::Disk).await.unwrap_err();
        assert!(why.contains(&alice) && why.contains(&raw_cid(&a)), "{why}");
        assert!(!why.contains(&raw_cid(&b)), "{why}");
        assert_eq!(moving(&dir).unwrap(), Some(Some(Place::Disk)));
        assert!(check_dir(&dir, false).is_err() && check_dir(&dir, true).is_err());
        fake.objects.lock().unwrap().insert(key(&alice, &a), saved);
        fake.fail(Some((Method::GET, 503)));
        let why = run(&dir, c(), Place::Disk).await.unwrap_err();
        assert!(why.contains("503"), "{why}");
        fake.fail(None);
        assert_eq!(run(&dir, c(), Place::Disk).await.unwrap(), moved(&[&a, &b]));
        let s = open_disk(&dir, &plc);
        assert_eq!(get(&s, &alice, &a).await, (200, a.clone()));
        drop(s);
        assert!(fake.keys().is_empty());
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&sink);
    }
}