chadtmiller.com/monolith
An atproto PDS
0
0
//! Blobs kept in a bucket of an object store that speaks S3's API, rather
//! than in the data directory (`spec/BLOBS.md`, "Where blobs are kept"):
//! the variables that say so, the mark the data directory keeps of where
//! its blobs are, the requests to the store, signed with AWS Signature
//! Version 4, and the deletion of the objects of blobs no longer held.
//!
//! Each blob an account holds is the object `blocks/<did>/<cid>`. The
//! account's log says which objects the store may keep (`Blobs::kept`): an
//! upload logs `blobPut` before it puts an object no entry names yet, so
//! that one the server is killed during is found again, and `blobDeleted`
//! is logged once the store has answered an object's delete. The objects it
//! may keep of blobs the account does not hold, and no upload is putting,
//! are its strays: the deleter deletes them once what made them is on disk,
//! each in its object's turn (`Bucket::turn`), which an upload holds from
//! before it notes its object until it is logged, so that no object is
//! deleted of a blob held again by then. A delete that fails is tried again
//! on the next pass, a second after the first and at most `RETRY_MAX`
//! after the last.
//!
//! An account with strays, or an upload on its way, is marked for a start
//! to load (`cache::Eager`) before the entry that makes them reaches its
//! log: an upload's `blobPut`, a commit that stops it holding a blob
//! (`log`), a change to its space repos while they reference blobs
//! (`space_repo`, `space_authority`), and uploads past their time
//! (`blobs::sweep`). A start so finds every stray, and the deleter unmarks
//! an account once it has none. A deleted account's objects are named in
//! the service log with its deletion (`blobsDoomed`), and leave it as their
//! deletes are answered (`blobsDeleted`).
//!
//! The data directory keeps its blobs where its first start put them: a
//! start with a bucket on a directory that holds nothing yet leaves the mark
//! `blobstore` in it, and a start otherwise than the mark says is refused
//! (`check_dir`). `pds migrate-blobs` moves them from one place to the other
//! (`migrate_blobs`): the mark `blobstore-moving`, naming where they go,
//! is in the directory from its start until it finishes, and a start is
//! refused while it is; `blobstore` is made or taken away, each durably, as
//! the move switches where they are kept.
use crate::ops::Target;
use crate::{Account, Shared, Store, XrpcError, err, internal, read, write};
use axum::body::Bytes;
use hmac::{Hmac, Mac};
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
use reqwest::header::{AUTHORIZATION, CONTENT_LENGTH, HOST, HeaderName, HeaderValue};
use serde_json::{Value as Json, json};
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
const BUCKET: &str = "PDS_BLOBSTORE_S3_BUCKET";
const REGION: &str = "PDS_BLOBSTORE_S3_REGION";
const ENDPOINT: &str = "PDS_BLOBSTORE_S3_ENDPOINT";
const PATH_STYLE: &str = "PDS_BLOBSTORE_S3_FORCE_PATH_STYLE";
const KEY_ID: &str = "PDS_BLOBSTORE_S3_ACCESS_KEY_ID";
const SECRET: &str = "PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY";
const UPLOAD_TIMEOUT: &str = "PDS_BLOBSTORE_S3_UPLOAD_TIMEOUT_MS";
const REQUEST_TIMEOUT: &str = "PDS_BLOBSTORE_S3_REQUEST_TIMEOUT_MS";
/// The blobstore's variables but the bucket: none may be set without it.
const OTHERS: [&str; 7] = [
REGION,
ENDPOINT,
PATH_STYLE,
KEY_ID,
SECRET,
UPLOAD_TIMEOUT,
REQUEST_TIMEOUT,
];
/// The mark in the data directory of a start that kept blobs in a bucket.
pub(crate) const MARK: &str = "blobstore";
/// What the mark holds.
const MARKED: &[u8] = b"s3\n";
/// The mark of a move of the blobs unfinished, holding where they go.
const MOVING: &str = "blobstore-moving";
/// The service log's entry naming a deleted account's objects, to delete.
pub(crate) const DOOMED: &str = "blobsDoomed";
/// The service log's entry of those of them deleted.
pub(crate) const DELETED: &str = "blobsDeleted";
/// The kinds of entry the service log has of the bucket.
pub(crate) const ENTRIES: &[&str] = &[DOOMED, DELETED];
/// How long the deleter waits, with nothing failing, before it looks
/// again unwoken.
const IDLE: Duration = Duration::from_secs(30);
/// The longest it waits after a pass with a delete that failed.
const RETRY_MAX: Duration = Duration::from_secs(30);
/// How many deletes it has in flight at once.
const AT_ONCE: usize = 16;
fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
m.lock().unwrap_or_else(PoisonError::into_inner)
}
// ---- Settings ---------------------------------------------------------------
/// The credentials requests are signed with.
#[derive(Clone)]
pub(crate) struct Keys {
pub id: String,
pub secret: String,
/// `AWS_SESSION_TOKEN`, with the keys of `AWS_*`.
pub token: Option<String>,
}
/// Where blobs are kept, as the variables say: with no bucket, in the data
/// directory.
#[derive(Clone)]
pub(crate) struct Settings {
pub bucket: String,
pub region: String,
/// The store's base URL.
pub endpoint: url::Url,
/// Whether the bucket is the path's first segment, rather than the
/// host's first label.
pub path_style: bool,
pub keys: Keys,
/// How long storing one upload may take.
pub upload_timeout: Duration,
/// How long any other request may take.
pub request_timeout: Duration,
}
impl Settings {
/// From the environment: none without a bucket.
pub(crate) fn from_env() -> Result<Option<Settings>, String> {
Settings::parse(|n| std::env::var(n).ok())
}
/// As `vars` gives the variables, an empty one as if unset.
fn parse(vars: impl Fn(&str) -> Option<String>) -> Result<Option<Settings>, String> {
let var = |n: &str| vars(n).filter(|v| !v.is_empty());
let path_style = match var(PATH_STYLE).as_deref() {
None | Some("false") => false,
Some("true") => true,
Some(v) => return Err(format!("{PATH_STYLE} is {v:?}, not true or false")),
};
let timeout = |n: &str, default: u64| match var(n) {
None => Ok(Duration::from_millis(default)),
Some(v) => v
.parse::<u64>()
.ok()
.filter(|ms| *ms > 0)
.map(Duration::from_millis)
.ok_or_else(|| format!("{n} is {v:?}, not a whole number of milliseconds above 0")),
};
let upload_timeout = timeout(UPLOAD_TIMEOUT, 20_000)?;
let request_timeout = timeout(REQUEST_TIMEOUT, 10_000)?;
let given = var(ENDPOINT).map(|e| endpoint(&e)).transpose()?;
let keys = match (var(KEY_ID), var(SECRET)) {
(Some(id), Some(secret)) => Some(Keys {
id,
secret,
token: None,
}),
(None, None) => None,
_ => {
return Err(format!(
"one of {KEY_ID} and {SECRET} is set without the other"
));
}
};
let Some(bucket) = var(BUCKET) else {
return match OTHERS.iter().find(|n| var(n).is_some()) {
Some(n) => Err(format!("{n} is set, but {BUCKET} is not")),
None => Ok(None),
};
};
let region = var(REGION).unwrap_or_else(|| "us-east-1".into());
let keys = keys
.or_else(|| {
Some(Keys {
id: var("AWS_ACCESS_KEY_ID")?,
secret: var("AWS_SECRET_ACCESS_KEY")?,
token: var("AWS_SESSION_TOKEN"),
})
})
.ok_or_else(|| {
format!(
"{BUCKET} is set, but no credentials are: neither {KEY_ID} and {SECRET}, nor AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY"
)
})?;
let endpoint = match given {
Some(e) => e,
None => endpoint(&format!("https://s3.{region}.amazonaws.com"))
.map_err(|_| format!("{REGION} is {region:?}, which makes no endpoint"))?,
};
Ok(Some(Settings {
bucket,
region,
endpoint,
path_style,
keys,
upload_timeout,
request_timeout,
}))
}
}
/// An endpoint: an `http` or `https` URL with a host, and no query or fragment.
fn endpoint(v: &str) -> Result<url::Url, String> {
let bad = || format!("{ENDPOINT} is {v:?}, not an http or https URL with no query or fragment");
let u = url::Url::parse(v).map_err(|_| bad())?;
if !matches!(u.scheme(), "http" | "https")
|| u.host_str().is_none_or(str::is_empty)
|| u.query().is_some()
|| u.fragment().is_some()
{
return Err(bad());
}
Ok(u)
}
/// Where a data directory keeps its blobs.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Place {
/// In itself, each a file.
Disk,
/// In the bucket its variables name.
Bucket,
}
impl Place {
/// The place `pds migrate-blobs --to` names: `disk` or `bucket`.
pub fn named(name: &str) -> Option<Place> {
match name {
"disk" => Some(Place::Disk),
"bucket" => Some(Place::Bucket),
_ => None,
}
}
pub fn name(self) -> &'static str {
match self {
Place::Disk => "disk",
Place::Bucket => "bucket",
}
}
pub(crate) fn other(self) -> Place {
match self {
Place::Disk => Place::Bucket,
Place::Bucket => Place::Disk,
}
}
/// Where a move into it goes.
pub(crate) fn towards(self) -> &'static str {
match self {
Place::Disk => "into the data directory",
Place::Bucket => "into the bucket",
}
}
}
/// Where `dir` keeps its blobs, as its mark says.
pub(crate) fn kept_in(dir: &Path) -> Result<Place, String> {
let mark = dir.join(MARK);
match std::fs::read(&mark) {
Ok(b) if b == MARKED => Ok(Place::Bucket),
Ok(_) => Err(format!("{}: not a mark this server knows", mark.display())),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(Place::Disk),
Err(e) => Err(format!("{}: {e}", mark.display())),
}
}
/// Whether a move of `dir`'s blobs is unfinished, as its mark says: none
/// if none is; if one is, where it takes them, or none if a power cut lost
/// the mark's bytes (`mark_moving`).
pub(crate) fn moving(dir: &Path) -> Result<Option<Option<Place>>, String> {
let mark = dir.join(MOVING);
match std::fs::read(&mark) {
Ok(b) => Ok(Some(
std::str::from_utf8(&b)
.ok()
.and_then(|s| s.strip_suffix('\n'))
.and_then(Place::named),
)),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(format!("{}: {e}", mark.display())),
}
}
/// Make `dir` keep its blobs in `place`, durably: its mark made, or taken
/// away. The mark is never there without its bytes.
pub(crate) fn keep_in(dir: &Path, place: Place) -> Result<(), String> {
let mark = dir.join(MARK);
let at = |e: io::Error| format!("{}: {e}", mark.display());
match place {
Place::Bucket => {
let (tmp, f) = beside(dir, MARK, MARKED)?;
pds_store::sync_file(&f)
.and_then(|()| std::fs::rename(&tmp, &mark))
.and_then(|()| pds_store::sync_dir(dir))
.map_err(at)
}
Place::Disk => take_mark(dir, MARK),
}
}
/// Mark `dir` as moving its blobs to `to`, or with none, as moving them no
/// more, durably, with what a mark's write killed before its rename left.
/// The mark is there as soon as it can be, its bytes synced after: what it
/// holds only says where the move goes.
pub(crate) fn mark_moving(dir: &Path, to: Option<Place>) -> Result<(), String> {
let Some(to) = to else {
for name in [MARK, MOVING] {
let tmp = dir.join(format!("{name}.tmp"));
match std::fs::remove_file(&tmp) {
Err(e) if e.kind() != io::ErrorKind::NotFound => {
return Err(format!("{}: {e}", tmp.display()));
}
_ => {}
}
}
return take_mark(dir, MOVING);
};
let mark = dir.join(MOVING);
let (tmp, f) = beside(dir, MOVING, format!("{}\n", to.name()).as_bytes())?;
std::fs::rename(&tmp, &mark)
.and_then(|()| pds_store::sync_file(&f))
.and_then(|()| pds_store::sync_dir(dir))
.map_err(|e| format!("{}: {e}", mark.display()))
}
/// `bytes` written to a file beside the mark `name` in `dir`, to rename
/// over it: its path, and the file.
fn beside(dir: &Path, name: &str, bytes: &[u8]) -> Result<(PathBuf, std::fs::File), String> {
pds_store::make_dirs(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
let tmp = dir.join(format!("{name}.tmp"));
let f = pds_store::open_private(
&tmp,
std::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(true),
)
.and_then(|mut f| f.write_all(bytes).map(|()| f))
.map_err(|e| format!("{}: {e}", tmp.display()))?;
Ok((tmp, f))
}
/// Take the mark `name` out of `dir`, if it is there, synced.
fn take_mark(dir: &Path, name: &str) -> Result<(), String> {
let mark = dir.join(name);
let at = |e: io::Error| format!("{}: {e}", mark.display());
match std::fs::remove_file(&mark) {
Err(e) if e.kind() != io::ErrorKind::NotFound => Err(at(e)),
_ => pds_store::sync_dir(dir).map_err(at),
}
}
/// Whether a start on `dir`, with a bucket or not, keeps blobs where the
/// directory's first start put them, with no move of them unfinished: a
/// start with a bucket on a directory that holds nothing yet marks it so,
/// synced, before anything else is written there.
pub(crate) fn check_dir(dir: &Path, bucket: bool) -> Result<(), String> {
match moving(dir)? {
Some(Some(to)) => {
return Err(format!(
"{} is moving its blobs {}, unfinished: `pds migrate-blobs --to {}` finishes the move, or `pds migrate-blobs --to {}` moves them back, given the bucket's variables either way; then it starts",
dir.display(),
to.towards(),
to.name(),
to.other().name(),
));
}
Some(None) => {
return Err(format!(
"{} is moving its blobs, unfinished: `pds migrate-blobs --to bucket` or `pds migrate-blobs --to disk`, given the bucket's variables, finishes the move or moves them back; then it starts",
dir.display(),
));
}
None => {}
}
match (bucket, kept_in(dir)?) {
(true, Place::Bucket) | (false, Place::Disk) => Ok(()),
(false, Place::Bucket) => Err(format!(
"{} keeps its blobs in a bucket, and {BUCKET} is not set: `pds migrate-blobs --to disk`, given the bucket's variables, moves them into it",
dir.display()
)),
(true, Place::Disk) if crate::store::holds_data(dir) => Err(format!(
"{} keeps its blobs in itself, and {BUCKET} is set: `pds migrate-blobs --to bucket` moves them into the bucket",
dir.display()
)),
(true, Place::Disk) => keep_in(dir, Place::Bucket),
}
}
// ---- Signing ------------------------------------------------------------------
type HmacSha256 = Hmac<Sha256>;
fn hmac(key: &[u8], data: &[u8]) -> [u8; 32] {
let mut m = HmacSha256::new_from_slice(key).expect("HMAC takes a key of any length");
m.update(data);
m.finalize().into_bytes().into()
}
/// What SigV4 leaves as it is in a path: letters, digits, `-`, `_`, `.` and `~`.
const UNRESERVED: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~');
/// A path segment as SigV4 has it, and as it is sent.
fn uri_encode(s: &str) -> String {
utf8_percent_encode(s, UNRESERVED).to_string()
}
/// The time `ms` after the epoch as SigV4 dates a request,
/// `YYYYMMDDTHHMMSSZ`, in UTC.
fn amz_date(ms: i64) -> String {
let secs = ms.div_euclid(1000);
let (y, m, d) = pds_repo::data::civil_from_days(secs.div_euclid(86_400));
let t = secs.rem_euclid(86_400);
format!(
"{y:04}{m:02}{d:02}T{:02}{:02}{:02}Z",
t / 3600,
t / 60 % 60,
t % 60
)
}
/// The `Authorization` of a request signed with AWS Signature Version 4 for
/// the service `s3` in `region`, at `date` (`amz_date`): `headers`, each by
/// its lowercase name, are those it signs; `path` and `query` are as sent,
/// already canonical; `payload` is the hex SHA-256 of its body.
#[allow(clippy::too_many_arguments)]
fn authorization(
keys: &Keys,
region: &str,
method: &str,
path: &str,
query: &str,
headers: &[(&str, &str)],
payload: &str,
date: &str,
) -> String {
let mut signed = headers.to_vec();
signed.sort_unstable();
let names = signed.iter().map(|(n, _)| *n).collect::<Vec<_>>().join(";");
let canonical_headers: String = signed
.iter()
.map(|(n, v)| {
format!(
"{n}:{}\n",
v.split_whitespace().collect::<Vec<_>>().join(" ")
)
})
.collect();
let request = format!("{method}\n{path}\n{query}\n{canonical_headers}\n{names}\n{payload}");
let day = &date[..8];
let scope = format!("{day}/{region}/s3/aws4_request");
let to_sign = format!(
"AWS4-HMAC-SHA256\n{date}\n{scope}\n{}",
hex::encode(Sha256::digest(request.as_bytes()))
);
let mut key = hmac(format!("AWS4{}", keys.secret).as_bytes(), day.as_bytes());
for part in [region, "s3", "aws4_request"] {
key = hmac(&key, part.as_bytes());
}
format!(
"AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={names}, Signature={}",
keys.id,
hex::encode(hmac(&key, to_sign.as_bytes()))
)
}
// ---- The bucket -----------------------------------------------------------------
/// The bytes of an object as they come, for `getBlob` to send.
pub(crate) type Chunks = tokio::sync::mpsc::Receiver<io::Result<Bytes>>;
/// The turns of the objects some upload or delete waits on or holds, by
/// account and CID.
type Turns = HashMap<(String, String), Arc<tokio::sync::Mutex<()>>>;
/// An object's turn: held, no other upload or delete of it goes on.
pub(crate) struct Turn<'a> {
bucket: &'a Bucket,
key: (String, String),
held: Option<tokio::sync::OwnedMutexGuard<()>>,
}
impl Drop for Turn<'_> {
fn drop(&mut self) {
drop(self.held.take());
let mut turns = lock(&self.bucket.turns);
if turns
.get(&self.key)
.is_some_and(|m| Arc::strong_count(m) == 1)
{
turns.remove(&self.key);
}
}
}
/// The bucket, and the deletes it is owed.
pub(crate) struct Bucket {
s: Settings,
/// Keeps a connection unused for up to 90 seconds.
http: reqwest::Client,
/// Keeps none: what a request that failed on its way is sent again with.
fresh: reqwest::Client,
/// Each object's turn while some upload or delete waits on it or holds it.
turns: Mutex<Turns>,
/// The accounts whose strays the deleter is to look at.
due: Mutex<BTreeSet<String>>,
/// Deleted accounts' objects not yet deleted.
doomed: Mutex<Doomed>,
wake: tokio::sync::Notify,
}
impl Bucket {
pub(crate) fn new(s: Settings) -> Result<Bucket, String> {
let builder = || {
reqwest::Client::builder()
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.connect_timeout(s.request_timeout.min(Duration::from_secs(5)))
};
let built =
|b: reqwest::ClientBuilder| b.build().map_err(|e| format!("an HTTP client: {e}"));
let http = built(builder().pool_idle_timeout(Duration::from_secs(90)))?;
let fresh = built(builder().pool_max_idle_per_host(0))?;
Ok(Bucket {
s,
http,
fresh,
turns: Mutex::default(),
due: Mutex::default(),
doomed: Mutex::default(),
wake: tokio::sync::Notify::new(),
})
}
/// Send `req` over a connection an earlier request left open, if one is;
/// one that fails on its way, as when the store closes that connection
/// just as it goes out, once more over a connection of its own. Every
/// request to the store may be made twice to the same effect
/// (`spec/BLOBS.md`, "Requests to the store").
async fn send(&self, req: reqwest::Request) -> reqwest::Result<reqwest::Response> {
let again = req.try_clone();
match crate::ops::execute(Target::Blobstore, &self.http, req).await {
Err(e) if !e.is_connect() && !e.is_timeout() => match again {
Some(req) => crate::ops::execute(Target::Blobstore, &self.fresh, req).await,
None => Err(e),
},
sent => sent,
}
}
/// Wait for the turn of `did`'s object `cid`.
pub(crate) async fn turn(&self, did: &str, cid: &str) -> Turn<'_> {
let key = (did.to_string(), cid.to_string());
let m = lock(&self.turns).entry(key.clone()).or_default().clone();
Turn {
bucket: self,
key,
held: Some(m.lock_owned().await),
}
}
/// Have the deleter look at `did`'s strays, and at whether its mark may go.
pub(crate) fn wake(&self, did: &str) {
lock(&self.due).insert(did.to_string());
self.wake.notify_one();
}
/// A deleted account's objects to delete, its deletion on disk.
pub(crate) fn doom(&self, did: &str, cids: Vec<String>) {
if cids.is_empty() {
return;
}
lock(&self.doomed)
.0
.entry(did.to_string())
.or_default()
.extend(cids);
self.wake.notify_one();
}
/// The deleted accounts' objects the service log names, as a start replayed it.
pub(crate) fn doomed(&self, d: Doomed) {
*lock(&self.doomed) = d;
}
/// The deleted accounts' objects not yet deleted, by DID.
pub(crate) fn owed(&self) -> Vec<(String, Vec<String>)> {
lock(&self.doomed)
.0
.iter()
.map(|(did, cids)| (did.clone(), cids.iter().cloned().collect()))
.collect()
}
/// A deleted account's objects deleted, their entry logged: owed no more.
pub(crate) fn forget_doomed(&self, did: &str, cids: &[String]) {
let mut d = lock(&self.doomed);
if let Some(left) = d.0.get_mut(did) {
for c in cids {
left.remove(c);
}
if left.is_empty() {
d.0.remove(did);
}
}
}
/// An object's URL, the `Host` it is sent with, and its path, as sent and
/// as SigV4 has it.
fn locate(&self, did: &str, cid: &str) -> Result<(url::Url, String, String), String> {
let e = &self.s.endpoint;
let mut path: String = e
.path_segments()
.into_iter()
.flatten()
.filter(|p| !p.is_empty())
.map(|p| {
format!(
"/{}",
uri_encode(&percent_decode_str(p).decode_utf8_lossy())
)
})
.collect();
let mut host = e.host_str().unwrap_or_default().to_string();
if self.s.path_style {
path.push('/');
path.push_str(&uri_encode(&self.s.bucket));
} else {
host = format!("{}.{host}", self.s.bucket);
}
for p in ["blocks", did, cid] {
path.push('/');
path.push_str(&uri_encode(p));
}
if let Some(port) = e.port() {
host = format!("{host}:{port}");
}
let url = url::Url::parse(&format!("{}://{host}{path}", e.scheme()))
.map_err(|why| format!("the URL of {did}'s object {cid}: {why}"))?;
Ok((url, host, path))
}
/// A request for `did`'s object `cid`, signed, with `body` if given.
fn request(
&self,
method: reqwest::Method,
did: &str,
cid: &str,
body: Option<Bytes>,
) -> Result<reqwest::Request, String> {
let (url, host, path) = self.locate(did, cid)?;
let payload = hex::encode(Sha256::digest(body.as_deref().unwrap_or_default()));
let date = amz_date(crate::events::now_ms() as i64);
let mut signed = vec![
("host", host.as_str()),
("x-amz-content-sha256", payload.as_str()),
("x-amz-date", date.as_str()),
];
if let Some(t) = &self.s.keys.token {
signed.push(("x-amz-security-token", t.as_str()));
}
let auth = authorization(
&self.s.keys,
&self.s.region,
method.as_str(),
&path,
"",
&signed,
&payload,
&date,
);
let mut req = reqwest::Request::new(method, url);
let h = req.headers_mut();
let value = |v: &str| HeaderValue::from_str(v).map_err(|e| format!("a header: {e}"));
for (n, v) in &signed {
let name = if *n == "host" {
HOST
} else {
HeaderName::from_bytes(n.as_bytes()).map_err(|e| format!("a header: {e}"))?
};
h.insert(name, value(v)?);
}
h.insert(AUTHORIZATION, value(&auth)?);
if let Some(b) = body {
h.insert(CONTENT_LENGTH, HeaderValue::from(b.len()));
*req.body_mut() = Some(b.into());
}
Ok(req)
}
/// Send `req`, and read its answer whole, within `wait`: the status the
/// store answered, or why it gave none.
async fn exchange(
&self,
req: reqwest::Request,
wait: Duration,
) -> Result<reqwest::StatusCode, String> {
let sent = tokio::time::timeout(wait, async {
let res = self.send(req).await?;
let status = res.status();
let _ = res.bytes().await;
Ok::<_, reqwest::Error>(status)
})
.await;
match sent {
Ok(Ok(s)) => Ok(s),
Ok(Err(e)) => Err(e.to_string()),
Err(_) => Err(format!("no answer within {} ms", wait.as_millis())),
}
}
/// Put `did`'s object `cid`, within `PDS_BLOBSTORE_S3_UPLOAD_TIMEOUT_MS`:
/// what `uploadBlob` answers if the store did not answer it with a 2xx.
pub(crate) async fn put(&self, did: &str, cid: &str, bytes: Bytes) -> Result<(), XrpcError> {
let req = match self.request(reqwest::Method::PUT, did, cid, Some(bytes)) {
Ok(r) => r,
Err(why) => {
failure("put", did, &why);
return internal();
}
};
match self.exchange(req, self.s.upload_timeout).await {
Ok(s) if s.is_success() => Ok(()),
Ok(_) => internal(),
Err(_) => err(504, "UpstreamTimeout", "The blob store did not answer"),
}
}
/// Put `did`'s object `cid` as `put` does: why not, if the store did not
/// answer it with a 2xx.
pub(crate) async fn put_object(
&self,
did: &str,
cid: &str,
bytes: Bytes,
) -> Result<(), String> {
let req = self
.request(reqwest::Method::PUT, did, cid, Some(bytes))
.inspect_err(|why| failure("put", did, why))?;
match self.exchange(req, self.s.upload_timeout).await? {
s if s.is_success() => Ok(()),
s => Err(format!("the store answered {s}")),
}
}
/// Read `did`'s object `cid`, of `size` bytes, as it comes: its first
/// bytes in hand within `PDS_BLOBSTORE_S3_REQUEST_TIMEOUT_MS`, the rest
/// each within that of the last, or none if the store failed before.
pub(crate) async fn get(&self, did: &str, cid: &str, size: u64) -> Option<Chunks> {
let req = match self.request(reqwest::Method::GET, did, cid, None) {
Ok(r) => r,
Err(why) => {
failure("get", did, &why);
return None;
}
};
let wait = self.s.request_timeout;
let first = tokio::time::timeout(wait, async {
let mut res = self.send(req).await.map_err(|_| None)?;
if res.status() != reqwest::StatusCode::OK {
return Err(None);
}
let first = match size {
0 => None,
_ => match res.chunk().await {
Ok(Some(b)) => Some(b),
Ok(None) => return Err(Some("the object ended before its first byte".into())),
Err(e) => return Err(Some(e.to_string())),
},
};
Ok((res, first))
})
.await;
let (mut res, first) = match first {
Ok(Ok(got)) => got,
// The request's own failure is logged as it is (`ops::execute`).
Ok(Err(None)) => return None,
Ok(Err(Some(why))) => {
failure("get", did, &why);
return None;
}
Err(_) => return None,
};
let (tx, chunks) = tokio::sync::mpsc::channel(4);
let did = did.to_string();
tokio::spawn(async move {
if let Some(b) = first
&& tx.send(Ok(b)).await.is_err()
{
return;
}
loop {
let why = match tokio::time::timeout(wait, res.chunk()).await {
Ok(Ok(Some(b))) => {
if tx.send(Ok(b)).await.is_err() {
return;
}
continue;
}
Ok(Ok(None)) => return,
Ok(Err(e)) => e.to_string(),
Err(_) => "timeout".to_string(),
};
failure("get", &did, &why);
let _ = tx.send(Err(io::Error::other(why))).await;
return;
}
});
Some(chunks)
}
/// Read `did`'s object `cid`, of `size` bytes, whole, as `get` reads it:
/// none if the store has no such object, a 404; why not if it failed
/// otherwise. Reading stops past `size` bytes.
pub(crate) async fn fetch(
&self,
did: &str,
cid: &str,
size: u64,
) -> Result<Option<Vec<u8>>, String> {
let req = self
.request(reqwest::Method::GET, did, cid, None)
.inspect_err(|why| failure("get", did, why))?;
let wait = self.s.request_timeout;
let late = || format!("no answer within {} ms", wait.as_millis());
let mut res = match tokio::time::timeout(wait, self.send(req)).await {
Ok(Ok(res)) => res,
Ok(Err(e)) => return Err(e.to_string()),
Err(_) => return Err(late()),
};
match res.status() {
reqwest::StatusCode::OK => {}
reqwest::StatusCode::NOT_FOUND => return Ok(None),
s => return Err(format!("the store answered {s}")),
}
let mut bytes = Vec::new();
while bytes.len() as u64 <= size {
let why = match tokio::time::timeout(wait, res.chunk()).await {
Ok(Ok(Some(b))) => {
bytes.extend_from_slice(&b);
continue;
}
Ok(Ok(None)) => break,
Ok(Err(e)) => e.to_string(),
Err(_) => late(),
};
failure("get", did, &why);
return Err(why);
}
Ok(Some(bytes))
}
/// Delete `did`'s object `cid`, within `PDS_BLOBSTORE_S3_REQUEST_TIMEOUT_MS`:
/// once the store answered that it is gone, a 2xx, or a 404 of a store
/// that answers so for a key it has not; why not otherwise.
pub(crate) async fn delete(&self, did: &str, cid: &str) -> Result<(), String> {
let req = self
.request(reqwest::Method::DELETE, did, cid, None)
.inspect_err(|why| failure("delete", did, why))?;
match self.exchange(req, self.s.request_timeout).await? {
s if s.is_success() || s == reqwest::StatusCode::NOT_FOUND => Ok(()),
s => Err(format!("the store answered {s}")),
}
}
}
/// A request to the bucket that failed where `ops::execute` does not see it:
/// one that could not be made, or an answer that broke off.
fn failure(op: &str, did: &str, why: &str) {
crate::ops::warn(
"blobstore failure",
&[
("op", json!(op)),
("error", json!(why)),
("did", json!(did)),
],
);
}
// ---- Deleted accounts' objects ------------------------------------------------------
/// Deleted accounts' objects not yet deleted, by DID, as the service log
/// has them.
#[derive(Default)]
pub(crate) struct Doomed(BTreeMap<String, BTreeSet<String>>);
impl Doomed {
/// Replay an entry of the kinds in `ENTRIES`.
pub(crate) fn replay(&mut self, e: &Json) -> Result<(), String> {
let (Some(did), Some(cids)) = (e["did"].as_str(), e["cids"].as_array()) else {
return Err(format!("an entry of objects with no did or cids: {e}"));
};
let cids = cids
.iter()
.map(|c| c.as_str().map(str::to_string))
.collect::<Option<Vec<String>>>()
.ok_or_else(|| format!("an entry of objects whose cids are not strings: {e}"))?;
if e["t"] == DOOMED {
self.0.entry(did.to_string()).or_default().extend(cids);
} else if let Some(left) = self.0.get_mut(did) {
for c in &cids {
left.remove(c);
}
if left.is_empty() {
self.0.remove(did);
}
}
Ok(())
}
/// One entry for each account with objects left: what a checkpoint keeps.
pub(crate) fn entries(&self) -> impl Iterator<Item = Json> + '_ {
self.0.iter().map(|(did, cids)| doomed_entry(did, cids))
}
}
/// The service log's entry naming a deleted account's objects, to delete.
pub(crate) fn doomed_entry<'a>(did: &str, cids: impl IntoIterator<Item = &'a String>) -> Json {
json!({ "t": DOOMED, "did": did, "cids": cids.into_iter().collect::<Vec<_>>() })
}
/// The service log's entry of a deleted account's objects deleted.
pub(crate) fn deleted_entry(did: &str, cids: &[String]) -> Json {
json!({ "t": DELETED, "did": did, "cids": cids })
}
// ---- Marks ---------------------------------------------------------------------------
/// With blobs in a bucket, mark the account of `st` for a start to load
/// before a change to its space repos is logged while their records
/// reference blobs: the change may stop it holding one.
pub(crate) fn mark_for(st: &Store) -> io::Result<()> {
if st.ctx.bucket.is_some() && st.blobs.space_held() {
st.ctx.eager.mark(&st.repo.did)
} else {
Ok(())
}
}
// ---- The deleter -------------------------------------------------------------------
/// Delete strays and deleted accounts' objects: at once, again whenever
/// woken or every `IDLE`, and after a pass with a delete that failed, a
/// second later, then twice as long each time up to `RETRY_MAX`.
pub(crate) fn delete_every(svc: &Shared) {
let Some(bucket) = svc.ctx.bucket.clone() else {
return;
};
let weak = Arc::downgrade(svc);
tokio::spawn(async move {
let mut retry = Duration::ZERO;
loop {
let Some(svc) = weak.upgrade() else { return };
let clean = pass(&svc, &bucket).await;
drop(svc);
if clean {
retry = Duration::ZERO;
tokio::select! {
() = bucket.wake.notified() => {}
() = tokio::time::sleep(IDLE) => {}
}
} else {
retry = (retry * 2).clamp(Duration::from_secs(1), RETRY_MAX);
tokio::time::sleep(retry).await;
}
}
});
}
/// What one delete of a pass came to.
enum Done {
Stray { did: String, ok: bool },
Doomed { did: String, cid: String, ok: bool },
}
/// One pass: every stray of the accounts due, and every deleted account's
/// object, deleted; the accounts left with none unmarked. Answers whether
/// every delete was answered.
pub(crate) async fn pass(svc: &Shared, bucket: &Arc<Bucket>) -> bool {
let dids: Vec<String> = std::mem::take(&mut *lock(&bucket.due))
.into_iter()
.collect();
// Read off the async threads: a store not held is loaded.
let found = {
let svc = svc.clone();
tokio::task::spawn_blocking(move || {
dids.iter()
.filter_map(|did| strays(&svc, did))
.collect::<Vec<_>>()
})
.await
.unwrap_or_default()
};
let doomed: Vec<(String, String)> = lock(&bucket.doomed)
.0
.iter()
.flat_map(|(did, cids)| cids.iter().map(|c| (did.clone(), c.clone())))
.collect();
let at_once = Arc::new(tokio::sync::Semaphore::new(AT_ONCE));
let mut jobs = tokio::task::JoinSet::new();
for (a, cids) in &found {
for cid in cids {
let (bucket, a, cid, at_once) =
(bucket.clone(), a.clone(), cid.clone(), at_once.clone());
jobs.spawn(async move {
let _turn = at_once.acquire().await;
let ok = delete_stray(&bucket, &a, &cid).await;
Done::Stray {
did: a.did.clone(),
ok,
}
});
}
}
for (did, cid) in doomed {
let (svc, bucket, at_once) = (svc.clone(), bucket.clone(), at_once.clone());
jobs.spawn(async move {
let _turn = at_once.acquire().await;
let ok = delete_doomed(&svc, &bucket, &did, &cid).await;
Done::Doomed { did, cid, ok }
});
}
let mut failed: BTreeSet<String> = BTreeSet::new();
let mut deleted: BTreeMap<String, Vec<String>> = BTreeMap::new();
let mut clean = true;
while let Some(done) = jobs.join_next().await {
match done {
Ok(Done::Stray { did, ok: false }) => {
failed.insert(did);
}
Ok(Done::Doomed { did, cid, ok: true }) => deleted.entry(did).or_default().push(cid),
Ok(Done::Stray { ok: true, .. }) => {}
Ok(Done::Doomed { ok: false, .. }) | Err(_) => clean = false,
}
}
for (did, cids) in deleted {
match svc.log.append(&[deleted_entry(&did, &cids)]) {
Ok(_) => bucket.forget_doomed(&did, &cids),
Err(e) => {
crate::ops::failed_for(
crate::ops::Level::Error,
"logging objects deleted failed",
&did,
e,
);
clean = false;
}
}
}
clean &= failed.is_empty();
for did in &failed {
lock(&bucket.due).insert(did.clone());
}
let settled: Vec<Arc<Account>> = found
.into_iter()
.map(|(a, _)| a)
.filter(|a| !failed.contains(&a.did))
.collect();
let _ = tokio::task::spawn_blocking(move || settled.iter().for_each(|a| unmark(a))).await;
clean
}
/// An account's strays, unless it is deleted, whose objects are doomed.
fn strays(svc: &crate::Service, did: &str) -> Option<(Arc<Account>, Vec<String>)> {
let a = svc.account(did)?;
if read(&a.state).deleted {
return None;
}
let cids = read(&a.store).blobs.strays().cloned().collect();
Some((a, cids))
}
/// Unmark an account a start need not load any more.
fn unmark(a: &Account) {
if read(&a.state).deleted {
return;
}
let st = read(&a.store);
if st.ctx.eager.is_marked(&a.did) && !st.needs_start() {
st.ctx.eager.relax(&a.did, Some(&a.handle()));
}
}
/// Delete a stray in its turn, once what made it is on disk, if it is a
/// stray still, and log that: whether nothing is left to do of it.
async fn delete_stray(bucket: &Bucket, a: &Account, cid: &str) -> bool {
let _turn = bucket.turn(&a.did, cid).await;
let on_disk = {
if read(&a.state).deleted {
return true;
}
let st = read(&a.store);
if !st.blobs.is_stray(cid) {
return true;
}
(st.wal.clone(), st.wal.written())
};
if crate::durable(on_disk.0, on_disk.1).await.is_err()
|| bucket.delete(&a.did, cid).await.is_err()
{
return false;
}
if read(&a.state).deleted {
return true;
}
let mut st = write(&a.store);
if !st.blobs.is_stray(cid) {
return true;
}
match st.wal.append(&[crate::blobs::deleted_entry(cid)]) {
Ok(_) => {
st.blobs.forget_object(cid);
true
}
Err(e) => {
crate::ops::failed_for(
crate::ops::Level::Error,
"logging an object deleted failed",
&a.did,
e,
);
false
}
}
}
/// Delete a deleted account's object in its turn: whether it is gone, or is
/// an account's of that DID made since, which keeps it.
async fn delete_doomed(svc: &Shared, bucket: &Bucket, did: &str, cid: &str) -> bool {
let _turn = bucket.turn(did, cid).await;
if let Some(a) = svc.account(did)
&& !read(&a.state).deleted
&& read(&a.store).blobs.keeps(cid)
{
return true;
}
bucket.delete(did, cid).await.is_ok()
}
/// A store as S3 answers it, in memory, and a server with its blobs there.
#[cfg(test)]
pub(crate) mod testing {
use super::*;
use crate::testing::*;
use axum::http::{Method, StatusCode};
use axum::response::IntoResponse;
/// A store as S3 answers, path-style, its bucket `bucket`, in memory:
/// each request's method, key, and whether it was signed as `request`
/// signs (and a `PUT` with its length), kept; failing a method as asked,
/// with a status, or with none for 5 seconds (0).
#[derive(Default)]
pub(crate) struct Fake {
pub objects: Mutex<BTreeMap<String, Vec<u8>>>,
pub fail: Mutex<Option<(Method, u16)>>,
pub seen: Mutex<Vec<(Method, String, bool)>>,
}
impl Fake {
pub fn has(&self, key: &str) -> bool {
lock(&self.objects).contains_key(key)
}
pub fn fail(&self, f: Option<(Method, u16)>) {
*lock(&self.fail) = f;
}
/// The keys it holds, in order.
pub fn keys(&self) -> Vec<String> {
lock(&self.objects).keys().cloned().collect()
}
}
pub(crate) async fn fake_store(fake: Arc<Fake>) -> String {
async fn answer(
axum::extract::State(f): axum::extract::State<Arc<Fake>>,
method: Method,
uri: axum::http::Uri,
headers: axum::http::HeaderMap,
body: Bytes,
) -> axum::response::Response {
let h = |n: &str| {
headers
.get(n)
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
};
let payload = hex::encode(Sha256::digest(&body));
let date = h("x-amz-date");
let keys = Keys {
id: "k".into(),
secret: "s".into(),
token: None,
};
let signed = authorization(
&keys,
"us-east-1",
method.as_str(),
uri.path(),
"",
&[
("host", &h("host")),
("x-amz-content-sha256", &h("x-amz-content-sha256")),
("x-amz-date", &date),
],
&payload,
&date,
);
let ok = h("authorization") == signed
&& h("x-amz-content-sha256") == payload
&& (method != Method::PUT || headers.contains_key("content-length"));
let key = uri
.path()
.strip_prefix("/bucket/")
.unwrap_or_default()
.split('/')
.map(|p| percent_decode_str(p).decode_utf8_lossy().into_owned())
.collect::<Vec<_>>()
.join("/");
lock(&f.seen).push((method.clone(), key.clone(), ok));
if !ok {
return StatusCode::FORBIDDEN.into_response();
}
let fail = lock(&f.fail).clone();
if let Some((m, status)) = fail
&& m == method
{
if status == 0 {
tokio::time::sleep(Duration::from_secs(5)).await;
}
return StatusCode::from_u16(status.max(500))
.unwrap()
.into_response();
}
let mut objects = lock(&f.objects);
match method {
Method::PUT => {
objects.insert(key, body.to_vec());
StatusCode::OK.into_response()
}
Method::GET => match objects.get(&key) {
Some(b) => b.clone().into_response(),
None => StatusCode::NOT_FOUND.into_response(),
},
Method::DELETE => {
objects.remove(&key);
StatusCode::NO_CONTENT.into_response()
}
_ => StatusCode::NOT_IMPLEMENTED.into_response(),
}
}
let app = axum::Router::new().fallback(answer).with_state(fake);
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
url
}
/// The bucket `bucket` of the store at `url`, path-style, with timeouts
/// of half a second.
pub(crate) fn settings(url: &str) -> Settings {
Settings {
bucket: "bucket".into(),
region: "us-east-1".into(),
endpoint: url::Url::parse(url).unwrap(),
path_style: true,
keys: Keys {
id: "k".into(),
secret: "s".into(),
token: None,
},
upload_timeout: Duration::from_millis(500),
request_timeout: Duration::from_millis(500),
}
}
/// The configuration of a server registering DIDs at `plc`, its blobs
/// in the bucket of the store at `url`, its mail in `sink`.
pub(crate) fn bucket_config(
plc: &str,
sink: &Path,
url: &str,
load_all: bool,
) -> crate::Config {
std::fs::create_dir_all(sink).unwrap();
crate::Config {
mailer: crate::email::Mailer::sink(sink),
blobstore: Some(settings(url)),
load_all,
..config(plc)
}
}
/// The server on `dir`, configured as `bucket_config` says.
pub(crate) fn open_bucket(
dir: &Path,
plc: &str,
sink: &Path,
url: &str,
load_all: bool,
) -> Shared {
check_dir(dir, true).unwrap();
Arc::new(crate::Service::open(dir, bucket_config(plc, sink, url, load_all)).unwrap())
}
pub(crate) fn body(n: u8) -> Vec<u8> {
format!("spec-pds blob {n} ").repeat(4).into_bytes()
}
pub(crate) fn raw_cid(b: &[u8]) -> String {
pds_repo::data::cid_string(&pds_repo::data::cid_of(0x55, b))
}
/// A record `rkey` referencing the blob `b`.
pub(crate) async fn put(
s: &Shared,
token: &str,
did: &str,
rkey: &str,
b: &[u8],
) -> Result<Json, String> {
let r = json!({ "$type": "blob", "ref": { "$link": raw_cid(b) }, "mimeType": "text/plain", "size": b.len() });
call(s, "com.atproto.repo.putRecord", Some(token), json!({ "repo": did, "collection": "com.example.blob", "rkey": rkey, "record": { "f": r } })).await
}
pub(crate) async fn unput(s: &Shared, token: &str, did: &str, rkey: &str) {
call(
s,
"com.atproto.repo.deleteRecord",
Some(token),
json!({ "repo": did, "collection": "com.example.blob", "rkey": rkey }),
)
.await
.unwrap();
}
/// getBlob through the router's handler: its status and bytes.
pub(crate) async fn get(s: &Shared, did: &str, b: &[u8]) -> (u16, Vec<u8>) {
let query = url::form_urlencoded::Serializer::new(String::new())
.append_pair("did", did)
.append_pair("cid", &raw_cid(b))
.finish();
let res = crate::handle(
axum::extract::State(s.clone()),
Method::GET,
format!("/xrpc/com.atproto.sync.getBlob?{query}")
.parse()
.unwrap(),
axum::http::HeaderMap::new(),
None,
axum::body::Body::empty(),
)
.await;
let status = res.status().as_u16();
let bytes = axum::body::to_bytes(res.into_body(), usize::MAX)
.await
.unwrap()
.to_vec();
(status, bytes)
}
}
#[cfg(test)]
mod tests {
use super::testing::*;
use super::*;
use crate::blobs::test_upload;
use crate::testing::*;
use axum::http::Method;
fn vars(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
let m: HashMap<String, String> = pairs
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
move |n| m.get(n).cloned()
}
#[test]
fn settings_parse_as_the_spec_says() {
assert!(Settings::parse(vars(&[])).unwrap().is_none());
// Empty is as unset.
assert!(
Settings::parse(vars(&[(BUCKET, ""), (REGION, "")]))
.unwrap()
.is_none()
);
let s = Settings::parse(vars(&[(BUCKET, "b"), (KEY_ID, "k"), (SECRET, "s")]))
.unwrap()
.unwrap();
assert_eq!(
(s.region.as_str(), s.endpoint.as_str(), s.path_style),
("us-east-1", "https://s3.us-east-1.amazonaws.com/", false)
);
assert_eq!(
(s.upload_timeout, s.request_timeout),
(Duration::from_secs(20), Duration::from_secs(10))
);
// The environment's keys, with its session token, when neither of the bucket's is set.
let s = Settings::parse(vars(&[
(BUCKET, "b"),
("AWS_ACCESS_KEY_ID", "ak"),
("AWS_SECRET_ACCESS_KEY", "as"),
("AWS_SESSION_TOKEN", "t"),
]))
.unwrap()
.unwrap();
assert_eq!(
(s.keys.id.as_str(), s.keys.token.as_deref()),
("ak", Some("t"))
);
let bad: &[&[(&str, &str)]] = &[
&[
(BUCKET, "b"),
(KEY_ID, "k"),
(SECRET, "s"),
(PATH_STYLE, "yes"),
],
&[
(BUCKET, "b"),
(KEY_ID, "k"),
(SECRET, "s"),
(UPLOAD_TIMEOUT, "soon"),
],
&[
(BUCKET, "b"),
(KEY_ID, "k"),
(SECRET, "s"),
(REQUEST_TIMEOUT, "0"),
],
&[
(BUCKET, "b"),
(KEY_ID, "k"),
(SECRET, "s"),
(ENDPOINT, "ftp://store.example"),
],
&[
(BUCKET, "b"),
(KEY_ID, "k"),
(SECRET, "s"),
(ENDPOINT, "http://store.example/?a=1"),
],
&[
(BUCKET, "b"),
(KEY_ID, "k"),
(SECRET, "s"),
(ENDPOINT, "http://store.example/#f"),
],
&[(BUCKET, "b"), (KEY_ID, "k")],
&[
(BUCKET, "b"),
(KEY_ID, "k"),
(SECRET, ""),
("AWS_ACCESS_KEY_ID", "ak"),
("AWS_SECRET_ACCESS_KEY", "as"),
],
&[(BUCKET, "b")],
&[(BUCKET, "b"), ("AWS_ACCESS_KEY_ID", "ak")],
&[(PATH_STYLE, "true")],
&[(PATH_STYLE, "yes")],
&[(KEY_ID, "k"), (SECRET, "s")],
];
for b in bad {
assert!(Settings::parse(vars(b)).is_err(), "{b:?}");
}
}
/// AWS's example of a signed `GET` of an object (Signature Version 4,
/// "Examples: Signature Calculations", Amazon S3).
#[test]
fn signs_as_aws_does() {
let keys = Keys {
id: "AKIAIOSFODNN7EXAMPLE".into(),
secret: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".into(),
token: None,
};
let empty = hex::encode(Sha256::digest(b""));
let got = authorization(
&keys,
"us-east-1",
"GET",
"/test.txt",
"",
&[
("host", "examplebucket.s3.amazonaws.com"),
("range", "bytes=0-9"),
("x-amz-content-sha256", &empty),
("x-amz-date", "20130524T000000Z"),
],
&empty,
"20130524T000000Z",
);
assert_eq!(
got,
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
);
// And its `PUT`, whose key needs encoding, with a body.
let payload = hex::encode(Sha256::digest(b"Welcome to Amazon S3."));
let got = authorization(
&keys,
"us-east-1",
"PUT",
&format!("/{}", uri_encode("test$file.text")),
"",
&[
("date", "Fri, 24 May 2013 00:00:00 GMT"),
("host", "examplebucket.s3.amazonaws.com"),
("x-amz-content-sha256", &payload),
("x-amz-date", "20130524T000000Z"),
("x-amz-storage-class", "REDUCED_REDUNDANCY"),
],
&payload,
"20130524T000000Z",
);
assert!(got.ends_with(
"Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"
));
assert_eq!(amz_date(1_369_353_600_000), "20130524T000000Z");
assert_eq!(amz_date(1_369_353_600_000 + 86_399_999), "20130524T235959Z");
}
/// Path-style and virtual-hosted URLs, each segment encoded once as
/// SigV4 has it, the endpoint's path kept.
#[test]
fn objects_are_addressed_as_the_spec_says() {
let mut s = Settings::parse(vars(&[
(BUCKET, "blobs"),
(KEY_ID, "k"),
(SECRET, "s"),
(ENDPOINT, "http://127.0.0.1:9000/base/"),
(PATH_STYLE, "true"),
]))
.unwrap()
.unwrap();
let b = Bucket::new(s.clone()).unwrap();
let (url, host, path) = b.locate("did:web:localhost%3A2583", "bafkrei").unwrap();
assert_eq!(host, "127.0.0.1:9000");
assert_eq!(
path,
"/base/blobs/blocks/did%3Aweb%3Alocalhost%253A2583/bafkrei"
);
assert_eq!(url.as_str(), format!("http://127.0.0.1:9000{path}"));
s.path_style = false;
s.endpoint = url::Url::parse("https://store.example").unwrap();
let b = Bucket::new(s).unwrap();
let (url, host, path) = b.locate("did:plc:abc", "bafkrei").unwrap();
assert_eq!(host, "blobs.store.example");
assert_eq!(path, "/blocks/did%3Aplc%3Aabc/bafkrei");
assert_eq!(
url.as_str(),
"https://blobs.store.example/blocks/did%3Aplc%3Aabc/bafkrei"
);
}
/// A directory keeps its blobs where its first start put them.
#[test]
fn a_data_directory_keeps_its_blobs_where_they_are() {
let dir = crate::testing::scratch("blobstore-mark");
// A start in itself on a directory holding nothing leaves no mark;
// a bucket's then may.
check_dir(&dir, false).unwrap();
assert!(!dir.join(MARK).exists());
check_dir(&dir, true).unwrap();
check_dir(&dir, true).unwrap();
assert!(check_dir(&dir, false).is_err());
// One whose first start kept them in itself.
let other = crate::testing::scratch("blobstore-unmarked");
std::fs::create_dir_all(&other).unwrap();
std::fs::write(crate::store::service_layout(&other).log(0), b"").unwrap();
check_dir(&other, false).unwrap();
assert!(check_dir(&other, true).is_err());
assert!(!other.join(MARK).exists());
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&other);
}
/// Each upload's object is put before it is answered, and no file of it
/// kept; a read the store fails, or of an object gone, is `500`; an
/// upload the store fails holds nothing, and its object, if any, is
/// deleted; a blob no longer held has its object deleted once the store
/// answers, the account marked for a start meanwhile, but not one held
/// again by then. Every request is signed.
#[tokio::test]
async fn objects_follow_what_is_held() {
let plc = plc().await;
let fake = Arc::new(Fake::default());
let url = fake_store(fake.clone()).await;
let (dir, sink) = (scratch("bucket-held"), scratch("bucket-held-mail"));
let s = open_bucket(&dir, &plc, &sink, &url, true);
let bucket = s.ctx.bucket.clone().unwrap();
let (did, token) = create(&s, "alice").await;
let t = Some(token.as_str());
let key = |b: &[u8]| format!("blocks/{did}/{}", raw_cid(b));
let (a, b, c) = (body(1), body(2), body(3));
test_upload(&s, t, Some("text/plain"), a.clone(), false)
.await
.unwrap();
assert!(fake.has(&key(&a)));
assert!(!crate::blobs::blob_dir(&dir, &did).exists());
put(&s, &token, &did, "a", &a).await.unwrap();
assert_eq!(get(&s, &did, &a).await, (200, a.clone()));
fake.fail(Some((Method::GET, 503)));
assert_eq!(get(&s, &did, &a).await.0, 500);
fake.fail(Some((Method::GET, 0)));
assert_eq!(get(&s, &did, &a).await.0, 500);
fake.fail(None);
let saved = lock(&fake.objects).remove(&key(&a)).unwrap();
assert_eq!(get(&s, &did, &a).await.0, 500);
lock(&fake.objects).insert(key(&a), saved);
assert_eq!(get(&s, &did, &a).await, (200, a.clone()));
// An upload the store fails: 500 for a status, 504 for none.
fake.fail(Some((Method::PUT, 500)));
assert_eq!(
test_upload(&s, t, Some("text/plain"), b.clone(), false).await,
Err((500, "InternalServerError".into()))
);
fake.fail(Some((Method::PUT, 0)));
assert_eq!(
test_upload(&s, t, Some("text/plain"), b.clone(), true).await,
Err((504, "UpstreamTimeout".into()))
);
fake.fail(None);
assert_eq!(
put(&s, &token, &did, "b", &b).await,
Err("BlobNotFound".into())
);
// As if the store kept it after all: the next pass deletes it, and
// the mark taken for it goes.
lock(&fake.objects).insert(key(&b), b.clone());
assert!(s.ctx.eager.is_marked(&did));
assert!(pass(&s, &bucket).await);
assert!(!fake.has(&key(&b)));
assert!(!s.ctx.eager.is_marked(&did));
// A blob no longer held, the store failing deletes: kept, the
// account marked, until the store answers.
fake.fail(Some((Method::DELETE, 500)));
unput(&s, &token, &did, "a").await;
assert!(s.ctx.eager.is_marked(&did));
assert!(!pass(&s, &bucket).await);
assert!(fake.has(&key(&a)));
fake.fail(None);
assert!(pass(&s, &bucket).await);
assert!(!fake.has(&key(&a)));
assert!(!s.ctx.eager.is_marked(&did));
// No longer held, and uploaded again at once: kept.
test_upload(&s, t, Some("text/plain"), c.clone(), false)
.await
.unwrap();
put(&s, &token, &did, "c", &c).await.unwrap();
unput(&s, &token, &did, "c").await;
test_upload(&s, t, Some("text/plain"), c.clone(), false)
.await
.unwrap();
assert!(pass(&s, &bucket).await);
put(&s, &token, &did, "c", &c).await.unwrap();
assert_eq!(get(&s, &did, &c).await, (200, c.clone()));
assert_eq!(
lock(&fake.objects).keys().cloned().collect::<Vec<_>>(),
vec![key(&c)]
);
assert!(lock(&fake.seen).iter().all(|(_, _, ok)| *ok));
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&sink);
}
/// Deletes not yet answered when the server stops are made after it
/// starts again, from its log or its checkpoint: a stray's, its account
/// loaded at the start for its mark alone; and a deleted account's, from
/// the service log. Once answered, they are made no more.
#[tokio::test]
async fn deletes_owed_survive_a_restart() {
let plc = plc().await;
let fake = Arc::new(Fake::default());
let url = fake_store(fake.clone()).await;
let (dir, sink) = (scratch("bucket-restart"), scratch("bucket-restart-mail"));
let open = |load_all| open_bucket(&dir, &plc, &sink, &url, load_all);
let s = open(true);
let (alice, token) = create(&s, "alice").await;
let (bob, bob_token) = create(&s, "bob").await;
let (x, y) = (body(10), body(11));
let (kx, ky) = (
format!("blocks/{alice}/{}", raw_cid(&x)),
format!("blocks/{bob}/{}", raw_cid(&y)),
);
test_upload(&s, Some(&token), Some("text/plain"), x.clone(), false)
.await
.unwrap();
put(&s, &token, &alice, "x", &x).await.unwrap();
test_upload(&s, Some(&bob_token), Some("text/plain"), y.clone(), false)
.await
.unwrap();
put(&s, &bob_token, &bob, "y", &y).await.unwrap();
let bucket = s.ctx.bucket.clone().unwrap();
assert!(pass(&s, &bucket).await);
fake.fail(Some((Method::DELETE, 503)));
unput(&s, &token, &alice, "x").await;
call(
&s,
"com.atproto.server.requestAccountDelete",
Some(&bob_token),
Json::Null,
)
.await
.unwrap();
let mailed = crate::email::testing::last_token(&sink);
call(
&s,
"com.atproto.server.deleteAccount",
None,
json!({ "did": bob, "password": "bob-pw", "token": mailed }),
)
.await
.unwrap();
assert!(!pass(&s, &bucket).await);
assert!(fake.has(&kx) && fake.has(&ky));
// Both logs compacted: what is owed is in their checkpoints.
let wal = read(&s.account(&alice).unwrap().store).wal.clone();
wal.wait_compacted();
wal.compact().unwrap();
s.log.wait_compacted();
s.log.compact().unwrap();
drop((s, bucket));
fake.fail(None);
// A start loads alice's store for her mark alone.
let s = open(false);
assert!(s.account(&alice).unwrap().store.is_loaded());
let bucket = s.ctx.bucket.clone().unwrap();
assert!(pass(&s, &bucket).await);
assert!(!fake.has(&kx) && !fake.has(&ky));
assert!(!s.ctx.eager.is_marked(&alice));
drop((s, bucket));
let deletes = |f: &Fake| {
lock(&f.seen)
.iter()
.filter(|(m, _, _)| *m == Method::DELETE)
.count()
};
let made = deletes(&fake);
let s = open(false);
assert!(!s.account(&alice).unwrap().store.is_loaded());
let bucket = s.ctx.bucket.clone().unwrap();
assert!(pass(&s, &bucket).await);
assert_eq!(deletes(&fake), made);
assert_eq!(
read(&s.account(&alice).unwrap().store)
.blobs
.strays()
.count(),
0
);
assert!(lock(&fake.seen).iter().all(|(_, _, ok)| *ok));
drop((s, bucket));
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&sink);
}
}