0
2
server: new photos, stories and avatars warm the CDN, and what goes is purged
crates/grain-serve/src/classify/mod.rs+4−8
| @@ -35,21 +35,21 @@ | ||
//! Clef. | ||
//! | ||
//! Images go to Clef inline, at most four a request, each the CDN's | ||
//! thumbnail of its blob made at most 512 pixels on its longer side and | ||
//! thumbnail of its blob, at the preset kept JPEG for grain-serve | ||
//! ([`crate::cdn::THUMBNAIL_OG`]), made at most 512 pixels on its longer side and | ||
//! JPEG: Clef counts its context on the encoded bytes, and a larger photo | ||
//! can overflow it. | ||
| ||
pub mod classifiers; | ||
pub mod clef; | ||
| ||
use crate::cdn::{Cdn, THUMBNAIL}; | ||
use crate::cdn::{Cdn, THUMBNAIL_OG}; | ||
use crate::index::moderation::NewReport; | ||
use crate::index::{Index, record_uri, records}; | ||
use base64::Engine; | ||
use base64::engine::general_purpose::STANDARD; | ||
use classifiers::{ALL, BRIDGED_HIDDEN, Classifier, Kind, Subject}; | ||
use grain::env::Ctx; | ||
use serde_json::json; | ||
use sha2::{Digest, Sha256}; | ||
use std::collections::HashMap; | ||
use std::sync::{Arc, Mutex}; | ||
| @@ -264,11 +264,7 @@ impl Classify { | ||
} | ||
let mut images = vec![]; | ||
for (did, cid) in (c.images)(&state).into_iter().take(MOST_IMAGES) { | ||
let blob = json!({ "ref": { "$link": cid } }); | ||
let url = self | ||
.cdn | ||
.url(&did, &blob, THUMBNAIL) | ||
.expect("a blob that names a CID"); | ||
let url = self.cdn.at(&did, &cid, THUMBNAIL_OG); | ||
images.push(self.image(&url).await?); | ||
} | ||
let answers = clef::ask( | ||
crates/grain-serve/src/index/edge.rsnew+174−0
| @@ -0,0 +1,174 @@ | ||
//! What the index tells the keeper of grain's CDN ([`crate::edge`]): the | ||
//! records with an image that a live path (the stream, `refreshRecords` | ||
//! and the web's writes) put new or changed, for their images to be | ||
//! warmed, and the images of what leaves the index or is taken down, for | ||
//! the CDN to forget. Backfill tells it nothing: what it puts is not new. | ||
//! | ||
//! Neither waits on the keeper: it is told on a channel, and what it is | ||
//! behind by [`crate::edge::QUEUED`] changes is dropped, and logged. | ||
| ||
use super::records::Json; | ||
use super::schema::{shown, unhidden}; | ||
use super::{Applied, Index, record_uri}; | ||
use rusqlite::Connection; | ||
use tokio::sync::mpsc::{Sender, error::TrySendError}; | ||
| ||
/// An image, as the CDN names it: the account whose repo holds the blob, | ||
/// and the blob's CID. | ||
#[derive(Debug, Clone, PartialEq, Eq, Hash)] | ||
pub struct Image { | ||
pub did: String, | ||
pub cid: String, | ||
} | ||
| ||
/// What the CDN's keeper is told. | ||
#[derive(Debug, Clone, PartialEq)] | ||
pub enum Change { | ||
/// The record at this URI, of a collection with an image | ||
/// ([`IMAGES`]), is new or changed: its image is warmed. | ||
Warm(String), | ||
/// These images are gone from the index, or taken down: the CDN | ||
/// forgets them. | ||
Purge(Vec<Image>), | ||
} | ||
| ||
/// The collections whose records carry an image: each one's table, and | ||
/// the column of its blob. | ||
pub const IMAGES: &[(&str, &str, &str)] = &[ | ||
("social.grain.photo", "photo", "photo"), | ||
("social.grain.story", "story", "media"), | ||
("social.grain.actor.profile", "profile", "avatar"), | ||
]; | ||
| ||
/// The table and blob column of `collection`'s image, if it has one. | ||
fn image_column(collection: &str) -> Option<(&'static str, &'static str)> { | ||
let found = IMAGES.iter().find(|(c, _, _)| *c == collection); | ||
found.map(|(_, table, column)| (*table, *column)) | ||
} | ||
| ||
/// The images that `sql`, of one parameter, reads: each row an account's | ||
/// DID and a blob, or none ([`image`]). | ||
fn read(db: &Connection, sql: &str, param: &str) -> rusqlite::Result<Vec<Image>> { | ||
let mut stmt = db.prepare_cached(sql)?; | ||
let rows = stmt.query_map([param], image)?; | ||
rows.filter_map(Result::transpose).collect() | ||
} | ||
| ||
/// The image a row names, of the account its first column names, by the | ||
/// blob in its second; none where it has no blob. | ||
fn image(r: &rusqlite::Row) -> rusqlite::Result<Option<Image>> { | ||
let (did, blob): (String, Option<Json>) = (r.get(0)?, r.get(1)?); | ||
let cid = blob.as_ref().and_then(|b| b.0["ref"]["$link"].as_str()); | ||
Ok(cid.map(|cid| Image { | ||
did, | ||
cid: cid.to_string(), | ||
})) | ||
} | ||
| ||
/// The image of the record at `uri`, if it is of a collection with one | ||
/// and the index holds it. | ||
pub(super) fn record_images(db: &Connection, uri: &str) -> rusqlite::Result<Vec<Image>> { | ||
let imaged = record_uri(uri).and_then(|(_, collection, _)| image_column(collection)); | ||
let Some((table, column)) = imaged else { | ||
return Ok(Vec::new()); | ||
}; | ||
let sql = format!("SELECT did, {column} FROM {table} WHERE uri = ?1"); | ||
read(db, &sql, uri) | ||
} | ||
| ||
/// Every image of the account `did`: its photos', its stories' and its | ||
/// avatar. | ||
pub(super) fn account_images(db: &Connection, did: &str) -> rusqlite::Result<Vec<Image>> { | ||
let mut images = Vec::new(); | ||
for (_, table, column) in IMAGES { | ||
let sql = format!("SELECT did, {column} FROM {table} WHERE did = ?1"); | ||
images.extend(read(db, &sql, did)?); | ||
} | ||
Ok(images) | ||
} | ||
| ||
/// The images of `subject`, an account's DID or a record's URI: an | ||
/// account's every one ([`account_images`]), a gallery's photos', or a | ||
/// photo's, story's or profile's own. | ||
fn subject_images(db: &Connection, subject: &str) -> rusqlite::Result<Vec<Image>> { | ||
match record_uri(subject) { | ||
None => account_images(db, subject), | ||
Some((_, "social.grain.gallery", _)) => read( | ||
db, | ||
"SELECT p.did, p.photo FROM gallery_item i JOIN photo p ON p.uri = i.item | ||
WHERE i.gallery = ?1", | ||
subject, | ||
), | ||
Some(_) => record_images(db, subject), | ||
} | ||
} | ||
| ||
impl Index { | ||
/// Has what the CDN's keeper is told ([`Change`]) sent to `to`, from | ||
/// then on; whether none was named before. | ||
pub fn edge_to(&self, to: Sender<Change>) -> bool { | ||
self.edge.set(to).is_ok() | ||
} | ||
| ||
/// Says what a live path's put of the record at `uri` did: one new to | ||
/// the index arrived ([`Index::arrived`]), and one new or changed with | ||
/// an image has it warmed. | ||
pub fn kept(&self, uri: &str, applied: Applied) { | ||
if applied == Applied::Created { | ||
self.arrived(uri); | ||
} | ||
let changed = matches!(applied, Applied::Created | Applied::Updated); | ||
let imaged = record_uri(uri).is_some_and(|(_, c, _)| image_column(c).is_some()); | ||
if changed && imaged { | ||
self.to_edge(Change::Warm(uri.to_string())); | ||
} | ||
} | ||
| ||
/// The image of the record at `uri` as the index holds it, to warm: | ||
/// none where it holds no such record, its collection has no image, or | ||
/// it is not shown: its account taken down or gone, or a label that | ||
/// hides standing on it, its account or a gallery it is in. What a | ||
/// takedown purged, a later write of it does not bring back. | ||
pub fn shown_images(&self, uri: &str) -> rusqlite::Result<Vec<Image>> { | ||
let imaged = record_uri(uri).and_then(|(_, collection, _)| image_column(collection)); | ||
let Some((table, column)) = imaged else { | ||
return Ok(Vec::new()); | ||
}; | ||
let sql = format!( | ||
"SELECT t.did, t.{column} FROM {table} t WHERE t.uri = ?1 AND {} AND {} AND {} | ||
AND NOT EXISTS (SELECT 1 FROM gallery_item i WHERE i.item = t.uri AND NOT ({}))", | ||
shown("t.did"), | ||
unhidden("t.uri"), | ||
unhidden("t.did"), | ||
unhidden("i.gallery"), | ||
); | ||
self.read(|db| read(db, &sql, uri)) | ||
} | ||
| ||
/// Has the CDN forget the images of `subject`, an account's DID or a | ||
/// record's URI, which an admin took down: all of an account's, and a | ||
/// gallery's photos'. | ||
pub fn uncache(&self, subject: &str) -> rusqlite::Result<()> { | ||
let images = self.read(|db| subject_images(db, subject))?; | ||
self.to_purge(images); | ||
Ok(()) | ||
} | ||
| ||
/// Has the CDN forget `images`, if there are any. | ||
pub(super) fn to_purge(&self, images: Vec<Image>) { | ||
if !images.is_empty() { | ||
self.to_edge(Change::Purge(images)); | ||
} | ||
} | ||
| ||
fn to_edge(&self, change: Change) { | ||
let Some(to) = self.edge.get() else { | ||
return; | ||
}; | ||
match to.try_send(change) { | ||
Ok(()) => {} | ||
Err(TrySendError::Full(change)) => tracing::warn!(?change, "cdn: behind, dropped"), | ||
Err(TrySendError::Closed(change)) => tracing::warn!(?change, "cdn: stopped, dropped"), | ||
} | ||
} | ||
} | ||
crates/grain-serve/src/index/mod.rs+15−4
| @@ -15,6 +15,7 @@ | ||
| ||
pub mod backfill; | ||
pub mod classify; | ||
pub mod edge; | ||
pub mod graph; | ||
pub mod moderation; | ||
#[cfg(test)] | ||
| @@ -50,6 +51,9 @@ pub struct Index { | ||
/// Where the same records are told for the classifiers | ||
/// ([`Index::classify_to`]). | ||
classifying: std::sync::OnceLock<tokio::sync::mpsc::Sender<String>>, | ||
/// Where the CDN's keeper is told what to warm and forget | ||
/// ([`Index::edge_to`]). | ||
edge: std::sync::OnceLock<tokio::sync::mpsc::Sender<edge::Change>>, | ||
/// Its file, unless it is held in memory. | ||
path: Option<std::path::PathBuf>, | ||
/// The feeds that keep it current, as `/_status` tells of them. | ||
| @@ -261,6 +265,7 @@ impl Index { | ||
tracked: Mutex::new(tracked), | ||
arrivals: std::sync::OnceLock::new(), | ||
classifying: std::sync::OnceLock::new(), | ||
edge: std::sync::OnceLock::new(), | ||
path: None, | ||
feeds: crate::status::Feeds::default(), | ||
}) | ||
| @@ -383,7 +388,8 @@ impl Index { | ||
}) | ||
} | ||
| ||
/// Deletes the record at `uri`; whether the index had it. | ||
/// Deletes the record at `uri`; whether the index had it. The CDN | ||
/// forgets its image, if it has one ([`edge`]). | ||
pub fn delete(&self, uri: &str) -> rusqlite::Result<bool> { | ||
self.remove(uri, None) | ||
} | ||
| @@ -406,8 +412,10 @@ impl Index { | ||
} | ||
stamp(&tx, uri, rev, true)?; | ||
tx.execute("DELETE FROM self_label WHERE uri = ?1", [uri])?; | ||
let images = edge::record_images(&tx, uri)?; | ||
let deleted = tx.execute(&format!("DELETE FROM {table} WHERE uri = ?1"), [uri])?; | ||
tx.commit()?; | ||
self.to_purge(images); | ||
Ok(deleted > 0) | ||
} | ||
| ||
| @@ -490,9 +498,10 @@ impl Index { | ||
} | ||
| ||
/// Deletes every record `did` wrote, and what grain keeps for it | ||
/// ([`KEPT_FOR`]); how many records. What an admin holds of it stays: | ||
/// its takedown, the labels given out and the reports made of it, as | ||
/// do the mutes others made of it, which are theirs. | ||
/// ([`KEPT_FOR`]); how many records. The CDN forgets their images | ||
/// ([`edge`]). What an admin holds of it stays: its takedown, the | ||
/// labels given out and the reports made of it, as do the mutes others | ||
/// made of it, which are theirs. | ||
pub fn purge(&self, did: &str) -> rusqlite::Result<usize> { | ||
let mut db = self.db.lock().unwrap(); | ||
let tx = db.transaction()?; | ||
| @@ -506,6 +515,7 @@ impl Index { | ||
"DELETE FROM record_rev WHERE uri >= ?1 AND uri < ?2", | ||
[&from, &to], | ||
)?; | ||
let images = edge::account_images(&tx, did)?; | ||
let mut purged = 0; | ||
for (_, table) in records::COLLECTIONS { | ||
purged += tx.execute(&format!("DELETE FROM {table} WHERE did = ?1"), [did])?; | ||
| @@ -514,6 +524,7 @@ impl Index { | ||
tx.execute(sql, [did])?; | ||
} | ||
tx.commit()?; | ||
self.to_purge(images); | ||
Ok(purged) | ||
} | ||
| ||
crates/grain-serve/src/og/draw.rs+10−10
| @@ -12,6 +12,9 @@ use std::sync::{Arc, LazyLock}; | ||
| ||
pub const WIDTH: u32 = 1200; | ||
pub const HEIGHT: u32 = 630; | ||
/// The JPEG quality a card is encoded at: a card is mostly photos, which a | ||
/// PNG keeps at many times the bytes for no difference an unfurl shows. | ||
const QUALITY: u8 = 85; | ||
| ||
static INTER: &[u8] = include_bytes!("../../fonts/Inter-Regular.ttf"); | ||
static SYNE: &[u8] = include_bytes!("../../fonts/Syne-ExtraBold.ttf"); | ||
| @@ -307,7 +310,7 @@ fn kind(bytes: Arc<Vec<u8>>) -> Option<usvg::ImageKind> { | ||
} | ||
} | ||
| ||
/// `svg` drawn as a PNG, each image it names by key from `images`; one it | ||
/// `svg` drawn as a JPEG, each image it names by key from `images`; one it | ||
/// is not given, or cannot read, left out. | ||
pub fn render(svg: &str, images: HashMap<String, Arc<Vec<u8>>>) -> Vec<u8> { | ||
let resolve = move |href: &str, _: &usvg::Options| images.get(href).cloned().and_then(kind); | ||
| @@ -331,13 +334,10 @@ pub fn render(svg: &str, images: HashMap<String, Arc<Vec<u8>>>) -> Vec<u8> { | ||
.iter() | ||
.flat_map(|p| [p[0], p[1], p[2]]) | ||
.collect(); | ||
let mut out = Vec::with_capacity(rgb.len() / 4); | ||
let mut encoder = png::Encoder::new(&mut out, WIDTH, HEIGHT); | ||
encoder.set_color(png::ColorType::Rgb); | ||
encoder.set_depth(png::BitDepth::Eight); | ||
let mut writer = encoder.write_header().expect("writing to memory"); | ||
writer.write_image_data(&rgb).expect("writing to memory"); | ||
writer.finish().expect("writing to memory"); | ||
let mut out = Vec::with_capacity(rgb.len() / 8); | ||
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, QUALITY) | ||
.encode(&rgb, WIDTH, HEIGHT, image::ExtendedColorType::Rgb8) | ||
.expect("writing to memory"); | ||
out | ||
} | ||
| ||
| @@ -395,8 +395,8 @@ mod tests { | ||
for family in ["Syne", "Noto Sans JP", "Noto Emoji", "Inter"] { | ||
assert!(svg.contains(&format!(r#"font-family="{family}""#)), "{svg}"); | ||
} | ||
let png = render(&svg, images); | ||
assert!(png.starts_with(b"\x89PNG")); | ||
let jpeg = render(&svg, images); | ||
assert!(jpeg.starts_with(&[0xFF, 0xD8, 0xFF])); | ||
} | ||
| ||
#[test] | ||
crates/grain-serve/src/og/mod.rs+34−25
| @@ -7,8 +7,9 @@ | ||
//! galleries over the frame ([`collage`]), a gallery's up to six of its | ||
//! photos, each above a bar of the avatar, the name and handle, and | ||
//! grain's name; a story's is the story's photo, whole on black, its | ||
//! author along the foot. The images are the CDN's ([`crate::cdn`]), | ||
//! fetched as the card is drawn. | ||
//! author along the foot. The images are the CDN's ([`crate::cdn`]), at | ||
//! the presets kept JPEG for what grain-serve draws itself | ||
//! ([`crate::cdn::AVATAR_OG`] and its twins), fetched as the card is drawn. | ||
//! | ||
//! A card goes wherever the link does, and cannot blur or warn, so what a | ||
//! label touches is left out of it: an account taken down, or gone from | ||
| @@ -17,14 +18,15 @@ | ||
//! from grain's labeler or its author, shows no photos, and an account | ||
//! with a label shows neither its photos nor its avatar. | ||
//! | ||
//! Each card is drawn once in five minutes at most: kept here by its path | ||
//! for that long, and sent with `Cache-Control: public, max-age=300`. | ||
//! Each card is a JPEG, drawn once in five minutes at most: kept here by | ||
//! its path for that long, and sent with `Cache-Control: public, | ||
//! max-age=300`, which Cloudflare, in front, keeps it by too. | ||
| ||
pub mod collage; | ||
pub mod draw; | ||
| ||
use crate::App; | ||
use crate::cdn::{AVATAR, FULLSIZE, THUMBNAIL}; | ||
use crate::cdn::{AVATAR_OG, FULLSIZE_OG, THUMBNAIL_OG}; | ||
use crate::index::Row; | ||
use crate::index::records::{Gallery, Photo, Story}; | ||
use crate::read::Reader; | ||
| @@ -36,6 +38,7 @@ use axum::response::{IntoResponse, Response}; | ||
use axum::routing::get; | ||
use base64::Engine; | ||
use base64::engine::general_purpose::STANDARD; | ||
use bytes::Bytes; | ||
use draw::{Font, Style, Svg, fit}; | ||
use std::collections::{HashMap, VecDeque}; | ||
use std::sync::{Arc, LazyLock, Mutex}; | ||
| @@ -140,7 +143,7 @@ impl Reader<'_> { | ||
let profile = actor.profile.map(|p| p.record); | ||
let avatar = profile.as_ref().and_then(|p| p.avatar.as_ref()); | ||
let avatar = avatar.filter(|_| !labeled); | ||
let avatar = avatar.and_then(|a| self.cdn.url(did, &a.0, AVATAR)); | ||
let avatar = avatar.and_then(|a| self.cdn.url(did, &a.0, AVATAR_OG)); | ||
Ok(Author { | ||
did: did.to_string(), | ||
display_name: profile | ||
| @@ -163,7 +166,11 @@ impl Reader<'_> { | ||
/// Each photo of `rows`, at its URL in `preset`: the CDN's whole one | ||
/// for a photo on its own, its thumbnail for more. | ||
fn photo_urls(&self, rows: Vec<Row<Photo>>) -> Vec<(String, f64)> { | ||
let preset = if rows.len() == 1 { FULLSIZE } else { THUMBNAIL }; | ||
let preset = if rows.len() == 1 { | ||
FULLSIZE_OG | ||
} else { | ||
THUMBNAIL_OG | ||
}; | ||
rows.into_iter() | ||
.filter_map(|p| { | ||
let url = self.cdn.url(&p.did, &p.record.photo.0, preset)?; | ||
| @@ -234,7 +241,9 @@ impl Reader<'_> { | ||
let author = self.author(&story.did)?; | ||
let photo = match author.labeled || self.labeled(&uri)? { | ||
true => None, | ||
false => self.cdn.url(&author.did, &story.record.media.0, FULLSIZE), | ||
false => self | ||
.cdn | ||
.url(&author.did, &story.record.media.0, FULLSIZE_OG), | ||
}; | ||
Ok(Card::Story { | ||
name: (author.display_name).unwrap_or_else(|| format!("@{}", author.handle)), | ||
| @@ -431,21 +440,21 @@ pub struct Cards { | ||
| ||
#[derive(Default)] | ||
struct Kept { | ||
by_path: HashMap<String, (Instant, Arc<[u8]>)>, | ||
by_path: HashMap<String, (Instant, Bytes)>, | ||
order: VecDeque<String>, | ||
} | ||
| ||
impl Cards { | ||
fn get(&self, path: &str, now: Instant) -> Option<Arc<[u8]>> { | ||
fn get(&self, path: &str, now: Instant) -> Option<Bytes> { | ||
let kept = self.kept.lock().unwrap(); | ||
let (at, png) = kept.by_path.get(path)?; | ||
(now.duration_since(*at) < KEPT).then(|| png.clone()) | ||
let (at, card) = kept.by_path.get(path)?; | ||
(now.duration_since(*at) < KEPT).then(|| card.clone()) | ||
} | ||
| ||
fn put(&self, path: &str, png: Arc<[u8]>, now: Instant) { | ||
fn put(&self, path: &str, card: Bytes, now: Instant) { | ||
let mut kept = self.kept.lock().unwrap(); | ||
let Kept { by_path, order } = &mut *kept; | ||
if by_path.insert(path.to_string(), (now, png)).is_none() { | ||
if by_path.insert(path.to_string(), (now, card)).is_none() { | ||
order.push_back(path.to_string()); | ||
} | ||
while order.len() > KEEP { | ||
| @@ -489,8 +498,8 @@ async fn answer( | ||
return StatusCode::NOT_FOUND.into_response(); | ||
}; | ||
let path = uri.path().to_string(); | ||
let png = match app.cards.get(&path, Instant::now()) { | ||
Some(png) => png, | ||
let jpeg = match app.cards.get(&path, Instant::now()) { | ||
Some(jpeg) => jpeg, | ||
None => { | ||
let card = index.read(|db| { | ||
read(&Reader { | ||
| @@ -519,23 +528,23 @@ async fn answer( | ||
.collect(); | ||
let (svg, images) = svg(&card, &images).finish(); | ||
let _turn = DRAWING.acquire().await.expect("never closed"); | ||
let png: Arc<[u8]> = tokio::task::spawn_blocking(move || draw::render(&svg, images)) | ||
let jpeg: Bytes = tokio::task::spawn_blocking(move || draw::render(&svg, images)) | ||
.await | ||
.expect("drawing a card does not panic") | ||
.into(); | ||
app.cards.put(&path, png.clone(), Instant::now()); | ||
png | ||
app.cards.put(&path, jpeg.clone(), Instant::now()); | ||
jpeg | ||
} | ||
}; | ||
let headers = [ | ||
(header::CONTENT_TYPE, "image/png".to_string()), | ||
(header::CONTENT_TYPE, "image/jpeg".to_string()), | ||
( | ||
header::CACHE_CONTROL, | ||
format!("public, max-age={}", KEPT.as_secs()), | ||
), | ||
(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".to_string()), | ||
]; | ||
(headers, png.to_vec()).into_response() | ||
(headers, jpeg).into_response() | ||
} | ||
| ||
async fn profile(State(app): State<Arc<App>>, Path(actor): Path<String>, uri: Uri) -> Response { | ||
| @@ -565,15 +574,15 @@ mod tests { | ||
#[test] | ||
fn a_card_is_kept_for_five_minutes_and_the_newest_two_hundred() { | ||
let cards = Cards::default(); | ||
let png: Arc<[u8]> = Arc::from(&b"png"[..]); | ||
let card = Bytes::from_static(b"jpeg"); | ||
let now = Instant::now(); | ||
cards.put("/a", png.clone(), now); | ||
cards.put("/a", png.clone(), now); | ||
cards.put("/a", card.clone(), now); | ||
cards.put("/a", card.clone(), now); | ||
assert!(cards.get("/a", now).is_some()); | ||
assert!(cards.get("/a", now + KEPT).is_none()); | ||
assert!(cards.get("/none", now).is_none()); | ||
for i in 0..KEEP { | ||
cards.put(&format!("/{i}"), png.clone(), now); | ||
cards.put(&format!("/{i}"), card.clone(), now); | ||
} | ||
assert!(cards.get("/a", now).is_none(), "the oldest is dropped"); | ||
assert!(cards.get(&format!("/{}", KEEP - 1), now).is_some()); | ||
crates/grain-serve/src/cdn.rs+51−4
| @@ -19,6 +19,25 @@ pub const AVATAR: &str = "avatar"; | ||
pub const THUMBNAIL: &str = "feed_thumbnail"; | ||
pub const FULLSIZE: &str = "feed_fullsize"; | ||
| ||
/// The same sizes for what grain-serve draws itself, the OpenGraph cards | ||
/// and what the classifiers show Clef: each its namesake's twin in | ||
/// imgproxy's `IMGPROXY_PRESETS`, kept JPEG (`ext:jpeg`) whatever a | ||
/// browser's would become, as the `image` crate here reads no AVIF. | ||
/// Bluesky's CDN, which has no such presets, draws their namesakes. | ||
pub const AVATAR_OG: &str = "avatar_og"; | ||
pub const THUMBNAIL_OG: &str = "feed_thumbnail_og"; | ||
pub const FULLSIZE_OG: &str = "feed_fullsize_og"; | ||
| ||
/// Every preset grain signs a URL at: all an image may be cached at. | ||
pub const PRESETS: &[&str] = &[ | ||
AVATAR, | ||
THUMBNAIL, | ||
FULLSIZE, | ||
AVATAR_OG, | ||
THUMBNAIL_OG, | ||
FULLSIZE_OG, | ||
]; | ||
| ||
/// Where blob URLs point. | ||
#[derive(Debug, Clone, Default, PartialEq)] | ||
pub struct Cdn { | ||
| @@ -50,18 +69,32 @@ impl Cdn { | ||
}) | ||
} | ||
| ||
/// Whether these are grain's own CDN's URLs, which it may warm and | ||
/// purge ([`crate::edge`]), rather than Bluesky's. | ||
pub fn own(&self) -> bool { | ||
self.signing.is_some() | ||
} | ||
| ||
/// The URL of the blob `blob`, a record's `{ref: {$link}}`, in `did`'s | ||
/// repo, drawn at `preset`; none where it names no CID. | ||
pub fn url(&self, did: &str, blob: &Value, preset: &str) -> Option<String> { | ||
let cid = blob["ref"]["$link"].as_str()?; | ||
let path = format!("/{preset}/plain/{did}/{cid}"); | ||
Some(match &self.signing { | ||
Some(self.at(did, cid, preset)) | ||
} | ||
| ||
/// The URL of the blob `cid` in `did`'s repo, drawn at `preset`. | ||
pub fn at(&self, did: &str, cid: &str, preset: &str) -> String { | ||
match &self.signing { | ||
Some(s) => { | ||
let path = format!("/{preset}/plain/{did}/{cid}"); | ||
let mac = hmac_sha256(&s.key, &[&s.salt, path.as_bytes()]); | ||
format!("{}/{}{path}", s.url, URL_SAFE_NO_PAD.encode(mac)) | ||
} | ||
None => format!("https://cdn.bsky.app/img{path}@jpeg"), | ||
}) | ||
None => { | ||
let preset = preset.strip_suffix("_og").unwrap_or(preset); | ||
format!("https://cdn.bsky.app/img/{preset}/plain/{did}/{cid}@jpeg") | ||
} | ||
} | ||
} | ||
} | ||
| ||
| @@ -135,16 +168,30 @@ mod tests { | ||
let sig = URL_SAFE_NO_PAD.encode(mac); | ||
assert_eq!(url, format!("https://cdn.grain.social/{sig}{path}")); | ||
assert_eq!(cdn.url("did:plc:alice", &json!({"cid": "x"}), AVATAR), None); | ||
assert!(cdn.own()); | ||
// A twin preset is signed by its own name. | ||
let og = cdn.at("did:plc:alice", "bafkreiblob", AVATAR_OG); | ||
assert!( | ||
og.ends_with("/avatar_og/plain/did:plc:alice/bafkreiblob"), | ||
"{og}" | ||
); | ||
assert_ne!(og.replace("avatar_og", "avatar"), url); | ||
} | ||
| ||
#[test] | ||
fn with_no_cdn_of_its_own_bluesky_draws_them() { | ||
let cdn = Cdn::new(None, None, None).unwrap(); | ||
assert_eq!(cdn, Cdn::default()); | ||
assert!(!cdn.own()); | ||
assert_eq!( | ||
cdn.url("did:plc:alice", &blob(), THUMBNAIL).unwrap(), | ||
"https://cdn.bsky.app/img/feed_thumbnail/plain/did:plc:alice/bafkreiblob@jpeg" | ||
); | ||
// Bluesky's has no twins: their namesakes draw them. | ||
assert_eq!( | ||
cdn.at("did:plc:alice", "bafkreiblob", FULLSIZE_OG), | ||
"https://cdn.bsky.app/img/feed_fullsize/plain/did:plc:alice/bafkreiblob@jpeg" | ||
); | ||
} | ||
| ||
#[test] | ||
crates/grain-serve/src/edge.rsnew+250−0
| @@ -0,0 +1,250 @@ | ||
//! Keeping grain's CDN in step with the index: what arrives is warmed, | ||
//! and what leaves or is taken down is purged. The index says which | ||
//! ([`crate::index::edge`]); this keeps the CDN, off the path of the | ||
//! request or the stream that brought it, so that neither waits on it. | ||
//! | ||
//! **Warming.** A photo, story or avatar a live path puts new or changed in | ||
//! the index is fetched through the CDN at once, at each preset the views | ||
//! draw it at ([`shown_at`]), so that the first to see it is not the first | ||
//! to ask: a miss costs seconds, the PDS's blob fetched and imgproxy's | ||
//! encode, and a story lives a day, seen by few, each usually the first | ||
//! where they are. Each URL is asked once for each type in [`ACCEPTS`], as | ||
//! the CDN keeps a copy for each type it varies on. At most `at_once` | ||
//! requests go at a time; one that fails is logged and dropped, and the | ||
//! image is drawn when it is first asked for, as before. | ||
//! | ||
//! **Purging.** An account or a record an admin takes down | ||
//! ([`Index::uncache`]), and the image of a record or an account that | ||
//! leaves the index, is purged from Bunny, the CDN in front of imgproxy, | ||
//! at every preset grain signs a URL at ([`cdn::PRESETS`]), by Bunny's API | ||
//! with the key in `BUNNY_API_KEY`: the CDN keeps an image for a year, and | ||
//! one who kept its URL would see it there all that while. Purges go one | ||
//! at a time, as Bunny allows about five a second, and one Bunny refuses | ||
//! for that (429), or fails, is tried again after [`Timing::retry`], or | ||
//! `Retry-After`, up to [`Timing::attempts`] times, then logged and dropped. | ||
//! | ||
//! Only grain's own CDN is kept ([`Cdn::own`]): Bluesky's is not ours to | ||
//! warm or purge. | ||
| ||
use crate::cdn::{self, AVATAR, Cdn, FULLSIZE, THUMBNAIL}; | ||
use crate::index::edge::{Change, Image}; | ||
use crate::index::{Index, record_uri}; | ||
use reqwest::StatusCode; | ||
use reqwest::header::{ACCEPT, RETRY_AFTER}; | ||
use std::sync::Arc; | ||
use std::time::Duration; | ||
use tokio::sync::{Mutex, Semaphore, mpsc}; | ||
| ||
/// How many changes wait for the keeper before more are dropped. | ||
pub const QUEUED: usize = 10_000; | ||
/// How many requests warm the CDN at once, unless `--cdn-warm` says. | ||
pub const AT_ONCE: usize = 4; | ||
/// Bunny's API, which purges, unless `--bunny-api` says. | ||
pub const BUNNY_API: &str = "https://api.bunny.net"; | ||
/// The types each URL is warmed for, as `Accept` asks them. One today: | ||
/// every answer is JPEG, and the CDN keeps one copy of each. Once | ||
/// imgproxy answers AVIF to those that take it and the CDN varies on it, | ||
/// `image/avif` joins it, and each URL is warmed in both. | ||
pub const ACCEPTS: &[&str] = &["image/jpeg"]; | ||
| ||
/// The presets the views draw the images of `collection`'s records at | ||
/// ([`crate::read::views`]): an avatar's, and a photo's or a story's, as | ||
/// the feeds, grids and stories show it, thumbnail and whole. | ||
fn shown_at(collection: &str) -> &'static [&'static str] { | ||
match collection { | ||
"social.grain.actor.profile" => &[AVATAR], | ||
_ => &[THUMBNAIL, FULLSIZE], | ||
} | ||
} | ||
| ||
/// How long purging waits. | ||
#[derive(Debug, Clone, Copy)] | ||
pub struct Timing { | ||
/// How long a purge that failed waits to be tried again, unless Bunny | ||
/// says how long. | ||
pub retry: Duration, | ||
/// How many times a purge is tried before it is dropped. | ||
pub attempts: u32, | ||
} | ||
| ||
impl Default for Timing { | ||
fn default() -> Timing { | ||
Timing { | ||
retry: Duration::from_secs(1), | ||
attempts: 10, | ||
} | ||
} | ||
} | ||
| ||
/// Bunny's API, and the key it takes, to purge with. | ||
#[derive(Clone, PartialEq)] | ||
pub struct Purge { | ||
/// Such as [`BUNNY_API`]. | ||
pub api: String, | ||
pub key: String, | ||
} | ||
| ||
impl std::fmt::Debug for Purge { | ||
/// The API, never the key. | ||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | ||
f.debug_struct("Purge").field("api", &self.api).finish() | ||
} | ||
} | ||
| ||
/// The CDN's keeper, at work on an index. | ||
pub struct Edge { | ||
pub index: Arc<Index>, | ||
/// What the CDN and Bunny are asked with. | ||
pub http: reqwest::Client, | ||
/// grain's own CDN, whose URLs are warmed and purged. | ||
pub cdn: Cdn, | ||
/// How many requests warm it at once; none warms nothing. | ||
pub at_once: usize, | ||
/// Bunny's API, if purges go to it; none purges nothing. | ||
pub purge: Option<Purge>, | ||
pub timing: Timing, | ||
} | ||
| ||
impl Edge { | ||
/// Starts keeping the CDN as the index says, on the runtime this is | ||
/// called on; whether it was not kept already. | ||
pub fn start(self) -> bool { | ||
let (to, from) = mpsc::channel(QUEUED); | ||
if !self.index.edge_to(to) { | ||
return false; | ||
} | ||
tokio::spawn(Arc::new(self).run(from)); | ||
true | ||
} | ||
| ||
/// Takes each change the index tells of, until its sender is gone. | ||
async fn run(self: Arc<Self>, mut from: mpsc::Receiver<Change>) { | ||
let turns = Arc::new(Semaphore::new(self.at_once)); | ||
let purging = Arc::new(Mutex::new(())); | ||
while let Some(change) = from.recv().await { | ||
match change { | ||
Change::Warm(uri) if self.at_once > 0 => self.warm(&uri, &turns), | ||
Change::Warm(_) => {} | ||
Change::Purge(images) => { | ||
if let Some(purge) = self.purge.clone() { | ||
let (edge, purging) = (self.clone(), purging.clone()); | ||
tokio::spawn(async move { | ||
let _turn = purging.lock().await; | ||
edge.purge(&purge, &images).await; | ||
}); | ||
} | ||
} | ||
} | ||
} | ||
} | ||
| ||
/// Warms the image of the record at `uri`, which the index says is | ||
/// one with an image, each URL in turn of `turns`. | ||
fn warm(self: &Arc<Self>, uri: &str, turns: &Arc<Semaphore>) { | ||
let collection = record_uri(uri).map_or("", |(_, c, _)| c); | ||
let images = match self.index.shown_images(uri) { | ||
Ok(images) => images, | ||
Err(e) => { | ||
tracing::warn!(%uri, error = %e, "cdn: reading what to warm"); | ||
return; | ||
} | ||
}; | ||
for Image { did, cid } in images { | ||
for preset in shown_at(collection) { | ||
let url = self.cdn.at(&did, &cid, preset); | ||
for accept in ACCEPTS { | ||
let (edge, turns, url) = (self.clone(), turns.clone(), url.clone()); | ||
tokio::spawn(async move { | ||
let _turn = turns.acquire().await.expect("never closed"); | ||
if let Err(e) = edge.fetch(&url, accept).await { | ||
tracing::warn!(%url, accept, error = %e, "cdn: warming"); | ||
} | ||
}); | ||
} | ||
} | ||
} | ||
} | ||
| ||
/// Asks the CDN for `url` as a client taking `accept` does, reading | ||
/// the answer whole so that the CDN has all of it to keep. | ||
async fn fetch(&self, url: &str, accept: &str) -> anyhow::Result<()> { | ||
let mut res = self.http.get(url).header(ACCEPT, accept).send().await?; | ||
let status = res.status(); | ||
while res.chunk().await?.is_some() {} | ||
anyhow::ensure!(status.is_success(), "answered {status}"); | ||
Ok(()) | ||
} | ||
| ||
/// Purges every URL of `images` from Bunny, one after another. | ||
async fn purge(&self, purge: &Purge, images: &[Image]) { | ||
for Image { did, cid } in images { | ||
for preset in cdn::PRESETS { | ||
let url = self.cdn.at(did, cid, preset); | ||
if let Err(e) = self.purge_one(purge, &url).await { | ||
tracing::warn!(%url, error = %e, "cdn: purging, dropped"); | ||
} | ||
} | ||
} | ||
} | ||
| ||
/// Purges `url` from Bunny, trying again while it fails, until | ||
/// [`Timing::attempts`]. | ||
async fn purge_one(&self, purge: &Purge, url: &str) -> anyhow::Result<()> { | ||
let endpoint = format!("{}/purge", purge.api); | ||
let mut tried = 0; | ||
loop { | ||
tried += 1; | ||
let sent = self | ||
.http | ||
.post(&endpoint) | ||
.query(&[("url", url)]) | ||
.header("AccessKey", &purge.key) | ||
.send() | ||
.await; | ||
let (wait, why) = match sent { | ||
Ok(res) if res.status().is_success() => return Ok(()), | ||
Ok(res) if again(res.status()) => { | ||
let after = res.headers().get(RETRY_AFTER); | ||
let after = after.and_then(|v| v.to_str().ok()?.parse().ok()); | ||
let wait = after.map_or(self.timing.retry, Duration::from_secs); | ||
(wait, format!("answered {}", res.status())) | ||
} | ||
Ok(res) => anyhow::bail!("answered {}", res.status()), | ||
Err(e) => (self.timing.retry, e.to_string()), | ||
}; | ||
if tried >= self.timing.attempts { | ||
anyhow::bail!("{why}, {tried} times"); | ||
} | ||
tokio::time::sleep(wait).await; | ||
} | ||
} | ||
} | ||
| ||
/// Whether a purge Bunny answered with `status` may go if tried again: | ||
/// one it took too many of at once, or failed itself. | ||
fn again(status: StatusCode) -> bool { | ||
status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error() | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[tokio::test] | ||
async fn the_keeper_ends_when_its_sender_is_gone() { | ||
let edge = Edge { | ||
index: Arc::new(Index::memory().unwrap()), | ||
http: reqwest::Client::new(), | ||
cdn: Cdn::new(Some("https://cdn.grain.social/"), Some("00"), Some("00")).unwrap(), | ||
at_once: 0, | ||
purge: None, | ||
timing: Timing::default(), | ||
}; | ||
let (to, from) = mpsc::channel(1); | ||
to.send(Change::Warm("at://did:plc:alice/x/y".into())) | ||
.await | ||
.unwrap(); | ||
drop(to); | ||
Arc::new(edge).run(from).await; | ||
} | ||
} | ||
crates/grain-serve/src/jetstream.rs+4−4
| @@ -13,7 +13,8 @@ | ||
//! stream write and a backfill's read of the same repo leave the later of | ||
//! the two. An account newly tracked, or back after its host said it was | ||
//! gone, is queued for backfill ([`Index::resync`]) to be read whole. | ||
//! A record new to the index is pushed ([`Index::arrived`]). An account | ||
//! A record new to the index is pushed, and one new or changed with an | ||
//! image has it warmed on the CDN ([`Index::kept`]). An account | ||
//! its host says was deleted goes, records and all, once the host its | ||
//! identity names now says it has no such account | ||
//! ([`crate::accounts::deletions`]): one that moved stays. | ||
| @@ -25,7 +26,7 @@ | ||
| ||
use crate::accounts; | ||
use crate::backfill::SIGNALS; | ||
use crate::index::{Applied, Index, PutError, records}; | ||
use crate::index::{Index, PutError, records}; | ||
use grain::env::Ctx; | ||
use grain::jetstream::{Event, Follower, Timing, What}; | ||
use std::sync::Arc; | ||
| @@ -187,8 +188,7 @@ pub fn apply_with( | ||
None => index.put(&uri, &cid, &record), | ||
}; | ||
match put { | ||
Ok(Applied::Created) => index.arrived(&uri), | ||
Ok(_) => {} | ||
Ok(applied) => index.kept(&uri, applied), | ||
Err(PutError::Refused(why)) => { | ||
tracing::info!(%uri, %why, "jetstream: skipped a record"); | ||
} | ||
crates/grain-serve/src/lib.rs+69−3
| @@ -10,13 +10,15 @@ | ||
//! that go ([`accounts`]), draws the cards links unfurl into ([`og`]), | ||
//! and logs the web in ([`login`]), handing the rest to the web's pages | ||
//! ([`pages`]). Off the path of any request, its classifiers score what | ||
//! arrives for moderators to review ([`classify`]). | ||
//! arrives for moderators to review ([`classify`]), and its CDN is warmed | ||
//! with what arrives and purged of what goes ([`edge`]). | ||
| ||
pub mod accounts; | ||
pub mod backfill; | ||
pub mod cdn; | ||
pub mod classify; | ||
pub mod did; | ||
pub mod edge; | ||
pub mod gallery; | ||
pub mod graph; | ||
pub mod index; | ||
| @@ -63,6 +65,8 @@ pub const FLAGS: &[&str] = &[ | ||
"--backfill-skip", | ||
"--backfill-concurrency", | ||
"--cdn", | ||
"--cdn-warm", | ||
"--bunny-api", | ||
"--bsky-appview", | ||
"--sign-up-pds", | ||
"--group-host", | ||
| @@ -80,8 +84,8 @@ pub const FLAGS: &[&str] = &[ | ||
]; | ||
| ||
/// What `grain-serve` says of its flags when given one it does not take. | ||
pub const USAGE: &str = "usage: grain-serve [--listen <addr>] [--index <file>] [--public-url <https://host>] [--jetstream <wss://host>] [--backfill <https://directory>[,...]] [--backfill-mirror <https://mirror>] [--backfill-skip <host>[,...]] [--backfill-concurrency <n>] [--cdn <https://imgproxy>] [--bsky-appview <https://appview>] [--sign-up-pds <https://pds>] [--group-host <https://host>] [--carry-cookies-until <rfc3339 time>] [--apns-key-id <id>] [--apns-team-id <id>] [--apns-topic <bundle id>] [--apns-url <https://apns>] [--fcm-project-id <id>] [--clef-model <model>] [--clef-url <url>] [--check-index <file>] [--metrics <addr>] [--web <http://app>] | ||
environment: GRAIN_ADMIN_DIDS, GRAIN_CDN_KEY, GRAIN_CDN_SALT, GRAIN_APNS_KEY_FILE, GRAIN_FCM_KEY_FILE, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_AI_TOKEN, RUST_LOG"; | ||
pub const USAGE: &str = "usage: grain-serve [--listen <addr>] [--index <file>] [--public-url <https://host>] [--jetstream <wss://host>] [--backfill <https://directory>[,...]] [--backfill-mirror <https://mirror>] [--backfill-skip <host>[,...]] [--backfill-concurrency <n>] [--cdn <https://imgproxy>] [--cdn-warm <n>] [--bunny-api <https://api>] [--bsky-appview <https://appview>] [--sign-up-pds <https://pds>] [--group-host <https://host>] [--carry-cookies-until <rfc3339 time>] [--apns-key-id <id>] [--apns-team-id <id>] [--apns-topic <bundle id>] [--apns-url <https://apns>] [--fcm-project-id <id>] [--clef-model <model>] [--clef-url <url>] [--check-index <file>] [--metrics <addr>] [--web <http://app>] | ||
environment: GRAIN_ADMIN_DIDS, GRAIN_CDN_KEY, GRAIN_CDN_SALT, BUNNY_API_KEY, GRAIN_APNS_KEY_FILE, GRAIN_FCM_KEY_FILE, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_AI_TOKEN, RUST_LOG"; | ||
| ||
/// What the server is set to do, from its flags ([`FLAGS`]) and, for | ||
/// secrets, key files and who its admins are, its environment. | ||
| @@ -104,6 +108,13 @@ pub struct Settings { | ||
/// environment: grain's imgproxy, and the hex key and salt it signs | ||
/// paths with ([`cdn::Cdn`]). None draws images from Bluesky's CDN. | ||
pub cdn: [Option<String>; 3], | ||
/// `--cdn-warm`: how many requests warm grain's own CDN at once with | ||
/// what arrives ([`edge`]). None is [`edge::AT_ONCE`]; 0 warms none. | ||
pub cdn_warm: Option<usize>, | ||
/// `BUNNY_API_KEY` from the environment, and `--bunny-api`, None | ||
/// being [`edge::BUNNY_API`]: the API that purges grain's own CDN, Bunny, | ||
/// of what goes ([`edge`]). With no key, it is purged of nothing. | ||
pub purge: Option<edge::Purge>, | ||
/// `--bsky-appview`: the Bluesky appview whose typeahead | ||
/// `searchActorsTypeahead` fills out its own with, such as | ||
/// [`read::search::BSKY_APPVIEW`]. None asks none, as tests and dev | ||
| @@ -185,6 +196,14 @@ impl Settings { | ||
backfill: backfill::Config::given(given)?, | ||
jetstream: get("--jetstream"), | ||
cdn: [get("--cdn"), get("GRAIN_CDN_KEY"), get("GRAIN_CDN_SALT")], | ||
cdn_warm: match get("--cdn-warm") { | ||
Some(n) => Some(n.parse().map_err(|_| "--cdn-warm takes a number")?), | ||
None => None, | ||
}, | ||
purge: get("BUNNY_API_KEY").map(|key| edge::Purge { | ||
api: url("--bunny-api").unwrap_or_else(|| edge::BUNNY_API.into()), | ||
key, | ||
}), | ||
bsky_appview: url("--bsky-appview"), | ||
sign_up_pds: get("--sign-up-pds"), | ||
group_host: url("--group-host"), | ||
| @@ -269,6 +288,30 @@ impl Settings { | ||
classify.start() | ||
} | ||
| ||
/// Starts keeping `app`'s CDN as its index says ([`edge`]), on the | ||
/// runtime this is called on: warming it `--cdn-warm` requests at a | ||
/// time, and purging it by Bunny's API with `BUNNY_API_KEY`; whether | ||
/// it does either. Only grain's own CDN is kept, of an index. | ||
pub fn edge(&self, app: &App) -> bool { | ||
let Some(index) = app.index.clone().filter(|_| app.cdn.own()) else { | ||
return false; | ||
}; | ||
let at_once = self.cdn_warm.unwrap_or(edge::AT_ONCE); | ||
if at_once == 0 && self.purge.is_none() { | ||
return false; | ||
} | ||
tracing::info!(at_once, purge = ?self.purge, "keeping the CDN"); | ||
let edge = edge::Edge { | ||
index, | ||
http: app.ctx.http.clone(), | ||
cdn: app.cdn.clone(), | ||
at_once, | ||
purge: self.purge.clone(), | ||
timing: edge::Timing::default(), | ||
}; | ||
edge.start() | ||
} | ||
| ||
/// The server `ctx` and these settings make: its index opened, and | ||
/// its DID named. | ||
pub fn app(&self, ctx: Ctx) -> anyhow::Result<App> { | ||
| @@ -550,10 +593,15 @@ mod tests { | ||
jetstream::DEFAULT, | ||
"--web", | ||
"http://web:3000/", | ||
"--cdn-warm", | ||
"2", | ||
"--bunny-api", | ||
"https://bunny.test/", | ||
]), | ||
|name| match name { | ||
"GRAIN_ADMIN_DIDS" => Some("did:plc:alice, did:web:bob.test".into()), | ||
"GRAIN_CDN_KEY" => Some("00".into()), | ||
"BUNNY_API_KEY" => Some("sekrit".into()), | ||
// A setting is a flag: its variable is never read. | ||
"GRAIN_INDEX" => Some("/elsewhere.db".into()), | ||
_ => None, | ||
| @@ -570,6 +618,22 @@ mod tests { | ||
assert_eq!(set.jetstream.as_deref(), Some(jetstream::DEFAULT)); | ||
assert_eq!(set.cdn[0].as_deref(), Some("https://cdn.grain.social")); | ||
assert_eq!(set.cdn[1].as_deref(), Some("00")); | ||
assert_eq!(set.cdn_warm, Some(2)); | ||
let purge = set.purge.clone().unwrap(); | ||
assert_eq!( | ||
(purge.api.as_str(), purge.key.as_str()), | ||
("https://bunny.test", "sekrit") | ||
); | ||
assert!( | ||
!format!("{set:?}").contains("sekrit"), | ||
"the key is not shown" | ||
); | ||
// With no --bunny-api, Bunny's own; with no key, none. | ||
let key = |name: &str| (name == "BUNNY_API_KEY").then(|| "k".to_string()); | ||
let bunny = Settings::parse(&args(&[]), key).unwrap().purge.unwrap(); | ||
assert_eq!(bunny.api, edge::BUNNY_API); | ||
let keyless = Settings::parse(&args(&["--bunny-api", "https://b.test"]), none); | ||
assert_eq!(keyless.unwrap().purge, None); | ||
assert!( | ||
set.app(Ctx::new(Env::from_vars(|_| None))).is_err(), | ||
"a CDN with no salt" | ||
| @@ -591,6 +655,8 @@ mod tests { | ||
assert!(FLAGS.iter().all(|flag| USAGE.contains(flag))); | ||
let bare = Settings::parse(&args(&["--index"]), none).unwrap_err(); | ||
assert_eq!(bare, "--index takes a value"); | ||
let many = Settings::parse(&args(&["--cdn-warm", "many"]), none).unwrap_err(); | ||
assert_eq!(many, "--cdn-warm takes a number"); | ||
// A setting is a flag, not its old variable. | ||
let var = Settings::parse(&args(&["GRAIN_INDEX=/data/index.db"]), none); | ||
assert!(var.unwrap_err().starts_with("unexpected GRAIN_INDEX")); | ||
crates/grain-serve/src/main.rs+1−0
| @@ -48,6 +48,7 @@ async fn run(settings: grain_serve::Settings) -> anyhow::Result<()> { | ||
} | ||
settings.push(&app)?; | ||
settings.classify(&app); | ||
settings.edge(&app); | ||
settings.follow(&app)?; | ||
let addr = settings.listen.as_deref().unwrap_or(grain_serve::LISTEN); | ||
let listener = tokio::net::TcpListener::bind(addr) | ||
crates/grain-serve/src/moderation.rs+7−3
crates/grain-serve/src/sync.rs+4−5
crates/grain-serve/src/write.rs+2−3
crates/grain-serve/tests/cdn/mod.rsnew+205−0
crates/grain-serve/tests/classify.rs+1−1
crates/grain-serve/tests/deletes.rs+9−1
crates/grain-serve/tests/edge.rsnew+432−0
432 lines changed: not loaded until asked for.