~ knotless Search/
↑↓ move↵ open⇧↵ all results

grain.social/grain

photography-focused photo sharing platform

grain.social
1 collaborator
0
2

server: new photos, stories and avatars warm the CDN, and what goes is purged

Merged chadtmiller.com asked to merge into main from grain.social/grain:serve/image-warm
crates/grain-serve/src/classify/mod.rs+4−8
@@ -35,21 +35,21 @@
//! Clef.
//!
//! Images go to Clef inline, at most four a request, each the CDN's
//! thumbnail of its blob made at most 512 pixels on its longer side and
//! thumbnail of its blob, at the preset kept JPEG for grain-serve
//! ([`crate::cdn::THUMBNAIL_OG`]), made at most 512 pixels on its longer side and
//! JPEG: Clef counts its context on the encoded bytes, and a larger photo
//! can overflow it.
pub mod classifiers;
pub mod clef;
use crate::cdn::{Cdn, THUMBNAIL};
use crate::cdn::{Cdn, THUMBNAIL_OG};
use crate::index::moderation::NewReport;
use crate::index::{Index, record_uri, records};
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use classifiers::{ALL, BRIDGED_HIDDEN, Classifier, Kind, Subject};
use grain::env::Ctx;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
@@ -264,11 +264,7 @@ impl Classify {
}
let mut images = vec![];
for (did, cid) in (c.images)(&state).into_iter().take(MOST_IMAGES) {
let blob = json!({ "ref": { "$link": cid } });
let url = self
.cdn
.url(&did, &blob, THUMBNAIL)
.expect("a blob that names a CID");
let url = self.cdn.at(&did, &cid, THUMBNAIL_OG);
images.push(self.image(&url).await?);
}
let answers = clef::ask(
crates/grain-serve/src/index/edge.rsnew+174−0
@@ -0,0 +1,174 @@
//! What the index tells the keeper of grain's CDN ([`crate::edge`]): the
//! records with an image that a live path (the stream, `refreshRecords`
//! and the web's writes) put new or changed, for their images to be
//! warmed, and the images of what leaves the index or is taken down, for
//! the CDN to forget. Backfill tells it nothing: what it puts is not new.
//!
//! Neither waits on the keeper: it is told on a channel, and what it is
//! behind by [`crate::edge::QUEUED`] changes is dropped, and logged.
use super::records::Json;
use super::schema::{shown, unhidden};
use super::{Applied, Index, record_uri};
use rusqlite::Connection;
use tokio::sync::mpsc::{Sender, error::TrySendError};
/// An image, as the CDN names it: the account whose repo holds the blob,
/// and the blob's CID.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Image {
pub did: String,
pub cid: String,
}
/// What the CDN's keeper is told.
#[derive(Debug, Clone, PartialEq)]
pub enum Change {
/// The record at this URI, of a collection with an image
/// ([`IMAGES`]), is new or changed: its image is warmed.
Warm(String),
/// These images are gone from the index, or taken down: the CDN
/// forgets them.
Purge(Vec<Image>),
}
/// The collections whose records carry an image: each one's table, and
/// the column of its blob.
pub const IMAGES: &[(&str, &str, &str)] = &[
("social.grain.photo", "photo", "photo"),
("social.grain.story", "story", "media"),
("social.grain.actor.profile", "profile", "avatar"),
];
/// The table and blob column of `collection`'s image, if it has one.
fn image_column(collection: &str) -> Option<(&'static str, &'static str)> {
let found = IMAGES.iter().find(|(c, _, _)| *c == collection);
found.map(|(_, table, column)| (*table, *column))
}
/// The images that `sql`, of one parameter, reads: each row an account's
/// DID and a blob, or none ([`image`]).
fn read(db: &Connection, sql: &str, param: &str) -> rusqlite::Result<Vec<Image>> {
let mut stmt = db.prepare_cached(sql)?;
let rows = stmt.query_map([param], image)?;
rows.filter_map(Result::transpose).collect()
}
/// The image a row names, of the account its first column names, by the
/// blob in its second; none where it has no blob.
fn image(r: &rusqlite::Row) -> rusqlite::Result<Option<Image>> {
let (did, blob): (String, Option<Json>) = (r.get(0)?, r.get(1)?);
let cid = blob.as_ref().and_then(|b| b.0["ref"]["$link"].as_str());
Ok(cid.map(|cid| Image {
did,
cid: cid.to_string(),
}))
}
/// The image of the record at `uri`, if it is of a collection with one
/// and the index holds it.
pub(super) fn record_images(db: &Connection, uri: &str) -> rusqlite::Result<Vec<Image>> {
let imaged = record_uri(uri).and_then(|(_, collection, _)| image_column(collection));
let Some((table, column)) = imaged else {
return Ok(Vec::new());
};
let sql = format!("SELECT did, {column} FROM {table} WHERE uri = ?1");
read(db, &sql, uri)
}
/// Every image of the account `did`: its photos', its stories' and its
/// avatar.
pub(super) fn account_images(db: &Connection, did: &str) -> rusqlite::Result<Vec<Image>> {
let mut images = Vec::new();
for (_, table, column) in IMAGES {
let sql = format!("SELECT did, {column} FROM {table} WHERE did = ?1");
images.extend(read(db, &sql, did)?);
}
Ok(images)
}
/// The images of `subject`, an account's DID or a record's URI: an
/// account's every one ([`account_images`]), a gallery's photos', or a
/// photo's, story's or profile's own.
fn subject_images(db: &Connection, subject: &str) -> rusqlite::Result<Vec<Image>> {
match record_uri(subject) {
None => account_images(db, subject),
Some((_, "social.grain.gallery", _)) => read(
db,
"SELECT p.did, p.photo FROM gallery_item i JOIN photo p ON p.uri = i.item
WHERE i.gallery = ?1",
subject,
),
Some(_) => record_images(db, subject),
}
}
impl Index {
/// Has what the CDN's keeper is told ([`Change`]) sent to `to`, from
/// then on; whether none was named before.
pub fn edge_to(&self, to: Sender<Change>) -> bool {
self.edge.set(to).is_ok()
}
/// Says what a live path's put of the record at `uri` did: one new to
/// the index arrived ([`Index::arrived`]), and one new or changed with
/// an image has it warmed.
pub fn kept(&self, uri: &str, applied: Applied) {
if applied == Applied::Created {
self.arrived(uri);
}
let changed = matches!(applied, Applied::Created | Applied::Updated);
let imaged = record_uri(uri).is_some_and(|(_, c, _)| image_column(c).is_some());
if changed && imaged {
self.to_edge(Change::Warm(uri.to_string()));
}
}
/// The image of the record at `uri` as the index holds it, to warm:
/// none where it holds no such record, its collection has no image, or
/// it is not shown: its account taken down or gone, or a label that
/// hides standing on it, its account or a gallery it is in. What a
/// takedown purged, a later write of it does not bring back.
pub fn shown_images(&self, uri: &str) -> rusqlite::Result<Vec<Image>> {
let imaged = record_uri(uri).and_then(|(_, collection, _)| image_column(collection));
let Some((table, column)) = imaged else {
return Ok(Vec::new());
};
let sql = format!(
"SELECT t.did, t.{column} FROM {table} t WHERE t.uri = ?1 AND {} AND {} AND {}
AND NOT EXISTS (SELECT 1 FROM gallery_item i WHERE i.item = t.uri AND NOT ({}))",
shown("t.did"),
unhidden("t.uri"),
unhidden("t.did"),
unhidden("i.gallery"),
);
self.read(|db| read(db, &sql, uri))
}
/// Has the CDN forget the images of `subject`, an account's DID or a
/// record's URI, which an admin took down: all of an account's, and a
/// gallery's photos'.
pub fn uncache(&self, subject: &str) -> rusqlite::Result<()> {
let images = self.read(|db| subject_images(db, subject))?;
self.to_purge(images);
Ok(())
}
/// Has the CDN forget `images`, if there are any.
pub(super) fn to_purge(&self, images: Vec<Image>) {
if !images.is_empty() {
self.to_edge(Change::Purge(images));
}
}
fn to_edge(&self, change: Change) {
let Some(to) = self.edge.get() else {
return;
};
match to.try_send(change) {
Ok(()) => {}
Err(TrySendError::Full(change)) => tracing::warn!(?change, "cdn: behind, dropped"),
Err(TrySendError::Closed(change)) => tracing::warn!(?change, "cdn: stopped, dropped"),
}
}
}
crates/grain-serve/src/index/mod.rs+15−4
@@ -15,6 +15,7 @@
pub mod backfill;
pub mod classify;
pub mod edge;
pub mod graph;
pub mod moderation;
#[cfg(test)]
@@ -50,6 +51,9 @@ pub struct Index {
/// Where the same records are told for the classifiers
/// ([`Index::classify_to`]).
classifying: std::sync::OnceLock<tokio::sync::mpsc::Sender<String>>,
/// Where the CDN's keeper is told what to warm and forget
/// ([`Index::edge_to`]).
edge: std::sync::OnceLock<tokio::sync::mpsc::Sender<edge::Change>>,
/// Its file, unless it is held in memory.
path: Option<std::path::PathBuf>,
/// The feeds that keep it current, as `/_status` tells of them.
@@ -261,6 +265,7 @@ impl Index {
tracked: Mutex::new(tracked),
arrivals: std::sync::OnceLock::new(),
classifying: std::sync::OnceLock::new(),
edge: std::sync::OnceLock::new(),
path: None,
feeds: crate::status::Feeds::default(),
})
@@ -383,7 +388,8 @@ impl Index {
})
}
/// Deletes the record at `uri`; whether the index had it.
/// Deletes the record at `uri`; whether the index had it. The CDN
/// forgets its image, if it has one ([`edge`]).
pub fn delete(&self, uri: &str) -> rusqlite::Result<bool> {
self.remove(uri, None)
}
@@ -406,8 +412,10 @@ impl Index {
}
stamp(&tx, uri, rev, true)?;
tx.execute("DELETE FROM self_label WHERE uri = ?1", [uri])?;
let images = edge::record_images(&tx, uri)?;
let deleted = tx.execute(&format!("DELETE FROM {table} WHERE uri = ?1"), [uri])?;
tx.commit()?;
self.to_purge(images);
Ok(deleted > 0)
}
@@ -490,9 +498,10 @@ impl Index {
}
/// Deletes every record `did` wrote, and what grain keeps for it
/// ([`KEPT_FOR`]); how many records. What an admin holds of it stays:
/// its takedown, the labels given out and the reports made of it, as
/// do the mutes others made of it, which are theirs.
/// ([`KEPT_FOR`]); how many records. The CDN forgets their images
/// ([`edge`]). What an admin holds of it stays: its takedown, the
/// labels given out and the reports made of it, as do the mutes others
/// made of it, which are theirs.
pub fn purge(&self, did: &str) -> rusqlite::Result<usize> {
let mut db = self.db.lock().unwrap();
let tx = db.transaction()?;
@@ -506,6 +515,7 @@ impl Index {
"DELETE FROM record_rev WHERE uri >= ?1 AND uri < ?2",
[&from, &to],
)?;
let images = edge::account_images(&tx, did)?;
let mut purged = 0;
for (_, table) in records::COLLECTIONS {
purged += tx.execute(&format!("DELETE FROM {table} WHERE did = ?1"), [did])?;
@@ -514,6 +524,7 @@ impl Index {
tx.execute(sql, [did])?;
}
tx.commit()?;
self.to_purge(images);
Ok(purged)
}
crates/grain-serve/src/og/draw.rs+10−10
@@ -12,6 +12,9 @@ use std::sync::{Arc, LazyLock};
pub const WIDTH: u32 = 1200;
pub const HEIGHT: u32 = 630;
/// The JPEG quality a card is encoded at: a card is mostly photos, which a
/// PNG keeps at many times the bytes for no difference an unfurl shows.
const QUALITY: u8 = 85;
static INTER: &[u8] = include_bytes!("../../fonts/Inter-Regular.ttf");
static SYNE: &[u8] = include_bytes!("../../fonts/Syne-ExtraBold.ttf");
@@ -307,7 +310,7 @@ fn kind(bytes: Arc<Vec<u8>>) -> Option<usvg::ImageKind> {
}
}
/// `svg` drawn as a PNG, each image it names by key from `images`; one it
/// `svg` drawn as a JPEG, each image it names by key from `images`; one it
/// is not given, or cannot read, left out.
pub fn render(svg: &str, images: HashMap<String, Arc<Vec<u8>>>) -> Vec<u8> {
let resolve = move |href: &str, _: &usvg::Options| images.get(href).cloned().and_then(kind);
@@ -331,13 +334,10 @@ pub fn render(svg: &str, images: HashMap<String, Arc<Vec<u8>>>) -> Vec<u8> {
.iter()
.flat_map(|p| [p[0], p[1], p[2]])
.collect();
let mut out = Vec::with_capacity(rgb.len() / 4);
let mut encoder = png::Encoder::new(&mut out, WIDTH, HEIGHT);
encoder.set_color(png::ColorType::Rgb);
encoder.set_depth(png::BitDepth::Eight);
let mut writer = encoder.write_header().expect("writing to memory");
writer.write_image_data(&rgb).expect("writing to memory");
writer.finish().expect("writing to memory");
let mut out = Vec::with_capacity(rgb.len() / 8);
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, QUALITY)
.encode(&rgb, WIDTH, HEIGHT, image::ExtendedColorType::Rgb8)
.expect("writing to memory");
out
}
@@ -395,8 +395,8 @@ mod tests {
for family in ["Syne", "Noto Sans JP", "Noto Emoji", "Inter"] {
assert!(svg.contains(&format!(r#"font-family="{family}""#)), "{svg}");
}
let png = render(&svg, images);
assert!(png.starts_with(b"\x89PNG"));
let jpeg = render(&svg, images);
assert!(jpeg.starts_with(&[0xFF, 0xD8, 0xFF]));
}
#[test]
crates/grain-serve/src/og/mod.rs+34−25
@@ -7,8 +7,9 @@
//! galleries over the frame ([`collage`]), a gallery's up to six of its
//! photos, each above a bar of the avatar, the name and handle, and
//! grain's name; a story's is the story's photo, whole on black, its
//! author along the foot. The images are the CDN's ([`crate::cdn`]),
//! fetched as the card is drawn.
//! author along the foot. The images are the CDN's ([`crate::cdn`]), at
//! the presets kept JPEG for what grain-serve draws itself
//! ([`crate::cdn::AVATAR_OG`] and its twins), fetched as the card is drawn.
//!
//! A card goes wherever the link does, and cannot blur or warn, so what a
//! label touches is left out of it: an account taken down, or gone from
@@ -17,14 +18,15 @@
//! from grain's labeler or its author, shows no photos, and an account
//! with a label shows neither its photos nor its avatar.
//!
//! Each card is drawn once in five minutes at most: kept here by its path
//! for that long, and sent with `Cache-Control: public, max-age=300`.
//! Each card is a JPEG, drawn once in five minutes at most: kept here by
//! its path for that long, and sent with `Cache-Control: public,
//! max-age=300`, which Cloudflare, in front, keeps it by too.
pub mod collage;
pub mod draw;
use crate::App;
use crate::cdn::{AVATAR, FULLSIZE, THUMBNAIL};
use crate::cdn::{AVATAR_OG, FULLSIZE_OG, THUMBNAIL_OG};
use crate::index::Row;
use crate::index::records::{Gallery, Photo, Story};
use crate::read::Reader;
@@ -36,6 +38,7 @@ use axum::response::{IntoResponse, Response};
use axum::routing::get;
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use bytes::Bytes;
use draw::{Font, Style, Svg, fit};
use std::collections::{HashMap, VecDeque};
use std::sync::{Arc, LazyLock, Mutex};
@@ -140,7 +143,7 @@ impl Reader<'_> {
let profile = actor.profile.map(|p| p.record);
let avatar = profile.as_ref().and_then(|p| p.avatar.as_ref());
let avatar = avatar.filter(|_| !labeled);
let avatar = avatar.and_then(|a| self.cdn.url(did, &a.0, AVATAR));
let avatar = avatar.and_then(|a| self.cdn.url(did, &a.0, AVATAR_OG));
Ok(Author {
did: did.to_string(),
display_name: profile
@@ -163,7 +166,11 @@ impl Reader<'_> {
/// Each photo of `rows`, at its URL in `preset`: the CDN's whole one
/// for a photo on its own, its thumbnail for more.
fn photo_urls(&self, rows: Vec<Row<Photo>>) -> Vec<(String, f64)> {
let preset = if rows.len() == 1 { FULLSIZE } else { THUMBNAIL };
let preset = if rows.len() == 1 {
FULLSIZE_OG
} else {
THUMBNAIL_OG
};
rows.into_iter()
.filter_map(|p| {
let url = self.cdn.url(&p.did, &p.record.photo.0, preset)?;
@@ -234,7 +241,9 @@ impl Reader<'_> {
let author = self.author(&story.did)?;
let photo = match author.labeled || self.labeled(&uri)? {
true => None,
false => self.cdn.url(&author.did, &story.record.media.0, FULLSIZE),
false => self
.cdn
.url(&author.did, &story.record.media.0, FULLSIZE_OG),
};
Ok(Card::Story {
name: (author.display_name).unwrap_or_else(|| format!("@{}", author.handle)),
@@ -431,21 +440,21 @@ pub struct Cards {
#[derive(Default)]
struct Kept {
by_path: HashMap<String, (Instant, Arc<[u8]>)>,
by_path: HashMap<String, (Instant, Bytes)>,
order: VecDeque<String>,
}
impl Cards {
fn get(&self, path: &str, now: Instant) -> Option<Arc<[u8]>> {
fn get(&self, path: &str, now: Instant) -> Option<Bytes> {
let kept = self.kept.lock().unwrap();
let (at, png) = kept.by_path.get(path)?;
(now.duration_since(*at) < KEPT).then(|| png.clone())
let (at, card) = kept.by_path.get(path)?;
(now.duration_since(*at) < KEPT).then(|| card.clone())
}
fn put(&self, path: &str, png: Arc<[u8]>, now: Instant) {
fn put(&self, path: &str, card: Bytes, now: Instant) {
let mut kept = self.kept.lock().unwrap();
let Kept { by_path, order } = &mut *kept;
if by_path.insert(path.to_string(), (now, png)).is_none() {
if by_path.insert(path.to_string(), (now, card)).is_none() {
order.push_back(path.to_string());
}
while order.len() > KEEP {
@@ -489,8 +498,8 @@ async fn answer(
return StatusCode::NOT_FOUND.into_response();
};
let path = uri.path().to_string();
let png = match app.cards.get(&path, Instant::now()) {
Some(png) => png,
let jpeg = match app.cards.get(&path, Instant::now()) {
Some(jpeg) => jpeg,
None => {
let card = index.read(|db| {
read(&Reader {
@@ -519,23 +528,23 @@ async fn answer(
.collect();
let (svg, images) = svg(&card, &images).finish();
let _turn = DRAWING.acquire().await.expect("never closed");
let png: Arc<[u8]> = tokio::task::spawn_blocking(move || draw::render(&svg, images))
let jpeg: Bytes = tokio::task::spawn_blocking(move || draw::render(&svg, images))
.await
.expect("drawing a card does not panic")
.into();
app.cards.put(&path, png.clone(), Instant::now());
png
app.cards.put(&path, jpeg.clone(), Instant::now());
jpeg
}
};
let headers = [
(header::CONTENT_TYPE, "image/png".to_string()),
(header::CONTENT_TYPE, "image/jpeg".to_string()),
(
header::CACHE_CONTROL,
format!("public, max-age={}", KEPT.as_secs()),
),
(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".to_string()),
];
(headers, png.to_vec()).into_response()
(headers, jpeg).into_response()
}
async fn profile(State(app): State<Arc<App>>, Path(actor): Path<String>, uri: Uri) -> Response {
@@ -565,15 +574,15 @@ mod tests {
#[test]
fn a_card_is_kept_for_five_minutes_and_the_newest_two_hundred() {
let cards = Cards::default();
let png: Arc<[u8]> = Arc::from(&b"png"[..]);
let card = Bytes::from_static(b"jpeg");
let now = Instant::now();
cards.put("/a", png.clone(), now);
cards.put("/a", png.clone(), now);
cards.put("/a", card.clone(), now);
cards.put("/a", card.clone(), now);
assert!(cards.get("/a", now).is_some());
assert!(cards.get("/a", now + KEPT).is_none());
assert!(cards.get("/none", now).is_none());
for i in 0..KEEP {
cards.put(&format!("/{i}"), png.clone(), now);
cards.put(&format!("/{i}"), card.clone(), now);
}
assert!(cards.get("/a", now).is_none(), "the oldest is dropped");
assert!(cards.get(&format!("/{}", KEEP - 1), now).is_some());
crates/grain-serve/src/cdn.rs+51−4
@@ -19,6 +19,25 @@ pub const AVATAR: &str = "avatar";
pub const THUMBNAIL: &str = "feed_thumbnail";
pub const FULLSIZE: &str = "feed_fullsize";
/// The same sizes for what grain-serve draws itself, the OpenGraph cards
/// and what the classifiers show Clef: each its namesake's twin in
/// imgproxy's `IMGPROXY_PRESETS`, kept JPEG (`ext:jpeg`) whatever a
/// browser's would become, as the `image` crate here reads no AVIF.
/// Bluesky's CDN, which has no such presets, draws their namesakes.
pub const AVATAR_OG: &str = "avatar_og";
pub const THUMBNAIL_OG: &str = "feed_thumbnail_og";
pub const FULLSIZE_OG: &str = "feed_fullsize_og";
/// Every preset grain signs a URL at: all an image may be cached at.
pub const PRESETS: &[&str] = &[
AVATAR,
THUMBNAIL,
FULLSIZE,
AVATAR_OG,
THUMBNAIL_OG,
FULLSIZE_OG,
];
/// Where blob URLs point.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Cdn {
@@ -50,18 +69,32 @@ impl Cdn {
})
}
/// Whether these are grain's own CDN's URLs, which it may warm and
/// purge ([`crate::edge`]), rather than Bluesky's.
pub fn own(&self) -> bool {
self.signing.is_some()
}
/// The URL of the blob `blob`, a record's `{ref: {$link}}`, in `did`'s
/// repo, drawn at `preset`; none where it names no CID.
pub fn url(&self, did: &str, blob: &Value, preset: &str) -> Option<String> {
let cid = blob["ref"]["$link"].as_str()?;
let path = format!("/{preset}/plain/{did}/{cid}");
Some(match &self.signing {
Some(self.at(did, cid, preset))
}
/// The URL of the blob `cid` in `did`'s repo, drawn at `preset`.
pub fn at(&self, did: &str, cid: &str, preset: &str) -> String {
match &self.signing {
Some(s) => {
let path = format!("/{preset}/plain/{did}/{cid}");
let mac = hmac_sha256(&s.key, &[&s.salt, path.as_bytes()]);
format!("{}/{}{path}", s.url, URL_SAFE_NO_PAD.encode(mac))
}
None => format!("https://cdn.bsky.app/img{path}@jpeg"),
})
None => {
let preset = preset.strip_suffix("_og").unwrap_or(preset);
format!("https://cdn.bsky.app/img/{preset}/plain/{did}/{cid}@jpeg")
}
}
}
}
@@ -135,16 +168,30 @@ mod tests {
let sig = URL_SAFE_NO_PAD.encode(mac);
assert_eq!(url, format!("https://cdn.grain.social/{sig}{path}"));
assert_eq!(cdn.url("did:plc:alice", &json!({"cid": "x"}), AVATAR), None);
assert!(cdn.own());
// A twin preset is signed by its own name.
let og = cdn.at("did:plc:alice", "bafkreiblob", AVATAR_OG);
assert!(
og.ends_with("/avatar_og/plain/did:plc:alice/bafkreiblob"),
"{og}"
);
assert_ne!(og.replace("avatar_og", "avatar"), url);
}
#[test]
fn with_no_cdn_of_its_own_bluesky_draws_them() {
let cdn = Cdn::new(None, None, None).unwrap();
assert_eq!(cdn, Cdn::default());
assert!(!cdn.own());
assert_eq!(
cdn.url("did:plc:alice", &blob(), THUMBNAIL).unwrap(),
"https://cdn.bsky.app/img/feed_thumbnail/plain/did:plc:alice/bafkreiblob@jpeg"
);
// Bluesky's has no twins: their namesakes draw them.
assert_eq!(
cdn.at("did:plc:alice", "bafkreiblob", FULLSIZE_OG),
"https://cdn.bsky.app/img/feed_fullsize/plain/did:plc:alice/bafkreiblob@jpeg"
);
}
#[test]
crates/grain-serve/src/edge.rsnew+250−0
@@ -0,0 +1,250 @@
//! Keeping grain's CDN in step with the index: what arrives is warmed,
//! and what leaves or is taken down is purged. The index says which
//! ([`crate::index::edge`]); this keeps the CDN, off the path of the
//! request or the stream that brought it, so that neither waits on it.
//!
//! **Warming.** A photo, story or avatar a live path puts new or changed in
//! the index is fetched through the CDN at once, at each preset the views
//! draw it at ([`shown_at`]), so that the first to see it is not the first
//! to ask: a miss costs seconds, the PDS's blob fetched and imgproxy's
//! encode, and a story lives a day, seen by few, each usually the first
//! where they are. Each URL is asked once for each type in [`ACCEPTS`], as
//! the CDN keeps a copy for each type it varies on. At most `at_once`
//! requests go at a time; one that fails is logged and dropped, and the
//! image is drawn when it is first asked for, as before.
//!
//! **Purging.** An account or a record an admin takes down
//! ([`Index::uncache`]), and the image of a record or an account that
//! leaves the index, is purged from Bunny, the CDN in front of imgproxy,
//! at every preset grain signs a URL at ([`cdn::PRESETS`]), by Bunny's API
//! with the key in `BUNNY_API_KEY`: the CDN keeps an image for a year, and
//! one who kept its URL would see it there all that while. Purges go one
//! at a time, as Bunny allows about five a second, and one Bunny refuses
//! for that (429), or fails, is tried again after [`Timing::retry`], or
//! `Retry-After`, up to [`Timing::attempts`] times, then logged and dropped.
//!
//! Only grain's own CDN is kept ([`Cdn::own`]): Bluesky's is not ours to
//! warm or purge.
use crate::cdn::{self, AVATAR, Cdn, FULLSIZE, THUMBNAIL};
use crate::index::edge::{Change, Image};
use crate::index::{Index, record_uri};
use reqwest::StatusCode;
use reqwest::header::{ACCEPT, RETRY_AFTER};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{Mutex, Semaphore, mpsc};
/// How many changes wait for the keeper before more are dropped.
pub const QUEUED: usize = 10_000;
/// How many requests warm the CDN at once, unless `--cdn-warm` says.
pub const AT_ONCE: usize = 4;
/// Bunny's API, which purges, unless `--bunny-api` says.
pub const BUNNY_API: &str = "https://api.bunny.net";
/// The types each URL is warmed for, as `Accept` asks them. One today:
/// every answer is JPEG, and the CDN keeps one copy of each. Once
/// imgproxy answers AVIF to those that take it and the CDN varies on it,
/// `image/avif` joins it, and each URL is warmed in both.
pub const ACCEPTS: &[&str] = &["image/jpeg"];
/// The presets the views draw the images of `collection`'s records at
/// ([`crate::read::views`]): an avatar's, and a photo's or a story's, as
/// the feeds, grids and stories show it, thumbnail and whole.
fn shown_at(collection: &str) -> &'static [&'static str] {
match collection {
"social.grain.actor.profile" => &[AVATAR],
_ => &[THUMBNAIL, FULLSIZE],
}
}
/// How long purging waits.
#[derive(Debug, Clone, Copy)]
pub struct Timing {
/// How long a purge that failed waits to be tried again, unless Bunny
/// says how long.
pub retry: Duration,
/// How many times a purge is tried before it is dropped.
pub attempts: u32,
}
impl Default for Timing {
fn default() -> Timing {
Timing {
retry: Duration::from_secs(1),
attempts: 10,
}
}
}
/// Bunny's API, and the key it takes, to purge with.
#[derive(Clone, PartialEq)]
pub struct Purge {
/// Such as [`BUNNY_API`].
pub api: String,
pub key: String,
}
impl std::fmt::Debug for Purge {
/// The API, never the key.
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Purge").field("api", &self.api).finish()
}
}
/// The CDN's keeper, at work on an index.
pub struct Edge {
pub index: Arc<Index>,
/// What the CDN and Bunny are asked with.
pub http: reqwest::Client,
/// grain's own CDN, whose URLs are warmed and purged.
pub cdn: Cdn,
/// How many requests warm it at once; none warms nothing.
pub at_once: usize,
/// Bunny's API, if purges go to it; none purges nothing.
pub purge: Option<Purge>,
pub timing: Timing,
}
impl Edge {
/// Starts keeping the CDN as the index says, on the runtime this is
/// called on; whether it was not kept already.
pub fn start(self) -> bool {
let (to, from) = mpsc::channel(QUEUED);
if !self.index.edge_to(to) {
return false;
}
tokio::spawn(Arc::new(self).run(from));
true
}
/// Takes each change the index tells of, until its sender is gone.
async fn run(self: Arc<Self>, mut from: mpsc::Receiver<Change>) {
let turns = Arc::new(Semaphore::new(self.at_once));
let purging = Arc::new(Mutex::new(()));
while let Some(change) = from.recv().await {
match change {
Change::Warm(uri) if self.at_once > 0 => self.warm(&uri, &turns),
Change::Warm(_) => {}
Change::Purge(images) => {
if let Some(purge) = self.purge.clone() {
let (edge, purging) = (self.clone(), purging.clone());
tokio::spawn(async move {
let _turn = purging.lock().await;
edge.purge(&purge, &images).await;
});
}
}
}
}
}
/// Warms the image of the record at `uri`, which the index says is
/// one with an image, each URL in turn of `turns`.
fn warm(self: &Arc<Self>, uri: &str, turns: &Arc<Semaphore>) {
let collection = record_uri(uri).map_or("", |(_, c, _)| c);
let images = match self.index.shown_images(uri) {
Ok(images) => images,
Err(e) => {
tracing::warn!(%uri, error = %e, "cdn: reading what to warm");
return;
}
};
for Image { did, cid } in images {
for preset in shown_at(collection) {
let url = self.cdn.at(&did, &cid, preset);
for accept in ACCEPTS {
let (edge, turns, url) = (self.clone(), turns.clone(), url.clone());
tokio::spawn(async move {
let _turn = turns.acquire().await.expect("never closed");
if let Err(e) = edge.fetch(&url, accept).await {
tracing::warn!(%url, accept, error = %e, "cdn: warming");
}
});
}
}
}
}
/// Asks the CDN for `url` as a client taking `accept` does, reading
/// the answer whole so that the CDN has all of it to keep.
async fn fetch(&self, url: &str, accept: &str) -> anyhow::Result<()> {
let mut res = self.http.get(url).header(ACCEPT, accept).send().await?;
let status = res.status();
while res.chunk().await?.is_some() {}
anyhow::ensure!(status.is_success(), "answered {status}");
Ok(())
}
/// Purges every URL of `images` from Bunny, one after another.
async fn purge(&self, purge: &Purge, images: &[Image]) {
for Image { did, cid } in images {
for preset in cdn::PRESETS {
let url = self.cdn.at(did, cid, preset);
if let Err(e) = self.purge_one(purge, &url).await {
tracing::warn!(%url, error = %e, "cdn: purging, dropped");
}
}
}
}
/// Purges `url` from Bunny, trying again while it fails, until
/// [`Timing::attempts`].
async fn purge_one(&self, purge: &Purge, url: &str) -> anyhow::Result<()> {
let endpoint = format!("{}/purge", purge.api);
let mut tried = 0;
loop {
tried += 1;
let sent = self
.http
.post(&endpoint)
.query(&[("url", url)])
.header("AccessKey", &purge.key)
.send()
.await;
let (wait, why) = match sent {
Ok(res) if res.status().is_success() => return Ok(()),
Ok(res) if again(res.status()) => {
let after = res.headers().get(RETRY_AFTER);
let after = after.and_then(|v| v.to_str().ok()?.parse().ok());
let wait = after.map_or(self.timing.retry, Duration::from_secs);
(wait, format!("answered {}", res.status()))
}
Ok(res) => anyhow::bail!("answered {}", res.status()),
Err(e) => (self.timing.retry, e.to_string()),
};
if tried >= self.timing.attempts {
anyhow::bail!("{why}, {tried} times");
}
tokio::time::sleep(wait).await;
}
}
}
/// Whether a purge Bunny answered with `status` may go if tried again:
/// one it took too many of at once, or failed itself.
fn again(status: StatusCode) -> bool {
status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error()
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn the_keeper_ends_when_its_sender_is_gone() {
let edge = Edge {
index: Arc::new(Index::memory().unwrap()),
http: reqwest::Client::new(),
cdn: Cdn::new(Some("https://cdn.grain.social/"), Some("00"), Some("00")).unwrap(),
at_once: 0,
purge: None,
timing: Timing::default(),
};
let (to, from) = mpsc::channel(1);
to.send(Change::Warm("at://did:plc:alice/x/y".into()))
.await
.unwrap();
drop(to);
Arc::new(edge).run(from).await;
}
}
crates/grain-serve/src/jetstream.rs+4−4
@@ -13,7 +13,8 @@
//! stream write and a backfill's read of the same repo leave the later of
//! the two. An account newly tracked, or back after its host said it was
//! gone, is queued for backfill ([`Index::resync`]) to be read whole.
//! A record new to the index is pushed ([`Index::arrived`]). An account
//! A record new to the index is pushed, and one new or changed with an
//! image has it warmed on the CDN ([`Index::kept`]). An account
//! its host says was deleted goes, records and all, once the host its
//! identity names now says it has no such account
//! ([`crate::accounts::deletions`]): one that moved stays.
@@ -25,7 +26,7 @@
use crate::accounts;
use crate::backfill::SIGNALS;
use crate::index::{Applied, Index, PutError, records};
use crate::index::{Index, PutError, records};
use grain::env::Ctx;
use grain::jetstream::{Event, Follower, Timing, What};
use std::sync::Arc;
@@ -187,8 +188,7 @@ pub fn apply_with(
None => index.put(&uri, &cid, &record),
};
match put {
Ok(Applied::Created) => index.arrived(&uri),
Ok(_) => {}
Ok(applied) => index.kept(&uri, applied),
Err(PutError::Refused(why)) => {
tracing::info!(%uri, %why, "jetstream: skipped a record");
}
crates/grain-serve/src/lib.rs+69−3
@@ -10,13 +10,15 @@
//! that go ([`accounts`]), draws the cards links unfurl into ([`og`]),
//! and logs the web in ([`login`]), handing the rest to the web's pages
//! ([`pages`]). Off the path of any request, its classifiers score what
//! arrives for moderators to review ([`classify`]).
//! arrives for moderators to review ([`classify`]), and its CDN is warmed
//! with what arrives and purged of what goes ([`edge`]).
pub mod accounts;
pub mod backfill;
pub mod cdn;
pub mod classify;
pub mod did;
pub mod edge;
pub mod gallery;
pub mod graph;
pub mod index;
@@ -63,6 +65,8 @@ pub const FLAGS: &[&str] = &[
"--backfill-skip",
"--backfill-concurrency",
"--cdn",
"--cdn-warm",
"--bunny-api",
"--bsky-appview",
"--sign-up-pds",
"--group-host",
@@ -80,8 +84,8 @@ pub const FLAGS: &[&str] = &[
];
/// What `grain-serve` says of its flags when given one it does not take.
pub const USAGE: &str = "usage: grain-serve [--listen <addr>] [--index <file>] [--public-url <https://host>] [--jetstream <wss://host>] [--backfill <https://directory>[,...]] [--backfill-mirror <https://mirror>] [--backfill-skip <host>[,...]] [--backfill-concurrency <n>] [--cdn <https://imgproxy>] [--bsky-appview <https://appview>] [--sign-up-pds <https://pds>] [--group-host <https://host>] [--carry-cookies-until <rfc3339 time>] [--apns-key-id <id>] [--apns-team-id <id>] [--apns-topic <bundle id>] [--apns-url <https://apns>] [--fcm-project-id <id>] [--clef-model <model>] [--clef-url <url>] [--check-index <file>] [--metrics <addr>] [--web <http://app>]
environment: GRAIN_ADMIN_DIDS, GRAIN_CDN_KEY, GRAIN_CDN_SALT, GRAIN_APNS_KEY_FILE, GRAIN_FCM_KEY_FILE, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_AI_TOKEN, RUST_LOG";
pub const USAGE: &str = "usage: grain-serve [--listen <addr>] [--index <file>] [--public-url <https://host>] [--jetstream <wss://host>] [--backfill <https://directory>[,...]] [--backfill-mirror <https://mirror>] [--backfill-skip <host>[,...]] [--backfill-concurrency <n>] [--cdn <https://imgproxy>] [--cdn-warm <n>] [--bunny-api <https://api>] [--bsky-appview <https://appview>] [--sign-up-pds <https://pds>] [--group-host <https://host>] [--carry-cookies-until <rfc3339 time>] [--apns-key-id <id>] [--apns-team-id <id>] [--apns-topic <bundle id>] [--apns-url <https://apns>] [--fcm-project-id <id>] [--clef-model <model>] [--clef-url <url>] [--check-index <file>] [--metrics <addr>] [--web <http://app>]
environment: GRAIN_ADMIN_DIDS, GRAIN_CDN_KEY, GRAIN_CDN_SALT, BUNNY_API_KEY, GRAIN_APNS_KEY_FILE, GRAIN_FCM_KEY_FILE, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_AI_TOKEN, RUST_LOG";
/// What the server is set to do, from its flags ([`FLAGS`]) and, for
/// secrets, key files and who its admins are, its environment.
@@ -104,6 +108,13 @@ pub struct Settings {
/// environment: grain's imgproxy, and the hex key and salt it signs
/// paths with ([`cdn::Cdn`]). None draws images from Bluesky's CDN.
pub cdn: [Option<String>; 3],
/// `--cdn-warm`: how many requests warm grain's own CDN at once with
/// what arrives ([`edge`]). None is [`edge::AT_ONCE`]; 0 warms none.
pub cdn_warm: Option<usize>,
/// `BUNNY_API_KEY` from the environment, and `--bunny-api`, None
/// being [`edge::BUNNY_API`]: the API that purges grain's own CDN, Bunny,
/// of what goes ([`edge`]). With no key, it is purged of nothing.
pub purge: Option<edge::Purge>,
/// `--bsky-appview`: the Bluesky appview whose typeahead
/// `searchActorsTypeahead` fills out its own with, such as
/// [`read::search::BSKY_APPVIEW`]. None asks none, as tests and dev
@@ -185,6 +196,14 @@ impl Settings {
backfill: backfill::Config::given(given)?,
jetstream: get("--jetstream"),
cdn: [get("--cdn"), get("GRAIN_CDN_KEY"), get("GRAIN_CDN_SALT")],
cdn_warm: match get("--cdn-warm") {
Some(n) => Some(n.parse().map_err(|_| "--cdn-warm takes a number")?),
None => None,
},
purge: get("BUNNY_API_KEY").map(|key| edge::Purge {
api: url("--bunny-api").unwrap_or_else(|| edge::BUNNY_API.into()),
key,
}),
bsky_appview: url("--bsky-appview"),
sign_up_pds: get("--sign-up-pds"),
group_host: url("--group-host"),
@@ -269,6 +288,30 @@ impl Settings {
classify.start()
}
/// Starts keeping `app`'s CDN as its index says ([`edge`]), on the
/// runtime this is called on: warming it `--cdn-warm` requests at a
/// time, and purging it by Bunny's API with `BUNNY_API_KEY`; whether
/// it does either. Only grain's own CDN is kept, of an index.
pub fn edge(&self, app: &App) -> bool {
let Some(index) = app.index.clone().filter(|_| app.cdn.own()) else {
return false;
};
let at_once = self.cdn_warm.unwrap_or(edge::AT_ONCE);
if at_once == 0 && self.purge.is_none() {
return false;
}
tracing::info!(at_once, purge = ?self.purge, "keeping the CDN");
let edge = edge::Edge {
index,
http: app.ctx.http.clone(),
cdn: app.cdn.clone(),
at_once,
purge: self.purge.clone(),
timing: edge::Timing::default(),
};
edge.start()
}
/// The server `ctx` and these settings make: its index opened, and
/// its DID named.
pub fn app(&self, ctx: Ctx) -> anyhow::Result<App> {
@@ -550,10 +593,15 @@ mod tests {
jetstream::DEFAULT,
"--web",
"http://web:3000/",
"--cdn-warm",
"2",
"--bunny-api",
"https://bunny.test/",
]),
|name| match name {
"GRAIN_ADMIN_DIDS" => Some("did:plc:alice, did:web:bob.test".into()),
"GRAIN_CDN_KEY" => Some("00".into()),
"BUNNY_API_KEY" => Some("sekrit".into()),
// A setting is a flag: its variable is never read.
"GRAIN_INDEX" => Some("/elsewhere.db".into()),
_ => None,
@@ -570,6 +618,22 @@ mod tests {
assert_eq!(set.jetstream.as_deref(), Some(jetstream::DEFAULT));
assert_eq!(set.cdn[0].as_deref(), Some("https://cdn.grain.social"));
assert_eq!(set.cdn[1].as_deref(), Some("00"));
assert_eq!(set.cdn_warm, Some(2));
let purge = set.purge.clone().unwrap();
assert_eq!(
(purge.api.as_str(), purge.key.as_str()),
("https://bunny.test", "sekrit")
);
assert!(
!format!("{set:?}").contains("sekrit"),
"the key is not shown"
);
// With no --bunny-api, Bunny's own; with no key, none.
let key = |name: &str| (name == "BUNNY_API_KEY").then(|| "k".to_string());
let bunny = Settings::parse(&args(&[]), key).unwrap().purge.unwrap();
assert_eq!(bunny.api, edge::BUNNY_API);
let keyless = Settings::parse(&args(&["--bunny-api", "https://b.test"]), none);
assert_eq!(keyless.unwrap().purge, None);
assert!(
set.app(Ctx::new(Env::from_vars(|_| None))).is_err(),
"a CDN with no salt"
@@ -591,6 +655,8 @@ mod tests {
assert!(FLAGS.iter().all(|flag| USAGE.contains(flag)));
let bare = Settings::parse(&args(&["--index"]), none).unwrap_err();
assert_eq!(bare, "--index takes a value");
let many = Settings::parse(&args(&["--cdn-warm", "many"]), none).unwrap_err();
assert_eq!(many, "--cdn-warm takes a number");
// A setting is a flag, not its old variable.
let var = Settings::parse(&args(&["GRAIN_INDEX=/data/index.db"]), none);
assert!(var.unwrap_err().starts_with("unexpected GRAIN_INDEX"));
crates/grain-serve/src/main.rs+1−0
@@ -48,6 +48,7 @@ async fn run(settings: grain_serve::Settings) -> anyhow::Result<()> {
}
settings.push(&app)?;
settings.classify(&app);
settings.edge(&app);
settings.follow(&app)?;
let addr = settings.listen.as_deref().unwrap_or(grain_serve::LISTEN);
let listener = tokio::net::TcpListener::bind(addr)
crates/grain-serve/src/moderation.rs+7−3
crates/grain-serve/src/sync.rs+4−5
crates/grain-serve/src/write.rs+2−3
crates/grain-serve/tests/cdn/mod.rsnew+205−0
crates/grain-serve/tests/classify.rs+1−1
crates/grain-serve/tests/deletes.rs+9−1
crates/grain-serve/tests/edge.rsnew+432−0
432 lines changed: not loaded until asked for.
crates/grain-serve/tests/jetstream.rs+75−1
crates/grain-serve/tests/moderation.rs+32−1
crates/grain-serve/tests/og.rs+39−16
crates/grain-serve/tests/refresh_records.rs+30−1
crates/grain-serve/Cargo.toml+1−1
deploy/docker-compose.yml+6−4
deploy/README.md+4−1