~ knotless Search/
↑↓ move↵ open⇧↵ all results

grain.social/grain

photography-focused photo sharing platform

grain.social
1 collaborator
0
2

grain-serve writes for the web and indexes it before answering

Merged chadtmiller.com asked to merge into feature/rust from grain.social/grain:web/writes

Only what this pull adds. It is built on grain runs on a Rust server modelled on knotless, without hatk.

crates/grain-serve/src/index/mod.rs+25−12
@@ -234,6 +234,30 @@ impl Index {
self.write(uri, cid, record, &grain::keys::now_iso(), Some(rev))
}
/// Whether the index would put `record` as one of `collection`: one
/// it keeps, of that `$type`, that fits its lexicon. For a write to
/// refuse what the index would before it is made.
pub fn check(&self, collection: &str, record: &Value) -> Result<(), PutError> {
self.parsed(collection, record).map(drop)
}
/// `record`, of `collection`, as a row of its table, and the table.
fn parsed(&self, collection: &str, record: &Value) -> Result<(&'static str, Any), PutError> {
let Some(table) = records::table_of(collection) else {
return Err(refused("", format!("the index keeps no {collection}")));
};
if record["$type"] != collection {
return Err(refused("$type", format!("must be {collection}")));
}
let lexicons = &self.lexicons;
lexicons
.validate_record(collection, record)
.map_err(PutError::Refused)?;
let row = Any::parse(collection, record).expect("a collection with a table parses");
let row = row.map_err(|e| refused("", e.to_string()))?;
Ok((table, row))
}
fn write(
&self,
uri: &str,
@@ -244,21 +268,10 @@ impl Index {
) -> Result<Applied, PutError> {
let (did, collection, _) =
record_uri(uri).ok_or_else(|| refused("", "not a record URI"))?;
let Some(table) = records::table_of(collection) else {
return Err(refused("", format!("the index keeps no {collection}")));
};
if !lexicon::valid_cid(cid) {
return Err(refused("", format!("not a CID: {cid}")));
}
if record["$type"] != collection {
return Err(refused("$type", format!("must be {collection}")));
}
let lexicons = &self.lexicons;
lexicons
.validate_record(collection, record)
.map_err(PutError::Refused)?;
let row = Any::parse(collection, record).expect("a collection with a table parses");
let row = row.map_err(|e| refused("", e.to_string()))?;
let (table, row) = self.parsed(collection, record)?;
let mut db = self.db.lock().unwrap();
let tx = db.transaction()?;
if !newer(&tx, uri, rev)? {
crates/grain-serve/src/lib.rs+19−0
@@ -14,9 +14,11 @@ pub mod notification;
pub mod read;
pub mod sync;
pub mod viewer;
pub mod write;
pub mod xrpc;
use anyhow::bail;
use axum::extract::DefaultBodyLimit;
use axum::http::StatusCode;
use axum::{Json, Router, routing::any, routing::get, routing::post};
use grain::env::Ctx;
@@ -187,6 +189,23 @@ pub fn router(app: Arc<App>) -> Router {
&format!("/xrpc/{}", notification::UPDATE_SEEN),
post(notification::update_seen),
)
.route(
"/xrpc/social.grain.repo.createRecord",
post(write::create_record),
)
.route("/xrpc/social.grain.repo.putRecord", post(write::put_record))
.route(
"/xrpc/social.grain.repo.deleteRecord",
post(write::delete_record),
)
.route(
"/xrpc/social.grain.repo.applyWrites",
post(write::apply_writes),
)
.route(
"/xrpc/social.grain.repo.uploadBlob",
post(write::upload_blob).layer(DefaultBodyLimit::max(write::MOST_BYTES)),
)
.route(login::METADATA, get(login::client_metadata))
.route(login::JWKS, get(login::client_jwks))
.route("/oauth/login", post(login::login))
crates/grain-serve/src/login.rs+22−9
@@ -550,14 +550,28 @@ pub async fn whoami(State(app): State<Arc<App>>, headers: HeaderMap) -> Result<R
/// the authorization server refuses ends the session, and the account
/// logs in again; one that fails on the way keeps it, to try again.
pub async fn session(app: &App, did: &str) -> Result<Session, Failure> {
fresh(app, did, None).await
}
/// `did`'s web session as [`session`] finds it, refreshed whether or not
/// it is about to expire if its access token is still `refused`, one its
/// PDS just refused: whoever refreshed it meanwhile has made it last.
pub async fn refreshed(app: &App, did: &str, refused: &str) -> Result<Session, Failure> {
fresh(app, did, Some(refused)).await
}
/// The session ended, as a call that needed it answers.
pub fn ended() -> Failure {
let why = "the web session has ended: log in again";
Failure::new(StatusCode::UNAUTHORIZED, "AuthenticationRequired", why)
}
async fn fresh(app: &App, did: &str, refused: Option<&str>) -> Result<Session, Failure> {
let web = web(app)?;
let ended = || {
let why = "the web session has ended: log in again";
Failure::new(StatusCode::UNAUTHORIZED, "AuthenticationRequired", why)
};
let stale = |s: &Session| s.expiring() || refused == Some(s.access_jwt.as_str());
let index = index(app);
let kept = index.web_session(did)?.ok_or_else(ended)?;
if !kept.session.expiring() {
if !stale(&kept.session) {
return Ok(kept.session);
}
let lock = {
@@ -567,11 +581,10 @@ pub async fn session(app: &App, did: &str) -> Result<Session, Failure> {
};
let _turn = lock.lock().await;
// Whoever refreshed while this waited left a session that lasts.
let now = index.web_session(did)?.ok_or_else(ended)?;
if now.session.refresh_jwt != kept.session.refresh_jwt && !now.session.expiring() {
return Ok(now.session);
let stored = index.web_session(did)?.ok_or_else(ended)?.session;
if !stale(&stored) {
return Ok(stored);
}
let stored = now.session;
let Some(o) = &stored.oauth else {
index.end_web_session(did)?;
return Err(ended());
crates/grain-serve/src/write.rsnew+579−0
579 lines changed: not loaded until asked for.
crates/grain-serve/tests/common/mod.rs+14−0
@@ -92,6 +92,20 @@ impl Served {
self.answer(path, res).await
}
/// A procedure called with `body` as it is, and `headers`.
pub async fn post_bytes(
&self,
path: &str,
headers: &[(&str, &str)],
body: Vec<u8>,
) -> (u16, Value) {
let mut res = self.http.post(format!("{}{path}", self.base)).body(body);
for (k, v) in headers {
res = res.header(*k, *v);
}
self.answer(path, res).await
}
async fn answer(&self, path: &str, req: reqwest::RequestBuilder) -> (u16, Value) {
let res = req.send().await.unwrap();
let status = res.status().as_u16();
crates/grain-serve/tests/web/mod.rsnew+181−0
@@ -0,0 +1,181 @@
//! The site as a browser reaches it, for the tests of what the web does
//! logged in: grain-serve behind `harness/tls-proxy.mjs`, at an https
//! public URL the PDS fetches its client metadata from, and
//! `harness/authorize.mjs` as the browser, stopping at the redirect back
//! so that the test comes back to the callback with the cookies its
//! browser holds.
#![allow(dead_code)]
use crate::{common, net};
use grain_serve::index::Index;
use grain_serve::login::Web;
use grain_serve::{App, service_did};
use reqwest::header::{COOKIE, ORIGIN, SET_COOKIE};
use serde_json::{Value, json};
use std::collections::HashMap;
use std::sync::Arc;
pub struct Site {
pub app: Arc<App>,
pub index: Arc<Index>,
pub served: common::Served,
/// The public URL, the proxy's.
pub url: String,
pub http: reqwest::Client,
_proxy: net::TlsProxy,
_dir: tempfile::TempDir,
}
/// grain-serve with an index and a web login, at an https public URL.
pub async fn site() -> Site {
site_on(Index::memory().unwrap(), &net::net().pds).await
}
/// [`site`], on `index`, sending sign-ups to `sign_up_pds`.
pub async fn site_on(index: Index, sign_up_pds: &str) -> Site {
let dir = tempfile::tempdir().unwrap();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let proxy = net::tls_proxy(listener.local_addr().unwrap().port());
let index = Arc::new(index);
let web = Web::new(&proxy.url, &index, sign_up_pds).unwrap();
let app = Arc::new(App {
ctx: net::ctx(dir.path(), &[]),
index: Some(index.clone()),
did: Some(service_did(&proxy.url).unwrap()),
cdn: Default::default(),
bsky_appview: None,
web: Some(Arc::new(web)),
});
let served = common::served_on(listener, shared(&app));
let http = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.unwrap();
Site {
app,
index,
served,
url: proxy.url.clone(),
http,
_proxy: proxy,
_dir: dir,
}
}
/// The same server again, sharing `app`'s index and logins: what the
/// test calls directly and what it serves are one.
pub fn shared(app: &Arc<App>) -> App {
App {
ctx: app.ctx.clone(),
index: app.index.clone(),
did: app.did.clone(),
cdn: Default::default(),
bsky_appview: None,
web: app.web.clone(),
}
}
/// A browser's cookies for the site, by name.
#[derive(Default, Clone)]
pub struct Jar(pub HashMap<String, String>);
impl Jar {
pub fn header(&self) -> String {
let pairs = self.0.iter().map(|(k, v)| format!("{k}={v}"));
pairs.collect::<Vec<_>>().join("; ")
}
/// Takes in what `res` sets, as a browser does: one cleared goes.
pub fn keep(&mut self, res: &reqwest::Response) -> Vec<String> {
let mut said = Vec::new();
for set in res.headers().get_all(SET_COOKIE) {
let set = set.to_str().unwrap().to_string();
let (pair, _) = set.split_once(';').unwrap();
let (name, value) = pair.split_once('=').unwrap();
match set.contains("Max-Age=0") {
true => self.0.remove(name),
false => self.0.insert(name.into(), value.into()),
};
said.push(set);
}
said
}
pub fn get(&self, name: &str) -> Option<&str> {
self.0.get(name).map(String::as_str)
}
}
impl Site {
/// Posts `body` to `path` as a page of `origin` would, with `jar`'s
/// cookies, keeping what it sets.
pub async fn post(
&self,
jar: &mut Jar,
origin: Option<&str>,
path: &str,
body: Value,
) -> (u16, Value) {
let mut req = self
.http
.post(format!("{}{path}", self.served.base))
.json(&body);
req = req.header(COOKIE, jar.header());
if let Some(origin) = origin {
req = req.header(ORIGIN, origin);
}
let res = req.send().await.unwrap();
jar.keep(&res);
let status = res.status().as_u16();
(status, res.json().await.unwrap_or(Value::Null))
}
pub async fn get(&self, jar: &mut Jar, path: &str) -> reqwest::Response {
let req = self.http.get(format!("{}{path}", self.served.base));
let res = req.header(COOKIE, jar.header()).send().await.unwrap();
jar.keep(&res);
res
}
pub async fn whoami(&self, jar: &mut Jar) -> Value {
self.get(jar, "/oauth/session").await.json().await.unwrap()
}
/// Logs `jar`'s browser in as `account`, approved at its PDS, going
/// back to `/after`: the login, the approval and the way back.
pub async fn log_in(&self, jar: &mut Jar, account: &net::Account) {
let body = json!({ "handle": account.handle, "back": "/after" });
let (status, started) = self.post(jar, Some(&self.url), "/oauth/login", body).await;
assert_eq!(status, 200, "{started}");
let back = approve(account, "", started["url"].as_str().unwrap()).await;
let res = self.back(jar, &back).await;
assert_eq!(res.status().as_u16(), 303);
assert_eq!(res.headers()["location"], "/after");
}
/// Comes back to the callback the PDS sent the browser to, `back`,
/// at this server.
pub async fn back(&self, jar: &mut Jar, back: &str) -> reqwest::Response {
let path = back.strip_prefix(&self.url).expect("back to this site");
self.get(jar, path).await
}
}
/// Approves the login at `url` as `account`, with `extra`, and answers
/// where the PDS sends the browser back to.
pub async fn approve(account: &net::Account, extra: &str, url: &str) -> String {
let command = net::browser(account, &format!("--stop {extra}"));
let mut words = command.split_whitespace();
let out = tokio::process::Command::new(words.next().unwrap())
.args(words)
.arg(url)
.output()
.await
.unwrap();
assert!(
out.status.success(),
"{}",
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8(out.stdout).unwrap().trim().to_string()
}
crates/grain-serve/tests/login.rs+6−175
@@ -1,187 +1,18 @@
//! Logging in to the web against the reference PDS: grain-serve behind
//! `harness/tls-proxy.mjs`, at an https public URL the PDS fetches its
//! client metadata from, and `harness/authorize.mjs` as the browser,
//! stopping at the redirect back so that the test comes back to the
//! callback with the cookies its browser holds.
//! Logging in to the web against the reference PDS, through [`web`]'s
//! site and browser.
#[path = "../../grain/tests/common/mod.rs"]
mod net;
mod common;
mod web;
use grain::keys;
use grain_serve::index::Index;
use grain_serve::login::{self, Web};
use grain_serve::{App, service_did};
use reqwest::header::{COOKIE, ORIGIN, SET_COOKIE};
use reqwest::header::ORIGIN;
use serde_json::{Value, json};
use std::collections::HashMap;
use std::sync::Arc;
struct Site {
app: Arc<App>,
index: Arc<Index>,
served: common::Served,
/// The public URL, the proxy's.
url: String,
http: reqwest::Client,
_proxy: net::TlsProxy,
_dir: tempfile::TempDir,
}
/// grain-serve with an index and a web login, at an https public URL.
async fn site() -> Site {
site_on(Index::memory().unwrap(), &net::net().pds).await
}
/// [`site`], on `index`, sending sign-ups to `sign_up_pds`.
async fn site_on(index: Index, sign_up_pds: &str) -> Site {
let dir = tempfile::tempdir().unwrap();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let proxy = net::tls_proxy(listener.local_addr().unwrap().port());
let index = Arc::new(index);
let web = Web::new(&proxy.url, &index, sign_up_pds).unwrap();
let app = Arc::new(App {
ctx: net::ctx(dir.path(), &[]),
index: Some(index.clone()),
did: Some(service_did(&proxy.url).unwrap()),
cdn: Default::default(),
bsky_appview: None,
web: Some(Arc::new(web)),
});
let served = common::served_on(listener, shared(&app));
let http = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.unwrap();
Site {
app,
index,
served,
url: proxy.url.clone(),
http,
_proxy: proxy,
_dir: dir,
}
}
/// The same server again, sharing `app`'s index and logins: what the
/// test calls directly and what it serves are one.
fn shared(app: &Arc<App>) -> App {
App {
ctx: app.ctx.clone(),
index: app.index.clone(),
did: app.did.clone(),
cdn: Default::default(),
bsky_appview: None,
web: app.web.clone(),
}
}
/// A browser's cookies for the site, by name.
#[derive(Default, Clone)]
struct Jar(HashMap<String, String>);
impl Jar {
fn header(&self) -> String {
let pairs = self.0.iter().map(|(k, v)| format!("{k}={v}"));
pairs.collect::<Vec<_>>().join("; ")
}
/// Takes in what `res` sets, as a browser does: one cleared goes.
fn keep(&mut self, res: &reqwest::Response) -> Vec<String> {
let mut said = Vec::new();
for set in res.headers().get_all(SET_COOKIE) {
let set = set.to_str().unwrap().to_string();
let (pair, _) = set.split_once(';').unwrap();
let (name, value) = pair.split_once('=').unwrap();
match set.contains("Max-Age=0") {
true => self.0.remove(name),
false => self.0.insert(name.into(), value.into()),
};
said.push(set);
}
said
}
fn get(&self, name: &str) -> Option<&str> {
self.0.get(name).map(String::as_str)
}
}
impl Site {
/// Posts `body` to `path` as a page of `origin` would, with `jar`'s
/// cookies, keeping what it sets.
async fn post(
&self,
jar: &mut Jar,
origin: Option<&str>,
path: &str,
body: Value,
) -> (u16, Value) {
let mut req = self
.http
.post(format!("{}{path}", self.served.base))
.json(&body);
req = req.header(COOKIE, jar.header());
if let Some(origin) = origin {
req = req.header(ORIGIN, origin);
}
let res = req.send().await.unwrap();
jar.keep(&res);
let status = res.status().as_u16();
(status, res.json().await.unwrap_or(Value::Null))
}
async fn get(&self, jar: &mut Jar, path: &str) -> reqwest::Response {
let req = self.http.get(format!("{}{path}", self.served.base));
let res = req.header(COOKIE, jar.header()).send().await.unwrap();
jar.keep(&res);
res
}
async fn whoami(&self, jar: &mut Jar) -> Value {
self.get(jar, "/oauth/session").await.json().await.unwrap()
}
/// Logs `jar`'s browser in as `account`, approved at its PDS, going
/// back to `/after`: the login, the approval and the way back.
async fn log_in(&self, jar: &mut Jar, account: &net::Account) {
let body = json!({ "handle": account.handle, "back": "/after" });
let (status, started) = self.post(jar, Some(&self.url), "/oauth/login", body).await;
assert_eq!(status, 200, "{started}");
let back = approve(account, "", started["url"].as_str().unwrap()).await;
let res = self.back(jar, &back).await;
assert_eq!(res.status().as_u16(), 303);
assert_eq!(res.headers()["location"], "/after");
}
/// Comes back to the callback the PDS sent the browser to, `back`,
/// at this server.
async fn back(&self, jar: &mut Jar, back: &str) -> reqwest::Response {
let path = back.strip_prefix(&self.url).expect("back to this site");
self.get(jar, path).await
}
}
/// Approves the login at `url` as `account`, with `extra`, and answers
/// where the PDS sends the browser back to.
async fn approve(account: &net::Account, extra: &str, url: &str) -> String {
let command = net::browser(account, &format!("--stop {extra}"));
let mut words = command.split_whitespace();
let out = tokio::process::Command::new(words.next().unwrap())
.args(words)
.arg(url)
.output()
.await
.unwrap();
assert!(
out.status.success(),
"{}",
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8(out.stdout).unwrap().trim().to_string()
}
use web::{Jar, approve, site, site_on};
#[tokio::test]
async fn the_client_says_who_it_is() {
@@ -579,6 +410,6 @@ async fn a_server_with_no_login_says_so() {
assert_eq!(res.status().as_u16(), 501, "{path}");
}
let ctx = grain::env::Ctx::new(grain::env::Env::from_vars(|_| None));
let app = App::bare(ctx);
let app = grain_serve::App::bare(ctx);
assert!(login::session(&app, "did:plc:a").await.is_err());
}
crates/grain-serve/tests/write.rsnew+428−0
428 lines changed: not loaded until asked for.
harness/answers.mjs+3−2
@@ -17,7 +17,7 @@ import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const { Lexicons } = createRequire(import.meta.url)("@atproto/lexicon");
const { Lexicons, jsonToLex } = createRequire(import.meta.url)("@atproto/lexicon");
const [dir, lexiconDir = join(root, "lexicons")] = process.argv.slice(2);
if (!dir) {
@@ -87,8 +87,9 @@ for (const line of lines) {
// An answer that is not JSON, such as a blob, is the lexicon's encoding
// to say, not this script's.
if (body === undefined) continue;
// Read as atproto reads JSON: a {$link} is a CID, a blob's JSON a blob.
try {
lexicons.assertValidXrpcOutput(nsid, body);
lexicons.assertValidXrpcOutput(nsid, jsonToLex(body));
} catch (e) {
say(e.message);
}
lexicons/social/grain/repo/applyWrites.jsonnew+102−0
@@ -0,0 +1,102 @@
{
"lexicon": 1,
"id": "social.grain.repo.applyWrites",
"defs": {
"main": {
"type": "procedure",
"description": "Make several writes in the logged-in account's repo at once, as com.atproto.repo.applyWrites makes them at its PDS, and index them before answering. Called as social.grain.repo.createRecord is.",
"input": {
"encoding": "application/json",
"schema": {
"type": "object",
"required": ["writes"],
"properties": {
"writes": {
"type": "array",
"minLength": 1,
"maxLength": 200,
"items": {
"type": "union",
"refs": ["#create", "#update", "#delete"],
"closed": true
}
}
}
}
},
"output": {
"encoding": "application/json",
"schema": {
"type": "object",
"required": ["results"],
"properties": {
"results": {
"type": "array",
"description": "Each write's result, in the place of its write.",
"items": {
"type": "union",
"refs": ["#createResult", "#updateResult", "#deleteResult"],
"closed": true
}
}
}
}
},
"errors": [
{
"name": "UnknownCollection",
"description": "grain does not index a write's collection."
},
{ "name": "InvalidRecord", "description": "A write's value does not fit its lexicon." },
{ "name": "InvalidSwap" },
{
"name": "ScopeMissingError",
"description": "The account's PDS granted the session less than the writes need."
}
]
},
"create": {
"type": "object",
"required": ["collection", "value"],
"properties": {
"collection": { "type": "string", "format": "nsid" },
"rkey": { "type": "string", "format": "record-key", "maxLength": 512 },
"value": { "type": "unknown" }
}
},
"update": {
"type": "object",
"required": ["collection", "rkey", "value"],
"properties": {
"collection": { "type": "string", "format": "nsid" },
"rkey": { "type": "string", "format": "record-key", "maxLength": 512 },
"value": { "type": "unknown" }
}
},
"delete": {
"type": "object",
"required": ["collection", "rkey"],
"properties": {
"collection": { "type": "string", "format": "nsid" },
"rkey": { "type": "string", "format": "record-key", "maxLength": 512 }
}
},
"createResult": {
"type": "object",
"required": ["uri", "cid"],
"properties": {
"uri": { "type": "string", "format": "at-uri" },
"cid": { "type": "string", "format": "cid" }
}
},
"updateResult": {
"type": "object",
"required": ["uri", "cid"],
"properties": {
"uri": { "type": "string", "format": "at-uri" },
"cid": { "type": "string", "format": "cid" }
}
},
"deleteResult": { "type": "object", "properties": {} }
}
}
lexicons/social/grain/repo/createRecord.jsonnew+43−0
@@ -0,0 +1,43 @@
{
"lexicon": 1,
"id": "social.grain.repo.createRecord",
"defs": {
"main": {
"type": "procedure",
"description": "Create a record in the logged-in account's repo, at its PDS, and index it before answering. For the web, by its session cookie, from a page of the site; a service token is refused (the apps write to their own PDS, then call social.grain.sync.indexRecords).",
"input": {
"encoding": "application/json",
"schema": {
"type": "object",
"required": ["collection", "record"],
"properties": {
"collection": {
"type": "string",
"format": "nsid",
"description": "A collection grain indexes."
},
"rkey": { "type": "string", "format": "record-key", "maxLength": 512 },
"record": {
"type": "unknown",
"description": "The record. One without a $type is given the collection's."
},
"swapCommit": { "type": "string", "format": "cid" }
}
}
},
"output": {
"encoding": "application/json",
"schema": { "type": "ref", "ref": "social.grain.repo.defs#written" }
},
"errors": [
{ "name": "UnknownCollection", "description": "grain does not index the collection." },
{ "name": "InvalidRecord", "description": "The record does not fit its lexicon." },
{ "name": "InvalidSwap" },
{
"name": "ScopeMissingError",
"description": "The account's PDS granted the session less than the write needs."
}
]
}
}
}
lexicons/social/grain/repo/defs.jsonnew+15−0
@@ -0,0 +1,15 @@
{
"lexicon": 1,
"id": "social.grain.repo.defs",
"defs": {
"written": {
"type": "object",
"description": "A record written, as its PDS answered.",
"required": ["uri", "cid"],
"properties": {
"uri": { "type": "string", "format": "at-uri" },
"cid": { "type": "string", "format": "cid" }
}
}
}
}
lexicons/social/grain/repo/deleteRecord.jsonnew+39−0
lexicons/social/grain/repo/putRecord.jsonnew+44−0
lexicons/social/grain/repo/uploadBlob.jsonnew+25−0
lexicons/README.md+13−12
hatk.generated.client.ts+3−3
hatk.generated.ts+52−23