johnandersen777.bsky.social/knotless
git that lives in your Atmosphere account
Forked from chadtmiller.com/knotless
0
0
//! The appview's index: every public repository and every pull, issue,
//! comment and state change anyone has written about one, from the firehose
//! and a backfill. Private repositories never reach it; their records are
//! not on the firehose.
//!
//! It keeps what moderation needs too: whom owners block from all their
//! repositories, labels from the labelers the server follows, and, kept
//! when the rest is rebuilt, its own takedowns, the labels it gives out for
//! them, the reports it was sent, and the accounts whose hosts say they are
//! not active.
use jiff::{Timestamp, tz::TimeZone};
use knotless::aturi::{AtUri, SpaceRef};
use knotless::forge::{Entry, Item};
use knotless::records::{
FailedCheck, OrgMemberRecord, OrgRecord, ReleaseRecord, RepoRecord, RepoRef, SiteRecord, nsid,
};
use knotless::release::valid_sha256;
use knotless::secrets::Secret;
use knotless::usage::{Day, Network};
use rusqlite::functions::{Context, FunctionFlags};
use rusqlite::{Connection, OpenFlags, OptionalExtension, params};
use serde::Serialize;
use serde_json::Value;
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use std::time::{Duration, Instant};
use tokio::sync::{broadcast, mpsc};
mod shared;
pub use shared::{STEP, Shared, Version, kind_of};
/// The kinds of preference kept, in `pref`: how the pages are drawn, and
/// which notifications an account hears of, which the queries for them
/// read by name.
pub const APPEARANCE: &str = "appearance";
pub const NOTIFICATIONS: &str = "notifications";
pub const THEMES: &str = "themes";
/// The theme a choice names, and its palette, if it names one.
fn chosen(choice: &Value) -> Option<(String, Option<Value>)> {
let palette = choice.get("palette").cloned();
let theme = choice["theme"].as_str().map(str::to_string);
theme.map(|t| (t, palette))
}
/// The most connections reading at once: past it a read waits for one.
const READERS: usize = 8;
/// Connections to the index that only read, each opened as reads first
/// need it, and kept for the next read once one is done with it. Opened
/// as needed, not all at once, as a process with many indexes open, as
/// tests are, would run out of file descriptors.
struct Readers {
path: PathBuf,
/// Those not reading now.
free: Mutex<Vec<Connection>>,
/// How many are open, reading or not.
open: Mutex<usize>,
returned: Condvar,
}
impl Readers {
fn new(path: &Path) -> Readers {
Readers {
path: path.to_path_buf(),
free: Mutex::default(),
open: Mutex::default(),
returned: Condvar::new(),
}
}
/// A connection to read with, given back as it is dropped: a free one,
/// else one opened, else, past [`READERS`], once one is free. One that
/// cannot be opened reads through `writer` instead.
fn take<'a>(&'a self, writer: &'a Mutex<Connection>) -> Reader<'a> {
let mut open = self.open.lock().unwrap();
loop {
if let Some(db) = self.free.lock().unwrap().pop() {
return Reader::own(db, self);
}
if *open < READERS {
let Ok(db) = reader(&self.path) else {
let held = Held::Writer(writer.lock().unwrap());
return Reader {
held: Some(held),
from: self,
};
};
*open += 1;
return Reader::own(db, self);
}
open = self.returned.wait(open).unwrap();
}
}
}
/// A connection the index's tables are read with, as [`connect`] made
/// them: it cannot write.
fn reader(path: &Path) -> rusqlite::Result<Connection> {
let db = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)?;
let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC;
db.create_scalar_function("zone_day", 2, flags, zone_day)?;
db.busy_timeout(Duration::from_secs(5))?;
db.execute_batch("PRAGMA query_only = ON")?;
crate::telemetry::watch(&db);
Ok(db)
}
/// A connection taken from [`Readers`], given back as it is dropped.
struct Reader<'a> {
held: Option<Held<'a>>,
from: &'a Readers,
}
/// What a [`Reader`] reads through: a connection of its own, or the
/// writer's, when one of its own could not be opened.
enum Held<'a> {
Own(Connection),
Writer(std::sync::MutexGuard<'a, Connection>),
}
impl<'a> Reader<'a> {
fn own(db: Connection, from: &'a Readers) -> Reader<'a> {
let held = Some(Held::Own(db));
Reader { held, from }
}
}
impl std::ops::Deref for Reader<'_> {
type Target = Connection;
fn deref(&self) -> &Connection {
let held = self.held.as_ref();
match held.expect("a reader holds its connection until dropped") {
Held::Own(db) => db,
Held::Writer(db) => db,
}
}
}
impl Drop for Reader<'_> {
fn drop(&mut self) {
if let Some(Held::Own(db)) = self.held.take() {
// Held while it goes back, so a read waiting sees it there.
let open = self.from.open.lock().unwrap();
self.from.free.lock().unwrap().push(db);
drop(open);
self.from.returned.notify_one();
}
}
}
pub struct Index {
/// The one connection that writes, and reads what a write needs.
db: Mutex<Connection>,
/// The connections that only read: a page waits on no write, nor on
/// another page's read, as the database is in WAL mode.
readers: Readers,
/// Whether opening it built its cache anew, empty: a new file, or one
/// of another version of the tables.
pub rebuilt: bool,
/// The repositories written about, as `<did>/<key>`, as each write
/// lands: what the pages follow to show it ([`crate::live`]).
pub changes: broadcast::Sender<Arc<str>>,
/// The repositories asked to have their language counted, by owner and
/// key, until it is ([`Index::recount`]).
asked: Mutex<HashSet<(String, String)>>,
recount: mpsc::UnboundedSender<(String, String)>,
/// What counts them takes this, once ([`crate::languages::follow`]).
counting: Mutex<Option<mpsc::UnboundedReceiver<(String, String)>>>,
/// The public repositories asked to have their code indexed again, by
/// owner and key, until it is ([`Index::reindex`]).
coded: Mutex<HashSet<(String, String)>>,
recode: mpsc::UnboundedSender<(String, String)>,
/// What indexes their code takes this, once ([`crate::code::follow`]).
coding: Mutex<Option<mpsc::UnboundedReceiver<(String, String)>>>,
/// The releases of things shared whose files are to be read, by URI;
/// what reads them takes the other end, once ([`crate::shared::follow`]).
read: mpsc::UnboundedSender<String>,
reading: Mutex<Option<mpsc::UnboundedReceiver<String>>>,
/// When what each repository's pulls and issues are made of last
/// changed, for [`Index::assembled`] to know what it kept still holds.
stamps: Stamps,
/// Repositories' pulls and issues, as [`Index::assembled`] made them.
kept: Mutex<Kept>,
/// Held across a write weighed by its revision, from the weighing to
/// the stamp, so a backfill and the feed cannot both pass it at once
/// ([`Index::put_as_of`]).
revs: Mutex<()>,
/// A lock for each space, held across its sync, so that no two read
/// one repo at once and the slower puts back what it read over what
/// the other did ([`crate::spaces::sync_with`]).
syncing: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
}
/// What backfills made of an account ([`Index::backfilled`]).
#[derive(Debug, Default, PartialEq)]
pub struct Backfilled {
pub rev: Option<String>,
pub tries: u32,
pub retry_at: u64,
}
/// A repository's pulls and issues: the forge records about it that stand,
/// as [`Index::forge`] gives them, and the items they make.
#[derive(Debug, Default)]
pub struct Assembled {
pub entries: Vec<Entry>,
/// As [`knotless::forge::assemble`] makes them, newest first.
pub items: Vec<Item>,
}
/// How long a repository's pulls and issues are kept with nothing written
/// about them: a label from a labeler runs out, and one set a little ahead
/// of the clock comes in, with time alone.
const KEPT_FOR: Duration = Duration::from_secs(60);
/// The most forge records kept, over every repository: a busy repository
/// has thousands, each a few kilobytes once read.
const KEPT_ENTRIES: usize = 50_000;
/// The most scopes stamped before all are forgotten, and everything
/// stamped at once in their place.
const STAMPS_MAX: usize = 100_000;
/// When each scope last changed, on a clock of writes: a repository, as
/// `<did>/<key>`, or an owner's blocks, as their DID. A write stamps only
/// what it is about, so the network writing elsewhere leaves what was
/// kept of a repository standing; one whose reach is wide or unknown (a
/// takedown, a label, a move, an account forgotten) stamps everything.
struct Stamps {
/// The most scopes stamped before all are forgotten.
max: usize,
clock: AtomicU64,
/// When everything was last stamped.
all: AtomicU64,
by: Mutex<HashMap<String, u64>>,
/// The records of a repository written since, by scope, each with
/// when: what [`Index::assembled`] reads again to patch what it kept.
items: Mutex<HashMap<String, Vec<(u64, String)>>>,
}
/// The most records noted as changed in one repository before the whole of
/// it is stamped changed instead: a read patches that many, or builds anew.
const ITEMS_MAX: usize = 32;
impl Stamps {
fn new(max: usize) -> Stamps {
Stamps {
max,
clock: AtomicU64::default(),
all: AtomicU64::default(),
by: Mutex::default(),
items: Mutex::default(),
}
}
/// The clock: what is read after this is as new as it.
fn now(&self) -> u64 {
self.clock.load(Ordering::SeqCst)
}
/// Stamps `scope` as changed, once its write is done.
fn mark(&self, scope: &str) {
let at = self.clock.fetch_add(1, Ordering::SeqCst) + 1;
let mut by = self.by.lock().unwrap();
if by.len() >= self.max {
by.clear();
self.all.fetch_max(at, Ordering::SeqCst);
}
by.insert(scope.to_string(), at);
}
/// Stamps `uri`, one record of `scope`, as changed, once its write is
/// done: the scope's own stamp stays, so what was kept of it is patched
/// with that record, not built anew. Past [`ITEMS_MAX`] of them, or
/// `max` scopes, it is stamped as a whole.
fn mark_item(&self, scope: &str, uri: &str) {
let at = self.clock.fetch_add(1, Ordering::SeqCst) + 1;
let mut items = self.items.lock().unwrap();
if items.len() >= self.max && !items.contains_key(scope) {
items.clear();
self.all.fetch_max(at, Ordering::SeqCst);
}
let noted = items.entry(scope.to_string()).or_default();
noted.push((at, uri.to_string()));
if noted.len() > ITEMS_MAX {
items.remove(scope);
drop(items);
self.mark(scope);
}
}
/// The records of `scopes` changed after `after`, each once.
fn touched(&self, scopes: &[String], after: u64) -> Vec<String> {
let items = self.items.lock().unwrap();
let mut found: Vec<String> = Vec::new();
for (at, uri) in scopes.iter().filter_map(|s| items.get(s)).flatten() {
if *at > after && !found.contains(uri) {
found.push(uri.clone());
}
}
found
}
/// Stamps everything as changed.
fn mark_all(&self) {
let at = self.clock.fetch_add(1, Ordering::SeqCst) + 1;
self.all.fetch_max(at, Ordering::SeqCst);
}
/// When the latest of `scopes` changed.
fn since(&self, scopes: &[String]) -> u64 {
let by = self.by.lock().unwrap();
let each = scopes.iter().filter_map(|s| by.get(s).copied());
each.fold(self.all.load(Ordering::SeqCst), u64::max)
}
}
/// Repositories' pulls and issues, by `<did>/<key>`, the oldest made
/// dropped past a number of records in all.
struct Kept {
/// The most records kept.
cap: usize,
by: HashMap<String, Made>,
/// Oldest first.
order: VecDeque<String>,
entries: usize,
}
/// What [`Kept`] holds of one repository.
struct Made {
/// The clock when it was read, and when.
at: u64,
made: Instant,
/// The scopes it was read from: the repository, the one it moved
/// from, and its owner's blocks.
scopes: Vec<String>,
/// The owner's hides, which moderating takes out of the records, for
/// a patch to moderate again with.
hides: Vec<Entry>,
/// The record it was assembled with.
record: RepoRecord,
assembled: Arc<Assembled>,
}
impl Kept {
fn new(cap: usize) -> Kept {
Kept {
cap,
by: HashMap::new(),
order: VecDeque::new(),
entries: 0,
}
}
fn put(&mut self, key: String, made: Made) {
self.entries += made.assembled.entries.len();
if let Some(old) = self.by.insert(key.clone(), made) {
self.entries -= old.assembled.entries.len();
self.order.retain(|k| *k != key);
}
self.order.push_back(key);
while self.entries > self.cap && self.order.len() > 1 {
let oldest = self.order.pop_front().expect("more than one");
let gone = self.by.remove(&oldest).expect("kept in order");
self.entries -= gone.assembled.entries.len();
}
}
}
/// A repository as the index knows it.
#[derive(Serialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct Listed {
pub did: String,
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub fork_of: Option<String>,
pub created_at: String,
/// Its record, which pages are cut after.
#[serde(skip)]
pub uri: String,
}
/// A repository's pushes, as [`Index::actor_pushes`] counts them.
pub struct RepoPushes {
/// The repository's name now.
pub name: String,
/// When it was last pushed to.
pub pushed_at: String,
/// Each day it was pushed to, with how often.
pub days: Vec<(String, u64)>,
}
/// Which repositories [`Index::repos_page`] lists: each named, only those
/// that `owner` owns, that `not_owner` does not, not named `not_named`
/// (ignoring case), that name `runner` among their runners, `triager`
/// among their triagers or `topic` among their topics, or that
/// `followed_by` or one it follows owns.
#[derive(Debug, Clone, Copy, Default)]
pub struct Among<'a> {
pub owner: Option<&'a str>,
pub not_owner: Option<&'a str>,
pub not_named: Option<&'a str>,
pub runner: Option<&'a str>,
pub triager: Option<&'a str>,
pub followed_by: Option<&'a str>,
pub topic: Option<&'a str>,
}
/// A repository starred, as [`Index::stars`] finds it.
#[derive(Debug, Clone, PartialEq)]
pub struct Starred {
/// The star, which pages are cut after.
pub uri: String,
/// Who starred it.
pub did: String,
/// The repository's record.
pub repo: String,
/// The repository's owner.
pub owner: String,
/// The repository's name now.
pub name: String,
pub created_at: String,
}
/// A ref moving, as a claim recorded it.
#[derive(Debug, Clone, PartialEq)]
pub struct Push {
/// The claim, which pages are cut after.
pub uri: String,
/// The repository's record.
pub repo: String,
pub did: String,
/// The repository's name now.
pub name: String,
/// The repository it was forked from, if it is a fork.
pub fork_of: Option<String>,
/// The full ref name, such as `refs/heads/main`.
pub refname: String,
/// Where it was, when it was there before.
pub from: Option<String>,
/// Where it is now, unless it was deleted.
pub to: Option<String>,
pub created_at: String,
/// Who it is credited to: the owner, or the collaborator or runner who
/// pushed through the gateway, as [`BY`] says.
pub by: String,
}
/// Whether `$who` is a member in public of the organization `$org`, in
/// SQL over those two expressions: the organization's public records name
/// them, taken in (`joinedAt`), and their public records keep a record of
/// joining it. What may push to a public repository of `$org` besides its
/// collaborators, as [`knotless::org::fill`] reads it.
macro_rules! public_member {
($org:literal, $who:literal) => {
concat!(
"EXISTS (SELECT 1 FROM org_member m JOIN membership j
ON j.space = '' AND j.did = m.did AND j.org = m.org
WHERE m.space = '' AND m.org = ",
$org,
" AND m.did = ",
$who,
" AND m.joined_at IS NOT NULL)"
)
};
}
/// Who a row of `push p JOIN repo r` is credited to: the pusher it
/// names, while the repository names them among its collaborators or
/// runners, or they are a member in public of the organization its owner
/// is; otherwise the owner, whose account wrote the claim.
const BY: &str = concat!(
"CASE WHEN p.pusher IS NOT NULL AND (EXISTS (
SELECT 1 FROM json_each(r.value, '$.collaborators') c
WHERE json_extract(c.value, '$.did') = p.pusher
UNION ALL SELECT 1 FROM json_each(r.value, '$.runners') c
WHERE json_extract(c.value, '$.did') = p.pusher) OR ",
public_member!("r.did", "p.pusher"),
") THEN p.pusher ELSE p.did END"
);
/// An account an organization's records name, as [`Index::org_roster`]
/// finds it.
#[derive(Debug, Clone, PartialEq)]
pub struct Invitee {
pub did: String,
/// `admin` or `member`, as the record's roles give it.
pub role: Option<String>,
pub invited_at: String,
/// When the invitation lapses, when the record says.
pub expires_at: Option<String>,
/// When the organization took them in.
pub joined_at: Option<String>,
/// Whether they keep a record of joining it, in the same place as the
/// organization's record of them: its public records, or its space.
pub joined: bool,
}
/// Which checks of a repository [`Index::checks`] gives.
#[derive(Debug, Clone, Copy, Default)]
pub struct CheckFilter<'a> {
/// Only those on this ref, such as `refs/heads/main`.
pub refname: Option<&'a str>,
/// Only those of this commit.
pub commit: Option<&'a str>,
/// Only those of one of these commits.
pub commits: Option<&'a [String]>,
/// Only the runs of this pull's head, by its AT-URI.
pub pull: Option<&'a str>,
/// Only the runs of a merge queue's batches.
pub batch: bool,
/// Only the runs of this workflow, by name.
pub workflow: Option<&'a str>,
}
/// How many reacted to a subject with one content, as [`Index::reactions`]
/// counts them: `(subject, content, count, the viewer's reaction)`.
pub type Reacted = (String, String, u64, Option<String>);
/// A theme as the index keeps it: `(uri, the DID of who made it, record)`.
pub type Theme = (String, String, Value);
/// The shape of the tables. An index of another shape is dropped and
/// built again: it is a cache, and `--backfill` refills it, all but the
/// pushes, which outlive the claims they were read from.
pub const VERSION: i64 = 8;
/// The index's database at `path`, its tables made as this version has
/// them, and whether they were built anew: what [`Index::open`] opens, and
/// what [`crate::plans`] plans the index's queries against.
pub(crate) fn connect(path: &Path) -> rusqlite::Result<(Connection, bool)> {
let db = Connection::open(path)?;
let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC;
db.create_scalar_function("zone_day", 2, flags, zone_day)?;
let version: i64 = db.query_row("PRAGMA user_version", [], |row| row.get(0))?;
let rebuilt = version != VERSION;
if rebuilt {
db.execute_batch(CACHE)?;
}
let commented = db
.prepare("SELECT 1 FROM sqlite_master WHERE name = 'comment'")?
.exists([])?;
let reviewed = db
.prepare("SELECT 1 FROM sqlite_master WHERE name = 'reviewer'")?
.exists([])?;
let collaborated = db
.prepare("SELECT 1 FROM sqlite_master WHERE name = 'collaborator'")?
.exists([])?;
let shared = db
.prepare("SELECT 1 FROM sqlite_master WHERE name = 'shared'")?
.exists([])?;
let used = db
.prepare("SELECT 1 FROM sqlite_master WHERE name = 'shared_use'")?
.exists([])?;
// A repository is known by its owner and key; its name, and the
// URIs others wrote naming it, can change.
db.execute_batch(&format!(
"PRAGMA journal_mode = WAL;
PRAGMA synchronous = NORMAL;
PRAGMA user_version = {VERSION};
CREATE TABLE IF NOT EXISTS repo (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, name TEXT NOT NULL, key TEXT NOT NULL,
description TEXT, fork_of TEXT, fork_did TEXT, fork_key TEXT,
from_did TEXT, from_key TEXT, created_at TEXT NOT NULL, value TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS repo_key ON repo (did, key);
CREATE INDEX IF NOT EXISTS repo_fork ON repo (fork_did, fork_key);
CREATE INDEX IF NOT EXISTS repo_created ON repo (created_at);
CREATE INDEX IF NOT EXISTS repo_name ON repo (name COLLATE NOCASE);
CREATE TABLE IF NOT EXISTS collaborator (
uri TEXT NOT NULL, repo_did TEXT NOT NULL, repo_key TEXT NOT NULL,
did TEXT NOT NULL, PRIMARY KEY (uri, did));
CREATE INDEX IF NOT EXISTS collaborator_did ON collaborator (did);
CREATE TABLE IF NOT EXISTS forge (
uri TEXT PRIMARY KEY, author TEXT NOT NULL, repo_did TEXT NOT NULL,
repo_key TEXT NOT NULL, value TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS forge_repo ON forge (repo_did, repo_key);
CREATE INDEX IF NOT EXISTS forge_author ON forge (author);
CREATE TABLE IF NOT EXISTS check_run (
uri TEXT PRIMARY KEY, author TEXT NOT NULL, repo_did TEXT NOT NULL,
repo_key TEXT NOT NULL, ref TEXT NOT NULL, sha TEXT NOT NULL,
started_at TEXT NOT NULL, value TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS check_repo ON check_run (repo_did, repo_key, started_at);
CREATE INDEX IF NOT EXISTS check_sha ON check_run (repo_did, repo_key, sha);
CREATE INDEX IF NOT EXISTS check_author ON check_run (author);
CREATE TABLE IF NOT EXISTS minutes (
uri TEXT PRIMARY KEY, runner TEXT NOT NULL, owner TEXT NOT NULL, value TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS minutes_owner ON minutes (owner);
CREATE TABLE IF NOT EXISTS runner_key (
did TEXT PRIMARY KEY, uri TEXT NOT NULL, key TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS push (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, key TEXT NOT NULL, ref TEXT NOT NULL,
before TEXT, after TEXT, created_at TEXT NOT NULL, pusher TEXT);
CREATE INDEX IF NOT EXISTS push_created ON push (created_at);
CREATE INDEX IF NOT EXISTS push_did ON push (did, created_at);
CREATE TABLE IF NOT EXISTS language (
did TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL,
PRIMARY KEY (did, key));
CREATE TABLE IF NOT EXISTS moved (
uri TEXT PRIMARY KEY, from_did TEXT NOT NULL, from_key TEXT NOT NULL,
to_did TEXT NOT NULL, to_key TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS block (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, subject TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS block_did ON block (did);
CREATE TABLE IF NOT EXISTS star (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, repo_did TEXT NOT NULL,
repo_key TEXT NOT NULL, created_at TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS star_repo ON star (repo_did, repo_key);
CREATE INDEX IF NOT EXISTS star_did ON star (did, created_at);
CREATE INDEX IF NOT EXISTS star_created ON star (created_at);
CREATE TABLE IF NOT EXISTS comment (
uri TEXT PRIMARY KEY, author TEXT NOT NULL, subject TEXT NOT NULL,
created_at TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS comment_subject ON comment (subject, author, created_at);
CREATE INDEX IF NOT EXISTS comment_author ON comment (author, subject);
CREATE TABLE IF NOT EXISTS mention (
uri TEXT NOT NULL, did TEXT NOT NULL, at TEXT NOT NULL, PRIMARY KEY (uri, did));
CREATE INDEX IF NOT EXISTS mention_did ON mention (did);
CREATE TABLE IF NOT EXISTS reviewer (
uri TEXT NOT NULL, did TEXT NOT NULL, PRIMARY KEY (uri, did));
CREATE INDEX IF NOT EXISTS reviewer_did ON reviewer (did);
CREATE TABLE IF NOT EXISTS assigned (
uri TEXT NOT NULL, did TEXT NOT NULL, PRIMARY KEY (uri, did));
CREATE INDEX IF NOT EXISTS assigned_did ON assigned (did);
CREATE TABLE IF NOT EXISTS follow (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, subject TEXT NOT NULL,
created_at TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS follow_subject ON follow (subject, created_at);
CREATE INDEX IF NOT EXISTS follow_did ON follow (did, created_at);
CREATE TABLE IF NOT EXISTS reaction (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, subject TEXT NOT NULL,
content TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS reaction_subject ON reaction (subject, content);
CREATE TABLE IF NOT EXISTS release (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, repo_key TEXT NOT NULL,
created_at TEXT NOT NULL, value TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS release_repo ON release (did, repo_key, created_at);
CREATE TABLE IF NOT EXISTS release_yank (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, repo_key TEXT NOT NULL, release TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS release_yank_release ON release_yank (release);
CREATE INDEX IF NOT EXISTS release_yank_repo ON release_yank (did, repo_key);
CREATE TABLE IF NOT EXISTS site (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, repo_key TEXT NOT NULL, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS theme (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, created_at TEXT NOT NULL,
name TEXT NOT NULL, value TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS theme_made ON theme (created_at, uri);
CREATE INDEX IF NOT EXISTS theme_did ON theme (did, created_at, uri);
CREATE INDEX IF NOT EXISTS theme_name ON theme (name, uri);
CREATE TABLE IF NOT EXISTS org (
space TEXT NOT NULL, did TEXT NOT NULL, gateway TEXT, created_at TEXT NOT NULL,
PRIMARY KEY (space, did));
CREATE TABLE IF NOT EXISTS org_member (
space TEXT NOT NULL, org TEXT NOT NULL, did TEXT NOT NULL, role TEXT,
invited_at TEXT NOT NULL, expires_at TEXT, joined_at TEXT,
PRIMARY KEY (space, org, did));
CREATE INDEX IF NOT EXISTS org_member_did ON org_member (did);
CREATE TABLE IF NOT EXISTS membership (
space TEXT NOT NULL, did TEXT NOT NULL, org TEXT NOT NULL,
created_at TEXT NOT NULL, PRIMARY KEY (space, did, org));
CREATE TABLE IF NOT EXISTS label (
src TEXT NOT NULL, uri TEXT NOT NULL, val TEXT NOT NULL, exp TEXT,
PRIMARY KEY (src, uri, val));
CREATE TABLE IF NOT EXISTS labeler (src TEXT PRIMARY KEY, seq INTEGER NOT NULL);
CREATE TABLE IF NOT EXISTS stream (src TEXT PRIMARY KEY, seq INTEGER NOT NULL);
CREATE TABLE IF NOT EXISTS space (uri TEXT PRIMARY KEY, cursor TEXT);
CREATE TABLE IF NOT EXISTS space_syncer (uri TEXT PRIMARY KEY, did TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS space_gone (uri TEXT PRIMARY KEY);
DROP TABLE IF EXISTS space_repo;
CREATE TABLE IF NOT EXISTS space_writer (
space TEXT NOT NULL, did TEXT NOT NULL, rev TEXT NOT NULL,
set_hash BLOB NOT NULL, PRIMARY KEY (space, did));
CREATE TABLE IF NOT EXISTS space_reader (
uri TEXT NOT NULL, did TEXT NOT NULL, PRIMARY KEY (uri, did));
CREATE TABLE IF NOT EXISTS space_record (
space TEXT NOT NULL, did TEXT NOT NULL, collection TEXT NOT NULL,
rkey TEXT NOT NULL, PRIMARY KEY (space, did, collection, rkey));
CREATE TABLE IF NOT EXISTS seen (did TEXT PRIMARY KEY, at TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS push_device (
token TEXT PRIMARY KEY, did TEXT NOT NULL, environment TEXT NOT NULL,
at TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS push_device_did ON push_device (did, at);
DROP TABLE IF EXISTS notify_pref;
CREATE TABLE IF NOT EXISTS pref (
did TEXT NOT NULL, kind TEXT NOT NULL, value TEXT NOT NULL,
PRIMARY KEY (did, kind));
CREATE TABLE IF NOT EXISTS subscription (
did TEXT NOT NULL, subject TEXT NOT NULL, level TEXT NOT NULL, at TEXT NOT NULL,
PRIMARY KEY (did, subject));
CREATE TABLE IF NOT EXISTS takedown (
subject TEXT PRIMARY KEY, value TEXT NOT NULL, at TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS emitted (seq INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS profile_kept (did TEXT PRIMARY KEY);
CREATE TABLE IF NOT EXISTS bot (did TEXT PRIMARY KEY);
CREATE TABLE IF NOT EXISTS profile_record (
did TEXT NOT NULL, collection TEXT NOT NULL, rkey TEXT NOT NULL,
value TEXT NOT NULL, PRIMARY KEY (did, collection, rkey));
CREATE TABLE IF NOT EXISTS handle (
did TEXT PRIMARY KEY, handle TEXT, checked INTEGER NOT NULL);
CREATE INDEX IF NOT EXISTS handle_name ON handle (handle);
CREATE TABLE IF NOT EXISTS home (
did TEXT PRIMARY KEY, moved INTEGER NOT NULL, own INTEGER NOT NULL,
checked INTEGER NOT NULL);
CREATE TABLE IF NOT EXISTS commit_author (
sha TEXT NOT NULL, address TEXT NOT NULL, repo_did TEXT NOT NULL,
repo_key TEXT NOT NULL, did TEXT NOT NULL,
PRIMARY KEY (sha, address, repo_did, repo_key)) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS commit_author_repo ON commit_author (repo_did, repo_key);
CREATE TABLE IF NOT EXISTS authors_walked (
repo_did TEXT NOT NULL, repo_key TEXT NOT NULL, tip TEXT NOT NULL,
people TEXT NOT NULL, PRIMARY KEY (repo_did, repo_key));
CREATE TABLE IF NOT EXISTS report (
id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS invite (
id INTEGER PRIMARY KEY AUTOINCREMENT, runner TEXT NOT NULL, did TEXT NOT NULL,
name TEXT NOT NULL, at TEXT NOT NULL, UNIQUE (runner, did, name));
CREATE TABLE IF NOT EXISTS secret (
owner TEXT NOT NULL, repo TEXT NOT NULL, runner TEXT NOT NULL,
name TEXT NOT NULL, refs TEXT NOT NULL, key TEXT NOT NULL,
sealed BLOB NOT NULL, at TEXT NOT NULL,
PRIMARY KEY (owner, repo, runner, name));
CREATE TABLE IF NOT EXISTS inactive (did TEXT PRIMARY KEY, status TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS record_rev (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, rev TEXT NOT NULL, gone INTEGER NOT NULL);
CREATE INDEX IF NOT EXISTS record_rev_gone ON record_rev (did, rev) WHERE gone;
CREATE TABLE IF NOT EXISTS backfill (
did TEXT PRIMARY KEY, rev TEXT, tries INTEGER NOT NULL, retry_at INTEGER NOT NULL);
CREATE INDEX IF NOT EXISTS label_uri ON label (uri);
DROP VIEW IF EXISTS temp.down;
CREATE TABLE IF NOT EXISTS down (subject TEXT PRIMARY KEY);
{shared}
{triggers}",
shared = shared::TABLES,
triggers = down_triggers(),
))?;
// What is down, from what makes it so, as it stands now: the
// triggers keep it so from here, but for labels as they lapse.
db.execute_batch(&format!(
"BEGIN;
DELETE FROM down;
INSERT OR IGNORE INTO down SELECT subject FROM takedown
UNION ALL SELECT did FROM inactive
UNION ALL SELECT uri FROM label WHERE {LIVE};
COMMIT;"
))?;
// Indexes of what records say, apart: a table of another shape,
// with no `value`, is kept without them, as every use of it fails.
let _ = db.execute_batch(EXPRESSIONS);
// The push table outlives a rebuild, so a column it gains is added
// in place.
let pushers = db
.prepare("SELECT 1 FROM pragma_table_info('push') WHERE name = 'pusher'")?
.exists([])?;
if !pushers {
db.execute_batch("ALTER TABLE push ADD COLUMN pusher TEXT")?;
}
db.execute_batch("CREATE INDEX IF NOT EXISTS push_pusher ON push (pusher, created_at)")?;
// When a choice began to hear of a repository's releases, which a
// change between all it does and its releases alone keeps: added in
// place, as the table outlives a rebuild, and kept if it was there.
let _ = db.execute_batch("ALTER TABLE subscription ADD COLUMN releases_at TEXT");
// Comments, by what they are on, from the forge table, once: an
// index from before had them only there. One whose forge table is
// of another shape gets them as they come.
if !commented {
let _ = db.execute(
"INSERT OR IGNORE INTO comment
SELECT uri, author, json_extract(value, '$.subject'),
json_extract(value, '$.createdAt')
FROM forge WHERE instr(uri, '/dev.knotless.comment/')",
[],
);
}
// Whom pulls ask to review them, from the forge table, once: an index
// from before had them only there.
if !reviewed {
let _ = db.execute(REVIEWERS, []);
}
// Collaborators, from the repositories' records, once: an index
// from before had them only there.
if !collaborated {
let _ = db.execute(COLLABORATORS, [None::<&str>]);
}
// The releases of what is shared, from the releases, once: an index
// from before had them only there.
if !shared {
let _ = db.execute(shared::UNREAD, []);
}
// What each repository uses is read as what it is written in is
// counted: an index from before counted them without, so each is
// counted again as a list next shows it.
if !used {
let _ = db.execute("DELETE FROM language", []);
}
Ok((db, rebuilt))
}
/// The indexes of what records say: what a forge record is about, for
/// the merge queue's word on a pull; when it was made, for the feed,
/// newest first, for one account's, and for one repository's since a
/// time, as its watchers hear of it; the repository a pull's head is in, for a push to find
/// the pulls it moves; the merge queue's batch a pull landed in, for its
/// badges; the few blocks and hides owners write; and a
/// pull's runs, each workflow's latest last.
const EXPRESSIONS: &str =
"CREATE INDEX IF NOT EXISTS forge_subject ON forge (json_extract(value, '$.subject'));
CREATE INDEX IF NOT EXISTS forge_created ON forge (json_extract(value, '$.createdAt'), uri);
CREATE INDEX IF NOT EXISTS forge_author_created ON forge (author,
json_extract(value, '$.createdAt'), uri);
CREATE INDEX IF NOT EXISTS forge_repo_created ON forge (repo_did, repo_key,
json_extract(value, '$.createdAt'));
CREATE INDEX IF NOT EXISTS forge_head ON forge (json_extract(value, '$.head.repo.key'));
CREATE INDEX IF NOT EXISTS forge_batch ON forge (json_extract(value, '$.batch'));
CREATE INDEX IF NOT EXISTS forge_moderation ON forge (repo_did, repo_key)
WHERE instr(uri, '/dev.knotless.block/') OR instr(uri, '/dev.knotless.hide/');
CREATE INDEX IF NOT EXISTS check_pull ON check_run (json_extract(value, '$.pull'),
json_extract(value, '$.workflow'), started_at);
CREATE INDEX IF NOT EXISTS check_pull_at ON check_run (json_extract(value, '$.pull'),
started_at, uri);";
/// Drops the tables that are a cache. Takedowns, the labels given out for
/// them and reports are not, and stay; nor are pushes, since compaction
/// deletes the claims they came from and no backfill finds them again; nor
/// who was proven to read whose space, which only their reading proved. A
/// change to the push table's shape alters it in place.
const CACHE: &str = "DROP TABLE IF EXISTS repo; DROP TABLE IF EXISTS forge;
DROP TABLE IF EXISTS moved; DROP TABLE IF EXISTS block; DROP TABLE IF EXISTS label;
DROP TABLE IF EXISTS labeler; DROP TABLE IF EXISTS stream; DROP TABLE IF EXISTS check_run;
DROP TABLE IF EXISTS space; DROP TABLE IF EXISTS space_syncer; DROP TABLE IF EXISTS space_gone;
DROP TABLE IF EXISTS space_writer; DROP TABLE IF EXISTS space_record;
DROP TABLE IF EXISTS profile_kept; DROP TABLE IF EXISTS bot;
DROP TABLE IF EXISTS profile_record; DROP TABLE IF EXISTS handle;
DROP TABLE IF EXISTS home;
DROP TABLE IF EXISTS star; DROP TABLE IF EXISTS follow; DROP TABLE IF EXISTS comment;
DROP TABLE IF EXISTS reaction; DROP TABLE IF EXISTS release; DROP TABLE IF EXISTS release_yank; DROP TABLE IF EXISTS site; DROP TABLE IF EXISTS mention;
DROP TABLE IF EXISTS reviewer; DROP TABLE IF EXISTS assigned;
DROP TABLE IF EXISTS org; DROP TABLE IF EXISTS org_member; DROP TABLE IF EXISTS membership;
DROP TABLE IF EXISTS collaborator; DROP TABLE IF EXISTS shared;
DROP TABLE IF EXISTS record_rev; DROP TABLE IF EXISTS backfill;";
/// A label that has not lapsed.
const LIVE: &str = "(exp IS NULL OR exp > strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))";
/// Whether nothing keeps `down.subject` down any more: no takedown, no
/// host saying its account is inactive, and no label that has not lapsed.
fn up() -> String {
format!(
"NOT EXISTS (SELECT 1 FROM takedown t WHERE t.subject = down.subject)
AND NOT EXISTS (SELECT 1 FROM inactive i WHERE i.did = down.subject)
AND NOT EXISTS (SELECT 1 FROM label l WHERE l.uri = down.subject AND {})",
LIVE.replace("exp", "l.exp")
)
}
/// What keeps `down`, a table of every subject left out of reads, as
/// takedowns, accounts' hosts and labelers say: a table, not a view of
/// them, so that `NOT IN down` looks each up by its key rather than
/// building the whole list for every statement, as the network's
/// inactive accounts are many. A subject goes when nothing keeps it down.
fn down_triggers() -> String {
let unless = up();
let live = LIVE.replace("exp", "new.exp");
format!(
"CREATE TRIGGER IF NOT EXISTS takedown_down AFTER INSERT ON takedown
BEGIN INSERT OR IGNORE INTO down VALUES (new.subject); END;
CREATE TRIGGER IF NOT EXISTS inactive_down AFTER INSERT ON inactive
BEGIN INSERT OR IGNORE INTO down VALUES (new.did); END;
CREATE TRIGGER IF NOT EXISTS label_down AFTER INSERT ON label WHEN {live}
BEGIN INSERT OR IGNORE INTO down VALUES (new.uri); END;
CREATE TRIGGER IF NOT EXISTS takedown_up AFTER DELETE ON takedown
BEGIN DELETE FROM down WHERE subject = old.subject AND {unless}; END;
CREATE TRIGGER IF NOT EXISTS inactive_up AFTER DELETE ON inactive
BEGIN DELETE FROM down WHERE subject = old.did AND {unless}; END;
CREATE TRIGGER IF NOT EXISTS label_up AFTER DELETE ON label
BEGIN DELETE FROM down WHERE subject = old.uri AND {unless}; END;"
)
}
/// Who repositories' records name among their collaborators, a row each,
/// from their rows in `repo`: the one at `?1`, or every one when it is
/// null.
const COLLABORATORS: &str = "INSERT OR IGNORE INTO collaborator
SELECT r.uri, r.did, r.key, c.value ->> '$.did'
FROM repo r, json_each(r.value, '$.collaborators') c
WHERE c.value ->> '$.did' IS NOT NULL AND r.uri = coalesce(?1, r.uri)";
/// Whom each pull asks to review it, by its record: once, for an index
/// from before the table.
const REVIEWERS: &str = "INSERT OR IGNORE INTO reviewer
SELECT f.uri, j.value ->> '$.did' FROM forge f, json_each(f.value, '$.reviewers') j
-- scans: forge, once, for an index from before it kept reviewers
WHERE instr(f.uri, '/dev.knotless.pull/') AND j.value ->> '$.did' IS NOT NULL";
/// What [`Index::forget_repo`] runs, with `?1` the owner's DID and `?2` the
/// repository's key: what was said on it first, then what names it.
const FORGET_REPO: &[&str] = &[
"DELETE FROM reaction WHERE subject IN
(SELECT uri FROM forge WHERE repo_did = ?1 AND repo_key = ?2)",
"DELETE FROM comment WHERE uri IN
(SELECT uri FROM forge WHERE repo_did = ?1 AND repo_key = ?2)",
"DELETE FROM mention WHERE uri IN
(SELECT uri FROM forge WHERE repo_did = ?1 AND repo_key = ?2)",
"DELETE FROM reviewer WHERE uri IN
(SELECT uri FROM forge WHERE repo_did = ?1 AND repo_key = ?2)",
"DELETE FROM assigned WHERE uri IN
(SELECT uri FROM forge WHERE repo_did = ?1 AND repo_key = ?2)",
"DELETE FROM forge WHERE repo_did = ?1 AND repo_key = ?2",
"DELETE FROM check_run WHERE repo_did = ?1 AND repo_key = ?2",
"DELETE FROM release WHERE did = ?1 AND repo_key = ?2",
"DELETE FROM release_yank WHERE did = ?1 AND repo_key = ?2",
"DELETE FROM site WHERE uri = 'at://' || ?1 || '/dev.knotless.site/' || ?2",
"DELETE FROM shared WHERE did = ?1 AND repo_key = ?2",
"DELETE FROM shared_use WHERE did = ?1 AND key = ?2",
"DELETE FROM star WHERE repo_did = ?1 AND repo_key = ?2",
"DELETE FROM push WHERE did = ?1 AND key = ?2",
"DELETE FROM language WHERE did = ?1 AND key = ?2",
];
/// The repositories, as `<did>/<key>`, whose pulls and issues
/// [`Index::purge`] changes of the account `?1`: its own, those it moved
/// from and to, and those it wrote about or checked.
const PURGED_SCOPES: &[&str] = &[
"SELECT did || '/' || key FROM repo WHERE did = ?1",
"SELECT from_did || '/' || from_key FROM moved WHERE from_did = ?1",
"SELECT to_did || '/' || to_key FROM moved WHERE from_did = ?1",
"SELECT DISTINCT repo_did || '/' || repo_key FROM forge WHERE author = ?1",
"SELECT DISTINCT repo_did || '/' || repo_key FROM check_run WHERE author = ?1",
];
/// What [`Index::purge`] runs, with `?1` the account's DID, `?2` the
/// start of its records' URIs and `?3` the first URI past them: first the records of the spaces it owned,
/// whoever wrote them, then each table of what it wrote or was kept for it.
const PURGE: &[&str] = &[
"CREATE TEMP TABLE IF NOT EXISTS purged (uri TEXT PRIMARY KEY)",
"DELETE FROM purged",
"INSERT OR IGNORE INTO purged SELECT space || '/' || did || '/' || collection || '/' || rkey
FROM space_record WHERE substr(space, 1, length(?2)) = ?2",
"DELETE FROM collaborator WHERE repo_did = ?1 OR did = ?1 OR uri IN purged",
"DELETE FROM repo WHERE did = ?1 OR uri IN purged",
"DELETE FROM reviewer WHERE did = ?1 OR uri IN purged
OR uri IN (SELECT uri FROM forge WHERE author = ?1)",
"DELETE FROM assigned WHERE did = ?1 OR uri IN purged
OR uri IN (SELECT uri FROM forge WHERE author = ?1)",
"DELETE FROM forge WHERE author = ?1 OR uri IN purged",
"DELETE FROM comment WHERE author = ?1 OR uri IN purged",
"DELETE FROM mention WHERE did = ?1 OR (uri >= ?2 AND uri < ?3) OR uri IN purged",
"DELETE FROM check_run WHERE author = ?1 OR uri IN purged",
"DELETE FROM minutes WHERE runner = ?1 OR owner = ?1 -- scans: minutes, once, as an account is forgotten",
"DELETE FROM runner_key WHERE did = ?1",
"DELETE FROM push WHERE did = ?1",
"DELETE FROM language WHERE did = ?1",
"DELETE FROM moved WHERE from_did = ?1",
"DELETE FROM block WHERE did = ?1",
"DELETE FROM star WHERE did = ?1",
"DELETE FROM follow WHERE did = ?1",
"DELETE FROM reaction WHERE did = ?1 -- scans: reaction, once, as an account is forgotten",
"DELETE FROM release WHERE did = ?1",
"DELETE FROM release_yank WHERE did = ?1",
"DELETE FROM site WHERE uri >= ?2 AND uri < ?3",
"DELETE FROM shared WHERE did = ?1",
"DELETE FROM shared_use WHERE did = ?1",
"DELETE FROM theme WHERE did = ?1",
"DELETE FROM org WHERE did = ?1 OR substr(space, 1, length(?2)) = ?2",
"DELETE FROM org_member WHERE org = ?1 OR substr(space, 1, length(?2)) = ?2",
"DELETE FROM membership WHERE did = ?1 OR substr(space, 1, length(?2)) = ?2",
"DELETE FROM seen WHERE did = ?1",
"DELETE FROM push_device WHERE did = ?1",
"DELETE FROM pref WHERE did = ?1",
"DELETE FROM subscription WHERE did = ?1",
"DELETE FROM profile_kept WHERE did = ?1",
"DELETE FROM bot WHERE did = ?1",
"DELETE FROM profile_record WHERE did = ?1",
"DELETE FROM handle WHERE did = ?1",
"DELETE FROM invite WHERE did = ?1 OR runner = ?1",
"DELETE FROM secret WHERE owner = ?1 OR runner = ?1",
"DELETE FROM inactive WHERE did = ?1",
"DELETE FROM space_record WHERE did = ?1 OR substr(space, 1, length(?2)) = ?2 -- scans: space_record, once, as an account is forgotten",
"DELETE FROM space_writer WHERE did = ?1 OR substr(space, 1, length(?2)) = ?2",
"DELETE FROM space_reader WHERE did = ?1 OR substr(uri, 1, length(?2)) = ?2",
"DELETE FROM space_syncer WHERE did = ?1 OR substr(uri, 1, length(?2)) = ?2",
"DELETE FROM space WHERE substr(uri, 1, length(?2)) = ?2",
"DELETE FROM space_gone WHERE substr(uri, 1, length(?2)) = ?2",
];
/// The version of the tables in the index at `path`, read without
/// changing it, or none when there is no index there yet.
pub fn version_at(path: &Path) -> rusqlite::Result<Option<i64>> {
if !path.exists() {
return Ok(None);
}
let db = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
db.query_row("PRAGMA user_version", [], |row| row.get(0))
.map(Some)
}
impl Index {
pub fn open(path: &Path) -> rusqlite::Result<Index> {
let (db, rebuilt) = connect(path)?;
crate::telemetry::watch(&db);
let (recount, counting) = mpsc::unbounded_channel();
let (read, reading) = mpsc::unbounded_channel();
let (recode, coding) = mpsc::unbounded_channel();
let index = Index {
db: Mutex::new(db),
readers: Readers::new(path),
rebuilt,
changes: broadcast::channel(1024).0,
asked: Mutex::default(),
recount,
counting: Mutex::new(Some(counting)),
coded: Mutex::default(),
recode,
coding: Mutex::new(Some(coding)),
read,
reading: Mutex::new(Some(reading)),
stamps: Stamps::new(STAMPS_MAX),
kept: Mutex::new(Kept::new(KEPT_ENTRIES)),
revs: Mutex::default(),
syncing: Mutex::default(),
};
Ok(index)
}
/// Records the record at `uri`, by `did`, replacing what was there.
pub fn put(
&self,
did: &str,
collection: &str,
rkey: &str,
value: &Value,
) -> rusqlite::Result<()> {
self.put_at("", did, collection, rkey, value)
}
/// [`Index::put`], of the record as the commit at `rev` left it,
/// unless a later commit's write to it is indexed already: a backfill
/// reads a repository while the feed goes on, and must not put back
/// what the feed has since changed or deleted. With no revision it is
/// [`Index::put`].
pub fn put_as_of(
&self,
did: &str,
collection: &str,
rkey: &str,
value: &Value,
rev: Option<&str>,
) -> rusqlite::Result<()> {
self.as_of(did, collection, rkey, rev, false, &|| {
self.put(did, collection, rkey, value)
})
}
/// [`Index::delete`], as of `rev`, as [`Index::put_as_of`] puts. The
/// deletion is remembered until a backfill reads the account past it
/// ([`Index::read_through`]), so the record is not put back meanwhile.
pub fn delete_as_of(
&self,
did: &str,
collection: &str,
rkey: &str,
rev: Option<&str>,
) -> rusqlite::Result<()> {
self.as_of(did, collection, rkey, rev, true, &|| {
self.delete(did, collection, rkey)
})
}
/// Runs `write` unless the record's last indexed write came at a
/// revision after `rev`, then stamps `rev` on it, `gone` for a deletion.
fn as_of(
&self,
did: &str,
collection: &str,
rkey: &str,
rev: Option<&str>,
gone: bool,
write: &dyn Fn() -> rusqlite::Result<()>,
) -> rusqlite::Result<()> {
let Some(rev) = rev else {
return write();
};
let _held = self.revs.lock().unwrap();
let uri = record_uri("", did, collection, rkey);
let later = self.db.lock().unwrap().query_row(
"SELECT 1 FROM record_rev WHERE uri = ?1 AND rev > ?2",
params![uri, rev],
|_| Ok(()),
);
if later.optional()?.is_some() {
return Ok(());
}
write()?;
self.db.lock().unwrap().execute(
"INSERT INTO record_rev VALUES (?1, ?2, ?3, ?4)
ON CONFLICT (uri) DO UPDATE SET rev = excluded.rev, gone = excluded.gone",
params![uri, did, rev, gone],
)?;
Ok(())
}
/// What backfills made of `did`: the revision one last read it at, and
/// how many have failed since, the last of them asking for no other
/// before `retry_at`, in seconds since the epoch.
pub fn backfilled(&self, did: &str) -> rusqlite::Result<Backfilled> {
let db = self.readers.take(&self.db);
let found = db.query_row(
"SELECT rev, tries, retry_at FROM backfill WHERE did = ?1",
[did],
|row| {
Ok(Backfilled {
rev: row.get(0)?,
tries: row.get(1)?,
retry_at: row.get(2)?,
})
},
);
Ok(found.optional()?.unwrap_or_default())
}
/// Notes that a backfill read `did` as of `rev`.
pub fn backfill_read(&self, did: &str, rev: &str) -> rusqlite::Result<()> {
self.db.lock().unwrap().execute(
"INSERT OR REPLACE INTO backfill VALUES (?1, ?2, 0, 0)",
params![did, rev],
)?;
Ok(())
}
/// Notes that a backfill failed to read `did`, and that none is to try
/// again before `retry_at`.
pub fn backfill_failed(&self, did: &str, retry_at: u64) -> rusqlite::Result<()> {
self.db.lock().unwrap().execute(
"INSERT INTO backfill VALUES (?1, NULL, 1, ?2)
ON CONFLICT (did) DO UPDATE SET tries = tries + 1, retry_at = excluded.retry_at",
params![did, retry_at],
)?;
Ok(())
}
/// Forgets the deletions of `did`'s records made at or before `rev`,
/// once a backfill has read the account as of then: what it read no
/// longer holds them.
pub fn read_through(&self, did: &str, rev: &str) -> rusqlite::Result<()> {
self.db.lock().unwrap().execute(
"DELETE FROM record_rev WHERE did = ?1 AND rev <= ?2 AND gone",
params![did, rev],
)?;
Ok(())
}
/// [`Index::put`], of a record in `space`, or in public records when
/// it is empty: where an organization's record is kept tells what it
/// says apart, as one keeps a record of the same name in each.
fn put_at(
&self,
space: &str,
did: &str,
collection: &str,
rkey: &str,
value: &Value,
) -> rusqlite::Result<()> {
let uri = record_uri(space, did, collection, rkey);
let db = self.db.lock().unwrap();
let text = value.to_string();
// Whether an account says it is a bot, of every account, for the
// feed to leave out what bots say and push.
if collection == nsid::PROFILE {
let sql = match value["bot"] == true {
true => "INSERT OR IGNORE INTO bot VALUES (?1)",
false => "DELETE FROM bot WHERE did = ?1",
};
db.execute(sql, [did])?;
}
// A profile's records, kept only for the accounts whose profiles
// are: those of every other account on the network pass by.
if nsid::PROFILES.contains(&collection) {
db.execute(
"INSERT OR REPLACE INTO profile_record SELECT ?1, ?2, ?3, ?4
WHERE ?1 IN (SELECT did FROM profile_kept)",
params![did, collection, rkey, text],
)?;
return Ok(());
}
let named = |field: &str| serde_json::from_value::<RepoRef>(value[field].clone()).ok();
let key = value["key"].as_str().unwrap_or_default();
// An owner's word that the repository moved to another account:
// its offer, or the redirect left when the old copy went.
let away = match collection {
nsid::REPO => named("transfer").map(|to| (key.to_string(), to)),
nsid::REDIRECT => named("to").map(|to| (to.key.clone(), to)),
_ => None,
};
let unmoved = db.execute("DELETE FROM moved WHERE uri = ?1", [&uri])?;
let mut moves = unmoved > 0;
if let Some((from_key, to)) = away.filter(|(_, to)| to.did() != did) {
let moved = db.execute(
"INSERT INTO moved VALUES (?1, ?2, ?3, ?4, ?5)",
params![uri, did, from_key, to.did(), to.key],
);
moved?;
moves = true;
}
// Whose pulls and issues it changes: a repository's own record, or
// what is said about one, or an owner's blocks.
let whose = match named("repo") {
_ if collection == nsid::REPO => Some(format!("{did}/{key}")),
Some(repo) => Some(format!("{}/{}", repo.did(), repo.key)),
None if collection == nsid::BLOCK => Some(did.to_string()),
None => None,
};
// Bound, then checked: a `?` alone on a line counts as a line of its
// own for coverage.
let wrote = if collection == nsid::REPO {
let fork = named("forkOf");
let from = named("transferredFrom");
db.execute("DELETE FROM collaborator WHERE uri = ?1", [&uri])?;
let put = db.execute(
"INSERT OR REPLACE INTO repo VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
params![
uri,
did,
rkey,
key,
value["description"].as_str(),
fork.as_ref().map(|f| &f.uri),
fork.as_ref().map(RepoRef::did),
fork.as_ref().map(|f| &f.key),
from.as_ref().map(RepoRef::did),
from.as_ref().map(|f| &f.key),
value["createdAt"].as_str().unwrap_or_default(),
text
],
);
put.and_then(|_| db.execute(COLLABORATORS, [&uri]))
} else if collection == nsid::REF && made(value) {
// A new ref, which a push writes with no claim: its first
// version is its creation, kept under the key a claim on the
// generation before it would have had, which none ever has.
let before = value["generation"].as_u64().unwrap_or(1) - 1;
db.execute(
"INSERT OR IGNORE INTO push VALUES (?1, ?2, ?3, ?4, NULL, ?5, ?6, ?7)",
params![
record_uri(space, did, nsid::CLAIM, &format!("{rkey}.{before}")),
did,
value["repo"].as_str().unwrap_or_default(),
value["name"].as_str().unwrap_or_default(),
value["target"].as_str(),
value["updatedAt"].as_str().unwrap_or_default(),
value["pusher"].as_str()
],
)
} else if collection == nsid::CLAIM {
// A ref moving, kept after compaction drops its claim: the
// claims are the reflog, and this is the history of it.
db.execute(
"INSERT OR REPLACE INTO push VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![
uri,
did,
value["repo"].as_str().unwrap_or_default(),
value["name"].as_str().unwrap_or_default(),
value["from"].as_str(),
value["to"].as_str(),
value["createdAt"].as_str().unwrap_or_default(),
value["pusher"].as_str()
],
)
} else if collection == nsid::RERUN || collection == nsid::CANCEL {
// For the repository's runners alone, who read it from its PDS.
Ok(0)
} else if collection == nsid::CHECK {
// Whoever wrote it: which runners to show is the reader's to
// say, from the repository's record as it is when they read.
let repo = named("repo").unwrap_or_default();
db.execute(
"INSERT OR REPLACE INTO check_run VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![
uri,
did,
repo.did(),
repo.key,
value["ref"].as_str().unwrap_or_default(),
value["commit"].as_str().unwrap_or_default(),
value["startedAt"].as_str().unwrap_or_default(),
text
],
)
} else if collection == knotless::sealed::KEY_NSID && rkey == knotless::sealed::KEY_RKEY {
// The key a runner publishes for owners to seal its secrets to.
let key = value["key"].as_str().unwrap_or_default();
db.execute(
"INSERT OR REPLACE INTO runner_key VALUES (?1, ?2, ?3)",
params![did, uri, key],
)
} else if collection == nsid::MINUTES {
// Keyed by the owner whose minutes they are.
db.execute(
"INSERT OR REPLACE INTO minutes VALUES (?1, ?2, ?3, ?4)",
params![uri, did, rkey, text],
)
} else if collection == nsid::THEME {
// Kept as it was written: what of it is drawn is read as it is
// answered (web::themes::palette). Its name in lower case, to
// be found by how it starts.
let name = value["name"].as_str().unwrap_or_default();
db.execute(
"INSERT OR REPLACE INTO theme VALUES (?1, ?2, ?3, ?4, ?5)",
params![
uri,
did,
value["createdAt"].as_str().unwrap_or_default(),
name.trim().to_lowercase(),
text
],
)
} else if collection == nsid::SITE {
// A repository's site, keyed by the repository's key, and a
// private one's by its space too, as its URI names it: one that
// is not a site's record is passed by, as is one with a file
// whose hash is not one, as the hash names the file it is kept
// in.
let site = serde_json::from_value::<SiteRecord>(value.clone());
let hashed = |s: SiteRecord| s.files.iter().all(|f| valid_sha256(&f.sha256));
let kept = site.is_ok_and(hashed);
db.execute(
"INSERT OR REPLACE INTO site SELECT ?1, ?2, ?3, ?4 WHERE ?5",
params![uri, did, rkey, text, kept],
)
} else if collection == nsid::RELEASE {
// Kept by the owner and the key of the repository it is of, as
// its record names it: one that is not a release, by its names,
// is passed by.
let release = serde_json::from_value::<ReleaseRecord>(value.clone());
let release = release.ok().filter(|r| {
knotless::release::valid_package(&r.package)
&& knotless::release::version(&r.version).is_ok()
});
match release {
Some(release) => db
.execute(
"INSERT OR REPLACE INTO release VALUES (?1, ?2, ?3, ?4, ?5)",
params![uri, did, release.repo, release.created_at, text],
)
.and_then(|n| self.share(&db, &uri, did, &release).map(|_| n)),
None => Ok(0),
}
} else if collection == nsid::RELEASE_YANK {
// Found by the release it is keyed as, which is the same
// account's, so none but the owner can yank it.
db.execute(
"INSERT OR REPLACE INTO release_yank VALUES (?1, ?2, ?3, ?4)",
params![
uri,
did,
rkey.split('.').next().unwrap_or_default(),
record_uri(space, did, nsid::RELEASE, rkey)
],
)
} else if let Some(repo) = named("repo") {
if collection == nsid::COMMENT {
db.execute(
"INSERT OR REPLACE INTO comment VALUES (?1, ?2, ?3, ?4)",
params![
uri,
did,
value["subject"].as_str().unwrap_or_default(),
value["createdAt"].as_str().unwrap_or_default()
],
)?;
}
// Who a pull, issue or comment mentions, as its facets say, for
// them to hear of it: one it names first in an edit as of the
// edit, so it is not among what they saw already; one it named
// before, as before.
let named: Vec<String> = knotless::mentions::dids(value).into_iter().collect();
let created = value["createdAt"].as_str().unwrap_or_default();
let edited = value["updatedAt"].as_str().unwrap_or(created);
let known = db.query_row(
"SELECT EXISTS (SELECT 1 FROM forge WHERE uri = ?1)",
[&uri],
|row| row.get::<_, bool>(0),
);
let known = known?;
let at = if known { edited } else { created };
db.execute(
"DELETE FROM mention WHERE uri = ?1 AND did NOT IN (SELECT value FROM json_each(?2))",
params![uri, Value::from(named.clone()).to_string()],
)?;
for whom in named {
db.execute(
"INSERT OR IGNORE INTO mention VALUES (?1, ?2, ?3)",
params![uri, whom, at],
)?;
}
// Whom a pull asks to review it, to find what asks an account.
db.execute("DELETE FROM reviewer WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM assigned WHERE uri = ?1", [&uri])?;
let table = match collection == nsid::ASSIGNMENT {
true => "assigned",
false => "reviewer",
};
// And whom a maintainer's assignment record names, either way.
let assignees = match collection == nsid::ASSIGNMENT {
true => "'assignees'",
false => "NULL",
};
let named = format!(
"INSERT OR IGNORE INTO {table}
SELECT ?1, e.value ->> '$.did' FROM json_each(?2) k, json_each(k.value) e
WHERE k.key IN ('reviewers', {assignees}) AND e.value ->> '$.did' IS NOT NULL"
);
db.execute(&named, params![uri, text])?;
db.execute(
"INSERT OR REPLACE INTO forge VALUES (?1, ?2, ?3, ?4, ?5)",
params![uri, did, repo.did(), repo.key, text],
)
} else if collection == nsid::STAR {
// Counted once per account and repository, however many of
// an account's stars name it.
let repo = named("subject").unwrap_or_default();
db.execute(
"INSERT OR REPLACE INTO star VALUES (?1, ?2, ?3, ?4, ?5)",
params![
uri,
did,
repo.did(),
repo.key,
value["createdAt"].as_str().unwrap_or_default()
],
)
} else if collection == nsid::FOLLOW {
db.execute(
"INSERT OR REPLACE INTO follow VALUES (?1, ?2, ?3, ?4)",
params![
uri,
did,
value["subject"].as_str().unwrap_or_default(),
value["createdAt"].as_str().unwrap_or_default()
],
)
} else if collection == nsid::REACTION {
// Counted once per account, subject and content, however many
// of an account's reactions name them.
db.execute(
"INSERT OR REPLACE INTO reaction VALUES (?1, ?2, ?3, ?4)",
params![
uri,
did,
value["subject"]["uri"].as_str().unwrap_or_default(),
value["content"].as_str().unwrap_or_default()
],
)
} else if collection == nsid::BLOCK {
// A block naming no repository keeps its subject out of all the
// blocker's.
let subject = value["subject"].as_str().unwrap_or_default();
db.execute(
"INSERT OR REPLACE INTO block VALUES (?1, ?2, ?3)",
params![uri, did, subject],
)
} else if [nsid::ORG, nsid::ORG_MEMBER, nsid::ORG_ACCEPTANCE].contains(&collection) {
org_rows(&db, space, did, collection, rkey, Some(value))
} else {
Ok(0)
};
wrote?;
// Stamped once written, so what is read after the stamp has it.
if moves {
self.stamps.mark_all();
}
if let Some(whose) = whose {
// A record that is only one of a repository's items, with its
// own comments, changes that one for the next read to patch.
match collection == nsid::REPO || collection == nsid::BLOCK || collection == nsid::HIDE
{
true => self.stamps.mark(&whose),
false => self.stamps.mark_item(&whose, &uri),
}
}
// A ref that moved, or the record naming the default branch, has
// what the repository is written in counted again, if it was before.
let moved = match collection {
nsid::REF | nsid::CLAIM => value["repo"].as_str(),
nsid::REPO => Some(key),
_ => None,
};
// And a public one's code is indexed again, if it is searched.
if let Some(key) = moved.filter(|_| space.is_empty()) {
self.reindex(did, key);
}
let counted = "SELECT 1 FROM language WHERE did = ?1 AND key = ?2";
let kept = |k: &&str| db.query_row(counted, [did, k], |_| Ok(())).is_ok();
if let Some(key) = moved.filter(kept) {
self.recount(did, key);
}
self.tell(|| about(&db, did, collection, value));
Ok(())
}
/// Asks for what `did`'s repository with `key` is written in to be
/// counted again, once however often it is asked before it is.
pub fn recount(&self, did: &str, key: &str) {
let repo = (did.to_string(), key.to_string());
if self.asked.lock().unwrap().insert(repo.clone()) {
let _ = self.recount.send(repo);
}
}
/// Asks for the code of `did`'s public repository with `key` to be
/// indexed again, or forgotten if it is gone, once however often it is
/// asked before it is.
pub fn reindex(&self, did: &str, key: &str) {
let repo = (did.to_string(), key.to_string());
if self.coded.lock().unwrap().insert(repo.clone()) {
let _ = self.recode.send(repo);
}
}
/// What [`Index::reindex`] asks for, for the one that indexes code:
/// none after the first take.
pub fn indexing(&self) -> Option<mpsc::UnboundedReceiver<(String, String)>> {
self.coding.lock().unwrap().take()
}
/// Takes back the ask for `did`'s repository with `key`, as it is
/// indexed: one asked after is indexed again.
pub fn reindexed(&self, did: &str, key: &str) {
let repo = (did.to_string(), key.to_string());
self.coded.lock().unwrap().remove(&repo);
}
/// What [`Index::recount`] asks for, for the one that counts: none
/// after the first take.
pub fn counting(&self) -> Option<mpsc::UnboundedReceiver<(String, String)>> {
self.counting.lock().unwrap().take()
}
/// Takes back the ask for `did`'s repository with `key`, as it is
/// counted: one asked after is counted again.
pub fn answered(&self, did: &str, key: &str) {
let repo = (did.to_string(), key.to_string());
self.asked.lock().unwrap().remove(&repo);
}
/// What `did`'s repository with `key` is mostly written in, as JSON
/// (`null` for nothing counted), if it was ever counted.
pub fn language(&self, did: &str, key: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt =
db.prepare_cached("SELECT value FROM language WHERE did = ?1 AND key = ?2")?;
stmt.query_row([did, key], |row| row.get(0)).optional()
}
/// Keeps `value`, as [`Index::language`] gives it, for `did`'s
/// repository with `key`.
pub fn set_language(&self, did: &str, key: &str, value: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
let mut stmt = db.prepare_cached("INSERT OR REPLACE INTO language VALUES (?1, ?2, ?3)")?;
stmt.execute([did, key, value]).map(|_| ())
}
/// Tells whoever follows them that `repos` were written about: no one
/// asks which, when no one follows any.
fn tell(&self, repos: impl FnOnce() -> Vec<String>) {
if self.changes.receiver_count() == 0 {
return;
}
for repo in repos() {
let _ = self.changes.send(repo.into());
}
}
pub fn delete(&self, did: &str, collection: &str, rkey: &str) -> rusqlite::Result<()> {
self.delete_at("", did, collection, rkey)
}
/// [`Index::delete`], of a record in `space`, or in public records
/// when it is empty.
fn delete_at(
&self,
space: &str,
did: &str,
collection: &str,
rkey: &str,
) -> rusqlite::Result<()> {
let uri = record_uri(space, did, collection, rkey);
let db = self.db.lock().unwrap();
self.tell(|| deleted_about(&db, &uri));
org_rows(&db, space, did, collection, rkey, None)?;
// Whose pulls and issues it changes, stamped once all is deleted.
let scope = |sql: &str| db.query_row(sql, [&uri], |row| row.get(0)).optional();
let repo: Option<String> =
scope("DELETE FROM repo WHERE uri = ?1 RETURNING did || '/' || key")?;
// A public repository's code is forgotten.
let gone = repo.as_deref().and_then(|r| r.split_once('/'));
if let Some((owner, key)) = gone.filter(|_| space.is_empty()) {
self.reindex(owner, key);
}
db.execute("DELETE FROM collaborator WHERE uri = ?1", [&uri])?;
let said: Option<String> =
scope("DELETE FROM forge WHERE uri = ?1 RETURNING repo_did || '/' || repo_key")?;
let moved = db.execute("DELETE FROM moved WHERE uri = ?1", [&uri])?;
let blocked = db.execute("DELETE FROM block WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM star WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM follow WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM theme WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM reaction WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM comment WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM mention WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM reviewer WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM assigned WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM check_run WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM minutes WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM runner_key WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM release WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM release_yank WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM site WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM shared WHERE uri = ?1", [&uri])?;
db.execute(
"DELETE FROM profile_record WHERE did = ?1 AND collection = ?2 AND rkey = ?3",
[did, collection, rkey],
)?;
if collection == nsid::PROFILE {
db.execute("DELETE FROM bot WHERE did = ?1", [did])?;
}
if moved > 0 {
self.stamps.mark_all();
}
let blocks = (blocked > 0).then(|| did.to_string());
for whose in [repo, blocks].into_iter().flatten() {
self.stamps.mark(&whose);
}
// A hide taken back shows what it hid; any other record leaves
// only itself to read again.
if let Some(whose) = said {
match collection == nsid::HIDE {
true => self.stamps.mark(&whose),
false => self.stamps.mark_item(&whose, &uri),
}
}
Ok(())
}
/// The key each of `runners` publishes to seal its secrets to, by DID,
/// of those that publish one.
pub fn runner_keys(&self, runners: &[String]) -> rusqlite::Result<Vec<(String, String)>> {
let runners = serde_json::to_string(runners).unwrap_or_default();
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT k.did, k.key FROM json_each(?1) r JOIN runner_key k ON k.did = r.value
ORDER BY r.key",
)?;
let keys = stmt.query_map([runners], |row| Ok((row.get(0)?, row.get(1)?)))?;
keys.collect()
}
/// The minutes each of `runners` counts for `owner`'s public
/// repositories, by runner, in the order `runners` names them: none by
/// a runner taken down.
pub fn minutes(&self, owner: &str, runners: &[String]) -> rusqlite::Result<Vec<Entry>> {
let runners = serde_json::to_string(runners).unwrap_or_default();
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT m.uri, m.runner, m.value FROM json_each(?2) r
JOIN minutes m ON m.runner = r.value AND m.owner = ?1
WHERE m.uri NOT IN down AND m.runner NOT IN down
ORDER BY r.key",
)?;
let entries = stmt.query_map(params![owner, runners], entry)?;
entries.collect()
}
/// The repository records `did` holds, by record key, as the index
/// has them, each with its URI.
pub fn actor_repos(&self, did: &str) -> rusqlite::Result<Vec<(String, RepoRecord, String)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT name, value, uri FROM repo WHERE did = ?1")?;
let rows = stmt.query_map([did], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
let rows: Vec<(String, String, String)> = rows.collect::<rusqlite::Result<_>>()?;
let read = rows
.into_iter()
.map(|(rkey, value, uri)| (rkey, serde_json::from_str(&value), uri));
Ok(read
.filter_map(|(rkey, value, uri)| Some((rkey, value.ok()?, uri)))
.collect())
}
/// The check `uri`, as the index has it.
pub fn check_record(&self, uri: &str) -> rusqlite::Result<Option<Value>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT value FROM check_run WHERE uri = ?1")?;
let found: Option<String> = stmt.query_row([uri], |row| row.get(0)).optional()?;
Ok(found.and_then(|value| serde_json::from_str(&value).ok()))
}
/// The releases of `did`'s repository `key`, each by its record's
/// AT-URI and marked if yanked, the newest first: a repository publishes few enough that
/// they are read all at once, as its latest of each package is found
/// among them all.
pub fn releases(&self, did: &str, key: &str) -> rusqlite::Result<Vec<(String, ReleaseRecord)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare(
"SELECT r.uri, r.value, y.uri IS NOT NULL FROM release r
LEFT JOIN release_yank y ON y.release = r.uri
WHERE r.did = ?1 AND r.repo_key = ?2 AND r.uri NOT IN down
ORDER BY r.created_at DESC, r.uri DESC",
)?;
let rows = stmt.query_map([did, key], release_row)?;
Ok(rows.filter_map(Result::ok).flatten().collect())
}
/// The site of `did`'s repository `key`, as its record was last
/// indexed: none when it has none, or it is taken down.
pub fn site(&self, did: &str, key: &str) -> rusqlite::Result<Option<String>> {
self.site_at(&format!("at://{did}/{}/{key}", nsid::SITE))
}
/// The site of `did`'s private repository `key`, in the space `space`,
/// as [`Index::site`].
pub fn space_site(
&self,
space: &str,
did: &str,
key: &str,
) -> rusqlite::Result<Option<String>> {
self.site_at(&record_uri(space, did, nsid::SITE, key))
}
/// The private repository `did` owns with `key`, by its name: none
/// for one this index keeps no space of.
pub fn private_repo(&self, did: &str, key: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt =
db.prepare_cached("SELECT uri, name FROM repo WHERE did = ?1 AND key = ?2")?;
let rows = stmt.query_map([did, key], uri_and_name)?;
let private =
|(uri, _): &(String, String)| AtUri::parse(uri).is_ok_and(|at| at.space.is_some());
let found = rows.filter_map(Result::ok).find(private);
Ok(found.map(|(_, name)| name))
}
/// The site whose record is `uri`, as [`Index::site`].
fn site_at(&self, uri: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT value FROM site WHERE uri = ?1 AND uri NOT IN down AND did NOT IN down",
)?;
stmt.query_row([uri], |row| row.get(0)).optional()
}
/// How many accounts starred `did`'s repository with `key`, leaving
/// out those taken down.
pub fn star_count(&self, did: &str, key: &str) -> rusqlite::Result<u64> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT COUNT(DISTINCT did) FROM star
WHERE repo_did = ?1 AND repo_key = ?2 AND did NOT IN down",
)?;
stmt.query_row([did, key], |row| row.get(0))
}
/// `viewer`'s star naming `did`'s repository with `key`, by its URI,
/// if they starred it.
pub fn star_of(&self, viewer: &str, did: &str, key: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT uri FROM star WHERE did = ?1 AND repo_did = ?2 AND repo_key = ?3
ORDER BY uri LIMIT 1",
)?;
stmt.query_row([viewer, did, key], |row| row.get(0))
.optional()
}
/// The repositories `did` starred that the index knows, as
/// [`Index::repos`] lists them, the latest starred first.
pub fn actor_stars(&self, did: &str) -> rusqlite::Result<Vec<Listed>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT r.did, r.name, r.description, r.fork_of, r.created_at, r.uri
FROM star s JOIN repo r ON r.did = s.repo_did AND r.key = s.repo_key
WHERE s.did = ?1 AND r.did NOT IN down AND r.uri NOT IN down
GROUP BY r.uri ORDER BY MAX(s.created_at) DESC, r.uri",
)?;
stmt.query_map([did], listed)?.collect()
}
/// How many accounts follow `did`, and how many it follows, each once
/// however many of its follows name them, leaving out those taken down.
pub fn follow_counts(&self, did: &str) -> rusqlite::Result<(u64, u64)> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT
(SELECT COUNT(DISTINCT did) FROM follow
WHERE subject = ?1 AND did NOT IN down),
(SELECT COUNT(DISTINCT subject) FROM follow
WHERE did = ?1 AND subject NOT IN down)",
)?;
stmt.query_row([did], |row| Ok((row.get(0)?, row.get(1)?)))
}
/// What `did`'s badges count that the index can count alone: how many
/// releases it published, the most stars any one of its repositories
/// has, each star once per account, and how many repositories it has,
/// leaving out those taken down.
pub fn badge_counts(&self, did: &str) -> rusqlite::Result<(u64, u64, u64)> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT
(SELECT COUNT(*) FROM release WHERE did = ?1 AND uri NOT IN down),
(SELECT coalesce(MAX(n), 0) FROM (
SELECT COUNT(DISTINCT s.did) n
FROM star s JOIN repo r ON r.did = s.repo_did AND r.key = s.repo_key
WHERE s.repo_did = ?1 AND s.did NOT IN down AND r.uri NOT IN down
GROUP BY s.repo_key)),
(SELECT COUNT(*) FROM repo WHERE did = ?1 AND uri NOT IN down)",
)?;
stmt.query_row([did], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
}
/// The record of each repository holding a pull or an issue `did`
/// opened, is assigned or is asked to review, with its owner, leaving
/// out those taken down.
pub fn repos_involving(&self, did: &str) -> rusqlite::Result<Vec<(String, RepoRecord)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT r.did, r.value FROM (
SELECT repo_did, repo_key FROM forge WHERE author = ?1
AND (instr(uri, '/dev.knotless.pull/') OR instr(uri, '/dev.knotless.issue/'))
UNION SELECT f.repo_did, f.repo_key
FROM assigned a CROSS JOIN forge f ON f.uri = a.uri WHERE a.did = ?1
UNION SELECT f.repo_did, f.repo_key
FROM reviewer a CROSS JOIN forge f ON f.uri = a.uri WHERE a.did = ?1
) i CROSS JOIN repo r ON r.did = i.repo_did AND r.key = i.repo_key
WHERE r.uri NOT IN down",
)?;
let rows = stmt.query_map([did], owner_and_record)?;
let rows = rows.collect::<rusqlite::Result<Vec<_>>>()?;
let parsed = rows.into_iter().filter_map(|(owner, value)| {
let record = serde_json::from_str(&value).ok()?;
Some((owner, record))
});
Ok(parsed.collect())
}
/// The pulls `did` opened, on any repository.
pub fn pulls_by(&self, did: &str) -> rusqlite::Result<Vec<Pulls>> {
self.pulls_where("p.author = ?1", did)
}
/// The pulls of other accounts that `did` commented on.
pub fn pulls_commented(&self, did: &str) -> rusqlite::Result<Vec<Pulls>> {
self.pulls_where(
"p.uri IN (SELECT subject FROM comment WHERE author = ?1) AND p.author != ?1",
did,
)
}
/// When `did` first commented on each pull or issue it commented on,
/// by its URI.
pub fn first_comments(&self, did: &str) -> rusqlite::Result<HashMap<String, String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT subject, MIN(created_at) FROM comment
WHERE author = ?1 AND uri NOT IN down GROUP BY subject",
)?;
stmt.query_map([did], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect()
}
/// The pulls other accounts sent to `did`'s repositories from forks of
/// them.
pub fn pulls_from_forks(&self, did: &str) -> rusqlite::Result<Vec<Pulls>> {
self.pulls_where(
"p.repo_did = ?1 AND p.author != ?1 AND EXISTS (SELECT 1 FROM repo h
WHERE h.uri = p.value ->> '$.head.repo.uri'
AND h.fork_did = ?1 AND h.fork_key = p.repo_key)",
did,
)
}
/// The pulls `which` picks, with what changed their state and what the
/// merge queue did with them, by repository, for
/// [`knotless::forge::assemble`] to say which merged, as whoever may
/// merge each says. A repository the index does not have, or that was
/// taken down, is left out.
fn pulls_where(&self, which: &str, did: &str) -> rusqlite::Result<Vec<Pulls>> {
let db = self.readers.take(&self.db);
// The pulls, then what is about them by `forge_subject`, in this
// order (`CROSS JOIN`): a join on either, as one `OR`, reads every
// record of the repository for each pull.
let mut stmt = db.prepare_cached(&format!(
"SELECT p.uri, p.author, p.value, r.did, r.value, r.uri
FROM forge p
JOIN repo r ON r.did = p.repo_did AND r.key = p.repo_key
WHERE {which}
AND instr(p.uri, '/dev.knotless.pull/')
AND p.uri NOT IN down AND r.uri NOT IN down
UNION ALL
SELECT f.uri, f.author, f.value, r.did, r.value, r.uri
FROM forge p
CROSS JOIN repo r ON r.did = p.repo_did AND r.key = p.repo_key
CROSS JOIN forge f ON json_extract(f.value, '$.subject') = +p.uri
AND +f.repo_did = p.repo_did AND +f.repo_key = p.repo_key
WHERE {which}
AND instr(p.uri, '/dev.knotless.pull/')
AND p.uri NOT IN down AND r.uri NOT IN down
AND (instr(f.uri, '/dev.knotless.state/') OR instr(f.uri, '/dev.knotless.queue/'))
ORDER BY 6"
))?;
let rows = stmt.query_map([did], elsewhere)?;
let mut by_repo: Vec<(String, String, String, Vec<Entry>)> = Vec::new();
for row in rows {
let (e, owner, repo, uri) = row?;
match by_repo.last_mut() {
Some((_, _, last, entries)) if *last == uri => entries.push(e),
_ => by_repo.push((owner, repo, uri, vec![e])),
}
}
let read = by_repo
.into_iter()
.map(|(owner, repo, _, entries)| (owner, serde_json::from_str(&repo), entries));
Ok(read
.filter_map(|(owner, repo, entries)| {
let repo = repo.ok()?;
Some(Pulls {
owner,
repo,
entries,
})
})
.collect())
}
/// The states naming a batch of the merge queue, by its tip, as the
/// one they landed in, on the repository `owner` keeps at `key`.
pub fn batch_states(&self, owner: &str, key: &str) -> rusqlite::Result<Vec<Entry>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT uri, author, value FROM forge
WHERE repo_did = ?1 AND repo_key = ?2
AND instr(uri, '/dev.knotless.state/')
AND json_extract(value, '$.batch') IS NOT NULL
AND uri NOT IN down",
)?;
stmt.query_map([owner, key], entry)?.collect()
}
/// What was last found of where `did`'s account lives, and when, in
/// seconds: whether it moved from one PDS to another, and whether its
/// PDS is one of its own.
pub fn home(&self, did: &str) -> rusqlite::Result<Option<(Home, i64)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT moved, own, checked FROM home WHERE did = ?1")?;
stmt.query_row([did], |row| {
let home = Home {
moved: row.get(0)?,
own: row.get(1)?,
};
Ok((home, row.get(2)?))
})
.optional()
}
/// Keeps `home` as what was found of where `did`'s account lives at
/// `checked`.
pub fn set_home(&self, did: &str, home: Home, checked: i64) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO home (did, moved, own, checked) VALUES (?1, ?2, ?3, ?4)
ON CONFLICT (did) DO UPDATE SET moved = excluded.moved, own = excluded.own,
checked = excluded.checked",
params![did, home.moved, home.own, checked],
)?;
Ok(())
}
/// How many accounts reacted to each of `subjects` (pulls, issues and
/// comments, by URI) with each content, as `(subject, content, count,
/// viewer's reaction)`, by subject then content: each account once
/// however many of its reactions say so, and none by accounts taken
/// down or that the repository's owner blocked, from all of their
/// repositories or from this one. A subject the index does not have
/// has none.
pub fn reactions(
&self,
subjects: &[String],
viewer: Option<&str>,
) -> rusqlite::Result<Vec<Reacted>> {
let subjects = serde_json::to_string(subjects).unwrap_or_default();
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT x.subject, x.content, COUNT(DISTINCT x.did),
MIN(CASE WHEN x.did = ?2 THEN x.uri END)
FROM json_each(?1) s
JOIN reaction x ON x.subject = s.value
JOIN forge f ON f.uri = x.subject
WHERE x.did NOT IN down
AND x.did NOT IN (SELECT subject FROM block WHERE did = f.repo_did)
AND NOT EXISTS (SELECT 1 FROM forge b
WHERE b.author = f.repo_did AND b.repo_did = f.repo_did
AND b.repo_key = f.repo_key AND b.uri LIKE '%/dev.knotless.block/%'
AND json_extract(b.value, '$.subject') = x.did)
GROUP BY x.subject, x.content ORDER BY x.subject, x.content",
)?;
let rows = stmt.query_map(params![subjects, viewer], |row| {
Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))
});
rows?.collect()
}
/// Whether `did`'s knotless profile says it is a bot.
pub fn is_bot(&self, did: &str) -> rusqlite::Result<bool> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT 1 FROM bot WHERE did = ?1")?;
stmt.exists([did])
}
/// `viewer`'s follow naming `did`, by its URI, if they follow it.
pub fn follow_of(&self, viewer: &str, did: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT uri FROM follow WHERE did = ?1 AND subject = ?2 ORDER BY uri LIMIT 1",
)?;
stmt.query_row([viewer, did], |row| row.get(0)).optional()
}
/// `viewer`'s block keeping `did` out of all its repositories, by its
/// URI, if it has one.
pub fn block_of(&self, viewer: &str, did: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT uri FROM block WHERE did = ?1 AND subject = ?2 ORDER BY uri LIMIT 1",
)?;
stmt.query_row([viewer, did], |row| row.get(0)).optional()
}
/// A page of at most `limit` of the accounts following `did`, or with
/// `followers` false those it follows, each once and the latest
/// followed first, leaving out those taken down: each by DID, with
/// where the page ends if it is not the last. `after` is where the
/// page before ended.
pub fn follows(
&self,
did: &str,
followers: bool,
limit: u32,
after: Option<&str>,
) -> rusqlite::Result<(Vec<String>, Option<String>)> {
let (who, whom) = if followers {
("did", "subject")
} else {
("subject", "did")
};
let (at, uri) = after.and_then(|a| a.split_once(' ')).unwrap_or(("~", ""));
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(&format!(
"SELECT {who}, MAX(created_at) AS at, MAX(uri) AS u FROM follow
WHERE {whom} = ?1 AND {who} NOT IN down GROUP BY {who}
HAVING at < ?2 OR (at = ?2 AND u < ?3)
ORDER BY at DESC, u DESC LIMIT ?4"
))?;
let rows = stmt.query_map(params![did, at, uri, limit + 1], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
))
});
let mut rows = rows?.collect::<rusqlite::Result<Vec<_>>>()?;
let more = rows.len() > limit as usize;
rows.truncate(limit as usize);
let cursor = rows
.last()
.map(|(_, at, u)| format!("{at} {u}"))
.filter(|_| more);
Ok((rows.into_iter().map(|(did, ..)| did).collect(), cursor))
}
/// A page of at most `limit` of what others did that concerns `did`
/// (comments on its pulls and issues, and on those it commented on
/// after it did, a review's line comments heard of with it, pulls that
/// are not drafts asking it to review them, as of when they asked: one
/// on its own repository, or one mentioning it, is then that alone, and
/// invitations to organizations it has yet to take up), the latest
/// first, leaving out its own doings and the accounts it
/// blocked or that are taken down, and those it said it would rather
/// not hear of ([`Index::notification_preferences`]); with where the
/// page ends if it is not the last. `after` is where the page before
/// ended. Each says what state the pull or issue it is about is in, and
/// a runner's word on a pull that has since merged or closed, its
/// checks or the merge queue's, is left out: there is nothing to do.
/// What it watches, subscribed to and muted is as this index keeps it
/// ([`Index::subscriptions`]).
pub fn notifications(
&self,
did: &str,
orgs: &[String],
limit: u32,
after: Option<&str>,
kind: Option<&str>,
) -> rusqlite::Result<(Vec<Notified>, Option<String>)> {
let mut subs = self.subscriptions(did)?;
let owned = self.owned_watches(did, &subs)?;
subs.extend(owned);
self.notifications_as(did, orgs, &subs, limit, after, kind)
}
/// [`Index::notifications`], by `subs`, what `did` chose to hear of
/// as another index keeps it: the main index's, for this server's
/// index of a space, which keeps no choices of its own.
pub fn notifications_as(
&self,
did: &str,
orgs: &[String],
subs: &[Subscription],
limit: u32,
after: Option<&str>,
kind: Option<&str>,
) -> rusqlite::Result<(Vec<Notified>, Option<String>)> {
let (found, cursor) = self.notified(did, orgs, subs, limit, after, kind)?;
let mut states: HashMap<String, Option<String>> = HashMap::new();
let mut kept = Vec::with_capacity(found.len());
for mut n in found {
if let Some(item) = n.subject.clone().filter(|_| n.reason != "star") {
let state = match states.get(&item) {
Some(state) => state.clone(),
None => self.item_state(&item)?,
};
n.state = state.clone();
states.insert(item, state);
}
let runners = matches!(n.reason.as_str(), "checks-failed" | "ejected");
if !(runners && n.state.as_deref().is_some_and(|s| s != "open")) {
kept.push(n);
}
}
Ok((kept, cursor))
}
/// The state the pull or issue at `uri` is in, as its page says:
/// `open`, `closed` or `merged`, by the state records of those who may
/// set it. None when the index has not it, or its repository.
fn item_state(&self, uri: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT r.did, r.value FROM forge p
JOIN repo r ON r.did = p.repo_did AND r.key = p.repo_key WHERE p.uri = ?1",
)?;
let repo = stmt.query_row([uri], owner_record).optional()?;
let Some((owner, record)) = repo else {
return Ok(None);
};
let mut stmt = db.prepare_cached(
"SELECT uri, author, value FROM forge WHERE uri = ?1
UNION ALL
SELECT f.uri, f.author, f.value FROM forge f
WHERE json_extract(f.value, '$.subject') = ?1 AND instr(f.uri, '/dev.knotless.state/')",
)?;
let entries = stmt.query_map([uri], entry)?;
let entries = entries.collect::<rusqlite::Result<Vec<_>>>()?;
let record: RepoRecord = serde_json::from_str(&record).unwrap_or_default();
let items = knotless::forge::assemble(&record, &owner, &entries);
Ok(items.into_iter().find(|i| i.uri == uri).map(|i| i.state))
}
/// [`Index::notifications`], as the index finds them.
fn notified(
&self,
did: &str,
orgs: &[String],
subs: &[Subscription],
limit: u32,
after: Option<&str>,
kind: Option<&str>,
) -> rusqlite::Result<(Vec<Notified>, Option<String>)> {
let (at, uri) = after.unwrap_or("~ ").split_once(' ').unwrap_or(("~", ""));
let orgs = serde_json::to_string(orgs).unwrap_or_default();
let subs = serde_json::Value::Array(subs.iter().map(Subscription::row).collect());
let subs = subs.to_string();
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(&format!(
"WITH
-- The repositories the account looks after, by owner and, for
-- those it is a collaborator of, name: its own, and its
-- organizations'.
mine(did, key) AS (SELECT ?1, NULL
UNION SELECT repo_did, repo_key FROM collaborator WHERE did = ?1
UNION SELECT value, NULL FROM json_each(?5)),
-- What it chose to hear of: a repository it watches, by
-- owner and key, or a pull or an issue it subscribed to or
-- muted, by URI; each since when it chose. Read from its
-- JSON once: inlined, `json_each` parses it again for each
-- row heard of, and an owner's every repository is in it.
subs(subject, did, key, level, at, releases) AS MATERIALIZED (SELECT value ->> '$.subject',
value ->> '$.did', value ->> '$.key', value ->> '$.level', value ->> '$.at',
value ->> '$.releases'
FROM json_each(?7)),
-- The reasons it turned off.
off(reason) AS (SELECT j.value FROM pref p, json_each(p.value, '$.off') j
WHERE p.did = ?1 AND p.kind = 'notifications')
-- What is said of each is found for the page alone, outside
-- its limit: a column is worked out for every row before a
-- sort, and the rows sorted are all the account ever heard of.
SELECT n.uri, n.author, n.reason, n.shown, n.owner, n.name, n.title, n.at,
n.verdict,
-- Named in it too, though heard of for another reason.
n.reason != 'mention'
AND EXISTS (SELECT 1 FROM mention m WHERE m.uri = n.uri AND m.did = ?1),
-- A review's line comments, sent with it.
CASE WHEN instr(n.uri, '/dev.knotless.comment/') THEN
(SELECT count(*) FROM comment l JOIN forge lf ON lf.uri = l.uri
WHERE l.subject = n.subject AND l.author = n.author
AND lf.value ->> '$.review' = n.uri)
END,
n.role, n.checks, n.body, n.reviewed
FROM (
SELECT n.*, CASE WHEN n.reason = 'star' THEN r.uri ELSE n.subject END AS shown,
r.did AS owner, r.name AS name
FROM (
SELECT uri, did AS author, 'star' AS reason, NULL AS subject,
repo_did, repo_key, NULL AS title, created_at AS at, NULL AS verdict,
NULL AS role, NULL AS checks, NULL AS body, NULL AS reviewed
FROM star WHERE repo_did = ?1
UNION ALL
SELECT uri, did, 'follow', NULL, NULL, NULL, NULL, created_at, NULL, NULL, NULL,
NULL, NULL
FROM follow WHERE subject = ?1
UNION ALL
-- An invitation to an organization, until it is taken up or
-- lapses: the organization's record of the account.
SELECT 'at://' || org || '/{member}/' || did, org, 'invite', NULL, NULL, NULL,
NULL, invited_at, NULL, COALESCE(role, 'member'), NULL, NULL, NULL
FROM org_member WHERE did = ?1 AND joined_at IS NULL
AND julianday(COALESCE(expires_at,
datetime(invited_at, '+{lapse} seconds'))) > julianday('now')
UNION ALL
SELECT uri, author,
CASE WHEN instr(uri, '/dev.knotless.pull/') THEN 'pull' ELSE 'issue' END,
uri, repo_did, repo_key, json_extract(value, '$.title'),
json_extract(value, '$.createdAt'), NULL, NULL, NULL, value ->> '$.body', NULL
FROM forge WHERE repo_did IN (SELECT did FROM mine)
AND EXISTS (SELECT 1 FROM mine WHERE mine.did = forge.repo_did
AND coalesce(mine.key = forge.repo_key, 1))
AND (instr(uri, '/dev.knotless.pull/') OR instr(uri, '/dev.knotless.issue/'))
AND NOT EXISTS (SELECT 1 FROM json_each(forge.value, '$.reviewers') j
WHERE j.value ->> '$.did' = ?1 AND NOT coalesce(forge.value ->> '$.draft', 0))
UNION ALL
SELECT p.uri, p.author, 'review-requested', p.uri, p.repo_did, p.repo_key,
p.value ->> '$.title',
coalesce(r.value ->> '$.requestedAt', p.value ->> '$.createdAt'), NULL, NULL, NULL,
p.value ->> '$.body',
-- Whether it reviewed the pull since it was asked to.
EXISTS (SELECT 1 FROM comment a JOIN forge af ON af.uri = a.uri
WHERE a.subject = p.uri AND a.author = ?1
AND a.created_at >= coalesce(r.value ->> '$.requestedAt',
p.value ->> '$.createdAt')
AND af.value ->> '$.verdict' IS NOT NULL)
FROM reviewer v CROSS JOIN forge p ON p.uri = v.uri,
json_each(p.value, '$.reviewers') r
WHERE v.did = ?1 AND instr(p.uri, '/dev.knotless.pull/')
AND r.value ->> '$.did' = ?1 AND NOT coalesce(p.value ->> '$.draft', 0)
UNION ALL
-- Assigned a pull or an issue, or asked to review a pull, by
-- its repository's owner or a collaborator, as the latest
-- any of them said of it stands: a review asked of it as of
-- when, unless the pull's author asked it already, or it
-- opened the pull.
SELECT a.uri, a.author,
CASE k.key WHEN 'assignees' THEN 'assigned' ELSE 'review-requested' END,
i.uri, i.repo_did, i.repo_key, i.value ->> '$.title', e.value ->> '$.setAt',
NULL, NULL, NULL, i.value ->> '$.body',
CASE WHEN k.key = 'reviewers' THEN
EXISTS (SELECT 1 FROM comment c JOIN forge cf ON cf.uri = c.uri
WHERE c.subject = i.uri AND c.author = ?1
AND c.created_at >= e.value ->> '$.setAt'
AND cf.value ->> '$.verdict' IS NOT NULL) END
FROM assigned v CROSS JOIN forge a ON a.uri = v.uri
CROSS JOIN json_each(a.value) k CROSS JOIN json_each(k.value) e
JOIN forge i ON i.uri = a.value ->> '$.subject'
WHERE v.did = ?1 AND k.key IN ('assignees', 'reviewers')
AND e.value ->> '$.did' = ?1 AND NOT coalesce(e.value ->> '$.removed', 0)
AND (a.author = i.repo_did OR EXISTS (SELECT 1 FROM collaborator m
WHERE m.repo_did = i.repo_did AND m.repo_key = i.repo_key AND m.did = a.author))
AND (k.key = 'assignees' OR (instr(i.uri, '/dev.knotless.pull/')
AND i.author != ?1 AND NOT coalesce(i.value ->> '$.draft', 0)
AND NOT EXISTS (SELECT 1 FROM reviewer rq WHERE rq.uri = i.uri AND rq.did = ?1)))
-- Nothing later said of it, either way, by another of them.
AND NOT EXISTS (SELECT 1 FROM forge o, json_each(o.value, '$.' || k.key) oe
WHERE json_extract(o.value, '$.subject') = +i.uri
AND +o.repo_did = i.repo_did AND +o.repo_key = i.repo_key
AND instr(o.uri, '/dev.knotless.assignment/') AND o.uri != a.uri
AND oe.value ->> '$.did' = ?1
AND oe.value ->> '$.setAt' > e.value ->> '$.setAt'
AND (o.author = i.repo_did OR EXISTS (SELECT 1 FROM collaborator m
WHERE m.repo_did = i.repo_did AND m.repo_key = i.repo_key
AND m.did = o.author)))
UNION ALL
SELECT c.uri, c.author, 'comment', c.subject, i.repo_did, i.repo_key,
json_extract(i.value, '$.title'), c.created_at,
(SELECT json_extract(f.value, '$.verdict') FROM forge f WHERE f.uri = c.uri),
NULL, NULL, (SELECT f.value ->> '$.body' FROM forge f WHERE f.uri = c.uri), NULL
-- From what the account wrote and the threads it commented
-- on, rather than every comment.
FROM (SELECT uri AS subject FROM forge WHERE author = ?1
AND (instr(uri, '/dev.knotless.pull/') OR instr(uri, '/dev.knotless.issue/'))
UNION SELECT subject FROM comment WHERE author = ?1) t
CROSS JOIN comment c ON c.subject = t.subject
JOIN forge i ON i.uri = c.subject
WHERE (i.author = ?1 OR EXISTS (
SELECT 1 FROM comment m
WHERE m.subject = c.subject AND m.author = ?1 AND m.created_at < c.created_at))
-- A review's line comments are heard of with it.
AND NOT EXISTS (SELECT 1 FROM forge f
WHERE f.uri = c.uri AND json_extract(f.value, '$.review') IS NOT NULL)
UNION ALL
SELECT f.uri, f.author, 'mention', coalesce(f.value ->> '$.subject', f.uri),
f.repo_did, f.repo_key,
COALESCE(i.value ->> '$.title', f.value ->> '$.title'), m.at,
f.value ->> '$.verdict', NULL, NULL, f.value ->> '$.body', NULL
FROM mention m JOIN forge f ON f.uri = m.uri
LEFT JOIN forge i ON i.uri = f.value ->> '$.subject'
WHERE m.did = ?1
-- Heard of once, for its other reason, when it has one the
-- account hears of: a comment on its pull or issue, or on
-- one it commented on before; a pull or an issue on a
-- repository it looks after.
AND NOT (CASE WHEN instr(f.uri, '/dev.knotless.comment/') THEN
COALESCE(i.author = ?1, 0)
OR EXISTS (SELECT 1 FROM comment b WHERE b.subject = i.uri
AND b.author = ?1 AND b.created_at < f.value ->> '$.createdAt')
ELSE EXISTS (SELECT 1 FROM mine WHERE mine.did = f.repo_did
AND coalesce(mine.key = f.repo_key, 1)) END
AND CASE WHEN instr(f.uri, '/dev.knotless.comment/') THEN 'comment'
WHEN instr(f.uri, '/dev.knotless.pull/') THEN 'pull' ELSE 'issue' END
NOT IN (SELECT j.value FROM pref p, json_each(p.value, '$.off') j
WHERE p.did = ?1 AND p.kind = 'notifications'))
-- Or a pull, not a draft, that asks it for a review.
AND NOT (instr(f.uri, '/dev.knotless.pull/')
AND NOT coalesce(f.value ->> '$.draft', 0)
AND EXISTS (SELECT 1 FROM json_each(f.value, '$.reviewers') j
WHERE j.value ->> '$.did' = ?1)
AND 'review-requested' NOT IN (SELECT j.value
FROM pref p, json_each(p.value, '$.off') j
WHERE p.did = ?1 AND p.kind = 'notifications'))
AND f.author NOT IN (SELECT subject FROM block WHERE did = f.repo_did)
-- By `forge_moderation`, named, for its few rows: left to
-- choose, SQLite takes `forge_repo_created`, as good for its
-- columns, where it was made later, as on prod, and so reads
-- every record of the repository for each heard of. The
-- `instr` beside it is the index's own `WHERE`, to use it.
AND NOT EXISTS (SELECT 1 FROM forge o INDEXED BY forge_moderation
WHERE o.author = f.repo_did AND o.repo_did = f.repo_did
AND o.repo_key = f.repo_key
AND (instr(o.uri, '/dev.knotless.block/') OR instr(o.uri, '/dev.knotless.hide/'))
AND ((instr(o.uri, '/dev.knotless.block/')
AND o.value ->> '$.subject' = f.author)
OR (instr(o.uri, '/dev.knotless.hide/')
AND o.value ->> '$.subject' IN (f.uri, f.value ->> '$.subject'))))
UNION ALL
-- A comment on a pull or an issue it subscribed to, from
-- after it did, that it hears of for no other reason: not
-- one on what it wrote or commented on before, nor one
-- naming it, heard of as a mention.
SELECT c.uri, c.author, 'subscribed', c.subject, i.repo_did, i.repo_key,
i.value ->> '$.title', c.created_at, cf.value ->> '$.verdict', NULL, NULL,
cf.value ->> '$.body', NULL
FROM subs s CROSS JOIN comment c ON c.subject = s.subject
JOIN forge i ON i.uri = c.subject
JOIN forge cf ON cf.uri = c.uri
WHERE s.level = 'subscribed' AND c.created_at > s.at
AND cf.value ->> '$.review' IS NULL
AND i.author != ?1
AND NOT EXISTS (SELECT 1 FROM comment m
WHERE m.subject = c.subject AND m.author = ?1 AND m.created_at < c.created_at)
AND NOT (EXISTS (SELECT 1 FROM mention m WHERE m.uri = c.uri AND m.did = ?1)
AND 'mention' NOT IN off)
UNION ALL
-- What is said on a repository it watches, from after it
-- began to: pulls and issues opened, and comments, each
-- heard of for no other reason. Not a pull or an issue on
-- a repository it looks after, nor a pull asking it for a
-- review; not a comment on what it wrote, commented on
-- before or subscribed to; nor what names it.
SELECT w.uri, w.author, 'watching', coalesce(w.value ->> '$.subject', w.uri),
w.repo_did, w.repo_key, coalesce(i.value ->> '$.title', w.value ->> '$.title'),
w.value ->> '$.createdAt', w.value ->> '$.verdict', NULL, NULL,
w.value ->> '$.body', NULL
FROM subs s CROSS JOIN forge w ON w.repo_did = s.did AND w.repo_key = s.key
AND json_extract(w.value, '$.createdAt') > s.at
LEFT JOIN forge i ON i.uri = w.value ->> '$.subject'
WHERE s.level = 'all'
AND CASE WHEN instr(w.uri, '/dev.knotless.comment/') THEN
w.value ->> '$.review' IS NULL AND i.author != ?1
AND NOT EXISTS (SELECT 1 FROM comment m WHERE m.subject = i.uri
AND m.author = ?1 AND m.created_at < w.value ->> '$.createdAt')
AND NOT EXISTS (SELECT 1 FROM subs t WHERE t.subject = i.uri)
WHEN instr(w.uri, '/dev.knotless.pull/')
OR instr(w.uri, '/dev.knotless.issue/') THEN
NOT EXISTS (SELECT 1 FROM mine WHERE mine.did = w.repo_did
AND coalesce(mine.key = w.repo_key, 1))
AND NOT EXISTS (SELECT 1 FROM json_each(w.value, '$.reviewers') j
WHERE j.value ->> '$.did' = ?1 AND NOT coalesce(w.value ->> '$.draft', 0))
ELSE 0 END
AND NOT (EXISTS (SELECT 1 FROM mention m WHERE m.uri = w.uri AND m.did = ?1)
AND 'mention' NOT IN off)
AND w.author NOT IN (SELECT subject FROM block WHERE did = w.repo_did)
AND NOT EXISTS (SELECT 1 FROM forge o INDEXED BY forge_moderation
WHERE o.author = w.repo_did AND o.repo_did = w.repo_did
AND o.repo_key = w.repo_key
AND (instr(o.uri, '/dev.knotless.block/') OR instr(o.uri, '/dev.knotless.hide/'))
AND ((instr(o.uri, '/dev.knotless.block/')
AND o.value ->> '$.subject' = w.author)
OR (instr(o.uri, '/dev.knotless.hide/')
AND o.value ->> '$.subject' IN (w.uri, w.value ->> '$.subject'))))
UNION ALL
-- A release of a repository it watches, for all it does or
-- for its releases alone.
SELECT rl.uri, rl.did, 'watching', rl.uri, rl.did, rl.repo_key,
(rl.value ->> '$.package') || ' ' || (rl.value ->> '$.version'), rl.created_at,
NULL, NULL, NULL, rl.value ->> '$.notes', NULL
FROM subs s CROSS JOIN release rl ON rl.did = s.did AND rl.repo_key = s.key
AND rl.created_at > coalesce(s.releases, s.at)
WHERE s.level IN ('all', 'releases')
UNION ALL
-- The checks that failed at the latest run of each workflow
-- on its pull, by a runner the repository names: one for
-- each head and base they ran at, as of the first to end.
SELECT min(c.uri), c.author, 'checks-failed', p.uri, p.repo_did, p.repo_key,
p.value ->> '$.title',
min(coalesce(c.value ->> '$.finishedAt', c.started_at)), NULL, NULL,
json_group_array(json_object('workflow', c.value ->> '$.workflow',
'status', c.value ->> '$.status', 'check', c.uri)), NULL, NULL
-- In this order (`CROSS JOIN`), from the account's pulls to
-- their runs by the expression `check_pull` keeps, and never
-- by repository: a repository has many more runs than a pull.
-- A `+` drops a column's affinity, without which the
-- expression's index is not used, or keeps an index from
-- being used.
FROM forge p
CROSS JOIN repo rp ON rp.did = p.repo_did AND rp.key = p.repo_key
CROSS JOIN check_run c ON json_extract(c.value, '$.pull') = +p.uri
AND +c.repo_did = p.repo_did AND +c.repo_key = p.repo_key
WHERE p.author = ?1 AND instr(p.uri, '/dev.knotless.pull/')
AND c.value ->> '$.status' IN ('failure', 'error') AND c.uri NOT IN down
AND c.author IN (SELECT j.value ->> '$.did' FROM json_each(rp.value, '$.runners') j)
AND NOT EXISTS (SELECT 1 FROM check_run l
WHERE json_extract(l.value, '$.pull') = +p.uri
AND json_extract(l.value, '$.workflow') = json_extract(c.value, '$.workflow')
AND +l.repo_did = c.repo_did AND +l.repo_key = c.repo_key
-- By any runner it names: one runner's pass after
-- another's failure is the latest word.
AND l.author IN (SELECT j.value ->> '$.did'
FROM json_each(rp.value, '$.runners') j)
AND (l.started_at > c.started_at
OR (l.started_at = c.started_at AND l.uri > c.uri)))
GROUP BY p.uri, c.author, c.sha, c.value ->> '$.baseCommit'
UNION ALL
-- Its pull turned away by the merge queue's runner.
SELECT q.uri, q.author, 'ejected', p.uri, p.repo_did, p.repo_key,
p.value ->> '$.title', q.value ->> '$.createdAt', NULL, NULL,
q.value -> '$.checks', NULL, NULL
-- From the account's pulls to what is about them, by `forge_subject`.
FROM forge p
CROSS JOIN repo rp ON rp.did = p.repo_did AND rp.key = p.repo_key
CROSS JOIN forge q ON json_extract(q.value, '$.subject') = +p.uri
AND +q.repo_did = p.repo_did AND +q.repo_key = p.repo_key
WHERE p.author = ?1 AND instr(p.uri, '/dev.knotless.pull/')
AND instr(q.uri, '/dev.knotless.queue/') AND q.value ->> '$.state' = 'ejected'
AND q.author IN (SELECT j.value ->> '$.did' FROM json_each(rp.value, '$.runners') j)
) n
LEFT JOIN repo r ON r.did = n.repo_did AND r.key = n.repo_key
WHERE n.author != ?1 AND n.author NOT IN down
AND (n.repo_did IS NULL OR r.did IS NOT NULL)
AND n.author NOT IN (SELECT subject FROM block WHERE did = ?1)
AND n.reason NOT IN off
-- Nothing of a thread it muted, or a repository it
-- ignores, but a review asked of it, or what it is assigned.
AND (n.reason IN ('review-requested', 'assigned') OR NOT EXISTS (SELECT 1 FROM subs s
WHERE (s.level = 'muted' AND s.subject = n.subject)
OR (s.level = 'ignore' AND s.did = n.repo_did AND s.key = n.repo_key)))
-- A runner's word is no one's doing to follow.
AND (n.reason IN ('checks-failed', 'ejected')
OR n.author IN (SELECT subject FROM follow WHERE did = ?1)
OR NOT EXISTS (SELECT 1 FROM pref WHERE did = ?1 AND kind = 'notifications'
AND value ->> '$.followedOnly'))
AND (n.at < ?2 OR (n.at = ?2 AND n.uri < ?3))
-- Of one kind, when asked: a review is a comment with a
-- verdict, and a comment that names the account a mention.
AND (?6 IS NULL
OR (?6 = 'mentions' AND (n.reason = 'mention'
OR EXISTS (SELECT 1 FROM mention m WHERE m.uri = n.uri AND m.did = ?1)))
OR (?6 = 'reviews' AND (n.reason = 'review-requested'
OR (n.reason IN ('comment', 'subscribed', 'watching')
AND n.verdict IS NOT NULL)))
OR (?6 = 'comments' AND n.verdict IS NULL AND (n.reason IN ('comment', 'subscribed')
OR (n.reason = 'watching' AND instr(n.uri, '/dev.knotless.comment/'))))
OR (?6 = 'pulls-and-issues' AND (n.reason IN ('pull', 'issue', 'assigned')
OR (n.reason = 'watching' AND (instr(n.uri, '/dev.knotless.pull/')
OR instr(n.uri, '/dev.knotless.issue/')))))
OR (?6 = 'checks' AND n.reason IN ('checks-failed', 'ejected'))
OR (?6 = 'stars-and-follows' AND n.reason IN ('star', 'follow')))
ORDER BY n.at DESC, n.uri DESC LIMIT ?4
) n
ORDER BY n.at DESC, n.uri DESC",
member = nsid::ORG_MEMBER,
lapse = knotless::org::LAPSE_SECS,
))?;
let rows = stmt.query_map(params![did, at, uri, limit + 1, orgs, kind, subs], |row| {
Ok(Notified {
uri: row.get(0)?,
author: row.get(1)?,
reason: row.get(2)?,
subject: row.get(3)?,
repo: row.get::<_, Option<String>>(4)?.zip(row.get(5)?),
title: row.get(6)?,
at: row.get(7)?,
verdict: row.get(8)?,
mentioned: row.get(9)?,
line_comments: row.get(10)?,
role: row.get(11)?,
checks: failed(row.get(12)?),
text: clipped(row.get(13)?),
reviewed: row.get::<_, Option<bool>>(14)?.unwrap_or_default(),
state: None,
private: false,
})
});
let mut rows = rows?.collect::<rusqlite::Result<Vec<_>>>()?;
let more = rows.len() > limit as usize;
rows.truncate(limit as usize);
let cursor = rows
.last()
.map(|n| format!("{} {}", n.at, n.uri))
.filter(|_| more);
Ok((rows, cursor))
}
/// Those of `found`, notifications another index found for `did`, that
/// it would hear of as [`Index::notifications`] leaves them, by what
/// this one keeps: not of accounts it blocked or that are taken down,
/// nor for reasons it would rather not hear of.
pub fn heard(&self, did: &str, found: Vec<Notified>) -> rusqlite::Result<Vec<Notified>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT ?2 NOT IN down
AND ?2 NOT IN (SELECT subject FROM block WHERE did = ?1)
AND ?3 NOT IN (SELECT j.value FROM pref p, json_each(p.value, '$.off') j
WHERE p.did = ?1 AND p.kind = 'notifications')
AND (?3 IN ('checks-failed', 'ejected')
OR ?2 IN (SELECT subject FROM follow WHERE did = ?1)
OR NOT EXISTS (SELECT 1 FROM pref WHERE did = ?1 AND kind = 'notifications'
AND value ->> '$.followedOnly'))",
)?;
let mut kept = Vec::new();
for n in found {
let hears: bool = stmt.query_row(params![did, n.author, n.reason], |row| row.get(0))?;
kept.extend(hears.then_some(n));
}
Ok(kept)
}
/// When `did` last saw its notifications, if ever.
pub fn seen(&self, did: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT at FROM seen WHERE did = ?1")?;
stmt.query_row([did], |row| row.get(0)).optional()
}
/// Keeps a device `did` hears on, told of nothing before `at`. One held
/// already stays as far as it was told, unless another account held it;
/// those of `did` past the latest [`PUSH_DEVICES`] are let go.
pub fn register_push(
&self,
did: &str,
token: &str,
environment: &str,
at: &str,
) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO push_device VALUES (?1, ?2, ?3, ?4)
ON CONFLICT (token) DO UPDATE SET
environment = excluded.environment,
at = CASE WHEN did = excluded.did THEN at ELSE excluded.at END,
did = excluded.did",
[token, did, environment, at],
)?;
db.execute(
"DELETE FROM push_device WHERE did = ?1 AND token NOT IN (
SELECT token FROM push_device WHERE did = ?1 ORDER BY at DESC LIMIT ?2)",
params![did, PUSH_DEVICES],
)?;
Ok(())
}
/// Forgets a device of `did`.
pub fn unregister_push(&self, did: &str, token: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"DELETE FROM push_device WHERE token = ?1 AND did = ?2",
[token, did],
)?;
Ok(())
}
/// Every device, those of one account together.
pub fn push_devices(&self) -> rusqlite::Result<Vec<PushDevice>> {
let db = self.readers.take(&self.db);
let mut stmt =
db.prepare_cached("SELECT token, did, environment, at FROM push_device ORDER BY did")?;
let rows = stmt.query_map([], push_device);
rows?.collect()
}
/// Keeps that the device was told of what came up to `at`.
pub fn pushed(&self, token: &str, at: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"UPDATE push_device SET at = max(at, ?2) WHERE token = ?1",
[token, at],
)?;
Ok(())
}
/// Keeps that `did` saw its notifications up to `at`, unless it saw
/// them later already; gives when it saw them now.
pub fn see(&self, did: &str, at: &str) -> rusqlite::Result<String> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO seen VALUES (?1, ?2)
ON CONFLICT (did) DO UPDATE SET at = max(at, excluded.at)",
[did, at],
)?;
db.query_row("SELECT at FROM seen WHERE did = ?1", [did], |row| {
row.get(0)
})
}
/// The reasons `did` hears no notifications of, and whether it hears
/// only of what accounts it follows did: none and no, until it says.
pub fn notification_preferences(&self, did: &str) -> rusqlite::Result<(Vec<String>, bool)> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT value ->> '$.off', value ->> '$.followedOnly' FROM pref
WHERE did = ?1 AND kind = ?2",
)?;
let kept = stmt.query_row([did, NOTIFICATIONS], |row| {
let off: String = row.get(0)?;
Ok((serde_json::from_str(&off).unwrap_or_default(), row.get(1)?))
});
Ok(kept.optional()?.unwrap_or_default())
}
/// Keeps that `did` hears no notifications for the reasons `off`, and
/// with `followed_only`, only of what accounts it follows did.
pub fn set_notification_preferences(
&self,
did: &str,
off: &[String],
followed_only: bool,
) -> rusqlite::Result<()> {
let value = serde_json::json!({ "off": off, "followedOnly": followed_only });
self.set_preference(did, NOTIFICATIONS, &value)
}
/// What `did` chose to hear of, the latest choice first: the
/// repositories it watches or ignores, and the pulls and issues it
/// subscribed to or muted. What it never chose for is heard of as its
/// part in it says.
pub fn subscriptions(&self, did: &str) -> rusqlite::Result<Vec<Subscription>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT subject, level, at, releases_at FROM subscription WHERE did = ?1
ORDER BY at DESC, subject",
)?;
let rows = stmt.query_map([did], |row| {
Ok(Subscription {
subject: row.get(0)?,
level: row.get(1)?,
at: row.get(2)?,
releases_at: row.get(3)?,
})
});
rows?.collect()
}
/// What `did` hears of each repository of its own it chose nothing
/// for, among `chosen`, its choices: all it does, as an owner looks
/// after all of it, from when it was made or [`OWNED_SINCE`], the
/// later.
pub fn owned_watches(
&self,
did: &str,
chosen: &[Subscription],
) -> rusqlite::Result<Vec<Subscription>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT key, max(min(created_at), ?2) FROM repo WHERE did = ?1
GROUP BY key ORDER BY key",
)?;
let rows = stmt.query_map([did, OWNED_SINCE], |row| Ok((row.get(0)?, row.get(1)?)));
let rows: Vec<(String, String)> = rows?.collect::<rusqlite::Result<_>>()?;
let chose = |subject: &str| chosen.iter().any(|s| s.subject == subject);
let watched = rows.into_iter().map(|(key, at)| Subscription {
subject: Subscription::repo(did, &key),
level: "all".into(),
releases_at: Some(at.clone()),
at,
});
Ok(watched.filter(|s| !chose(&s.subject)).collect())
}
/// What `did` hears of `owner`'s repository `key`, past what it takes
/// part in: what it chose, else all of it, as its owner. None when
/// it hears only what it takes part in.
pub fn watching(&self, did: &str, owner: &str, key: &str) -> rusqlite::Result<Option<String>> {
let chosen = self.subscription(did, &Subscription::repo(owner, key))?;
let level = chosen.or_else(|| (did == owner).then(|| "all".to_string()));
Ok(level.filter(|l| l != PARTICIPATING))
}
/// What `did` chose to hear of `subject`, if it chose.
pub fn subscription(&self, did: &str, subject: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt =
db.prepare_cached("SELECT level FROM subscription WHERE did = ?1 AND subject = ?2")?;
stmt.query_row([did, subject], |row| row.get(0)).optional()
}
/// Keeps that `did` hears of `subject` as `level` says, from `at`:
/// [`PARTICIPATING`] forgets what it chose, so it hears as its part in
/// it says, but of a repository of its own, which it hears all of
/// otherwise, it is kept as a choice. Choosing the level it chose already keeps it as of when it
/// first did; and moving between all of a repository and its releases
/// alone keeps its releases as of when it began to hear of them. False,
/// keeping nothing, for a new choice past `max`.
pub fn subscribe(
&self,
did: &str,
subject: &str,
level: &str,
at: &str,
max: usize,
) -> rusqlite::Result<bool> {
let db = self.db.lock().unwrap();
// A repository's subject is `did/key`; a pull's or an issue's, an
// at:// URI, whose part before its first `/` is no DID.
let own = subject
.split_once('/')
.is_some_and(|(owner, _)| owner == did);
if level == PARTICIPATING && !own {
db.execute(
"DELETE FROM subscription WHERE did = ?1 AND subject = ?2",
[did, subject],
)?;
return Ok(true);
}
let mut stmt = db
.prepare_cached("SELECT count(*) FROM subscription WHERE did = ?1 AND subject != ?2")?;
let others: usize = stmt.query_row([did, subject], |row| row.get(0))?;
if others >= max {
return Ok(false);
}
db.execute(
"INSERT INTO subscription (did, subject, level, at, releases_at)
VALUES (?1, ?2, ?3, ?4, CASE WHEN ?3 IN ('all', 'releases') THEN ?4 END)
ON CONFLICT (did, subject) DO UPDATE SET level = excluded.level, at = excluded.at,
releases_at = CASE WHEN excluded.level NOT IN ('all', 'releases') THEN NULL
WHEN level IN ('all', 'releases') THEN coalesce(releases_at, at)
ELSE excluded.at END
WHERE level != excluded.level",
[did, subject, level, at],
)?;
Ok(true)
}
/// `did`'s preference of `kind`, if it set one.
pub fn preference(&self, did: &str, kind: &str) -> rusqlite::Result<Option<Value>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT value FROM pref WHERE did = ?1 AND kind = ?2")?;
let kept: Option<String> = stmt.query_row([did, kind], |row| row.get(0)).optional()?;
Ok(kept.and_then(|v| serde_json::from_str(&v).ok()))
}
/// The theme `did` chose for the pages, if it chose one, and the
/// palette of one of its own: what logging in copies to the browser.
pub fn theme(&self, did: &str) -> rusqlite::Result<Option<(String, Option<Value>)>> {
let appearance = self.preference(did, APPEARANCE)?.unwrap_or_default();
Ok(chosen(&appearance))
}
/// The theme `did` chose for its pages at night, as `theme` gives its
/// own: none for one theme alone.
pub fn night(&self, did: &str) -> rusqlite::Result<Option<(String, Option<Value>)>> {
let appearance = self.preference(did, APPEARANCE)?.unwrap_or_default();
Ok(chosen(&appearance["night"]))
}
/// At most `limit` themes whose name starts with `prefix`, in any case,
/// in the order of their names, as `(uri, did, record)`.
pub fn themes_named(&self, prefix: &str, limit: u32) -> rusqlite::Result<Vec<Theme>> {
let db = self.readers.take(&self.db);
// The names from the prefix up to the last that could start so.
let mut stmt = db.prepare_cached(
"SELECT uri, did, created_at, value FROM theme
WHERE name >= ?1 AND name < ?1 || char(1114111) AND did NOT IN down
ORDER BY name, uri LIMIT ?2",
)?;
let rows = stmt.query_map(params![prefix.trim().to_lowercase(), limit], theme_row);
let rows = rows?.collect::<rusqlite::Result<Vec<_>>>()?;
let themes = rows
.into_iter()
.map(|(uri, did, _, value)| (uri, did, value));
Ok(themes.collect())
}
/// A page of at most `limit` themes, `actor`'s or everyone's, the
/// latest made first, after the cursor `after` an earlier page gave,
/// as `(uri, did, record)`; and the cursor for the next, if any.
pub fn themes(
&self,
actor: Option<&str>,
limit: u32,
after: Option<&str>,
) -> rusqlite::Result<(Vec<Theme>, Option<String>)> {
let (at, uri) = after.and_then(|a| a.split_once(' ')).unwrap_or(("~", ""));
let db = self.readers.take(&self.db);
// Each way in by an index of its own: one account's, or everyone's.
let sql = match actor {
Some(_) => {
"SELECT uri, did, created_at, value FROM theme
WHERE did = ?4 AND (created_at, uri) < (?1, ?2) AND did NOT IN down
ORDER BY created_at DESC, uri DESC LIMIT ?3"
}
None => {
"SELECT uri, did, created_at, value FROM theme
WHERE ?4 IS NULL AND (created_at, uri) < (?1, ?2) AND did NOT IN down
ORDER BY created_at DESC, uri DESC LIMIT ?3"
}
};
let mut stmt = db.prepare_cached(sql)?;
let rows = stmt.query_map(params![at, uri, limit + 1, actor], theme_row);
let mut rows = rows?.collect::<rusqlite::Result<Vec<_>>>()?;
let more = rows.len() > limit as usize;
rows.truncate(limit as usize);
let cursor = rows
.last()
.map(|(uri, _, at, _)| format!("{at} {uri}"))
.filter(|_| more);
let themes = rows
.into_iter()
.map(|(uri, did, _, value)| (uri, did, value));
Ok((themes.collect(), cursor))
}
/// The themes `uris` names that the index has, of accounts not taken
/// down, in the order named, as `(uri, did, record)`.
pub fn themes_at(&self, uris: &[&str]) -> rusqlite::Result<Vec<Theme>> {
let db = self.readers.take(&self.db);
let named = serde_json::to_string(uris).unwrap_or_default();
let mut stmt = db.prepare_cached(
"SELECT t.uri, t.did, t.created_at, t.value FROM json_each(?1) AS n
JOIN theme AS t ON t.uri = n.value
WHERE t.did NOT IN down",
)?;
let rows = stmt.query_map([named], theme_row)?;
let each = rows.map(|row| row.map(|(uri, did, _, value)| (uri, did, value)));
let mut found = each.collect::<rusqlite::Result<Vec<_>>>()?;
found.sort_by_key(|(uri, _, _)| uris.iter().position(|u| u == uri));
Ok(found)
}
/// Keeps `value` as `did`'s preference of its `kind`, in place of the one before.
pub fn set_preference(&self, did: &str, kind: &str, value: &Value) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT OR REPLACE INTO pref VALUES (?1, ?2, ?3)",
params![did, kind, value.to_string()],
)?;
Ok(())
}
/// Whether `did`'s profile is kept: its records, as the firehose
/// changes them.
pub fn profile_kept(&self, did: &str) -> rusqlite::Result<bool> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT 1 FROM profile_kept WHERE did = ?1")?;
stmt.exists([did])
}
/// `did`'s profile records, by collection and record key, if its
/// profile is kept.
pub fn profile(&self, did: &str) -> rusqlite::Result<Option<Vec<(String, String, Value)>>> {
if !self.profile_kept(did)? {
return Ok(None);
}
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT collection, rkey, value FROM profile_record WHERE did = ?1
ORDER BY collection, rkey",
)?;
let records: rusqlite::Result<Vec<_>> = stmt.query_map([did], profile_record)?.collect();
records.map(Some)
}
/// Keeps `records` as all of `did`'s profile, and from now on as the
/// firehose changes it.
pub fn keep_profile(
&self,
did: &str,
records: &[(String, String, Value)],
) -> rusqlite::Result<()> {
let mut db = self.db.lock().unwrap();
let tx = db.transaction()?;
tx.execute("DELETE FROM profile_record WHERE did = ?1", [did])?;
for (collection, rkey, value) in records {
tx.execute(
"INSERT OR REPLACE INTO profile_record VALUES (?1, ?2, ?3, ?4)",
params![did, collection, rkey, value.to_string()],
)?;
}
tx.execute("INSERT OR IGNORE INTO profile_kept VALUES (?1)", [did])?;
tx.commit()
}
/// Every forge record about `did`'s repository `key` that stands: not
/// taken down, not by an account taken down, and not kept out by the
/// repository's owner.
pub fn forge(&self, did: &str, key: &str) -> rusqlite::Result<Vec<Entry>> {
let (entries, blocked) = self.forge_read(did, key)?;
Ok(knotless::forge::moderate(did, key, &blocked, entries))
}
/// [`Index::forge`] before the owner's hides and blocks are applied,
/// with the accounts they have blocked.
fn forge_read(&self, did: &str, key: &str) -> rusqlite::Result<(Vec<Entry>, Vec<String>)> {
let db = self.readers.take(&self.db);
// With those about the repository it moved from, when both owners
// say so: it names that one, and that one's owner said it moved here.
let mut stmt = db.prepare_cached(
"SELECT * FROM (
SELECT uri, author, value FROM forge WHERE repo_did = ?1 AND repo_key = ?2
UNION ALL
SELECT f.uri, f.author, f.value FROM repo r
JOIN moved m ON m.from_did = r.from_did AND m.from_key = r.from_key
AND m.to_did = r.did AND m.to_key = r.key
JOIN forge f ON f.repo_did = r.from_did AND f.repo_key = r.from_key
WHERE r.did = ?1 AND r.key = ?2
GROUP BY f.uri)
WHERE uri NOT IN down AND author NOT IN down
AND coalesce(json_extract(value, '$.subject'), '') NOT IN down",
)?;
let entries = stmt
.query_map([did, key], entry)?
.collect::<rusqlite::Result<_>>()?;
let mut stmt = db.prepare_cached("SELECT subject FROM block WHERE did = ?1")?;
let blocked = stmt
.query_map([did], |row| row.get(0))?
.collect::<rusqlite::Result<Vec<String>>>()?;
Ok((entries, blocked))
}
/// The pulls and issues of `did`'s repository `key`, as `record` says
/// who may do what on it: kept, as reading and assembling a busy
/// repository's takes a good part of a tenth of a second and every
/// page of it asks, until something is written about it, its record
/// differs, or [`KEPT_FOR`] passes.
pub fn assembled(
&self,
did: &str,
key: &str,
record: &RepoRecord,
) -> rusqlite::Result<Arc<Assembled>> {
let name = format!("{did}/{key}");
// The clock first, before what is kept is judged and anything is
// read, so a write that lands meanwhile stamps past what is made.
let at = self.stamps.now();
let fresh = |m: &&Made| {
m.record == *record
&& m.made.elapsed() < KEPT_FOR
&& self.stamps.since(&m.scopes) <= m.at
};
let kept = self.kept.lock().unwrap();
let found = kept.by.get(&name).filter(fresh).map(|m| {
(
m.assembled.clone(),
m.at,
m.made,
m.scopes.clone(),
m.hides.clone(),
)
});
drop(kept);
// Only records of it written since: those are read again, and the
// rest stands, where a write to the repository would have it all
// read and assembled anew.
let touched = found
.as_ref()
.map(|(_, at, _, scopes, _)| self.stamps.touched(scopes, *at))
.unwrap_or_default();
let standing = found
.as_ref()
.filter(|_| touched.is_empty())
.map(|f| f.0.clone());
if let Some(found) = knotless::telemetry::looked("assembled", standing) {
return Ok(found);
}
let began = Instant::now();
let how = if found.is_some() {
"patched"
} else {
"rebuilt"
};
let (entries, made, scopes, hides) = match found {
Some((was, _, made, scopes, hides)) => {
let entries = self.patched(did, key, &was.entries, &hides, &touched, &scopes)?;
(entries, made, scopes, hides)
}
None => {
let from = self.moved_from(did, key)?;
let (entries, blocked) = self.forge_read(did, key)?;
// Kept apart: moderating takes them out of the entries.
let hide = format!("/{}/", nsid::HIDE);
let is_hide = |e: &&Entry| e.uri.contains(&hide);
let hides: Vec<Entry> = entries.iter().filter(is_hide).cloned().collect();
let entries = knotless::forge::moderate(did, key, &blocked, entries);
let scopes = [Some(name.clone()), from, Some(did.to_string())];
(
entries,
Instant::now(),
scopes.into_iter().flatten().collect(),
hides,
)
}
};
knotless::telemetry::built("assembled", how, began.elapsed());
let items = knotless::forge::assemble(record, did, &entries);
let assembled = Arc::new(Assembled { entries, items });
let made = Made {
at,
made,
scopes,
hides,
record: record.clone(),
assembled: assembled.clone(),
};
self.kept.lock().unwrap().put(name, made);
Ok(assembled)
}
/// `was`, the records of `did`'s repository `key` as they stood, with
/// those of `uris` read again: replaced, added, or gone. `scopes` are
/// what `was` was read from, whose last is the owner's blocks, and a
/// record read that is about none of the others is not one of it.
/// Hides and blocks are never among `uris` (a write of one stamps the
/// repository whole), and `hides` are moderated with the rest, so what
/// was moderated before stays so.
fn patched(
&self,
did: &str,
key: &str,
was: &[Entry],
hides: &[Entry],
uris: &[String],
scopes: &[String],
) -> rusqlite::Result<Vec<Entry>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT f.uri, f.author, f.value, f.repo_did || '/' || f.repo_key
FROM json_each(?1) j JOIN forge f ON f.uri = j.value
WHERE f.uri NOT IN down AND f.author NOT IN down
AND coalesce(json_extract(f.value, '$.subject'), '') NOT IN down",
)?;
let read = |row: &rusqlite::Row| Ok((entry(row)?, row.get::<_, String>(3)?));
let again = stmt
.query_map([serde_json::to_string(uris).unwrap_or_default()], read)?
.collect::<rusqlite::Result<Vec<_>>>()?;
let mut stmt = db.prepare_cached("SELECT subject FROM block WHERE did = ?1")?;
let blocked = stmt
.query_map([did], |row| row.get(0))?
.collect::<rusqlite::Result<Vec<String>>>()?;
let ours = |scope: &String| scopes[..scopes.len() - 1].contains(scope);
let mut entries: Vec<Entry> = was
.iter()
.filter(|e| !uris.contains(&e.uri))
.cloned()
.collect();
entries.extend(again.into_iter().filter(|(_, s)| ours(s)).map(|(e, _)| e));
entries.extend(hides.iter().cloned());
Ok(knotless::forge::moderate(did, key, &blocked, entries))
}
/// The repository `did`'s `key` moved from, as `<did>/<key>`, whose
/// pulls and issues [`Index::forge`] reads with its own.
fn moved_from(&self, did: &str, key: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT from_did || '/' || from_key FROM repo
WHERE did = ?1 AND key = ?2 AND from_did IS NOT NULL AND from_key IS NOT NULL",
)?;
stmt.query_row([did, key], |row| row.get(0)).optional()
}
/// A page of the checks `runners` wrote about `did`'s repository `key`,
/// newest first, on `refname` or of `commit` or `commits` when given,
/// after `cursor` as [`cursor_at`] makes one; and the next page's
/// cursor when there may be one. None by a runner taken down, or taken down itself.
pub fn checks(
&self,
did: &str,
key: &str,
runners: &[String],
filter: CheckFilter,
limit: usize,
cursor: Option<&str>,
) -> rusqlite::Result<(Vec<Entry>, Option<String>)> {
let (at, uri) = after(cursor);
let runners = serde_json::to_string(runners).unwrap_or_default();
let commits = filter
.commits
.map(|c| serde_json::to_string(c).unwrap_or_default());
let db = self.readers.take(&self.db);
// A pull's checks are read from its own, newest first, rather than
// the repository's walked for the few that are its: `?9 IS NULL OR`
// would keep the planner from the index on the pull.
let pulled = match filter.pull {
Some(_) => {
"check_run INDEXED BY check_pull_at
WHERE json_extract(value, '$.pull') = ?9 AND +repo_did = ?1"
}
None => "check_run WHERE repo_did = ?1 AND ?9 IS NULL",
};
let mut stmt = db.prepare_cached(&format!(
"SELECT uri, author, value, started_at FROM {pulled}
AND repo_key = ?2
AND author IN (SELECT value FROM json_each(?3))
AND (?4 IS NULL OR ref = ?4) AND (?5 IS NULL OR sha = ?5)
AND (?12 IS NULL OR sha IN (SELECT value FROM json_each(?12)))
AND (NOT ?10 OR json_array_length(value, '$.batch') > 0)
AND (?11 IS NULL OR json_extract(value, '$.workflow') = ?11)
AND (started_at < ?6 OR (started_at = ?6 AND uri < ?7))
AND uri NOT IN down AND author NOT IN down
ORDER BY started_at DESC, uri DESC LIMIT ?8"
))?;
let rows: Vec<(Entry, String)> = stmt
.query_map(
params![
did,
key,
runners,
filter.refname,
filter.commit,
at,
uri,
limit as i64,
filter.pull,
filter.batch,
filter.workflow,
commits
],
dated,
)?
.collect::<rusqlite::Result<_>>()?;
let next = (rows.len() == limit)
.then(|| rows.last().map(|(e, at)| cursor_at(at, &e.uri)))
.flatten();
Ok((rows.into_iter().map(|(e, _)| e).collect(), next))
}
/// Whether any of `subjects`, accounts or records, is taken down here
/// or by a labeler this server follows.
pub fn down(&self, subjects: &[&str]) -> rusqlite::Result<bool> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT 1 FROM down WHERE subject = ?1")?;
for s in subjects {
if stmt.exists([s])? {
return Ok(true);
}
}
Ok(false)
}
/// Notes what `did`'s host says of it, from its `#account` events:
/// one not active (`deactivated`, `takendown`, `suspended` there) is
/// left out of every read, as one taken down here is, with what was
/// indexed of it kept for when it is active again.
pub fn set_status(&self, did: &str, status: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
let was = db.execute("DELETE FROM inactive WHERE did = ?1", [did])? > 0;
let is = status != "active";
if is {
let hidden = db.execute("INSERT INTO inactive VALUES (?1, ?2)", [did, status]);
hidden?;
}
// Most accounts the network says this of never wrote here: what
// was kept stands for those.
let wrote = "SELECT EXISTS (SELECT 1 FROM forge WHERE author = ?1)";
if was != is && db.query_row(wrote, [did], |row| row.get(0))? {
self.stamps.mark_all();
}
Ok(())
}
/// What `did`'s host last said of it, if it is not active.
pub fn status(&self, did: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT status FROM inactive WHERE did = ?1")?;
stmt.query_row([did], |row| row.get(0)).optional()
}
/// Drops everything indexed from `did`, whose account was deleted:
/// its repositories and pushes, every record it wrote, the spaces it
/// owned, what it read in others', its profile, handle, notification
/// state and invites. What this server holds about it, its takedowns,
/// the labels given out for them and reports, stays.
pub fn purge(&self, did: &str) -> rusqlite::Result<()> {
let mut db = self.db.lock().unwrap();
let tx = db.transaction()?;
let own = format!("at://{did}/");
// Past the last URI starting `own`: `/` is followed by `0`.
let past = format!("at://{did}0");
let args = [did, own.as_str(), past.as_str()];
// What it wrote about, read before it goes: only those
// repositories' pulls and issues are made again, not every one's.
let mut scopes = vec![did.to_string()];
for sql in PURGED_SCOPES {
let mut stmt = tx.prepare(sql)?;
let found = stmt.query_map([did], |row| row.get::<_, String>(0))?;
scopes.extend(found.collect::<rusqlite::Result<Vec<_>>>()?);
}
for sql in PURGE {
let mut stmt = tx.prepare(sql)?;
let given = args.iter().take(stmt.parameter_count());
stmt.execute(rusqlite::params_from_iter(given))?;
}
// The records of spaces it owned can be about any repository.
let spaced: bool =
tx.query_row("SELECT EXISTS (SELECT 1 FROM purged)", [], |r| r.get(0))?;
tx.commit()?;
if spaced {
self.stamps.mark_all();
}
for scope in &scopes {
self.stamps.mark(scope);
}
Ok(())
}
/// Drops what is indexed about `did`'s repository `key`, deleted: the
/// pulls, issues and comments on it, whoever wrote them, the reactions
/// to those, its checks, releases, stars and pushes. Not while a
/// repository record has the key, as one renamed does, nor once another
/// account took it up, whose repository shows what was said here.
/// Returns whether anything was dropped.
pub fn forget_repo(&self, did: &str, key: &str) -> rusqlite::Result<bool> {
let mut db = self.db.lock().unwrap();
let tx = db.transaction()?;
let stands: rusqlite::Result<bool> = tx.query_row(
"SELECT EXISTS (SELECT 1 FROM repo WHERE did = ?1 AND key = ?2)
OR EXISTS (SELECT 1 FROM repo WHERE from_did = ?1 AND from_key = ?2)
OR EXISTS (SELECT 1 FROM moved WHERE from_did = ?1 AND from_key = ?2)",
[did, key],
|row| row.get(0),
);
if stands? {
return Ok(false);
}
for sql in FORGET_REPO {
tx.execute(sql, [did, key])?;
}
tx.commit()?;
self.stamps.mark(&format!("{did}/{key}"));
Ok(true)
}
/// Takes `subject` down, or puts it back: `value` is how the takedown
/// names it. Returns the label that says so, and its place among those
/// this server has given out.
pub fn take_down(
&self,
src: &str,
subject: &str,
value: &Value,
applied: bool,
) -> rusqlite::Result<(i64, Value)> {
let db = self.db.lock().unwrap();
let at = knotless::records::now();
if applied {
db.execute(
"INSERT OR REPLACE INTO takedown VALUES (?1, ?2, ?3)",
params![subject, value.to_string(), at],
)?;
} else {
db.execute("DELETE FROM takedown WHERE subject = ?1", [subject])?;
}
self.stamps.mark_all();
let mut label = serde_json::json!({
"ver": 1, "src": src, "uri": subject, "val": knotless::moderation::TAKEDOWN, "cts": at,
});
if !applied {
label["neg"] = true.into();
}
db.execute(
"INSERT INTO emitted (label) VALUES (?1)",
[label.to_string()],
)?;
Ok((db.last_insert_rowid(), label))
}
/// How a takedown of `subject` named it, if it is taken down.
pub fn taken_down(&self, subject: &str) -> rusqlite::Result<Option<Value>> {
let db = self.readers.take(&self.db);
db.query_row(
"SELECT value FROM takedown WHERE subject = ?1",
[subject],
json,
)
.optional()
}
/// The labels this server gave out after `seq`, oldest first.
pub fn emitted(&self, seq: i64) -> rusqlite::Result<Vec<(i64, Value)>> {
let db = self.readers.take(&self.db);
let mut stmt =
db.prepare_cached("SELECT seq, label FROM emitted WHERE seq > ?1 ORDER BY seq")?;
stmt.query_map([seq], numbered)?.collect()
}
/// Takedowns standing whose subject starts with `prefix`, as labels
/// from `src`.
pub fn labels(&self, src: &str, prefix: &str) -> rusqlite::Result<Vec<Value>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT subject, at FROM takedown WHERE substr(subject, 1, length(?1)) = ?1
ORDER BY subject LIMIT 250",
)?;
let rows = stmt
.query_map([prefix], pair)?
.collect::<rusqlite::Result<Vec<(String, String)>>>()?;
let label = |(uri, at)| {
serde_json::json!({
"ver": 1, "src": src, "uri": uri, "val": knotless::moderation::TAKEDOWN, "cts": at,
})
};
Ok(rows.into_iter().map(label).collect())
}
/// Takes in a label from the labeler `src`, seen at `seq` of its stream.
/// Only those that take things out of view are kept.
pub fn label(&self, src: &str, seq: i64, label: &Value) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
let val = label["val"].as_str().unwrap_or_default();
let uri = label["uri"].as_str().unwrap_or_default();
let known = [knotless::moderation::TAKEDOWN, knotless::moderation::HIDE];
if label["src"] == src && known.contains(&val) {
if label["neg"] == true {
db.execute(
"DELETE FROM label WHERE src = ?1 AND uri = ?2 AND val = ?3",
[src, uri, val],
)?;
} else {
db.execute(
"INSERT OR REPLACE INTO label VALUES (?1, ?2, ?3, ?4)",
params![src, uri, val, label["exp"].as_str()],
)?;
}
self.stamps.mark_all();
}
db.execute(
"INSERT OR REPLACE INTO labeler VALUES (?1, ?2)",
params![src, seq],
)?;
Ok(())
}
/// Puts back what was down only for labels that have since lapsed,
/// as `down` keeps them until this is asked.
pub fn lapse(&self) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
let lapsed = db.execute(
&format!(
"DELETE FROM down WHERE subject IN (SELECT uri FROM label WHERE NOT {LIVE})
AND {}",
up()
),
[],
)?;
if lapsed > 0 {
self.stamps.mark_all();
}
Ok(())
}
/// Where the labeler `src`'s stream was last read to.
pub fn labeler_seq(&self, src: &str) -> rusqlite::Result<Option<i64>> {
let db = self.readers.take(&self.db);
db.query_row("SELECT seq FROM labeler WHERE src = ?1", [src], |row| {
row.get(0)
})
.optional()
}
/// Whether the index answers: a read of one of its tables.
pub fn check(&self) -> rusqlite::Result<i64> {
let db = self.readers.take(&self.db);
db.query_row("SELECT count(*) FROM stream", [], |row| row.get(0))
}
/// What the network wrote on each of `days`, into it, and how many
/// accounts write: ever, and since the days `week` and `month`. An
/// author is an account that made a repository, pushed, or wrote in a
/// thread; a runner writing checks is not one.
pub fn activity(
&self,
days: &mut BTreeMap<String, Day>,
week: &str,
month: &str,
) -> rusqlite::Result<Network> {
let since = days.keys().next().cloned().unwrap_or_default();
let db = self.readers.take(&self.db);
let acts = format!(
"-- scans: forge, check_run, push, for the operator's usage report alone
-- sorts: forge, check_run, push, for the same
WITH act (d, did, kind) AS (
SELECT substr(created_at, 1, 10), did, 'repos' FROM repo
UNION ALL SELECT substr(p.created_at, 1, 10), {BY}, 'pushes'
FROM push p JOIN repo r ON r.did = p.did AND r.key = p.key
UNION ALL SELECT substr(json_extract(value, '$.createdAt'), 1, 10), author,
CASE WHEN instr(uri, '/dev.knotless.pull/') THEN 'pulls'
WHEN instr(uri, '/dev.knotless.issue/') THEN 'issues'
WHEN instr(uri, '/dev.knotless.comment/') THEN 'comments'
ELSE 'other' END
FROM forge f
WHERE EXISTS (SELECT 1 FROM repo r WHERE r.did = f.repo_did AND r.key = f.repo_key))"
);
let mut stmt = db.prepare(&format!(
"{acts} SELECT d, kind, count(*) FROM act WHERE d >= ?1 GROUP BY 1, 2
UNION ALL SELECT d, 'authors', count(DISTINCT did) FROM act WHERE d >= ?1 GROUP BY 1
UNION ALL SELECT substr(started_at, 1, 10), 'checks', count(*) FROM check_run c
WHERE started_at >= ?1
AND EXISTS (SELECT 1 FROM repo r WHERE r.did = c.repo_did AND r.key = c.repo_key)
GROUP BY 1"
))?;
for row in stmt.query_map([&since], kinds)? {
let (d, kind, n) = row?;
let Some(day) = days.get_mut(&d) else {
continue;
};
let slot = match kind.as_str() {
"repos" => &mut day.repos,
"pushes" => &mut day.pushes,
"pulls" => &mut day.pulls,
"issues" => &mut day.issues,
"comments" => &mut day.comments,
"checks" => &mut day.checks,
"authors" => &mut day.authors,
_ => continue,
};
*slot = n;
}
db.query_row(
&format!(
"{acts} SELECT count(DISTINCT did), (SELECT count(*) FROM repo),
count(DISTINCT CASE WHEN d >= ?1 THEN did END),
count(DISTINCT CASE WHEN d >= ?2 THEN did END) FROM act"
),
[week, month],
network,
)
}
/// Follows the space `uri`, a private repository's, for this
/// server to sync as `syncer`, a member: from the start, if it is new.
pub fn follow_space(&self, uri: &str, syncer: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute("INSERT OR IGNORE INTO space (uri) VALUES (?1)", [uri])?;
db.execute("DELETE FROM space_gone WHERE uri = ?1", [uri])?;
db.execute(
"INSERT INTO space_syncer (uri, did) VALUES (?1, ?2)
ON CONFLICT (uri) DO UPDATE SET did = excluded.did",
[uri, syncer],
)?;
Ok(())
}
/// Keeps an index of the space `uri`, with no one's session to sync it
/// with: a member with a delegation token syncs it as they read it.
pub fn keep_space(&self, uri: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute("INSERT OR IGNORE INTO space (uri) VALUES (?1)", [uri])?;
Ok(())
}
/// The member whose session the space `uri` is synced with, if any.
pub fn syncer_of(&self, uri: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
db.query_row(
"SELECT did FROM space_syncer WHERE uri = ?1",
[uri],
|row| row.get(0),
)
.optional()
}
/// The spaces to sync, each with the member whose session it is synced
/// with.
pub fn syncers(&self) -> rusqlite::Result<Vec<(String, String)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT uri, did FROM space_syncer ORDER BY uri")?;
stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect()
}
/// Syncs nothing more as `did`, who logged out: each space they synced
/// waits for the next member to read it here.
pub fn drop_syncer(&self, did: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute("DELETE FROM space_syncer WHERE did = ?1", [did])?;
Ok(())
}
/// Notes that `did` was proven a member of the space `uri`: its
/// repository is listed to them from now on.
pub fn read_by(&self, uri: &str, did: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT OR IGNORE INTO space_reader (uri, did) VALUES (?1, ?2)",
[uri, did],
)?;
Ok(())
}
/// Lists the space `uri`'s repository to `did` no more: they were
/// refused it.
pub fn unread_by(&self, uri: &str, did: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"DELETE FROM space_reader WHERE uri = ?1 AND did = ?2",
[uri, did],
)?;
Ok(())
}
/// The records of the private repositories `did` was proven to read
/// in someone else's space, by URI.
pub fn read_of(&self, did: &str) -> rusqlite::Result<HashSet<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT r.space || '/' || r.did || '/' || r.collection || '/' || r.rkey
FROM space_reader s JOIN space_record r ON r.space = s.uri
WHERE s.did = ?1 AND r.collection = ?2",
)?;
stmt.query_map([did, nsid::REPO], |row| row.get(0))?
.collect()
}
/// The spaces, someone else's, that `did` may be a member of: followed
/// ones where a repository names them a collaborator, where the
/// organization's record there names them, or where they wrote, and
/// those they were proven to read before, followed or not.
pub fn maybe_member(&self, did: &str) -> rusqlite::Result<Vec<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT uri FROM space WHERE uri NOT LIKE ?2 AND (
EXISTS (SELECT 1 FROM space_record r JOIN repo ON repo.uri =
r.space || '/' || r.did || '/' || r.collection || '/' || r.rkey
WHERE r.space = space.uri AND r.collection = ?3
AND EXISTS (SELECT 1 FROM json_each(repo.value, '$.collaborators') c
WHERE json_extract(c.value, '$.did') = ?1))
OR EXISTS (SELECT 1 FROM org_member m WHERE m.space = space.uri AND m.did = ?1)
OR EXISTS (SELECT 1 FROM space_writer w WHERE w.space = space.uri AND w.did = ?1))
UNION SELECT uri FROM space_reader WHERE did = ?1 AND uri NOT LIKE ?2
ORDER BY uri",
)?;
let own = format!("at://{did}/%");
stmt.query_map(params![did, own, nsid::REPO], |row| row.get(0))?
.collect()
}
/// Stops following the space `uri`, and takes out every record it gave.
pub fn forget_space(&self, uri: &str) -> rusqlite::Result<()> {
let gave: Vec<(String, String, String)> = {
let db = self.db.lock().unwrap();
let mut stmt = db.prepare_cached(
"SELECT did, collection, rkey FROM space_record WHERE space = ?1",
)?;
let rows = stmt.query_map([uri], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)));
rows?.collect::<rusqlite::Result<_>>()?
};
for (did, collection, rkey) in gave {
self.delete_from_space(uri, &did, &collection, &rkey)?;
}
let db = self.db.lock().unwrap();
db.execute("DELETE FROM space_writer WHERE space = ?1", [uri])?;
db.execute("DELETE FROM space_reader WHERE uri = ?1", [uri])?;
db.execute("DELETE FROM space_syncer WHERE uri = ?1", [uri])?;
db.execute("DELETE FROM space WHERE uri = ?1", [uri])?;
Ok(())
}
/// Notes that the authority of the space `uri` said it is deleted, or
/// has no such space: an owner's PDS can go on naming it in
/// `listSpaces`, and it is not followed again as they log in, only as
/// it is followed for some other reason ([`Index::follow_space`]).
pub fn space_gone(&self, uri: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute("INSERT OR IGNORE INTO space_gone (uri) VALUES (?1)", [uri])?;
Ok(())
}
/// The spaces [`Index::space_gone`] noted.
pub fn gone_spaces(&self) -> rusqlite::Result<HashSet<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT uri FROM space_gone")?;
stmt.query_map([], |row| row.get(0))?.collect()
}
/// The spaces followed, each with where its sync is, if it has begun:
/// the space revision of the newest write it has taken in.
pub fn spaces(&self) -> rusqlite::Result<Vec<(String, Option<String>)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT uri, cursor FROM space ORDER BY uri")?;
stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect()
}
/// The lock held across a sync of the space `uri`.
pub fn syncing(&self, uri: &str) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = self.syncing.lock().unwrap();
locks.entry(uri.to_string()).or_default().clone()
}
/// Records that the space `uri` is synced to the space revision `cursor`.
pub fn set_space_cursor(&self, uri: &str, cursor: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO space (uri, cursor) VALUES (?1, ?2)
ON CONFLICT (uri) DO UPDATE SET cursor = excluded.cursor",
params![uri, cursor],
)?;
Ok(())
}
/// Whether the space `uri` has been synced at all.
pub fn space_synced(&self, uri: &str) -> rusqlite::Result<bool> {
let db = self.readers.take(&self.db);
let found = db.query_row("SELECT cursor FROM space WHERE uri = ?1", [uri], |row| {
row.get::<_, Option<String>>(0)
});
Ok(found.optional()?.flatten().is_some())
}
/// Where `did`'s repo in the space `uri` was last synced to: its
/// revision, and its set hash's state there.
pub fn space_writer(
&self,
uri: &str,
did: &str,
) -> rusqlite::Result<Option<(String, Vec<u8>)>> {
let db = self.readers.take(&self.db);
db.query_row(
"SELECT rev, set_hash FROM space_writer WHERE space = ?1 AND did = ?2",
[uri, did],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.optional()
}
/// Records that `did`'s repo in the space `uri` is synced to `rev`,
/// where its set hash's state is `set_hash`.
pub fn set_space_writer(
&self,
uri: &str,
did: &str,
rev: &str,
set_hash: &[u8],
) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO space_writer (space, did, rev, set_hash) VALUES (?1, ?2, ?3, ?4)
ON CONFLICT (space, did) DO UPDATE SET rev = excluded.rev, set_hash = excluded.set_hash",
params![uri, did, rev, set_hash],
)?;
Ok(())
}
/// The records `did`'s repo in the space `uri` gave this index, by
/// collection and rkey: what a sync that reads it whole takes out if
/// it no longer has them.
pub fn space_records(&self, uri: &str, did: &str) -> rusqlite::Result<Vec<(String, String)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT collection, rkey FROM space_record WHERE space = ?1 AND did = ?2",
)?;
stmt.query_map([uri, did], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect()
}
/// Puts a record of `did`'s from the space `uri`, as [`Index::put`],
/// noting the space it came from.
pub fn put_from_space(
&self,
uri: &str,
did: &str,
collection: &str,
rkey: &str,
value: &Value,
) -> rusqlite::Result<()> {
self.put_in(Some(uri), did, collection, rkey, value)
}
/// Takes out a record of `did`'s from the space `uri`, as
/// [`Index::delete`].
pub fn delete_from_space(
&self,
uri: &str,
did: &str,
collection: &str,
rkey: &str,
) -> rusqlite::Result<()> {
self.delete_in(Some(uri), did, collection, rkey)
}
/// Puts a record of `did`'s, as [`Index::put`]: from the space `space`,
/// noted so, if it is in one.
pub fn put_in(
&self,
space: Option<&str>,
did: &str,
collection: &str,
rkey: &str,
value: &Value,
) -> rusqlite::Result<()> {
self.put_at(space.unwrap_or_default(), did, collection, rkey, value)?;
let db = self.db.lock().unwrap();
// With no space, the row is NULL where none may be, so none is kept.
db.execute(
"INSERT OR IGNORE INTO space_record VALUES (?1, ?2, ?3, ?4)",
params![space, did, collection, rkey],
)?;
Ok(())
}
/// Takes out a record of `did`'s, as [`Index::delete`], and from the
/// space `space`'s if it is in one.
pub fn delete_in(
&self,
space: Option<&str>,
did: &str,
collection: &str,
rkey: &str,
) -> rusqlite::Result<()> {
self.delete_at(space.unwrap_or_default(), did, collection, rkey)?;
let db = self.db.lock().unwrap();
db.execute(
"DELETE FROM space_record WHERE space = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4",
params![space, did, collection, rkey],
)?;
Ok(())
}
/// The accounts the organization `org`'s records in `space` (its
/// public records when empty) name, the first invited first, each with
/// whether they keep a record of joining it there.
pub fn org_roster(&self, org: &str, space: &str) -> rusqlite::Result<Vec<Invitee>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT m.did, m.role, m.invited_at, m.expires_at, m.joined_at,
EXISTS (SELECT 1 FROM membership j
WHERE j.space = m.space AND j.did = m.did AND j.org = m.org)
FROM org_member m WHERE m.space = ?1 AND m.org = ?2
ORDER BY m.invited_at, m.did",
)?;
let rows = stmt.query_map([space, org], invitee)?;
rows.collect()
}
/// Whether `did` is an organization: its public records say it is one.
pub fn is_org(&self, did: &str) -> rusqlite::Result<bool> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT 1 FROM org WHERE space = '' AND did = ?1")?;
stmt.exists([did])
}
/// The gateway the organization `org`'s public record names.
pub fn org_gateway(&self, org: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let found = db.query_row(
"SELECT gateway FROM org WHERE space = '' AND did = ?1",
[org],
|row| row.get(0),
);
Ok(found.optional()?.flatten())
}
/// The organizations `did` is a member of, by DID: in public, or with
/// `hidden` those whose spaces name them taken in, where they keep a
/// record of joining.
pub fn orgs_of(&self, did: &str, hidden: bool) -> rusqlite::Result<Vec<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(&format!(
"SELECT m.org FROM org_member m
JOIN membership j ON j.space = m.space AND j.did = m.did AND j.org = m.org
WHERE m.did = ?1 AND m.joined_at IS NOT NULL
AND m.space = CASE WHEN ?2 THEN 'at://' || m.org || '/space/{}/self' ELSE '' END
ORDER BY m.org",
knotless::org::SPACE_TYPE
))?;
stmt.query_map(params![did, hidden], |row| row.get(0))?
.collect()
}
/// Where the index was last fed to from the event stream `src`.
pub fn stream_seq(&self, src: &str) -> rusqlite::Result<Option<i64>> {
let db = self.readers.take(&self.db);
db.query_row("SELECT seq FROM stream WHERE src = ?1", [src], |row| {
row.get(0)
})
.optional()
}
/// Records that the index was fed to `seq` from the event stream `src`.
pub fn set_stream_seq(&self, src: &str, seq: i64) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO stream (src, seq) VALUES (?1, ?2)
ON CONFLICT (src) DO UPDATE SET seq = excluded.seq",
params![src, seq],
)?;
Ok(())
}
/// Keeps a report, as `com.atproto.moderation.createReport` answers it
/// without its `id`, and returns it with the `id` it was given.
pub fn report(&self, mut report: Value) -> rusqlite::Result<Value> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO report (value) VALUES (?1)",
[report.to_string()],
)?;
report["id"] = db.last_insert_rowid().into();
Ok(report)
}
/// Reports, newest first, from before `before` when it is given.
pub fn reports(&self, limit: usize, before: Option<i64>) -> rusqlite::Result<Vec<Value>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT id, value FROM report WHERE id < ?1 ORDER BY id DESC LIMIT ?2",
)?;
let before = before.unwrap_or(i64::MAX);
stmt.query_map(params![before, limit as i64], numbered)?
.map(|r| {
r.map(|(id, mut report): (i64, Value)| {
report["id"] = id.into();
report
})
})
.collect()
}
/// Leaves `runner` an invite to `did`'s private repository `name`,
/// after any it has: one there already moves to the end.
pub fn invite(&self, runner: &str, did: &str, name: &str) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT OR REPLACE INTO invite (runner, did, name, at) VALUES (?1, ?2, ?3, ?4)",
params![runner, did, name, knotless::records::now()],
)?;
Ok(())
}
/// Whether `did` invited `runner` to its private repository `name`.
pub fn invited(&self, runner: &str, did: &str, name: &str) -> rusqlite::Result<bool> {
let db = self.readers.take(&self.db);
let mut stmt =
db.prepare_cached("SELECT 1 FROM invite WHERE runner = ?1 AND did = ?2 AND name = ?3")?;
stmt.exists(params![runner, did, name])
}
/// The invites left for `runner` after `after`, oldest first, each
/// with its id.
pub fn invites(
&self,
runner: &str,
after: i64,
limit: usize,
) -> rusqlite::Result<Vec<(i64, Value)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT id, did, name, at FROM invite WHERE runner = ?1 AND id > ?2 ORDER BY id LIMIT ?3",
)?;
stmt.query_map(params![runner, after, limit as i64], invite_row)?
.collect()
}
/// Keeps `secret`, sealed, for `owner`'s repository `repo`, in place
/// of any of its name sealed to the same runner.
pub fn put_secret(&self, owner: &str, repo: &str, secret: &Secret) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
let refs = serde_json::to_string(&secret.refs).expect("refs are strings");
db.execute(
"INSERT OR REPLACE INTO secret VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![
owner,
repo,
secret.runner,
secret.name,
refs,
secret.key,
secret.sealed,
secret.updated_at
],
)?;
Ok(())
}
/// Deletes `owner`'s repository `repo`'s secrets of `name`, sealed to
/// `runner`, or both; with neither, all of them. Says how many.
pub fn delete_secrets(
&self,
owner: &str,
repo: &str,
name: Option<&str>,
runner: Option<&str>,
) -> rusqlite::Result<usize> {
let db = self.db.lock().unwrap();
db.execute(
"DELETE FROM secret WHERE owner = ?1 AND repo = ?2
AND (?3 IS NULL OR name = ?3) AND (?4 IS NULL OR runner = ?4)",
params![owner, repo, name, runner],
)
}
/// `owner`'s repository `repo`'s secrets, by name, then runner.
pub fn secrets(&self, owner: &str, repo: &str) -> rusqlite::Result<Vec<Secret>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT name, runner, refs, key, sealed, at FROM secret
WHERE owner = ?1 AND repo = ?2 ORDER BY name, runner",
)?;
stmt.query_map([owner, repo], secret_row)?.collect()
}
/// The name `did`'s repository `key` has now.
pub fn name(&self, did: &str, key: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
db.query_row(
"SELECT name FROM repo WHERE did = ?1 AND key = ?2",
[did, key],
|row| row.get(0),
)
.optional()
}
/// `did`'s repository `key`, unless it or its owner is taken down.
pub fn listed(&self, did: &str, key: &str) -> rusqlite::Result<Option<Listed>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT did, name, description, fork_of, created_at, uri FROM repo
WHERE did = ?1 AND key = ?2 AND did NOT IN down AND uri NOT IN down",
)?;
stmt.query_row([did, key], listed).optional()
}
/// Every public repository, by owner and key, but those taken down.
pub fn code_repos(&self) -> rusqlite::Result<Vec<(String, String)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"-- scans: repo, as code search reads it once, as it starts
SELECT did, key FROM repo WHERE did NOT IN down AND uri NOT IN down",
)?;
let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?;
rows.collect()
}
/// How far the authors `did`'s repository `key` vouches for were
/// walked: the tip its default branch was at, and who it believed
/// then, as [`Index::vouch_authors`] kept them.
pub fn authors_walked(
&self,
did: &str,
key: &str,
) -> rusqlite::Result<Option<(String, String)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT tip, people FROM authors_walked WHERE repo_did = ?1 AND repo_key = ?2",
)?;
stmt.query_row([did, key], |row| Ok((row.get(0)?, row.get(1)?)))
.optional()
}
/// Keeps that `did`'s repository `key` vouches for `rows`, each a
/// commit, an address of it ([`address`]) and the account it is,
/// walked to `tip` believing `people`. `afresh` forgets what it
/// vouched for before, as who it believes has changed.
pub fn vouch_authors(
&self,
(did, key): (&str, &str),
(tip, people): (&str, &str),
rows: &[(String, String, String)],
afresh: bool,
) -> rusqlite::Result<()> {
let mut db = self.db.lock().unwrap();
let tx = db.transaction()?;
if afresh {
tx.execute(
"DELETE FROM commit_author WHERE repo_did = ?1 AND repo_key = ?2",
[did, key],
)?;
}
{
let mut put = tx.prepare_cached(
"INSERT OR IGNORE INTO commit_author VALUES (?1, ?2, ?3, ?4, ?5)",
)?;
for (sha, address, by) in rows {
put.execute([sha.as_str(), address, did, key, by])?;
}
}
tx.execute(
"INSERT OR REPLACE INTO authors_walked VALUES (?1, ?2, ?3, ?4)",
[did, key, tip, people],
)?;
tx.commit()
}
/// The accounts the repositories the index knows, other than `did`'s
/// `key`, vouch for as `sha`'s at `address`.
pub fn authors_elsewhere(
&self,
sha: &str,
address: &str,
(did, key): (&str, &str),
) -> rusqlite::Result<Vec<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT DISTINCT c.did FROM commit_author c
JOIN repo r ON r.did = c.repo_did AND r.key = c.repo_key
WHERE c.sha = ?1 AND c.address = ?2
AND NOT (c.repo_did = ?3 AND c.repo_key = ?4)
AND r.uri NOT IN down AND c.did NOT IN down",
)?;
let found = stmt.query_map([sha, address, did, key], |row| row.get(0))?;
found.collect()
}
/// The record of `did`'s repository `key`.
pub fn repo_of(&self, did: &str, key: &str) -> rusqlite::Result<Option<Value>> {
let db = self.readers.take(&self.db);
let found = db.query_row(
"SELECT value FROM repo WHERE did = ?1 AND key = ?2",
[did, key],
|row| row.get::<_, String>(0),
);
let found = found.optional()?;
Ok(found.and_then(|v| serde_json::from_str(&v).ok()))
}
/// The repository record at `uri`, in either form: in the public
/// form, an account's record of that name, wherever the index keeps it,
/// as an account has one repository of a name.
pub fn repo(&self, uri: &str) -> rusqlite::Result<Option<Value>> {
let public = AtUri::parse(uri).ok().filter(|at| at.space.is_none());
let (did, name) = public.map(|at| (at.author, at.rkey)).unwrap_or_default();
let db = self.readers.take(&self.db);
db.query_row(
"SELECT value FROM repo WHERE uri = ?1 OR (did = ?2 AND name = ?3) LIMIT 1",
params![uri, did, name],
|row| row.get::<_, String>(0),
)
.optional()
.map(|v| v.and_then(|v| serde_json::from_str(&v).ok()))
}
/// Repositories, newest first, whose name or description holds `query`.
pub fn repos(&self, query: &str, limit: usize) -> rusqlite::Result<Vec<Listed>> {
self.repos_page(query, limit, None, Among::default())
.map(|(repos, _)| repos)
}
/// A page of [`Index::repos`] after `cursor`, and the cursor of the
/// next page when there may be one: of `owner`'s alone, and only those
/// naming `runner` among their runners and `triager` among their
/// triagers, when given; or those of `followed_by` and whom it follows,
/// if named.
pub fn repos_page(
&self,
query: &str,
limit: usize,
cursor: Option<&str>,
among: Among,
) -> rusqlite::Result<(Vec<Listed>, Option<String>)> {
let db = self.readers.take(&self.db);
let pattern = format!("%{}%", query.replace(['%', '_'], ""));
let (at, uri) = after(cursor);
let mut stmt = db.prepare_cached(
"SELECT did, name, description, fork_of, created_at, uri FROM repo
WHERE (name LIKE ?1 OR description LIKE ?1)
AND (created_at < ?2 OR (created_at = ?2 AND uri < ?3))
AND did NOT IN down AND uri NOT IN down
AND (?5 IS NULL OR did = ?5)
AND (?6 IS NULL OR EXISTS (SELECT 1 FROM json_each(repo.value, '$.runners') r
WHERE json_extract(r.value, '$.did') = ?6))
AND (?7 IS NULL OR EXISTS (SELECT 1 FROM json_each(repo.value, '$.triagers') t
WHERE json_extract(t.value, '$.did') = ?7))
AND (?8 IS NULL OR did = ?8 OR did IN (SELECT subject FROM follow WHERE did = ?8))
AND (?9 IS NULL OR EXISTS (SELECT 1 FROM json_each(repo.value, '$.topics') t
WHERE json_extract(t.value, '$.name') = ?9))
AND (?10 IS NULL OR did != ?10)
AND (?11 IS NULL OR name != ?11 COLLATE NOCASE)
ORDER BY created_at DESC, uri DESC LIMIT ?4",
)?;
let Among {
owner,
not_owner,
not_named,
runner,
triager,
followed_by,
topic,
} = among;
let found = params![
pattern,
at,
uri,
limit as i64,
owner,
runner,
triager,
followed_by,
topic,
not_owner,
not_named
];
let rows: Vec<Listed> = stmt
.query_map(found, listed)?
.collect::<rusqlite::Result<_>>()?;
let next = (rows.len() == limit)
.then(|| rows.last().map(|r| cursor_at(&r.created_at, &r.uri)))
.flatten();
Ok((rows, next))
}
/// At most `limit` repositories named `name`, ignoring case: the
/// originals before the forks, each newest first. Those a search for
/// the name means, before the rest that hold it.
pub fn named(&self, name: &str, limit: usize) -> rusqlite::Result<Vec<Listed>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT did, name, description, fork_of, created_at, uri FROM repo
WHERE name = ?1 COLLATE NOCASE AND did NOT IN down AND uri NOT IN down
ORDER BY fork_of IS NOT NULL, created_at DESC, uri DESC LIMIT ?2",
)?;
stmt.query_map(params![name, limit as i64], listed)?
.collect()
}
/// The handle `did` was last found to have, `None` for none that
/// resolves back to it, and when it was looked up, in seconds since the
/// epoch: if it ever was.
pub fn handle(&self, did: &str) -> rusqlite::Result<Option<(Option<String>, i64)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached("SELECT handle, checked FROM handle WHERE did = ?1")?;
stmt.query_row([did], |row| Ok((row.get(0)?, row.get(1)?)))
.optional()
}
/// The account whose handle was last found to be `handle`, if any was.
pub fn did_of_handle(&self, handle: &str) -> rusqlite::Result<Option<String>> {
let db = self.db.lock().unwrap();
let mut stmt = db.prepare_cached(
"SELECT did FROM handle WHERE handle = ?1 ORDER BY checked DESC LIMIT 1",
)?;
stmt.query_row([handle], |row| row.get(0)).optional()
}
/// Keeps `handle` as what `did`'s handle was found to be at `checked`.
pub fn set_handle(
&self,
did: &str,
handle: Option<&str>,
checked: i64,
) -> rusqlite::Result<()> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO handle (did, handle, checked) VALUES (?1, ?2, ?3)
ON CONFLICT (did) DO UPDATE SET handle = excluded.handle, checked = excluded.checked",
params![did, handle, checked],
)?;
Ok(())
}
/// The accounts owning repositories, as [`Index::repos`] lists them,
/// whose handles were never looked up.
pub fn unnamed_owners(&self) -> rusqlite::Result<Vec<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT DISTINCT did FROM repo
WHERE did NOT IN down AND uri NOT IN down AND did NOT IN (SELECT did FROM handle)",
)?;
stmt.query_map([], |row| row.get(0))?.collect()
}
/// The accounts owning repositories, as [`Index::repos`] lists them,
/// whose handle holds `query`, ignoring case: those it starts with
/// first, then the one with the newest repository. Each with its handle
/// and how many repositories it owns.
pub fn owners(
&self,
query: &str,
limit: usize,
) -> rusqlite::Result<Vec<(String, String, i64)>> {
let db = self.readers.take(&self.db);
let query = query.replace(['%', '_'], "");
let mut stmt = db.prepare_cached(
"SELECT r.did, h.handle, COUNT(*) FROM repo r JOIN handle h ON h.did = r.did
WHERE r.did NOT IN down AND r.uri NOT IN down AND h.handle LIKE ?1
GROUP BY r.did
ORDER BY h.handle LIKE ?2 DESC, MAX(r.created_at) DESC, r.did LIMIT ?3",
)?;
let found = params![format!("%{query}%"), format!("{query}%"), limit as i64];
stmt.query_map(found, |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
.collect()
}
/// Pulls, issues, comments and changes of state on any repository,
/// newest first: not labels, the merge queue, or a state record that
/// only sets `base`, which change no state, with the owner and key of the repository each is on:
/// without what its owner hides, with what was said about that, or
/// what those its owner blocks wrote, as `forge` leaves them out. After
/// `cursor`, as [`cursor_at`] makes one, if given. The blocks and hides
/// owners wrote are gathered once, being few, rather than looked for
/// among all a repository's records for each record that might be shown.
/// `author`'s alone, if named; or with `followed_by`, what it and whom
/// it follows wrote, and anything on its repositories and those it
/// starred. With `only`, `pull` or `issue`, those and changes of their
/// state alone; `comment`, comments alone. With `quiet`, of a
/// repository's bots ([`bot_on`]), and of accounts that say they are
/// bots, changes of state alone.
pub fn recent(
&self,
limit: usize,
cursor: Option<&str>,
author: Option<&str>,
followed_by: Option<&str>,
only: Option<&str>,
quiet: bool,
) -> rusqlite::Result<Vec<(Entry, String, String)>> {
let (at, uri) = after(cursor);
// One account's by forge_author_created, which a test of whether
// one was named would keep SQLite from using.
let by = match author {
Some(_) => "f.author = ?4",
None => "?4 IS NULL",
};
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
&"-- scans: forge, the blocks and hides by forge_moderation, few, and the rest
-- newest first by forge_created, or forge_author_created, to the page's limit
WITH owned AS MATERIALIZED (
SELECT repo_did, repo_key, uri LIKE '%/dev.knotless.block/%' blocks,
json_extract(value, '$.subject') subject
FROM forge INDEXED BY forge_moderation WHERE author = repo_did
AND (instr(uri, '/dev.knotless.block/') OR instr(uri, '/dev.knotless.hide/')))
SELECT f.uri, f.author, f.value, f.repo_did, f.repo_key FROM forge f
WHERE (f.uri LIKE '%/dev.knotless.pull/%' OR f.uri LIKE '%/dev.knotless.issue/%'
OR f.uri LIKE '%/dev.knotless.comment/%'
OR (f.uri LIKE '%/dev.knotless.state/%'
AND coalesce(json_extract(f.value, '$.state'), '') != ''
AND json_extract(f.value, '$.subject') NOT LIKE '%/dev.knotless.comment/%'))
AND (json_extract(f.value, '$.createdAt') < ?2
OR (json_extract(f.value, '$.createdAt') = ?2 AND f.uri < ?3))
AND EXISTS (SELECT 1 FROM repo r WHERE r.did = f.repo_did AND r.key = f.repo_key)
AND f.uri NOT IN down AND f.author NOT IN down
AND {BY}
AND (?5 IS NULL OR f.author = ?5 OR f.repo_did = ?5
OR f.author IN (SELECT subject FROM follow WHERE did = ?5)
OR EXISTS (SELECT 1 FROM star s WHERE s.did = ?5
AND s.repo_did = f.repo_did AND s.repo_key = f.repo_key))
AND (?6 IS NULL
OR (?6 = 'comment' AND f.uri LIKE '%/dev.knotless.comment/%')
OR (?6 IN ('pull', 'issue')
AND (f.uri LIKE '%/dev.knotless.' || ?6 || '/%'
OR (f.uri LIKE '%/dev.knotless.state/%'
AND json_extract(f.value, '$.subject') LIKE '%/dev.knotless.' || ?6 || '/%'))))
AND coalesce(json_extract(f.value, '$.subject'), '') NOT IN down
AND f.author NOT IN (SELECT subject FROM block WHERE did = f.repo_did)
AND (NOT ?7 OR f.uri LIKE '%/dev.knotless.state/%'
OR NOT ({BOT} OR EXISTS (SELECT 1 FROM bot WHERE did = f.author)))
AND NOT EXISTS (SELECT 1 FROM owned o
WHERE o.repo_did = f.repo_did AND o.repo_key = f.repo_key
AND ((o.blocks AND o.subject = f.author)
OR (NOT o.blocks AND o.subject
IN (f.uri, coalesce(json_extract(f.value, '$.subject'), '')))))
ORDER BY json_extract(f.value, '$.createdAt') DESC, f.uri DESC LIMIT ?1"
.replace("{BY}", by)
.replace("{BOT}", &bot_on("f.author", "f.repo_did", "f.repo_key")),
)?;
let found = params![limit as i64, at, uri, author, followed_by, only, quiet];
stmt.query_map(found, recent)?.collect()
}
/// Refs that moved on repositories that stand, newest first; after
/// `cursor`, as [`cursor_at`] makes one, if given: `pusher`'s alone,
/// or those of `followed_by` and whom it follows, if named; with
/// `quiet`, none by the repository's bots ([`bot_on`]), nor by an
/// account that says it is a bot.
pub fn pushes(
&self,
limit: usize,
cursor: Option<&str>,
pusher: Option<&str>,
followed_by: Option<&str>,
quiet: bool,
) -> rusqlite::Result<Vec<Push>> {
let db = self.readers.take(&self.db);
let (at, uri) = after(cursor);
// From the pushes, newest first by `push_created`, or one
// account's by what it pushed and what it owns: never every
// repository's pushes, sorted.
let whose = match pusher {
Some(_) => "(p.pusher = ?4 OR p.did = ?4)",
None => "?4 IS NULL",
};
// A pusher is credited only while the repository names them, so
// one it names as a bot is the one credited.
let bot = bot_on("coalesce(p.pusher, p.did)", "p.did", "p.key");
let mut stmt = db.prepare_cached(&format!(
"SELECT * FROM (
SELECT r.uri, p.did, r.name, r.fork_of, p.ref, p.before, p.after, p.created_at,
p.uri, {BY} AS by
FROM push p CROSS JOIN repo r ON r.did = p.did AND r.key = p.key
WHERE p.did NOT IN down AND r.uri NOT IN down AND {whose}
AND (p.created_at < ?2 OR (p.created_at = ?2 AND p.uri < ?3))
AND (NOT ?6 OR NOT {bot}))
WHERE by NOT IN down AND (?4 IS NULL OR by = ?4)
AND (?5 IS NULL OR by = ?5 OR by IN (SELECT subject FROM follow WHERE did = ?5))
AND (NOT ?6 OR NOT EXISTS (SELECT 1 FROM bot WHERE did = by))
ORDER BY created_at DESC, uri DESC LIMIT ?1"
))?;
let found = params![limit as i64, at, uri, pusher, followed_by, quiet];
stmt.query_map(found, push)?.collect()
}
/// Stars of repositories that stand, by accounts that do, newest
/// first; after `cursor`, as [`cursor_at`] makes one, if given:
/// `by`'s alone, or those of `followed_by` and whom it follows, if
/// named.
pub fn stars(
&self,
limit: usize,
cursor: Option<&str>,
by: Option<&str>,
followed_by: Option<&str>,
) -> rusqlite::Result<Vec<Starred>> {
let db = self.readers.take(&self.db);
let (at, uri) = after(cursor);
// From the stars, newest first by `star_created`, or one
// account's by `star_did`: never every repository's, sorted.
let whose = match by {
Some(_) => "s.did = ?4",
None => "?4 IS NULL",
};
let mut stmt = db.prepare_cached(&format!(
"SELECT s.uri, s.did, r.uri, r.did, r.name, s.created_at
FROM star s CROSS JOIN repo r ON r.did = s.repo_did AND r.key = s.repo_key
WHERE s.did NOT IN down AND r.did NOT IN down AND r.uri NOT IN down
AND (s.created_at < ?2 OR (s.created_at = ?2 AND s.uri < ?3))
AND {whose}
AND (?5 IS NULL OR s.did = ?5 OR s.did IN (SELECT subject FROM follow WHERE did = ?5))
ORDER BY s.created_at DESC, s.uri DESC LIMIT ?1"
))?;
let found = params![limit as i64, at, uri, by, followed_by];
stmt.query_map(found, starred)?.collect()
}
/// How many of each kind of thing `did` did on each day in the zone
/// `tz` (UTC if it is none) from `since`'s (a year before now, if
/// none), oldest first: repositories made, pushes, and pulls, issues
/// and comments written. Nothing, if the account is taken down;
/// nothing on a repository that is.
pub fn actor_days(
&self,
did: &str,
since: Option<&str>,
tz: &str,
) -> rusqlite::Result<Vec<(String, String, u64)>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(&format!(
"SELECT d, kind, count(*) FROM (
SELECT zone_day(created_at, ?3) d, 'repos' kind FROM repo
WHERE did = ?1 AND uri NOT IN down
-- Its pushes are those it made or owns, by `push_pusher`
-- and `push_did`, not every repository's.
UNION ALL SELECT zone_day(p.created_at, ?3), 'pushes'
FROM push p CROSS JOIN repo r ON r.did = p.did AND r.key = p.key
WHERE (p.pusher = ?1 OR p.did = ?1)
AND {BY} = ?1 AND p.did NOT IN down AND r.uri NOT IN down
UNION ALL SELECT zone_day(json_extract(value, '$.createdAt'), ?3),
CASE WHEN instr(uri, '/dev.knotless.pull/') THEN 'pulls'
WHEN instr(uri, '/dev.knotless.issue/') THEN 'issues'
WHEN instr(uri, '/dev.knotless.comment/') THEN 'comments' END
FROM forge WHERE author = ?1 AND uri NOT IN down)
WHERE kind IS NOT NULL AND ?1 NOT IN down
AND d >= zone_day(coalesce(?2, strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-1 year')), ?3)
GROUP BY 1, 2 ORDER BY 1, 2"
))?;
stmt.query_map(params![did, since, tz], kinds)?.collect()
}
/// Each repository of `did`'s that stands and was ever pushed to, by
/// name: when it last was, and how often on each day in the zone `tz`
/// from `since`'s (a year before now, if none), oldest first.
pub fn actor_pushes(
&self,
did: &str,
since: Option<&str>,
tz: &str,
) -> rusqlite::Result<Vec<RepoPushes>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT r.name, zone_day(p.created_at, ?3), count(*), max(p.created_at),
zone_day(p.created_at, ?3) >= zone_day(coalesce(?2, strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-1 year')), ?3)
FROM push p JOIN repo r ON r.did = p.did AND r.key = p.key
WHERE p.did = ?1 AND r.uri NOT IN down AND ?1 NOT IN down
GROUP BY r.uri, 2 ORDER BY r.uri, 2",
)?;
let mut out: Vec<RepoPushes> = Vec::new();
for row in stmt.query_map(params![did, since, tz], pushed_on)? {
let (name, day, n, at, counted) = row?;
if out.last().is_none_or(|last| last.name != name) {
out.push(RepoPushes {
name,
pushed_at: String::new(),
days: Vec::new(),
});
}
let last = out.last_mut().unwrap();
last.pushed_at = at.max(std::mem::take(&mut last.pushed_at));
if counted {
last.days.push((day, n));
}
}
Ok(out)
}
/// Where the repository at `repo` had `refname` just before `at`, as
/// far as the index saw.
pub fn tip_before(
&self,
repo: &str,
refname: &str,
at: &str,
) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT p.after FROM push p JOIN repo r ON r.did = p.did AND r.key = p.key
WHERE r.uri = ?1 AND p.ref = ?2 AND p.created_at < ?3
ORDER BY p.created_at DESC LIMIT 1",
)?;
let tip = stmt
.query_row([repo, refname, at], |row| row.get(0))
.optional()?;
Ok(tip.flatten())
}
/// Where each of the repository at `repo`'s refs but `refname` was
/// just before the push `uri` at `at`, as far as the index saw; a ref
/// deleted by then, where it was when it was.
pub fn tips_before(
&self,
repo: &str,
refname: &str,
at: &str,
uri: &str,
) -> rusqlite::Result<Vec<String>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT COALESCE(p.after, p.before), MAX(p.created_at || ' ' || p.uri)
FROM push p JOIN repo r ON r.did = p.did AND r.key = p.key
WHERE r.uri = ?1 AND p.ref != ?2
AND (p.created_at < ?3 OR (p.created_at = ?3 AND p.uri < ?4))
GROUP BY p.ref",
)?;
let tips: Vec<Option<String>> = stmt
.query_map([repo, refname, at, uri], tip)?
.collect::<rusqlite::Result<_>>()?;
Ok(tips.into_iter().flatten().collect())
}
/// How many forks of `did`'s repository `key` there are, leaving out
/// those taken down, as [`Index::forks`] lists them.
pub fn fork_count(&self, did: &str, key: &str) -> rusqlite::Result<u64> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT COUNT(*) FROM repo
WHERE fork_did = ?1 AND fork_key = ?2 AND did NOT IN down AND uri NOT IN down",
)?;
stmt.query_row([did, key], |row| row.get(0))
}
/// Forks of `did`'s repository `key`.
pub fn forks(&self, did: &str, key: &str) -> rusqlite::Result<Vec<Listed>> {
let db = self.readers.take(&self.db);
let mut stmt = db.prepare_cached(
"SELECT did, name, description, fork_of, created_at, uri FROM repo
WHERE fork_did = ?1 AND fork_key = ?2 AND did NOT IN down AND uri NOT IN down
ORDER BY created_at",
)?;
stmt.query_map([did, key], listed)?.collect()
}
}
/// A number and the JSON after it.
/// The repositories `did`'s record of `collection`, `value`, is about, as
/// `<did>/<key>`: its own, the one it names, or for a ref that moved, its
/// repository and those it is the head of a pull into.
fn about(db: &Connection, did: &str, collection: &str, value: &Value) -> Vec<String> {
let named = |field: &str| serde_json::from_value::<RepoRef>(value[field].clone()).ok();
let one = |r: RepoRef| vec![format!("{}/{}", r.did(), r.key)];
match collection {
nsid::REPO => vec![format!(
"{did}/{}",
value["key"].as_str().unwrap_or_default()
)],
nsid::RELEASE | nsid::RELEASE_YANK => vec![format!(
"{did}/{}",
value["repo"].as_str().unwrap_or_default()
)],
nsid::REF | nsid::CLAIM => {
let key = value["repo"].as_str().unwrap_or_default();
let mut repos = pulled_into(db, did, key).unwrap_or_default();
repos.insert(0, format!("{did}/{key}"));
repos
}
nsid::STAR => named("subject").map(one).unwrap_or_default(),
_ => named("repo").map(one).unwrap_or_default(),
}
}
/// Whether a ref record is its ref as made, before it first moved: one
/// made since refs start at the time ([`knotless::keys::first_generation`])
/// says so by a `createdAt` equal to its `updatedAt`, an older one by its
/// generation of 1.
fn made(value: &Value) -> bool {
let created = value["createdAt"].as_str();
value["generation"].as_u64() == Some(1)
|| created.is_some_and(|c| Some(c) == value["updatedAt"].as_str())
}
/// The repositories `did`'s repository `key` is the head of a pull into.
fn pulled_into(db: &Connection, did: &str, key: &str) -> rusqlite::Result<Vec<String>> {
let stmt = db.prepare_cached(
"SELECT DISTINCT repo_did || '/' || repo_key FROM forge
WHERE instr(uri, '/dev.knotless.pull/')
AND json_extract(value, '$.head.repo.key') = ?2
AND json_extract(value, '$.head.repo.uri') LIKE 'at://' || ?1 || '/%'",
);
let mut stmt = stmt?;
stmt.query_map(params![did, key], |row| row.get(0))?
.collect()
}
/// The repository the record at `uri`, about to go, was about, if any.
fn deleted_about(db: &Connection, uri: &str) -> Vec<String> {
let mut stmt = db
.prepare_cached(
"SELECT repo_did || '/' || repo_key FROM forge WHERE uri = ?1
UNION SELECT repo_did || '/' || repo_key FROM check_run WHERE uri = ?1
UNION SELECT did || '/' || key FROM repo WHERE uri = ?1
UNION SELECT did || '/' || repo_key FROM release WHERE uri = ?1",
)
.expect("a fixed query");
let rows = stmt.query_map([uri], |row| row.get(0));
rows.expect("a fixed query").flatten().collect()
}
/// Keeps `did`'s record `collection`/`rkey` as `space` has it (its public
/// records when empty), or with no `value` takes it out: an organization's,
/// keyed `self`; its record of an account it invited, keyed by the DID it
/// names; or a member's record of joining one, keyed by the organization's
/// DID, which it names.
/// The URI the index keeps `did`'s record under: in full, in `space`,
/// for one from a space, or in the public form when `space` is empty.
fn record_uri(space: &str, did: &str, collection: &str, rkey: &str) -> String {
let at = AtUri::public(did, collection, rkey);
at.resolved(SpaceRef::parse(space).ok().as_ref())
.to_string()
}
fn org_rows(
db: &Connection,
space: &str,
did: &str,
collection: &str,
rkey: &str,
value: Option<&Value>,
) -> rusqlite::Result<usize> {
match collection {
// Only the one keyed self is the organization's.
nsid::ORG if rkey != "self" => Ok(0),
nsid::ORG => {
let dropped = "DELETE FROM org WHERE space = ?1 AND did = ?2";
db.execute(dropped, [space, did])?;
let found = value.cloned().map(serde_json::from_value::<OrgRecord>);
let Some(Ok(org)) = found else {
return Ok(0);
};
let kept = "INSERT INTO org VALUES (?1, ?2, ?3, ?4)";
db.execute(kept, params![space, did, org.gateway, org.created_at])
}
nsid::ORG_MEMBER => {
let gone = "DELETE FROM org_member WHERE space = ?1 AND org = ?2 AND did = ?3";
db.execute(gone, [space, did, rkey])?;
let found = value
.cloned()
.map(serde_json::from_value::<OrgMemberRecord>);
let found = found.and_then(Result::ok);
// Keyed other than by the DID it names, it names no one.
let Some(m) = found.filter(|m| m.member == rkey) else {
return Ok(0);
};
let role = knotless::org::Role::of_all(&m.roles).as_str();
db.execute(
"INSERT INTO org_member VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
params![
space,
did,
m.member,
role,
m.invited_at,
m.expires_at,
m.joined_at
],
)
}
nsid::ORG_ACCEPTANCE => {
let gone = "DELETE FROM membership WHERE space = ?1 AND did = ?2 AND org = ?3";
db.execute(gone, [space, did, rkey])?;
let subject = value.and_then(|v| v["subject"].as_str());
let at = value
.and_then(|v| v["createdAt"].as_str())
.unwrap_or_default();
match subject.filter(|org| *org == rkey) {
Some(org) => db.execute(
"INSERT INTO membership VALUES (?1, ?2, ?3, ?4)",
params![space, did, org, at],
),
None => Ok(0),
}
}
_ => Ok(0),
}
}
/// A row's first two columns, a repository's URI and its name.
fn uri_and_name(row: &rusqlite::Row) -> rusqlite::Result<(String, String)> {
Ok((row.get(0)?, row.get(1)?))
}
fn secret_row(row: &rusqlite::Row) -> rusqlite::Result<Secret> {
let refs: String = row.get(2)?;
Ok(Secret {
name: row.get(0)?,
runner: row.get(1)?,
refs: serde_json::from_str(&refs).unwrap_or_default(),
key: row.get(3)?,
sealed: row.get(4)?,
updated_at: row.get(5)?,
})
}
fn invitee(row: &rusqlite::Row) -> rusqlite::Result<Invitee> {
Ok(Invitee {
did: row.get(0)?,
role: row.get(1)?,
invited_at: row.get(2)?,
expires_at: row.get(3)?,
joined_at: row.get(4)?,
joined: row.get(5)?,
})
}
fn numbered(row: &rusqlite::Row) -> rusqlite::Result<(i64, Value)> {
Ok((row.get(0)?, json(row)?))
}
/// An invite's id, and the invite as `dev.knotless.listRunnerInvites`
/// gives it.
fn invite_row(row: &rusqlite::Row) -> rusqlite::Result<(i64, Value)> {
let (did, name, at): (String, String, String) = (row.get(1)?, row.get(2)?, row.get(3)?);
let invite = serde_json::json!({"did": did, "name": name, "createdAt": at});
Ok((row.get(0)?, invite))
}
fn pair(row: &rusqlite::Row) -> rusqlite::Result<(String, String)> {
Ok((row.get(0)?, row.get(1)?))
}
/// The JSON in a row's last column.
fn json(row: &rusqlite::Row) -> rusqlite::Result<Value> {
let text: String = row.get(row.as_ref().column_count() - 1)?;
Ok(serde_json::from_str(&text).unwrap_or_default())
}
/// A repository's owner and its record, as JSON.
fn owner_record(row: &rusqlite::Row) -> rusqlite::Result<(String, String)> {
Ok((row.get(0)?, row.get(1)?))
}
fn entry(row: &rusqlite::Row) -> rusqlite::Result<Entry> {
let value: String = row.get(2)?;
Ok(Entry {
uri: row.get(0)?,
author: row.get(1)?,
value: serde_json::from_str(&value).unwrap_or_default(),
})
}
/// A row of [`Index::pulls_where`]: a record, and the owner, record and
/// URI of the repository it is on.
fn elsewhere(row: &rusqlite::Row) -> rusqlite::Result<(Entry, String, String, String)> {
Ok((entry(row)?, row.get(3)?, row.get(4)?, row.get(5)?))
}
/// Pulls on one repository, with the records about them, as
/// [`Index::pulls_where`] finds them: the repository's owner and record.
pub struct Pulls {
pub owner: String,
pub repo: RepoRecord,
pub entries: Vec<Entry>,
}
/// A repository's owner and its record, as a row has them.
fn owner_and_record(row: &rusqlite::Row) -> rusqlite::Result<(String, String)> {
Ok((row.get(0)?, row.get(1)?))
}
/// Where an account lives, as its badges count it: whether its DID
/// document ever named another PDS, and whether its PDS is its own.
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct Home {
pub moved: bool,
pub own: bool,
}
fn dated(row: &rusqlite::Row) -> rusqlite::Result<(Entry, String)> {
Ok((entry(row)?, row.get(3)?))
}
fn push(row: &rusqlite::Row) -> rusqlite::Result<Push> {
Ok(Push {
repo: row.get(0)?,
did: row.get(1)?,
name: row.get(2)?,
fork_of: row.get(3)?,
refname: row.get(4)?,
from: row.get(5)?,
to: row.get(6)?,
created_at: row.get(7)?,
uri: row.get(8)?,
by: row.get(9)?,
})
}
/// A row of `theme`, as [`Index::themes`] reads it: a record that is not
/// JSON, which none is as written, as null.
fn theme_row(row: &rusqlite::Row) -> rusqlite::Result<(String, String, String, Value)> {
let value: String = row.get(3)?;
let value = serde_json::from_str(&value).unwrap_or_default();
Ok((row.get(0)?, row.get(1)?, row.get(2)?, value))
}
fn release_row(row: &rusqlite::Row) -> rusqlite::Result<Option<(String, ReleaseRecord)>> {
let text: String = row.get(1)?;
let yanked: bool = row.get(2)?;
Ok(serde_json::from_str(&text).ok().map(|r| {
(
row.get(0).unwrap_or_default(),
ReleaseRecord { yanked, ..r },
)
}))
}
fn starred(row: &rusqlite::Row) -> rusqlite::Result<Starred> {
Ok(Starred {
uri: row.get(0)?,
did: row.get(1)?,
repo: row.get(2)?,
owner: row.get(3)?,
name: row.get(4)?,
created_at: row.get(5)?,
})
}
fn tip(row: &rusqlite::Row) -> rusqlite::Result<Option<String>> {
row.get(0)
}
fn recent(row: &rusqlite::Row) -> rusqlite::Result<(Entry, String, String)> {
Ok((entry(row)?, row.get(3)?, row.get(4)?))
}
/// SQL saying whether the account `author` is a bot on the repository of
/// owner `did` and key `key`: one its record names as a runner or a
/// triager that is neither its owner nor a collaborator, who are people
/// a bot may act as.
fn bot_on(author: &str, did: &str, key: &str) -> String {
format!(
"EXISTS (SELECT 1 FROM repo b WHERE b.did = {did} AND b.key = {key} AND b.did != {author}
AND {author} IN (SELECT value ->> '$.did' FROM json_each(b.value, '$.runners')
UNION ALL SELECT value ->> '$.did' FROM json_each(b.value, '$.triagers'))
AND {author} NOT IN (SELECT value ->> '$.did' FROM json_each(b.value, '$.collaborators')))"
)
}
/// How [`Index::vouch_authors`] keeps a commit's address: hashed, as
/// git has it trimmed and lowercased, so the index names no address.
pub fn address(email: &str) -> String {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(email.trim().to_lowercase());
digest.iter().map(|b| format!("{b:02x}")).collect()
}
/// Where a page cut after the record `uri`, made at `at`, starts the next:
/// newest first, ties broken by URI, so the next starts strictly after it.
pub fn cursor_at(at: &str, uri: &str) -> String {
format!("{at} {uri}")
}
/// The time and URI a cursor is after, or past everything without one.
fn after(cursor: Option<&str>) -> (&str, &str) {
cursor
.and_then(|c| c.split_once(' '))
.unwrap_or(("\u{10ffff}", ""))
}
/// When owners began to hear all of their own repositories by default:
/// what was said on one before then they hear of as they did, so the
/// default brings no past along, as no watch does.
pub const OWNED_SINCE: &str = "2026-11-08T00:00:00Z";
/// The level that is no choice at all: hearing of a repository, a pull or
/// an issue as one's part in it says.
pub const PARTICIPATING: &str = "participating";
/// What an account chose to hear of a repository, a pull or an issue.
#[derive(Debug, Clone, PartialEq)]
pub struct Subscription {
/// A repository, as its owner's DID and its key, `did/key`, which a
/// rename keeps; or a pull's or an issue's URI.
pub subject: String,
/// For a repository, `all`, `releases` or `ignore`; for a pull or an
/// issue, `subscribed` or `muted`.
pub level: String,
/// When it chose.
pub at: String,
/// For a repository, when it began to hear of its releases, which may
/// be before it chose the level it has now.
pub releases_at: Option<String>,
}
impl Subscription {
/// A repository's subject, `did/key`.
pub fn repo(did: &str, key: &str) -> String {
format!("{did}/{key}")
}
/// The repository it is of, by owner and key, if it is of one.
pub fn repo_of(&self) -> Option<(&str, &str)> {
let repo = (!self.subject.starts_with("at://")).then_some(&self.subject);
repo.and_then(|s| s.split_once('/'))
}
/// As [`Index::notifications`] reads it, in JSON.
fn row(&self) -> Value {
let (did, key) = self.repo_of().unzip();
serde_json::json!({ "subject": self.subject, "did": did, "key": key,
"level": self.level, "at": self.at, "releases": self.releases_at })
}
}
/// Something another account did that concerns one, as
/// [`Index::notifications`] finds it.
#[derive(Debug, Clone, PartialEq)]
pub struct Notified {
/// The record it wrote.
pub uri: String,
pub author: String,
/// `star`, `follow`, `pull`, `issue`, `comment`, `mention`,
/// `review-requested`, `invite`, `checks-failed`, `ejected`,
/// `subscribed` or `watching`.
pub reason: String,
/// The pull or issue it is about: one a comment is on, else the pull
/// or issue itself; for a star, the repository's record; for a
/// release, itself.
pub subject: Option<String>,
/// The repository it is about, by owner and name now, if the index
/// knows it.
pub repo: Option<(String, String)>,
pub title: Option<String>,
pub at: String,
/// For a comment that reviews a pull, what it found.
pub verdict: Option<String>,
/// Whether it mentions the one it concerns, who hears of it for
/// another reason.
pub mentioned: bool,
/// For a comment, how many line comments were sent with it, as a
/// review's are.
pub line_comments: Option<u32>,
/// For an invitation, the role it offers.
pub role: Option<String>,
/// For checks that failed, or a pull the merge queue turned away for
/// its checks, those that did not pass.
pub checks: Vec<FailedCheck>,
/// For a pull, an issue or a comment, the start of what it says, as
/// Markdown: at most [`TEXT_MAX`] characters.
pub text: Option<String>,
/// For a review asked for, whether the one it concerns has reviewed
/// the pull since.
pub reviewed: bool,
/// The state the pull or issue it is about is in: `open`, `closed` or
/// `merged`.
pub state: Option<String>,
/// Whether it is about a private repository: found in the index of its
/// space, and so kept from what a push sends through Apple.
pub private: bool,
}
/// A device an account hears on, by Apple's push service.
#[derive(Clone, Debug, PartialEq)]
pub struct PushDevice {
pub token: String,
pub did: String,
/// `production` or `sandbox`: which of Apple's services it is known to.
pub environment: String,
/// The time of the latest notification it was told of, or when it was
/// registered.
pub at: String,
}
/// A row of `push_device`, as [`Index::push_devices`] reads it.
fn push_device(row: &rusqlite::Row) -> rusqlite::Result<PushDevice> {
Ok(PushDevice {
token: row.get(0)?,
did: row.get(1)?,
environment: row.get(2)?,
at: row.get(3)?,
})
}
/// The most devices an account hears on.
pub const PUSH_DEVICES: u32 = 20;
/// The most of what a notification's record says that it carries.
pub const TEXT_MAX: usize = 1000;
/// `text` cut to [`TEXT_MAX`] characters; none when it says nothing.
fn clipped(text: Option<String>) -> Option<String> {
let text = text?;
let text = text.trim();
let cut = text
.char_indices()
.nth(TEXT_MAX)
.map_or(text.len(), |(i, _)| i);
(!text.is_empty()).then(|| text[..cut].to_string())
}
/// The checks a notification names, as JSON in a row, by workflow: none
/// for a notification of another reason.
fn failed(json: Option<String>) -> Vec<FailedCheck> {
let json = json.unwrap_or_default();
let mut checks: Vec<FailedCheck> = serde_json::from_str(&json).unwrap_or_default();
checks.sort_by(|a, b| a.workflow.cmp(&b.workflow));
checks
}
fn listed(row: &rusqlite::Row) -> rusqlite::Result<Listed> {
Ok(Listed {
did: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
fork_of: row.get(3)?,
created_at: row.get(4)?,
uri: row.get(5)?,
})
}
/// `zone_day(at, tz)` in SQL: the day the time `at` falls on in the zone
/// `tz`, as `2026-10-25`; UTC if `tz` is no zone, and null if `at` is no time.
fn zone_day(ctx: &Context) -> rusqlite::Result<Option<String>> {
let at: Option<String> = ctx.get(0)?;
let tz = TimeZone::get(&ctx.get::<String>(1)?).unwrap_or(TimeZone::UTC);
let at = at.and_then(|at| at.parse::<Timestamp>().ok());
Ok(at.map(|at| at.to_zoned(tz).date().to_string()))
}
fn profile_record(row: &rusqlite::Row) -> rusqlite::Result<(String, String, Value)> {
let value: String = row.get(2)?;
let value = serde_json::from_str(&value).unwrap_or_default();
Ok((row.get(0)?, row.get(1)?, value))
}
fn pushed_on(row: &rusqlite::Row) -> rusqlite::Result<(String, String, u64, String, bool)> {
Ok((
row.get(0)?,
row.get(1)?,
row.get(2)?,
row.get(3)?,
row.get(4)?,
))
}
fn kinds(row: &rusqlite::Row) -> rusqlite::Result<(String, String, u64)> {
Ok((row.get(0)?, row.get(1)?, row.get(2)?))
}
fn network(row: &rusqlite::Row) -> rusqlite::Result<Network> {
Ok(Network {
accounts: row.get(0)?,
repos: row.get(1)?,
authors_week: row.get(2)?,
authors_month: row.get(3)?,
})
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn not_a_database() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("i.db");
std::fs::write(&path, vec![7u8; 4096]).unwrap();
assert!(Index::open(&path).is_err());
}
/// did:a's repository `r`, keyed `k`, which the notifications
/// below are about.
fn owned() -> serde_json::Value {
json!({"name": "r", "key": "k", "createdAt": "2026-01-01T00:00:00Z"})
}
#[test]
fn a_backfill_never_undoes_a_later_write() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let uri = "at://did:a/dev.knotless.repo/r";
let named = |about: &str| json!({"name": "r", "key": "k", "description": about});
let about = || index.repo(uri).unwrap().map(|r| r["description"].clone());
// The feed's write at rev 3 stands over a backfill's read as of 2.
let (fed, read) = (named("fed"), named("read"));
index
.put_as_of("did:a", nsid::REPO, "r", &fed, Some("3"))
.unwrap();
index
.put_as_of("did:a", nsid::REPO, "r", &read, Some("2"))
.unwrap();
assert_eq!(about(), Some(json!("fed")));
// One as of a later commit replaces it.
index
.put_as_of("did:a", nsid::REPO, "r", &read, Some("4"))
.unwrap();
assert_eq!(about(), Some(json!("read")));
// A deletion at 5 keeps a read as of 4 from putting it back, and
// one from before it changes nothing.
index
.delete_as_of("did:a", nsid::REPO, "r", Some("5"))
.unwrap();
index
.delete_as_of("did:a", nsid::REPO, "r", Some("1"))
.unwrap();
index
.put_as_of("did:a", nsid::REPO, "r", &read, Some("4"))
.unwrap();
assert_eq!(about(), None);
// A read past it forgets it, and what was read before it then lands.
index.read_through("did:a", "4").unwrap();
index
.put_as_of("did:a", nsid::REPO, "r", &read, Some("4"))
.unwrap();
assert_eq!(about(), None);
index.read_through("did:a", "5").unwrap();
index
.put_as_of("did:a", nsid::REPO, "r", &read, Some("4"))
.unwrap();
assert_eq!(about(), Some(json!("read")));
index.delete_as_of("did:a", nsid::REPO, "r", None).unwrap();
assert_eq!(about(), None);
}
#[test]
fn pulls_and_issues_are_kept_until_written_about() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
let record: RepoRecord = serde_json::from_value(owned()).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let issue = json!({"repo": repo, "title": "T", "createdAt": "2026-01-02T00:00:00Z"});
index.put("did:b", nsid::ISSUE, "1", &issue).unwrap();
let mut was = index.assembled("did:a", "k", &record).unwrap();
assert_eq!(was.items.len(), 1);
// Whether what is asked now is what was kept, and keeps it.
let kept = |was: &mut Arc<Assembled>, record: &RepoRecord| {
let now = index.assembled("did:a", "k", record).unwrap();
let same = Arc::ptr_eq(was, &now);
*was = now;
same
};
assert!(kept(&mut was, &record));
// What is written elsewhere leaves it kept.
let other = json!({"uri": "at://did:c/dev.knotless.repo/x", "key": "x"});
let there = json!({"repo": other, "subject": "at://did:c/dev.knotless.issue/1",
"body": "hi", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:c", nsid::COMMENT, "1", &there).unwrap();
index.set_status("did:z", "deactivated").unwrap();
assert!(kept(&mut was, &record));
// A comment on it, and taking it back.
let said = json!({"repo": repo, "subject": "at://did:b/dev.knotless.issue/1",
"body": "hi", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:b", nsid::COMMENT, "2", &said).unwrap();
assert!(!kept(&mut was, &record));
assert_eq!(was.entries.len(), 2);
index.delete("did:b", nsid::COMMENT, "2").unwrap();
assert!(!kept(&mut was, &record));
// Its author gone inactive, and back; another status of inactive
// changes nothing.
index.set_status("did:b", "deactivated").unwrap();
assert!(!kept(&mut was, &record));
assert!(was.items.is_empty());
index.set_status("did:b", "suspended").unwrap();
assert!(kept(&mut was, &record));
index.set_status("did:b", "active").unwrap();
assert!(!kept(&mut was, &record));
// Its owner blocking, and not.
let block = json!({"subject": "did:b", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:a", nsid::BLOCK, "b", &block).unwrap();
assert!(!kept(&mut was, &record));
assert!(was.items.is_empty());
index.delete("did:a", nsid::BLOCK, "b").unwrap();
assert!(!kept(&mut was, &record));
// Its record saying otherwise, and its own record written.
let described = RepoRecord {
description: Some("d".into()),
..record.clone()
};
assert!(!kept(&mut was, &described));
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
assert!(!kept(&mut was, &record));
// A move anywhere, offered and taken back.
let offer = json!({"name": "s", "key": "s", "createdAt": "2026-01-01T00:00:00Z",
"transfer": {"uri": "at://did:t/dev.knotless.repo/s", "key": "s"}});
index.put("did:a", nsid::REPO, "s", &offer).unwrap();
assert!(!kept(&mut was, &record));
index.delete("did:a", nsid::REPO, "s").unwrap();
assert!(!kept(&mut was, &record));
// An account forgotten that never wrote about it leaves it kept;
// its issue's author forgotten does not.
index.purge("did:z").unwrap();
assert!(kept(&mut was, &record));
index.purge("did:b").unwrap();
assert!(!kept(&mut was, &record));
assert!(was.items.is_empty());
// Time alone, for a label to run out.
let mut made = index.kept.lock().unwrap();
made.by.get_mut("did:a/k").unwrap().made -= KEPT_FOR;
drop(made);
assert!(!kept(&mut was, &record));
}
#[test]
fn a_record_written_patches_what_is_kept_of_its_repository() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
let record: RepoRecord = serde_json::from_value(owned()).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let issue = |title: &str| json!({"repo": repo, "title": title, "createdAt": "2026-01-02T00:00:00Z"});
index.put("did:b", nsid::ISSUE, "1", &issue("One")).unwrap();
index.put("did:b", nsid::ISSUE, "2", &issue("Two")).unwrap();
// The owner blocking another account, whose writes are left out.
let block = json!({"subject": "did:e", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:a", nsid::BLOCK, "e", &block).unwrap();
let was = index.assembled("did:a", "k", &record).unwrap();
assert_eq!(was.items.len(), 2);
// What the whole read gives, to hold a patch to.
let same = |now: &Assembled| {
let sorted = |mut e: Vec<Entry>| {
e.sort_by(|a, b| a.uri.cmp(&b.uri));
e
};
let all = index.forge("did:a", "k").unwrap();
assert_eq!(sorted(now.entries.clone()), sorted(all));
};
// One edited, one added, and a comment, patched in.
index.put("did:b", nsid::ISSUE, "1", &issue("Uno")).unwrap();
index
.put("did:b", nsid::ISSUE, "3", &issue("Three"))
.unwrap();
let said = json!({"repo": repo, "subject": "at://did:b/dev.knotless.issue/1",
"body": "hi", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:b", nsid::COMMENT, "c", &said).unwrap();
index
.put("did:e", nsid::ISSUE, "1", &issue("Blocked"))
.unwrap();
let now = index.assembled("did:a", "k", &record).unwrap();
assert!(!Arc::ptr_eq(&was, &now));
assert_eq!(now.items.len(), 3);
assert_eq!(now.entries.len(), 4);
same(&now);
// One deleted, and the same twice.
index.delete("did:b", nsid::ISSUE, "2").unwrap();
index.delete("did:b", nsid::ISSUE, "2").unwrap();
let now = index.assembled("did:a", "k", &record).unwrap();
assert_eq!(now.items.len(), 2);
same(&now);
// One of another repository's, written between, is no part of it.
let other = json!({"uri": "at://did:c/dev.knotless.repo/x", "key": "x"});
let there = json!({"repo": other, "title": "T", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:c", nsid::ISSUE, "9", &there).unwrap();
assert!(Arc::ptr_eq(
&now,
&index.assembled("did:a", "k", &record).unwrap()
));
// A hide hides the issue, and taking it back shows it.
let hide = json!({"repo": repo, "subject": "at://did:b/dev.knotless.issue/3",
"createdAt": "2026-01-04T00:00:00Z"});
index.put("did:a", nsid::HIDE, "h", &hide).unwrap();
let now = index.assembled("did:a", "k", &record).unwrap();
assert_eq!(now.items.len(), 1);
index
.put("did:b", nsid::ISSUE, "3", &issue("Three!"))
.unwrap();
let now = index.assembled("did:a", "k", &record).unwrap();
assert_eq!(now.items.len(), 1, "still hidden when written again");
index.delete("did:a", nsid::HIDE, "h").unwrap();
let now = index.assembled("did:a", "k", &record).unwrap();
assert_eq!(now.items.len(), 2);
same(&now);
// A write by an author gone inactive is left out, and a flood of
// them is read whole.
index.set_status("did:d", "deactivated").unwrap();
index
.put("did:d", nsid::ISSUE, "1", &issue("Gone"))
.unwrap();
let now = index.assembled("did:a", "k", &record).unwrap();
assert_eq!(now.items.len(), 2);
for n in 0..=ITEMS_MAX {
let rkey = format!("m{n}");
index.put("did:b", nsid::ISSUE, &rkey, &issue("M")).unwrap();
}
let now = index.assembled("did:a", "k", &record).unwrap();
assert_eq!(now.items.len(), 2 + ITEMS_MAX + 1);
same(&now);
}
#[test]
fn stamps_of_records_note_each_once_and_forget_past_their_number() {
let stamps = Stamps::new(2);
let scopes = ["a".to_string()];
stamps.mark_item("a", "u");
stamps.mark_item("a", "u");
stamps.mark_item("a", "v");
assert_eq!(stamps.touched(&scopes, 0), ["u", "v"]);
assert!(stamps.touched(&scopes, stamps.now()).is_empty());
assert_eq!(stamps.since(&scopes), 0);
// Two more repositories, past the number of them.
stamps.mark_item("b", "w");
stamps.mark_item("c", "x");
assert!(stamps.since(&scopes) > 0);
assert!(stamps.touched(&scopes, 0).is_empty());
}
#[test]
fn pulls_and_issues_of_a_repository_moved_here_are_kept_with_it() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let moved = json!({"name": "r", "key": "k", "createdAt": "2026-01-01T00:00:00Z",
"transferredFrom": {"uri": "at://did:o/dev.knotless.repo/r", "key": "ok"}});
index.put("did:a", nsid::REPO, "r", &moved).unwrap();
let record: RepoRecord = serde_json::from_value(moved).unwrap();
let was = index.assembled("did:a", "k", &record).unwrap();
// Said on the one it moved from, after.
let from = json!({"uri": "at://did:o/dev.knotless.repo/r", "key": "ok"});
let issue = json!({"repo": from, "title": "T", "createdAt": "2026-01-02T00:00:00Z"});
index.put("did:b", nsid::ISSUE, "1", &issue).unwrap();
let now = index.assembled("did:a", "k", &record).unwrap();
assert!(!Arc::ptr_eq(&was, &now));
}
#[test]
fn what_is_kept_stays_within_its_records() {
let made = |n: usize| Made {
at: 0,
made: Instant::now(),
scopes: Vec::new(),
hides: Vec::new(),
record: RepoRecord::default(),
assembled: Arc::new(Assembled {
entries: vec![
Entry {
uri: String::new(),
author: String::new(),
value: Value::Null
};
n
],
items: Vec::new(),
}),
};
let mut kept = Kept::new(3);
kept.put("a".into(), made(2));
kept.put("a".into(), made(1));
assert_eq!((kept.entries, kept.order.len()), (1, 1));
kept.put("b".into(), made(2));
kept.put("c".into(), made(2));
assert_eq!(kept.order, ["c"]);
assert_eq!(kept.entries, 2);
// One larger than the whole is kept alone.
kept.put("d".into(), made(5));
assert_eq!(kept.order, ["d"]);
}
#[test]
fn a_read_waits_for_a_reader_to_be_free() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let mut taken: Vec<Reader> = (0..READERS)
.map(|_| index.readers.take(&index.db))
.collect();
let (said, heard) = std::sync::mpsc::channel();
std::thread::scope(|s| {
s.spawn(|| {
let db = index.readers.take(&index.db);
let one: i64 = db.query_row("SELECT 1", [], |r| r.get(0)).unwrap();
said.send(one).unwrap();
});
let waited = heard.recv_timeout(Duration::from_millis(100));
assert!(waited.is_err(), "read with every reader taken");
taken.pop();
assert_eq!(heard.recv().unwrap(), 1);
});
// A reader cannot write.
let db = index.readers.take(&index.db);
assert!(db.execute("DELETE FROM repo", []).is_err());
drop(db);
// One that cannot be opened reads through the writer.
let nowhere = Readers::new(&dir.path().join("none").join("i.db"));
let db = nowhere.take(&index.db);
let one: i64 = db.query_row("SELECT 1", [], |r| r.get(0)).unwrap();
assert_eq!(one, 1);
}
#[test]
fn stamps_past_their_number_stamp_everything() {
let stamps = Stamps::new(2);
stamps.mark("a");
stamps.mark("b");
let before = stamps.now();
assert_eq!(stamps.since(&["c".into()]), 0);
stamps.mark("c");
assert!(stamps.since(&["a".into()]) >= before);
assert_eq!(stamps.by.lock().unwrap().len(), 1);
}
#[test]
fn what_was_said_on_a_deleted_repository() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let issue = json!({"repo": repo, "title": "T", "createdAt": "2026-01-02T00:00:00Z"});
index.put("did:b", nsid::ISSUE, "1", &issue).unwrap();
let star = json!({"subject": repo, "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:b", nsid::STAR, "2", &star).unwrap();
let claim = json!({"repo": "k", "name": "refs/heads/main", "to": "a1",
"createdAt": "2026-01-02T00:00:00Z"});
index.put("did:a", nsid::CLAIM, "k.m.1", &claim).unwrap();
let mut days = BTreeMap::new();
days.insert("2026-01-02".to_string(), Day::default());
let seen = |days: &mut BTreeMap<String, Day>| {
let recent = index
.recent(10, None, None, None, None, false)
.unwrap()
.len();
let heard = index
.notifications("did:a", &[], 10, None, None)
.unwrap()
.0
.len();
index.activity(days, "2026-01-01", "2026-01-01").unwrap();
let day = &days["2026-01-02"];
(recent, heard, day.issues, day.pushes)
};
assert_eq!(seen(&mut days), (1, 2, 1, 1));
// Its record gone, nothing said on it shows, though it is kept.
index.delete("did:a", nsid::REPO, "r").unwrap();
let mut days = BTreeMap::new();
days.insert("2026-01-02".to_string(), Day::default());
assert_eq!(seen(&mut days), (0, 0, 0, 0));
// Under another name, as a rename leaves it, it is not forgotten.
let renamed = json!({"name": "s", "key": "k", "createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::REPO, "s", &renamed).unwrap();
assert!(!index.forget_repo("did:a", "k").unwrap());
index.delete("did:a", nsid::REPO, "s").unwrap();
// Nor once another account took it up, whose it now is.
let taken = json!({"name": "t", "key": "k", "createdAt": "2026-01-04T00:00:00Z",
"transferredFrom": {"uri": "at://did:a/dev.knotless.repo/s", "key": "k"}});
index.put("did:c", nsid::REPO, "t", &taken).unwrap();
assert!(!index.forget_repo("did:a", "k").unwrap());
index.delete("did:c", nsid::REPO, "t").unwrap();
// Gone for good, what was said on it goes, whoever said it.
assert!(index.forget_repo("did:a", "k").unwrap());
assert!(index.forge("did:a", "k").unwrap().is_empty());
assert_eq!(index.star_count("did:a", "k").unwrap(), 0);
let db = index.db.lock().unwrap();
let pushes: i64 = db
.query_row(
"SELECT count(*) FROM push -- scans: push, a test's count",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(pushes, 0);
}
#[test]
fn comments_from_before_are_found() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("i.db");
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let issue = json!({"repo": repo, "title": "T", "createdAt": "2026-01-01T00:00:00Z"});
let subject = "at://did:a/dev.knotless.issue/1";
let comment = json!({"repo": repo, "subject": subject, "body": "b", "createdAt": "2026-01-02T00:00:00Z"});
{
let index = Index::open(&path).unwrap();
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
index.put("did:a", nsid::ISSUE, "1", &issue).unwrap();
index.put("did:b", nsid::COMMENT, "2", &comment).unwrap();
// As an index from before kept it: comments only in forge.
let db = index.db.lock().unwrap();
db.execute_batch("DROP TABLE comment").unwrap();
}
let index = Index::open(&path).unwrap();
let (found, _) = index.notifications("did:a", &[], 10, None, None).unwrap();
assert_eq!(found.len(), 1);
assert_eq!(found[0].reason, "comment");
assert_eq!(found[0].subject.as_deref(), Some(subject));
assert_eq!(found[0].title.as_deref(), Some("T"));
assert_eq!(found[0].verdict, None);
}
#[test]
fn those_who_look_after_a_repository_hear_of_its_pulls() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("i.db");
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let other = json!({"uri": "at://did:a/dev.knotless.repo/s", "key": "j"});
let shared = json!({"name": "r", "key": "k", "createdAt": "2026-01-01T00:00:00Z",
"collaborators": [{"did": "did:c"}, {"handle": "no.did"}]});
// It names the collaborator, who hears of it once, as the pull.
let named = json!([{"index": {"byteStart": 0, "byteEnd": 1},
"features": [{"$type": "dev.knotless.defs#mention", "did": "did:c"}]}]);
let pull = json!({"repo": repo, "title": "P", "createdAt": "2026-01-02T00:00:00Z",
"facets": named});
let issue = json!({"repo": other, "title": "I", "createdAt": "2026-01-03T00:00:00Z"});
let heard = |index: &Index, did: &str, orgs: &[String]| {
let (found, _) = index.notifications(did, orgs, 10, None, None).unwrap();
found
.into_iter()
.map(|n| (n.reason, n.mentioned))
.collect::<Vec<_>>()
};
{
let index = Index::open(&path).unwrap();
index.put("did:a", nsid::REPO, "r", &shared).unwrap();
let alone = json!({"name": "s", "key": "j", "createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::REPO, "s", &alone).unwrap();
index.put("did:b", nsid::PULL, "1", &pull).unwrap();
index.put("did:b", nsid::ISSUE, "2", &issue).unwrap();
// A collaborator, of the one repository and not the other.
assert_eq!(heard(&index, "did:c", &[]), [("pull".into(), true)]);
// A member of the organization that owns them both.
let orgs = ["did:a".to_string()];
let both = [("issue".into(), false), ("pull".into(), false)];
assert_eq!(heard(&index, "did:m", &orgs), both);
assert_eq!(heard(&index, "did:m", &[]), []);
// As an index from before kept it: collaborators only in the
// repositories' records.
let db = index.db.lock().unwrap();
db.execute_batch("DROP TABLE collaborator").unwrap();
}
let index = Index::open(&path).unwrap();
assert_eq!(heard(&index, "did:c", &[]), [("pull".into(), true)]);
// Taken out, they hear of it as they would anyone's: by name.
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
assert_eq!(heard(&index, "did:c", &[]), [("mention".into(), false)]);
index.put("did:a", nsid::REPO, "r", &shared).unwrap();
index.delete("did:a", nsid::REPO, "r").unwrap();
assert_eq!(heard(&index, "did:c", &[]), []);
}
#[test]
fn reviews_say_what_they_found() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let pull = json!({"repo": repo, "title": "P", "createdAt": "2026-01-01T00:00:00Z"});
let subject = "at://did:a/dev.knotless.pull/1";
// It names the pull's author, who hears of it once, as a review
// that mentions them.
let named = json!([{"index": {"byteStart": 0, "byteEnd": 1},
"features": [{"$type": "dev.knotless.defs#mention", "did": "did:a"}]}]);
let review = json!({"repo": repo, "subject": subject, "body": "b", "verdict": "changes",
"createdAt": "2026-01-02T00:00:00Z", "facets": named});
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
index.put("did:a", nsid::PULL, "1", &pull).unwrap();
index.put("did:b", nsid::COMMENT, "2", &review).unwrap();
// Its line comments are heard of with it, counted.
for rkey in ["3", "4"] {
let line = json!({"repo": repo, "subject": subject, "body": "here", "line": {"path": "a", "line": 1}, "review": "at://did:b/dev.knotless.comment/2", "createdAt": "2026-01-02T00:00:00Z", "facets": named});
index.put("did:b", nsid::COMMENT, rkey, &line).unwrap();
}
let (found, _) = index.notifications("did:a", &[], 10, None, None).unwrap();
assert_eq!(found.len(), 1);
assert_eq!(found[0].reason, "comment");
assert_eq!(found[0].subject.as_deref(), Some(subject));
assert_eq!(found[0].verdict.as_deref(), Some("changes"));
assert!(found[0].mentioned);
assert_eq!(found[0].line_comments, Some(2));
// A star is about the repository, by its record.
let star = json!({"subject": repo, "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:b", nsid::STAR, "5", &star).unwrap();
let (found, _) = index.notifications("did:a", &[], 10, None, None).unwrap();
assert_eq!(found[0].reason, "star");
let record = "at://did:a/dev.knotless.repo/r";
assert_eq!(found[0].subject.as_deref(), Some(record));
assert_eq!((found[0].mentioned, found[0].line_comments), (false, None));
}
#[test]
fn an_owner_hears_all_of_its_own_repository_until_it_chooses() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let at = |day: u32| format!("2026-01-{day:02}T00:00:00Z");
let watched = Subscription::repo("did:a", "k");
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
let issue = json!({"repo": repo, "title": "I", "createdAt": at(2)});
index.put("did:b", nsid::ISSUE, "1", &issue).unwrap();
let on = |day: u32| {
json!({"repo": repo, "subject": "at://did:b/dev.knotless.issue/1",
"body": "hi", "createdAt": format!("2026-12-{day:02}T00:00:00Z")})
};
// One said before owners heard all by default, and one after.
let mut before = on(1);
before["createdAt"] = json!(at(3));
index.put("did:c", nsid::COMMENT, "2", &before).unwrap();
index.put("did:c", nsid::COMMENT, "3", &on(1)).unwrap();
let reasons = || {
let (found, _) = index.notifications("did:a", &[], 20, None, None).unwrap();
let each = found.into_iter().map(|n| n.reason);
each.collect::<Vec<_>>()
};
// A comment on a thread it never joined, as watching, from when
// owners began to: and so its pages say.
assert_eq!(reasons(), ["watching", "issue"]);
assert_eq!(
index.watching("did:a", "did:a", "k").unwrap().as_deref(),
Some("all")
);
assert_eq!(index.watching("did:b", "did:a", "k").unwrap(), None);
// Participating is kept, as it is no longer the default.
index
.subscribe("did:a", &watched, PARTICIPATING, &at(4), 9)
.unwrap();
assert_eq!(reasons(), ["issue"]);
assert_eq!(index.watching("did:a", "did:a", "k").unwrap(), None);
let kept = index.subscriptions("did:a").unwrap();
assert_eq!(kept[0].level, PARTICIPATING);
assert!(index.owned_watches("did:a", &kept).unwrap().is_empty());
// Another's participating is forgotten, as ever.
index
.subscribe("did:b", &watched, "all", &at(4), 9)
.unwrap();
index
.subscribe("did:b", &watched, PARTICIPATING, &at(5), 9)
.unwrap();
assert!(index.subscriptions("did:b").unwrap().is_empty());
}
#[test]
fn what_an_account_watches_subscribed_to_and_muted() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let at = |day: u32| format!("2026-01-{day:02}T00:00:00Z");
let watched = Subscription::repo("did:a", "k");
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
let item =
|title: &str, day: u32| json!({"repo": repo, "title": title, "createdAt": at(day)});
let on = |subject: &str, day: u32| json!({"repo": repo, "subject": subject, "body": "hi", "createdAt": at(day)});
let issue = "at://did:b/dev.knotless.issue/2";
let naming = json!([{"index": {"byteStart": 0, "byteEnd": 1},
"features": [{"$type": "dev.knotless.defs#mention", "did": "did:c"}]}]);
// Before it watched, an issue; after, an issue, a comment on it, a
// review's line comment, a release, and a pull asking it to review.
index
.put("did:b", nsid::ISSUE, "1", &item("old", 1))
.unwrap();
index
.subscribe("did:c", &watched, "all", &at(2), 9)
.unwrap();
index.put("did:b", nsid::ISSUE, "2", &item("I", 3)).unwrap();
index
.put("did:b", nsid::COMMENT, "3", &on(issue, 4))
.unwrap();
let mut line = on(issue, 5);
line["review"] = json!("at://did:b/dev.knotless.comment/3");
index.put("did:b", nsid::COMMENT, "4", &line).unwrap();
let release = json!({"repo": "k", "package": "pkg", "version": "1.0.0",
"notes": "new", "files": [], "createdAt": at(6)});
index.put("did:a", nsid::RELEASE, "5", &release).unwrap();
let mut asks = item("P", 7);
asks["reviewers"] = json!([{"did": "did:c", "requestedAt": at(7)}]);
index.put("did:b", nsid::PULL, "6", &asks).unwrap();
// One naming it is a mention; once it commented, a comment.
let mut named = on(issue, 8);
named["facets"] = naming.clone();
index.put("did:b", nsid::COMMENT, "7", &named).unwrap();
index
.put("did:c", nsid::COMMENT, "8", &on(issue, 9))
.unwrap();
index
.put("did:b", nsid::COMMENT, "9", &on(issue, 10))
.unwrap();
let heard = |did: &str, kind: Option<&str>| {
let (found, _) = index.notifications(did, &[], 20, None, kind).unwrap();
let each = found
.into_iter()
.map(|n| (n.reason, n.uri.rsplit('/').next().unwrap().to_string()));
each.collect::<Vec<_>>()
};
let reasons = |did: &str| {
let mut all: Vec<String> = heard(did, None)
.into_iter()
.map(|(r, k)| format!("{r} {k}"))
.collect();
all.sort();
all
};
assert_eq!(
reasons("did:c"),
[
"comment 9",
"mention 7",
"review-requested 6",
"watching 2",
"watching 3",
"watching 5"
]
);
let (all, _) = index.notifications("did:c", &[], 20, None, None).unwrap();
let release = all.iter().find(|n| n.uri.ends_with("/5")).unwrap();
assert_eq!(release.title.as_deref(), Some("pkg 1.0.0"));
assert_eq!(release.text.as_deref(), Some("new"));
assert_eq!(
release.subject.as_deref(),
Some("at://did:a/dev.knotless.release/5")
);
let kind = |kind| {
heard("did:c", Some(kind))
.into_iter()
.map(|(_, k)| k)
.collect::<Vec<_>>()
};
assert_eq!(kind("comments"), ["9", "3"]);
assert_eq!(kind("pulls-and-issues"), ["2"]);
// Its releases alone, still from when it began to hear of them;
// then nothing of it, but a review asked of it.
index
.subscribe("did:c", &watched, "releases", &at(6), 9)
.unwrap();
assert_eq!(
reasons("did:c"),
["comment 9", "mention 7", "review-requested 6", "watching 5"]
);
index
.subscribe("did:c", &watched, "ignore", &at(11), 9)
.unwrap();
assert_eq!(reasons("did:c"), ["review-requested 6"]);
index
.subscribe("did:c", &watched, PARTICIPATING, &at(11), 9)
.unwrap();
assert_eq!(
reasons("did:c"),
["comment 9", "mention 7", "review-requested 6"]
);
assert!(index.subscriptions("did:c").unwrap().is_empty());
// Its owner watching hears of an issue as the issue, as before.
index
.subscribe("did:a", &watched, "all", &at(1), 9)
.unwrap();
assert!(reasons("did:a").contains(&"issue 2".to_string()));
assert!(!reasons("did:a").contains(&"watching 2".to_string()));
// Subscribed to the issue: its comments from then on, as such.
index
.subscribe("did:d", issue, "subscribed", &at(4), 9)
.unwrap();
assert_eq!(
reasons("did:d"),
["subscribed 7", "subscribed 8", "subscribed 9"]
);
// Subscribing again keeps it as of when it first did.
index
.subscribe("did:d", issue, "subscribed", &at(9), 9)
.unwrap();
assert_eq!(heard("did:d", Some("comments")).len(), 3);
let subs = index.subscriptions("did:d").unwrap();
assert_eq!(
(subs[0].at.as_str(), subs[0].repo_of()),
(at(4).as_str(), None)
);
assert_eq!(
index.subscription("did:d", issue).unwrap().as_deref(),
Some("subscribed")
);
// As another index's choices are given to this one.
let theirs = index
.notifications_as("did:x", &[], &subs, 20, None, None)
.unwrap()
.0;
assert_eq!(theirs.len(), 3);
// Watching the repository too, its comments are heard once.
index
.subscribe("did:d", &watched, "all", &at(1), 9)
.unwrap();
let mut once = reasons("did:d");
once.retain(|r| r.ends_with(" 7") || r.ends_with(" 9"));
assert_eq!(once, ["subscribed 7", "subscribed 9"]);
// Muted, the thread's comments and mentions go: its author's too.
index.subscribe("did:b", issue, "muted", &at(1), 9).unwrap();
assert!(reasons("did:b").is_empty());
index
.subscribe("did:c", issue, "muted", &at(11), 9)
.unwrap();
assert_eq!(reasons("did:c"), ["review-requested 6"]);
// A pull muted still says a review is asked of it.
index
.subscribe(
"did:c",
"at://did:b/dev.knotless.pull/6",
"muted",
&at(11),
9,
)
.unwrap();
assert_eq!(reasons("did:c"), ["review-requested 6"]);
assert_eq!(
index.subscription("did:c", "at://no/such/thing").unwrap(),
None
);
// A new choice past the most is kept not; one changed is.
assert!(
!index
.subscribe(
"did:c",
"at://did:b/dev.knotless.issue/9",
"muted",
&at(1),
2
)
.unwrap()
);
assert!(
index
.subscribe("did:c", issue, "subscribed", &at(12), 2)
.unwrap()
);
}
#[test]
fn notifications_of_a_kind_with_what_they_say() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let at = |day: u32| format!("2026-01-{day:02}T00:00:00Z");
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
// An issue opened on the account's repository, saying much.
let long = format!(" {} ", "é".repeat(TEXT_MAX + 5));
let issue = json!({"repo": repo, "title": "I", "body": long, "createdAt": at(2)});
index.put("did:b", nsid::ISSUE, "1", &issue).unwrap();
// A pull asking it for a review, saying nothing.
let asked = json!([{"did": "did:a", "requestedAt": at(3)}]);
let pull = json!({"repo": repo, "title": "P", "body": " ", "createdAt": at(3), "reviewers": asked});
index.put("did:b", nsid::PULL, "2", &pull).unwrap();
// On its own issue and pull: a comment, one naming it, and a review.
let mine = json!({"repo": repo, "title": "M", "createdAt": at(1)});
index.put("did:a", nsid::ISSUE, "3", &mine).unwrap();
index.put("did:a", nsid::PULL, "4", &mine).unwrap();
let on = |subject: &str, day: u32| json!({"repo": repo, "subject": subject, "body": "hi", "createdAt": at(day)});
index
.put(
"did:b",
nsid::COMMENT,
"5",
&on("at://did:a/dev.knotless.issue/3", 4),
)
.unwrap();
let mut naming = on("at://did:a/dev.knotless.issue/3", 5);
naming["facets"] = json!([{"index": {"byteStart": 0, "byteEnd": 1},
"features": [{"$type": "dev.knotless.defs#mention", "did": "did:a"}]}]);
index.put("did:b", nsid::COMMENT, "6", &naming).unwrap();
let mut review = on("at://did:a/dev.knotless.pull/4", 6);
review["verdict"] = json!("approve");
index.put("did:b", nsid::COMMENT, "7", &review).unwrap();
let star = json!({"subject": repo, "createdAt": at(7)});
index.put("did:b", nsid::STAR, "8", &star).unwrap();
let follow = json!({"subject": "did:a", "createdAt": at(8)});
index.put("did:b", nsid::FOLLOW, "9", &follow).unwrap();
// A mention on an issue the index has not: in no state it knows.
let mut elsewhere = on("at://did:z/dev.knotless.issue/9", 9);
elsewhere["facets"] = naming["facets"].clone();
index.put("did:b", nsid::COMMENT, "13", &elsewhere).unwrap();
let of = |kind: Option<&str>| index.notifications("did:a", &[], 10, None, kind).unwrap().0;
let reasons = |kind: &str| {
let mut found: Vec<_> = of(Some(kind)).into_iter().map(|n| n.reason).collect();
found.sort();
found
};
assert_eq!(of(None).len(), 8);
assert_eq!(reasons("mentions"), ["comment", "mention"]);
assert_eq!(reasons("reviews"), ["comment", "review-requested"]);
assert_eq!(reasons("comments"), ["comment", "comment"]);
assert_eq!(reasons("pulls-and-issues"), ["issue"]);
assert_eq!(reasons("stars-and-follows"), ["follow", "star"]);
assert!(reasons("checks").is_empty());
assert!(reasons("no-such-kind").is_empty());
// What each says, trimmed and cut; none when it says nothing.
let all = of(None);
let said = |reason: &str| all.iter().find(|n| n.reason == reason).unwrap().clone();
let text = said("issue").text.unwrap();
assert_eq!(text, "é".repeat(TEXT_MAX));
assert_eq!(said("comment").text.as_deref(), Some("hi"));
let asked = said("review-requested");
assert_eq!((asked.text, asked.reviewed), (None, false));
assert_eq!(asked.state.as_deref(), Some("open"));
assert_eq!((said("mention").state, said("star").state), (None, None));
assert_eq!(said("follow").text, None);
// Reviewed once it approves the pull, or asks for changes, after
// it was asked; a plain comment is no review.
let reviewed = || {
let all = of(Some("reviews")).into_iter();
all.filter(|n| n.reason == "review-requested")
.all(|n| n.reviewed)
};
let theirs = "at://did:b/dev.knotless.pull/2";
index
.put("did:a", nsid::COMMENT, "10", &on(theirs, 4))
.unwrap();
assert!(!reviewed());
let mut early = on(theirs, 2);
early["verdict"] = json!("changes");
index.put("did:a", nsid::COMMENT, "11", &early).unwrap();
assert!(!reviewed());
let mut given = on(theirs, 9);
given["verdict"] = json!("changes");
index.put("did:a", nsid::COMMENT, "12", &given).unwrap();
assert!(reviewed());
}
#[test]
fn invitations_until_taken_up_or_lapsed() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let now = knotless::records::now();
let org = json!({"createdAt": now});
index.put("did:o", nsid::ORG, "self", &org).unwrap();
let member = |did: &str, fields: Value| {
let mut m = json!({"member": did, "invitedAt": now});
m.as_object_mut()
.unwrap()
.extend(fields.as_object().unwrap().clone());
index.put("did:o", nsid::ORG_MEMBER, did, &m).unwrap();
};
member("did:a", json!({"roles": [{"role": "admin"}]}));
member("did:b", json!({}));
member("did:c", json!({"invitedAt": "2026-01-01T00:00:00.000Z"}));
member("did:d", json!({"joinedAt": now}));
member("did:e", json!({"expiresAt": "2026-01-01T00:00:00.000Z"}));
member(
"did:f",
json!({"invitedAt": "2026-01-01T00:00:00.000Z",
"expiresAt": "2999-01-01T00:00:00.000Z"}),
);
let heard = |did: &str| index.notifications(did, &[], 10, None, None).unwrap().0;
let invited = heard("did:a");
assert_eq!(invited.len(), 1);
let n = &invited[0];
assert_eq!(n.uri, "at://did:o/dev.knotless.orgMember/did:a");
assert_eq!((n.author.as_str(), n.reason.as_str()), ("did:o", "invite"));
assert_eq!((n.role.as_deref(), n.repo.as_ref()), (Some("admin"), None));
assert_eq!(heard("did:b")[0].role.as_deref(), Some("member"));
// Lapsed, by its age or by when it says, or taken up: nothing to
// answer. One that says it stands longer stands.
assert!(heard("did:c").is_empty());
assert!(heard("did:d").is_empty());
assert!(heard("did:e").is_empty());
assert_eq!(heard("did:f").len(), 1);
}
#[test]
fn reviewers_hear_once_a_pull_is_ready() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let asked = |at: &str| json!({"did": "did:c", "requestedAt": at});
let owner = json!({"did": "did:a"});
// It mentions whom it asks, who hears of it once, as asked.
let named = json!([{"index": {"byteStart": 0, "byteEnd": 1},
"features": [{"$type": "dev.knotless.defs#mention", "did": "did:c"}]}]);
let ready = json!({"repo": repo, "title": "P", "createdAt": "2026-01-01T00:00:00Z",
"reviewers": [asked("2026-01-02T00:00:00Z"), owner], "facets": named});
let draft = json!({"repo": repo, "title": "D", "createdAt": "2026-01-03T00:00:00Z",
"draft": true, "reviewers": [asked("2026-01-03T00:00:00Z"), owner]});
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
index.put("did:b", nsid::PULL, "1", &ready).unwrap();
index.put("did:b", nsid::PULL, "2", &draft).unwrap();
let heard = |did: &str| {
let (found, _) = index.notifications(did, &[], 10, None, None).unwrap();
let said = found
.into_iter()
.map(|n| (n.reason, n.title.unwrap(), n.at));
said.collect::<Vec<_>>()
};
let said = |reason: &str, title: &str, at: &str| {
(reason.to_string(), title.to_string(), at.to_string())
};
assert_eq!(
heard("did:c"),
[said("review-requested", "P", "2026-01-02T00:00:00Z")]
);
assert_eq!(
heard("did:a"),
[
said("pull", "D", "2026-01-03T00:00:00Z"),
said("review-requested", "P", "2026-01-01T00:00:00Z"),
]
);
}
#[test]
fn a_pulls_author_hears_its_checks_failed() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let ran = json!({"name": "r", "key": "k", "runners": [{"did": "did:ci"}, {"did": "did:ci2"}],
"createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::REPO, "r", &ran).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let pull = json!({"repo": repo, "title": "P", "createdAt": "2026-01-01T00:00:00Z"});
index.put("did:b", nsid::PULL, "1", &pull).unwrap();
let pull = "at://did:b/dev.knotless.pull/1";
let check = |by: &str, rkey: &str, workflow: &str, status: &str, at: &str| {
let run = json!({"repo": repo, "ref": "refs/heads/x", "commit": "c",
"baseCommit": "b", "pull": pull, "workflow": workflow, "status": status,
"startedAt": at, "finishedAt": at});
index.put(by, nsid::CHECK, rkey, &run).unwrap();
};
check("did:ci", "1", "test", "failure", "2026-01-02T00:00:00Z");
check("did:ci", "2", "web", "error", "2026-01-03T00:00:00Z");
check("did:ci", "3", "check", "success", "2026-01-02T00:00:00Z");
// Failed, then passed when run again: no longer heard of.
check("did:ci", "4", "docs", "failure", "2026-01-02T00:00:00Z");
check("did:ci", "5", "docs", "success", "2026-01-04T00:00:00Z");
// A runner the repository does not name is not heard.
check("did:x", "6", "lint", "failure", "2026-01-02T00:00:00Z");
// Failed on one runner it names, then passed on another.
check("did:ci", "10", "e2e", "failure", "2026-01-02T00:00:00Z");
check("did:ci2", "11", "e2e", "success", "2026-01-03T00:00:00Z");
let eject = |by: &str, rkey: &str| {
let q = json!({"repo": repo, "subject": pull, "state": "ejected",
"checks": [{"workflow": "test", "status": "failure"}],
"createdAt": "2026-01-05T00:00:00Z"});
index.put(by, nsid::QUEUE, rkey, &q).unwrap();
};
eject("did:ci", "7");
eject("did:x", "8");
// Heard of as well by one who hears only of whom it follows.
index
.set_notification_preferences("did:b", &[], true)
.unwrap();
let (heard, _) = index.notifications("did:b", &[], 10, None, None).unwrap();
let said: Vec<_> = heard
.iter()
.map(|n| (n.reason.as_str(), n.author.as_str(), n.at.as_str()))
.collect();
assert_eq!(
said,
[
("ejected", "did:ci", "2026-01-05T00:00:00Z"),
("checks-failed", "did:ci", "2026-01-02T00:00:00Z"),
]
);
fn named(n: &Notified) -> Vec<&str> {
n.checks.iter().map(|c| c.workflow.as_str()).collect()
}
assert_eq!(named(&heard[0]), ["test"]);
assert_eq!(named(&heard[1]), ["test", "web"]);
let failed = &heard[1];
assert_eq!(failed.subject.as_deref(), Some(pull));
assert_eq!(failed.uri, "at://did:ci/dev.knotless.check/1");
assert_eq!(failed.checks[1].status.as_deref(), Some("error"));
// The repository's owner hears of the pull, not of its checks.
let (owner, _) = index.notifications("did:a", &[], 10, None, None).unwrap();
let reasons: Vec<_> = owner.iter().map(|n| n.reason.as_str()).collect();
assert_eq!(reasons, ["pull"]);
let off = ["checks-failed".to_string(), "ejected".to_string()];
index
.set_notification_preferences("did:b", &off, false)
.unwrap();
assert!(
index
.notifications("did:b", &[], 10, None, None)
.unwrap()
.0
.is_empty()
);
let kept = index.heard("did:b", heard).unwrap();
assert!(kept.is_empty());
// Each says the pull is open; once it merges, a runner's word on
// it is no longer heard of, and what is about it says so.
index
.set_notification_preferences("did:b", &[], false)
.unwrap();
let states = |did: &str| {
let (found, _) = index.notifications(did, &[], 10, None, None).unwrap();
found
.into_iter()
.map(|n| (n.reason, n.state))
.collect::<Vec<_>>()
};
let open = Some("open".to_string());
assert_eq!(
states("did:b"),
[
("ejected".into(), open.clone()),
("checks-failed".into(), open)
]
);
let merged = json!({"repo": repo, "subject": pull, "state": "merged",
"createdAt": "2026-01-06T00:00:00Z"});
index.put("did:a", nsid::STATE, "3n", &merged).unwrap();
assert!(states("did:b").is_empty());
assert_eq!(states("did:a"), [("pull".into(), Some("merged".into()))]);
}
#[test]
fn devices_an_account_hears_on() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let (early, late) = ("2026-01-01T00:00:00.000Z", "2026-02-01T00:00:00.000Z");
let at = |token: &str| {
let all = index.push_devices().unwrap();
let found = all.into_iter().find(|d| d.token == token);
found.map(|d| (d.did, d.environment, d.at))
};
let held = |did: &str, env: &str, at: &str| Some((did.into(), env.into(), at.into()));
index
.register_push("did:a", "aa", "sandbox", early)
.unwrap();
assert_eq!(at("aa"), held("did:a", "sandbox", early));
// Registered again, as the app does each launch, it stays as far as
// it was told; its time only goes on.
index.pushed("aa", late).unwrap();
index.pushed("aa", early).unwrap();
let later = "2026-03-01T00:00:00.000Z";
index
.register_push("did:a", "aa", "production", later)
.unwrap();
assert_eq!(at("aa"), held("did:a", "production", late));
// Another account's now, it hears nothing from before.
index
.register_push("did:b", "aa", "production", later)
.unwrap();
assert_eq!(at("aa"), held("did:b", "production", later));
// Only its own account forgets it.
index.unregister_push("did:a", "aa").unwrap();
assert!(at("aa").is_some());
index.unregister_push("did:b", "aa").unwrap();
assert_eq!(at("aa"), None);
// The oldest of too many go.
for n in 0..=PUSH_DEVICES {
let when = format!("2026-01-01T00:00:{n:02}.000Z");
let token = format!("{n:02}");
index
.register_push("did:a", &token, "sandbox", &when)
.unwrap();
}
assert_eq!(index.push_devices().unwrap().len(), PUSH_DEVICES as usize);
assert_eq!(at("00"), None);
}
#[test]
fn notifications_as_their_account_prefers() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
let star = |did: &str, rkey: &str, at: &str| {
let subject = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let star = json!({"subject": subject, "createdAt": at});
index.put(did, nsid::STAR, rkey, &star).unwrap();
};
star("did:b", "1", "2026-01-01T00:00:00Z");
star("did:c", "2", "2026-01-02T00:00:00Z");
let follow = json!({"subject": "did:a", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:c", nsid::FOLLOW, "3", &follow).unwrap();
let heard = || {
let (found, _) = index.notifications("did:a", &[], 10, None, None).unwrap();
found.into_iter().map(|n| n.uri).collect::<Vec<_>>()
};
assert_eq!(heard().len(), 3);
assert_eq!(
index.notification_preferences("did:a").unwrap(),
(vec![], false)
);
// No stars, then only from those it follows: none yet.
let off = vec!["star".to_string()];
index
.set_notification_preferences("did:a", &off, false)
.unwrap();
assert_eq!(heard(), ["at://did:c/dev.knotless.follow/3"]);
index
.set_notification_preferences("did:a", &[], true)
.unwrap();
assert_eq!(heard(), Vec::<String>::new());
let follow = json!({"subject": "did:b", "createdAt": "2026-01-04T00:00:00Z"});
index.put("did:a", nsid::FOLLOW, "4", &follow).unwrap();
assert_eq!(heard(), ["at://did:b/dev.knotless.star/1"]);
assert_eq!(
index.notification_preferences("did:a").unwrap(),
(vec![], true)
);
// Another account's are its own.
assert_eq!(
index.notification_preferences("did:b").unwrap(),
(vec![], false)
);
}
#[test]
fn mentions_are_heard_of_by_whom_they_name() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
index.put("did:a", nsid::REPO, "r", &owned()).unwrap();
let repo = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let naming = |dids: &[&str]| {
let features: Vec<Value> = dids
.iter()
.map(|d| json!({"$type": "dev.knotless.defs#mention", "did": d}))
.collect();
json!([{"index": {"byteStart": 0, "byteEnd": 1}, "features": features}])
};
let issue = json!({"repo": repo, "title": "Hi @b.test", "createdAt": "2026-01-01T00:00:00Z",
"titleFacets": naming(&["did:b", "did:c"])});
index.put("did:c", nsid::ISSUE, "1", &issue).unwrap();
let issue_uri = "at://did:c/dev.knotless.issue/1";
let comment = |at: &str, dids: &[&str]| {
json!({"repo": repo, "subject": issue_uri, "body": "@b.test", "createdAt": at,
"verdict": null, "facets": naming(dids)})
};
index
.put(
"did:d",
nsid::COMMENT,
"2",
&comment("2026-01-02T00:00:00Z", &["did:b"]),
)
.unwrap();
let heard = |did: &str| {
let (found, _) = index.notifications(did, &[], 10, None, None).unwrap();
let found = found.into_iter().filter(|n| n.reason == "mention");
found
.map(|n| (n.uri, n.subject, n.title))
.collect::<Vec<_>>()
};
let comment_uri = "at://did:d/dev.knotless.comment/2".to_string();
let title = Some("Hi @b.test".to_string());
// b hears of both, the comment by the issue it is on; c, who wrote
// the issue, hears not of naming itself.
assert_eq!(
heard("did:b"),
[
(
comment_uri.clone(),
Some(issue_uri.to_string()),
title.clone()
),
(
issue_uri.to_string(),
Some(issue_uri.to_string()),
title.clone()
),
]
);
assert!(heard("did:c").is_empty());
// Changed, it is heard of once, as it names now.
index
.put(
"did:d",
nsid::COMMENT,
"2",
&comment("2026-01-02T00:00:00Z", &["did:b", "did:b"]),
)
.unwrap();
assert_eq!(heard("did:b").len(), 2);
// Not of what the owner hid, nor of whom it blocked.
let hide = json!({"repo": repo, "subject": issue_uri, "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:a", nsid::HIDE, "3", &hide).unwrap();
assert!(heard("did:b").is_empty());
index.delete("did:a", nsid::HIDE, "3").unwrap();
let block = json!({"repo": repo, "subject": "did:d", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:a", nsid::BLOCK, "4", &block).unwrap();
assert_eq!(heard("did:b").len(), 1);
index.delete("did:a", nsid::BLOCK, "4").unwrap();
let everywhere = json!({"subject": "did:c", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:a", nsid::BLOCK, "5", &everywhere).unwrap();
assert_eq!(
heard("did:b"),
[(comment_uri, Some(issue_uri.to_string()), title)]
);
// Taken back, it names no one.
index.delete("did:d", nsid::COMMENT, "2").unwrap();
assert!(heard("did:b").is_empty());
// Named first in an edit, as of the edit, so it is not among what
// was seen already; named again, as of when it first was.
let said = |at: &str, updated: Option<&str>, dids: &[&str]| {
let mut c = comment(at, dids);
c["updatedAt"] = updated.into();
index.put("did:d", nsid::COMMENT, "6", &c).unwrap();
};
let when = |did: &str| {
let (found, _) = index.notifications(did, &[], 10, None, None).unwrap();
let found = found.into_iter().map(|n| (n.reason, n.at));
found.collect::<Vec<_>>()
};
let mention = |at: &str| ("mention".to_string(), at.to_string());
said("2026-02-01T00:00:00Z", None, &[]);
assert!(when("did:e").is_empty());
said(
"2026-02-01T00:00:00Z",
Some("2026-02-02T00:00:00Z"),
&["did:e"],
);
assert_eq!(when("did:e"), [mention("2026-02-02T00:00:00Z")]);
said(
"2026-02-01T00:00:00Z",
Some("2026-02-03T00:00:00Z"),
&["did:e", "did:f"],
);
assert_eq!(when("did:e"), [mention("2026-02-02T00:00:00Z")]);
assert_eq!(when("did:f"), [mention("2026-02-03T00:00:00Z")]);
// A record heard of for another reason is heard of once, for that
// one, unless it is turned off: a comment naming the author of
// what it is on, and an issue naming the owner of its repository.
said(
"2026-02-01T00:00:00Z",
Some("2026-02-04T00:00:00Z"),
&["did:c"],
);
let comment_at = ("comment".to_string(), "2026-02-01T00:00:00Z".to_string());
assert_eq!(when("did:c"), [comment_at]);
let off = vec!["comment".to_string()];
index
.set_notification_preferences("did:c", &off, false)
.unwrap();
assert_eq!(when("did:c"), [mention("2026-02-04T00:00:00Z")]);
let naming_owner = json!({"repo": repo, "title": "@a.test", "createdAt": "2026-02-05T00:00:00Z",
"titleFacets": naming(&["did:a"])});
index.put("did:e", nsid::ISSUE, "7", &naming_owner).unwrap();
let first = &when("did:a")[0];
assert_eq!(first.0, "issue");
assert_eq!(
when("did:a").iter().filter(|(r, _)| r == "mention").count(),
0
);
}
#[test]
fn what_those_one_follows_did() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
// b and c each own a repository; a follows b, and stars c's.
for (did, at) in [("did:b", "01"), ("did:c", "02")] {
let repo = json!({"key": "r", "createdAt": format!("2026-01-{at}T00:00:00Z")});
index.put(did, nsid::REPO, "r", &repo).unwrap();
}
let follow = json!({"subject": "did:b", "createdAt": "2026-01-03T00:00:00Z"});
index.put("did:a", nsid::FOLLOW, "1", &follow).unwrap();
let star = |did: &str, rkey: &str, owner: &str, at: &str| {
let subject = json!({"uri": format!("at://{owner}/dev.knotless.repo/r"), "key": "r"});
let star = json!({"subject": subject, "createdAt": at});
index.put(did, nsid::STAR, rkey, &star).unwrap();
};
star("did:a", "2", "did:c", "2026-01-04T00:00:00Z");
star("did:b", "3", "did:c", "2026-01-05T00:00:00Z");
star("did:c", "4", "did:b", "2026-01-06T00:00:00Z");
// c opens an issue on its own repository, which a starred, and
// on b's, by someone a does not follow.
for (rkey, owner) in [("5", "did:c"), ("6", "did:b")] {
let repo = json!({"uri": format!("at://{owner}/dev.knotless.repo/r"), "key": "r"});
let issue = json!({"repo": repo, "title": "T", "createdAt": "2026-01-07T00:00:00Z"});
index.put("did:c", nsid::ISSUE, rkey, &issue).unwrap();
}
let a = Some("did:a");
let owners = |page: Vec<Listed>| page.into_iter().map(|r| r.did).collect::<Vec<_>>();
let (made, _) = index
.repos_page(
"",
10,
None,
Among {
followed_by: a,
..Among::default()
},
)
.unwrap();
assert_eq!(owners(made), ["did:b"]);
let stars = |by, followed_by| {
let found = index.stars(10, None, by, followed_by).unwrap();
found.into_iter().map(|s| s.uri).collect::<Vec<_>>()
};
// Its own and those of whom it follows, newest first; or one's.
assert_eq!(
stars(None, a),
[
"at://did:b/dev.knotless.star/3",
"at://did:a/dev.knotless.star/2"
]
);
assert_eq!(
stars(Some("did:c"), None),
["at://did:c/dev.knotless.star/4"]
);
let all = index.stars(10, None, None, None).unwrap();
assert_eq!(
(all.len(), all[0].owner.as_str(), all[0].name.as_str()),
(3, "did:b", "r")
);
let at = cursor_at(&all[0].created_at, &all[0].uri);
assert_eq!(index.stars(10, Some(&at), None, None).unwrap(), all[1..]);
// What anyone wrote on a repository it starred, but not elsewhere
// by those it does not follow.
let said = index.recent(10, None, None, a, None, false).unwrap();
let said: Vec<_> = said.into_iter().map(|(e, ..)| e.uri).collect();
assert_eq!(said, ["at://did:c/dev.knotless.issue/5"]);
// Issues alone are both; pulls and comments, none.
let only = |kind| {
index
.recent(10, None, None, None, Some(kind), false)
.unwrap()
.len()
};
assert_eq!((only("issue"), only("pull"), only("comment")), (2, 0, 0));
assert_eq!(
index
.recent(10, None, None, None, None, false)
.unwrap()
.len(),
2
);
assert!(index.pushes(10, None, None, a, false).unwrap().is_empty());
}
#[test]
fn a_database_of_another_shape() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("i.db");
// Claiming to be this version, so it is kept: every use fails.
Connection::open(&path)
.unwrap()
.execute_batch(&format!(
"PRAGMA user_version = {VERSION};
CREATE TABLE repo (did, key, name, fork_did, fork_key, created_at);
CREATE TABLE forge (author, repo_did, repo_key);"
))
.unwrap();
let index = Index::open(&path).unwrap();
assert!(index.put("did:a", nsid::REPO, "r", &json!({})).is_err());
let issue = json!({"repo": {"uri": "at://did:a/dev.knotless.repo/r", "key": "k"}});
assert!(index.put("did:a", nsid::ISSUE, "1", &issue).is_err());
assert!(index.repos("", 1).is_err());
assert!(index.forks("did:a", "k").is_err());
assert!(index.forge("did:a", "k").is_err());
assert!(index.recent(1, None, None, None, None, false).is_err());
assert!(index.pushes(1, None, None, None, false).is_err());
assert!(index.actor_days("did:a", None, "UTC").is_err());
assert!(index.actor_pushes("did:a", None, "UTC").is_err());
}
#[test]
fn profiles() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let bsky = json!({"displayName": "A"});
let link = json!({"url": "https://a.example"});
// An account whose profile is not kept: its records pass by.
index
.put("did:a", nsid::BSKY_PROFILE, "self", &bsky)
.unwrap();
assert!(!index.profile_kept("did:a").unwrap());
assert_eq!(index.profile("did:a").unwrap(), None);
// Kept, as read, then as the firehose changes it.
let read = [(nsid::BSKY_PROFILE.into(), "self".into(), bsky)];
index.keep_profile("did:a", &read).unwrap();
index.put("did:a", nsid::BSKY_LINK, "l", &link).unwrap();
index.delete("did:a", nsid::BSKY_PROFILE, "self").unwrap();
let kept = [(nsid::BSKY_LINK.to_string(), "l".to_string(), link)];
assert_eq!(index.profile("did:a").unwrap(), Some(kept.to_vec()));
// Read again, it is all there is.
index.keep_profile("did:a", &[]).unwrap();
assert_eq!(index.profile("did:a").unwrap(), Some(vec![]));
// Nothing of it is in what the feed shows.
assert_eq!(
index
.recent(10, None, None, None, None, false)
.unwrap()
.len(),
0
);
}
#[test]
fn moderation() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let named = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let issue = |did: &str, rkey: &str| {
index
.put(
did,
nsid::ISSUE,
rkey,
&json!({"repo": named, "title": "t"}),
)
.unwrap();
format!("at://{did}/dev.knotless.issue/{rkey}")
};
let forge = || -> Vec<String> {
index
.forge("did:a", "k")
.unwrap()
.into_iter()
.map(|e| e.uri)
.collect()
};
// Blocks from all the owner's repositories, and from this one.
let (first, second) = (issue("did:b", "1"), issue("did:c", "2"));
index
.put("did:a", nsid::BLOCK, "b", &json!({"subject": "did:c"}))
.unwrap();
assert_eq!(forge(), std::slice::from_ref(&first));
index.delete("did:a", nsid::BLOCK, "b").unwrap();
assert_eq!(forge().len(), 2);
index
.put(
"did:a",
nsid::BLOCK,
"c",
&json!({"subject": "did:c", "repo": named, "createdAt": "t"}),
)
.unwrap();
assert_eq!(
forge(),
[first.clone(), "at://did:a/dev.knotless.block/c".into()]
);
index.delete("did:a", nsid::BLOCK, "c").unwrap();
assert_eq!(forge().len(), 2);
// Takedowns, and the labels that say so.
let subject = json!({"$type": "com.atproto.repo.strongRef", "uri": first});
let (seq, label) = index
.take_down("did:web:me", &first, &subject, true)
.unwrap();
assert_eq!((seq, label["val"].as_str()), (1, Some("!takedown")));
assert_eq!(forge(), std::slice::from_ref(&second));
assert!(index.down(&["did:x", &first]).unwrap());
assert_eq!(index.taken_down(&first).unwrap(), Some(subject.clone()));
assert_eq!(index.labels("did:web:me", "at://did:b/").unwrap().len(), 1);
assert!(index.labels("did:web:me", "did:").unwrap().is_empty());
let (_, undone) = index
.take_down("did:web:me", &first, &subject, false)
.unwrap();
assert_eq!(undone["neg"], true);
assert_eq!(index.taken_down(&first).unwrap(), None);
assert_eq!(index.emitted(0).unwrap().len(), 2);
assert_eq!(index.emitted(1).unwrap()[0].1, undone);
// Labels from a labeler: only its own, only those that hide, and
// not past their expiry.
let label = |src: &str, uri: &str, val: &str, extra: Value| {
let mut l = json!({"src": src, "uri": uri, "val": val});
l.as_object_mut()
.unwrap()
.extend(extra.as_object().unwrap().clone());
l
};
assert_eq!(index.labeler_seq("did:web:l").unwrap(), None);
let none = json!({});
index
.label(
"did:web:l",
3,
&label("did:web:other", "did:c", "!takedown", none.clone()),
)
.unwrap();
index
.label(
"did:web:l",
4,
&label("did:web:l", "did:c", "porn", none.clone()),
)
.unwrap();
index
.label(
"did:web:l",
5,
&label(
"did:web:l",
"did:b",
"!hide",
json!({"exp": "2000-01-01T00:00:00Z"}),
),
)
.unwrap();
assert_eq!(forge().len(), 2);
index
.label(
"did:web:l",
6,
&label("did:web:l", "did:c", "!hide", none.clone()),
)
.unwrap();
assert_eq!(forge(), std::slice::from_ref(&first));
index
.label(
"did:web:l",
7,
&label("did:web:l", "did:c", "!hide", json!({"neg": true})),
)
.unwrap();
assert_eq!(forge().len(), 2);
assert_eq!(index.labeler_seq("did:web:l").unwrap(), Some(7));
// One that lapses hides until a sweep puts it back.
for (seq, exp) in [(8, "2999-01-01T00:00:00Z"), (9, "2000-01-01T00:00:00Z")] {
let hide = label("did:web:l", "did:c", "!hide", json!({"exp": exp}));
index.label("did:web:l", seq, &hide).unwrap();
assert_eq!(forge(), std::slice::from_ref(&first));
}
index.lapse().unwrap();
assert_eq!(forge().len(), 2);
// Reports, newest first, a page at a time.
for n in 0..3 {
let report = index.report(json!({"reason": n})).unwrap();
assert_eq!(report["id"], n + 1);
}
let page = index.reports(2, None).unwrap();
assert_eq!(
(page[0]["id"].as_i64(), page[1]["reason"].as_i64()),
(Some(3), Some(1))
);
assert_eq!(index.reports(2, Some(2)).unwrap()[0]["id"], 1);
// Takedowns, reports and pushes outlive a rebuild; the cache does
// not.
let claim = json!({"repo": "k", "name": "refs/heads/main", "to": "a1",
"createdAt": "2026-01-02T00:00:00Z"});
index.put("did:a", nsid::CLAIM, "k.m.1", &claim).unwrap();
index.delete("did:a", nsid::CLAIM, "k.m.1").unwrap();
drop(index);
let db = Connection::open(dir.path().join("i.db")).unwrap();
db.execute_batch("PRAGMA user_version = 1").unwrap();
drop(db);
let index = Index::open(&dir.path().join("i.db")).unwrap();
assert_eq!(index.reports(10, None).unwrap().len(), 3);
assert_eq!(index.emitted(0).unwrap().len(), 2);
assert_eq!(index.labeler_seq("did:web:l").unwrap(), None);
let pushes: i64 = index
.db
.lock()
.unwrap()
.query_row(
"SELECT count(*) FROM push -- scans: push, a test's count",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(pushes, 1);
}
#[test]
fn a_ref_made_again_keeps_each_making() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo = json!({"name": "r", "key": "k", "createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::REPO, "r", &repo).unwrap();
// Made twice, deleted between: each starts at the time, and says
// it was made by a createdAt as its updatedAt. A move is no making.
let at = |generation: u64, made: &str, moved: &str| {
json!({"repo": "k", "name": "refs/heads/x", "target": "a1",
"generation": generation, "createdAt": made, "updatedAt": moved})
};
let first = at(
1_800_000_000_000,
"2026-01-02T00:00:00Z",
"2026-01-02T00:00:00Z",
);
index.put("did:a", nsid::REF, "k.x", &first).unwrap();
let moved = at(
1_800_000_000_001,
"2026-01-02T00:00:00Z",
"2026-01-03T00:00:00Z",
);
index.put("did:a", nsid::REF, "k.x", &moved).unwrap();
index.delete("did:a", nsid::REF, "k.x").unwrap();
let again = at(
1_900_000_000_000,
"2026-01-04T00:00:00Z",
"2026-01-04T00:00:00Z",
);
index.put("did:a", nsid::REF, "k.x", &again).unwrap();
let uris: Vec<String> = index
.pushes(10, None, None, None, false)
.unwrap()
.into_iter()
.map(|p| p.uri)
.collect();
assert_eq!(
uris,
[
"at://did:a/dev.knotless.claim/k.x.1899999999999",
"at://did:a/dev.knotless.claim/k.x.1799999999999",
]
);
}
#[test]
fn pushers() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("i.db");
// A push table from before pushers were kept gains the column.
let db = Connection::open(&path).unwrap();
db.execute_batch(&format!(
"PRAGMA user_version = {VERSION};
CREATE TABLE push (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, key TEXT NOT NULL, ref TEXT NOT NULL,
before TEXT, after TEXT, created_at TEXT NOT NULL);"
))
.unwrap();
drop(db);
let index = Index::open(&path).unwrap();
let repo = json!({"name": "r", "key": "k", "createdAt": "2026-01-01T00:00:00Z",
"collaborators": [{"did": "did:b"}],
"runners": [{"did": "did:r"}, {"did": "did:b"}, {"did": "did:a"}]});
index.put("did:a", nsid::REPO, "r", &repo).unwrap();
// did:b made main, and moved it; did:r landed a batch on it; did:c,
// whom the repository does not name, moved it; and the owner did.
let made = json!({"repo": "k", "name": "refs/heads/main", "target": "a1",
"generation": 1, "updatedAt": "2026-01-02T00:00:00Z", "pusher": "did:b"});
index.put("did:a", nsid::REF, "k.m", &made).unwrap();
let claim = |rkey: &str, at: &str, pusher: Option<&str>| {
let mut c = json!({"repo": "k", "name": "refs/heads/main", "generation": 1,
"from": "a1", "to": "b2", "createdAt": at});
if let Some(p) = pusher {
c["pusher"] = json!(p);
}
index.put("did:a", nsid::CLAIM, rkey, &c).unwrap();
};
claim("k.m.1", "2026-01-03T00:00:00Z", Some("did:b"));
claim("k.m.2", "2026-01-04T00:00:00Z", Some("did:r"));
claim("k.m.3", "2026-01-05T00:00:00Z", Some("did:c"));
claim("k.m.4", "2026-01-06T00:00:00Z", None);
let by: Vec<String> = index
.pushes(10, None, None, None, false)
.unwrap()
.into_iter()
.map(|p| p.by)
.collect();
assert_eq!(by, ["did:a", "did:a", "did:r", "did:b", "did:b"]);
// Quiet, none of the runner's: the owner and a collaborator it
// names as runners too are people.
let by: Vec<String> = index
.pushes(10, None, None, None, true)
.unwrap()
.into_iter()
.map(|p| p.by)
.collect();
assert_eq!(by, ["did:a", "did:a", "did:b", "did:b"]);
// Nor of an account that says it is a bot.
let bot = json!({"bot": true});
index.put("did:b", nsid::PROFILE, "self", &bot).unwrap();
let by: Vec<String> = index
.pushes(10, None, None, None, true)
.unwrap()
.into_iter()
.map(|p| p.by)
.collect();
assert_eq!(by, ["did:a", "did:a"]);
assert!(index.is_bot("did:b").unwrap() && !index.is_bot("did:a").unwrap());
index.delete("did:b", nsid::PROFILE, "self").unwrap();
assert!(!index.is_bot("did:b").unwrap());
let of = |did| {
index
.pushes(10, None, Some(did), None, false)
.unwrap()
.len()
};
assert_eq!((of("did:a"), of("did:b"), of("did:c")), (2, 2, 0));
// Followers of did:b see its pushes to the owner's repository.
let follow = json!({"subject": "did:b", "createdAt": "2026-01-01T00:00:00Z"});
index.put("did:f", nsid::FOLLOW, "f", &follow).unwrap();
assert_eq!(
index
.pushes(10, None, None, Some("did:f"), false)
.unwrap()
.len(),
2
);
// Each account's days count the pushes it made; the owner's
// repository counts them all.
let since = Some("2026-01-01T00:00:00Z");
let pushes = |did| {
let days = index.actor_days(did, since, "UTC").unwrap();
days.iter()
.filter(|d| d.1 == "pushes")
.map(|d| d.2)
.sum::<u64>()
};
assert_eq!(
(pushes("did:a"), pushes("did:b"), pushes("did:c")),
(2, 2, 0)
);
let repos = index.actor_pushes("did:a", since, "UTC").unwrap();
assert_eq!(repos[0].days.iter().map(|d| d.1).sum::<u64>(), 5);
// Taken down, a pusher's pushes go from the feed.
index
.take_down("did:web:me", "did:b", &json!({"did": "did:b"}), true)
.unwrap();
assert_eq!(index.pushes(10, None, None, None, false).unwrap().len(), 3);
}
#[test]
fn what_happened() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo = json!({"name": "r", "key": "k", "createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::REPO, "r", &repo).unwrap();
// Found by its key, not its record's.
assert_eq!(index.repo_of("did:a", "k").unwrap(), Some(repo.clone()));
assert_eq!(index.repo_of("did:a", "r").unwrap(), None);
let claim = |rkey: &str, from: Option<&str>, to: Option<&str>, at: &str| {
let mut c =
json!({"repo": "k", "name": "refs/heads/main", "generation": 0, "createdAt": at});
if let Some(f) = from {
c["from"] = json!(f);
}
if let Some(t) = to {
c["to"] = json!(t);
}
index.put("did:a", nsid::CLAIM, rkey, &c).unwrap();
};
claim("k.m.0", None, Some("a1"), "2026-01-02T00:00:00Z");
claim("k.m.1", Some("a1"), Some("b2"), "2026-01-03T00:00:00Z");
// Of a repository the index does not know.
index
.put(
"did:a",
nsid::CLAIM,
"x.m.0",
&json!({"repo": "x", "name": "refs/heads/main", "createdAt": "2026-01-04T00:00:00Z"}),
)
.unwrap();
// The account's pushes, by repository and day: the last ever, and
// those since a day.
let pushed = index
.actor_pushes("did:a", Some("2026-01-03T00:00:00Z"), "UTC")
.unwrap();
assert_eq!(pushed.len(), 1);
assert_eq!(pushed[0].name, "r");
assert_eq!(pushed[0].pushed_at, "2026-01-03T00:00:00Z");
assert_eq!(pushed[0].days, [("2026-01-03".to_string(), 1)]);
// Midnight in UTC is the evening before in New York, and so is the
// day it is counted from.
let pushed = index
.actor_pushes("did:a", Some("2026-01-03T00:00:00Z"), "America/New_York")
.unwrap();
assert_eq!(pushed[0].days, [("2026-01-02".to_string(), 1)]);
// A zone that is none is UTC, and a time that is none is no day.
let days: (Option<String>, Option<String>) = index
.db
.lock()
.unwrap()
.query_row(
"SELECT zone_day('2026-01-03T00:00:00Z', 'Nowhere/Else'), zone_day('soon', 'UTC')",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.unwrap();
assert_eq!(days, (Some("2026-01-03".into()), None));
let pushes = index.pushes(10, None, None, None, false).unwrap();
assert_eq!(
pushes,
[
Push {
uri: "at://did:a/dev.knotless.claim/k.m.1".into(),
repo: "at://did:a/dev.knotless.repo/r".into(),
did: "did:a".into(),
name: "r".into(),
fork_of: None,
refname: "refs/heads/main".into(),
from: Some("a1".into()),
to: Some("b2".into()),
created_at: "2026-01-03T00:00:00Z".into(),
by: "did:a".into(),
},
Push {
uri: "at://did:a/dev.knotless.claim/k.m.0".into(),
repo: "at://did:a/dev.knotless.repo/r".into(),
did: "did:a".into(),
name: "r".into(),
fork_of: None,
refname: "refs/heads/main".into(),
from: None,
to: Some("a1".into()),
created_at: "2026-01-02T00:00:00Z".into(),
by: "did:a".into(),
},
]
);
let uri = "at://did:a/dev.knotless.repo/r";
let main = "refs/heads/main";
let tip = |at| index.tip_before(uri, main, at).unwrap();
assert_eq!(tip("2026-01-03T00:00:00Z"), Some("a1".into()));
assert_eq!(tip("2026-01-02T00:00:00Z"), None);
claim("k.m.2", Some("b2"), None, "2026-01-03T12:00:00Z");
assert_eq!(tip("2026-01-04T00:00:00Z"), None);
index.delete("did:a", nsid::CLAIM, "k.m.2").unwrap();
// Compaction drops claims; the history stays.
index.delete("did:a", nsid::CLAIM, "k.m.0").unwrap();
assert_eq!(index.pushes(10, None, None, None, false).unwrap().len(), 3);
assert_eq!(index.pushes(1, None, None, None, false).unwrap().len(), 1);
let all = index.pushes(10, None, None, None, false).unwrap();
let at = cursor_at(&all[0].created_at, &all[0].uri);
assert_eq!(
index.pushes(10, Some(&at), None, None, false).unwrap(),
all[1..]
);
// The other refs' tips, a deleted one's where it was.
let side = |rkey: &str, c: Value| {
let mut c = c;
c["repo"] = json!("k");
c["name"] = json!("refs/heads/side");
index.put("did:a", nsid::CLAIM, rkey, &c).unwrap();
};
side(
"k.s.0",
json!({"to": "s1", "createdAt": "2026-01-02T00:00:00Z"}),
);
let others = |at| index.tips_before(uri, main, at, "z").unwrap();
assert_eq!(others("2026-01-02T00:00:00Z"), ["s1"]);
assert!(others("2026-01-01T00:00:00Z").is_empty());
assert!(
index
.tips_before(uri, "refs/heads/side", "2026-01-02T00:00:00Z", "a")
.unwrap()
.is_empty()
);
side(
"k.s.1",
json!({"from": "s1", "createdAt": "2026-01-03T06:00:00Z"}),
);
assert_eq!(others("2026-01-04T00:00:00Z"), ["s1"]);
// A new ref, pushed with no claim: its first version is its push,
// under the key its claim would have had, so a ref an older push
// claimed is not counted twice. A later version is no push.
let version = |rkey: &str, name: &str, generation: u64, target: &str| {
let r = json!({"repo": "k", "name": name, "target": target,
"generation": generation, "updatedAt": "2026-01-04T00:00:00Z"});
index.put("did:a", nsid::REF, rkey, &r).unwrap();
};
version("k.t", "refs/tags/v1", 1, "t1");
version("k.t", "refs/tags/v1", 2, "t2");
version("k.m", main, 1, "a1");
let pushes = index.pushes(10, None, None, None, false).unwrap();
let tagged: Vec<_> = pushes
.iter()
.filter(|p| p.refname == "refs/tags/v1")
.collect();
assert_eq!(tagged.len(), 1);
assert_eq!(tagged[0].uri, "at://did:a/dev.knotless.claim/k.t.0");
assert_eq!(
(tagged[0].from.as_deref(), tagged[0].to.as_deref()),
(None, Some("t1"))
);
let first = pushes.iter().find(|p| p.uri.ends_with("/k.m.0")).unwrap();
assert_eq!(first.created_at, "2026-01-02T00:00:00Z");
// Pulls, issues and what was said, newest first, without what the
// owner hides or blocks, or those records themselves.
let named = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let at = |collection: &str, rkey: &str| format!("at://did:b/{collection}/{rkey}");
for (did, collection, rkey, at, subject) in [
("did:b", nsid::ISSUE, "1", "2026-01-05", None),
(
"did:b",
nsid::COMMENT,
"2",
"2026-01-07",
Some(at(nsid::ISSUE, "1")),
),
("did:b", nsid::PULL, "4", "2026-01-06", None),
("did:b", nsid::ISSUE, "5", "2026-01-09", None),
(
"did:b",
nsid::COMMENT,
"6",
"2026-01-10",
Some(at(nsid::ISSUE, "5")),
),
(
"did:a",
nsid::HIDE,
"7",
"2026-01-11",
Some(at(nsid::ISSUE, "5")),
),
("did:d", nsid::ISSUE, "8", "2026-01-12", None),
(
"did:a",
nsid::BLOCK,
"9",
"2026-01-13",
Some("did:d".into()),
),
("did:e", nsid::ISSUE, "10", "2026-01-14", None),
// Only the owner's are heeded.
(
"did:b",
nsid::HIDE,
"11",
"2026-01-15",
Some(at(nsid::PULL, "4")),
),
] {
let value = json!({"repo": named, "createdAt": at, "subject": subject});
index.put(did, collection, rkey, &value).unwrap();
}
index
.put("did:a", nsid::BLOCK, "12", &json!({"subject": "did:e"}))
.unwrap();
// A change of state shows; labels, the queue and a state record
// that only sets `base` change none.
let issue = at(nsid::ISSUE, "1");
for (collection, rkey, value) in [
(nsid::STATE, "13", json!({"state": "closed"})),
(nsid::STATE, "14", json!({"base": "trunk"})),
(nsid::STATE, "15", json!({"state": ""})),
(nsid::LABEL, "16", json!({"labels": [{"name": "area/web"}]})),
(nsid::QUEUE, "17", json!({"state": "queued"})),
] {
let mut value = value;
value["repo"] = named.clone();
value["subject"] = json!(issue);
let at = if rkey == "13" {
"2026-01-04"
} else {
"2026-01-08"
};
value["createdAt"] = json!(at);
index.put("did:b", collection, rkey, &value).unwrap();
}
// Nor does a thread on a pull's lines resolved.
let thread = at(nsid::COMMENT, "3");
let resolved = json!({"repo": named, "subject": thread, "state": "resolved", "createdAt": "2026-01-08"});
index.put("did:b", nsid::STATE, "18", &resolved).unwrap();
let recent = index.recent(10, None, None, None, None, false).unwrap();
let kinds: Vec<&str> = recent
.iter()
.map(|(e, _, _)| e.uri.rsplit('/').nth(1).unwrap())
.collect();
assert_eq!(kinds, [nsid::COMMENT, nsid::PULL, nsid::ISSUE, nsid::STATE]);
assert_eq!((recent[0].1.as_str(), recent[0].2.as_str()), ("did:a", "k"));
assert_eq!(
index
.recent(1, None, None, None, None, false)
.unwrap()
.len(),
1
);
// Paged by time and URI, the next page starting strictly after.
let first = &index.recent(1, None, None, None, None, false).unwrap()[0].0;
let at = cursor_at(first.value["createdAt"].as_str().unwrap(), &first.uri);
let rest = index
.recent(10, Some(&at), None, None, None, false)
.unwrap();
assert_eq!(rest.len(), 3);
assert_eq!(rest[0].0.uri, recent[1].0.uri);
// One account's alone.
let theirs = index
.recent(10, None, Some("did:b"), None, None, false)
.unwrap();
let theirs: Vec<_> = theirs.iter().map(|(e, ..)| e.uri.as_str()).collect();
let all: Vec<_> = recent.iter().map(|(e, ..)| e.uri.as_str()).collect();
let by_b: Vec<_> = all
.into_iter()
.filter(|u| u.starts_with("at://did:b/"))
.collect();
assert!(!by_b.is_empty());
assert_eq!(theirs, by_b);
// Quiet, of a bot of the repository, a triager it names, changes of
// state alone; of its owner, a runner it names too, all.
let repo = json!({"name": "r", "key": "k", "createdAt": "2026-01-01T00:00:00Z",
"runners": [{"did": "did:a"}], "triagers": [{"did": "did:b"}]});
index.put("did:a", nsid::REPO, "r", &repo).unwrap();
let quiet = index.recent(10, None, None, None, None, true).unwrap();
let quiet: Vec<_> = quiet.iter().map(|(e, ..)| e.uri.as_str()).collect();
let kept: Vec<_> = recent
.iter()
.map(|(e, ..)| e.uri.as_str())
.filter(|u| !u.starts_with("at://did:b/") || u.contains("/dev.knotless.state/"))
.collect();
assert!(kept.len() < recent.len());
assert_eq!(quiet, kept);
// As of an account that says it is a bot, wherever it writes,
// until it says so no more.
let plain = json!({"name": "r", "key": "k", "createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::REPO, "r", &plain).unwrap();
let says = |bot: bool| {
let profile = json!({"bot": bot});
index.put("did:b", nsid::PROFILE, "self", &profile).unwrap();
let quiet = index.recent(10, None, None, None, None, true).unwrap();
quiet.into_iter().map(|(e, ..)| e.uri).collect::<Vec<_>>()
};
assert_eq!(says(true), kept);
assert_eq!(says(false).len(), recent.len());
says(true);
index.delete("did:b", nsid::PROFILE, "self").unwrap();
let quiet = index.recent(10, None, None, None, None, true).unwrap();
assert_eq!(quiet.len(), recent.len());
// Gone with the repository.
index.delete("did:a", nsid::REPO, "r").unwrap();
assert!(
index
.pushes(10, None, None, None, false)
.unwrap()
.is_empty()
);
}
#[test]
fn an_index_of_another_version_is_rebuilt() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("i.db");
Connection::open(&path)
.unwrap()
.execute_batch("CREATE TABLE repo (did, created_at); CREATE TABLE forge (repo);")
.unwrap();
let index = Index::open(&path).unwrap();
index
.put("did:a", nsid::REPO, "r", &json!({"key": "k"}))
.unwrap();
assert_eq!(index.name("did:a", "k").unwrap().as_deref(), Some("r"));
}
/// A record from a space is kept by its URI in full there, with the
/// references it holds as written.
#[test]
fn records_in_a_space_are_kept_by_their_full_uri() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let space = knotless::space::uri("did:o", "r");
let forks = knotless::space::uri("did:f", "f");
let full =
|did: &str, collection: &str, rkey: &str| format!("{space}/{did}/{collection}/{rkey}");
let head = format!("{forks}/did:f/{}/f", nsid::REPO);
let fork = json!({"name": "f", "key": "fk", "createdAt": "2026-01-01"});
index
.put_from_space(&forks, "did:f", nsid::REPO, "f", &fork)
.unwrap();
let repo = json!({"name": "r", "key": "k", "createdAt": "2026-01-01"});
index
.put_from_space(&space, "did:o", nsid::REPO, "r", &repo)
.unwrap();
let named = json!({"uri": full("did:o", nsid::REPO, "r"), "key": "k"});
let pull = json!({"repo": named, "title": "t", "base": "main",
"head": {"repo": {"uri": head, "key": "fk"}, "branch": "b"},
"createdAt": "2026-01-02T00:00:00Z"});
index
.put_from_space(&space, "did:a", nsid::PULL, "p", &pull)
.unwrap();
let said = json!({"repo": named, "subject": full("did:a", nsid::PULL, "p"),
"body": "b", "createdAt": "2026-01-03T00:00:00Z"});
index
.put_from_space(&space, "did:b", nsid::COMMENT, "c", &said)
.unwrap();
let entries = index.forge("did:o", "k").unwrap();
let uris: Vec<&str> = entries.iter().map(|e| e.uri.as_str()).collect();
let (pull_uri, comment_uri) = (
full("did:a", nsid::PULL, "p"),
full("did:b", nsid::COMMENT, "c"),
);
assert!(
uris.contains(&pull_uri.as_str()) && uris.contains(&comment_uri.as_str()),
"{uris:?}"
);
let comment = entries.iter().find(|e| e.uri == comment_uri).unwrap();
assert_eq!(comment.value["subject"], pull_uri);
let opened = entries.iter().find(|e| e.uri == pull_uri).unwrap();
assert_eq!(opened.value["head"]["repo"]["uri"], head);
// A repository is found by its record's URI in either form.
let short = "at://did:o/dev.knotless.repo/r";
assert!(index.repo(short).unwrap().is_some());
assert!(
index
.repo(&full("did:o", nsid::REPO, "r"))
.unwrap()
.is_some()
);
assert!(
index
.repo("at://did:x/dev.knotless.repo/r")
.unwrap()
.is_none()
);
// A head in no space the index knows is left as it was written.
let elsewhere = json!({"repo": named, "head": {"repo": {"uri": "at://did:g/dev.knotless.repo/g", "key": "gk"}}});
index
.put_from_space(&space, "did:a", nsid::PULL, "q", &elsewhere)
.unwrap();
let entries = index.forge("did:o", "k").unwrap();
let q = entries
.iter()
.find(|e| e.uri == full("did:a", nsid::PULL, "q"))
.unwrap();
assert_eq!(
q.value["head"]["repo"]["uri"],
"at://did:g/dev.knotless.repo/g"
);
// Taken out by the same name.
index
.delete_from_space(&space, "did:b", nsid::COMMENT, "c")
.unwrap();
let entries = index.forge("did:o", "k").unwrap();
assert!(!entries.iter().any(|e| e.uri == comment_uri));
}
#[test]
fn accounts_away_and_gone() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo = json!({"name": "r", "key": "k", "createdAt": "2026-01-01"});
index.put("did:a", nsid::REPO, "r", &repo).unwrap();
let named = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let issue = json!({"repo": named, "title": "t"});
index.put("did:a", nsid::ISSUE, "1", &issue).unwrap();
index.put("did:b", nsid::ISSUE, "2", &issue).unwrap();
let claim = json!({"repo": "k", "name": "refs/heads/main", "to": "a1",
"createdAt": "2026-01-02T00:00:00Z"});
index.put("did:a", nsid::CLAIM, "k.m.1", &claim).unwrap();
index
.put("did:a", nsid::STAR, "s", &json!({"subject": named}))
.unwrap();
index.set_handle("did:a", Some("a.test"), 0).unwrap();
index.see("did:a", "2026-01-03").unwrap();
// In a space it owns, a collaborator's record.
let space = knotless::space::uri("did:a", "p");
index.follow_space(&space, "did:a").unwrap();
let private = json!({"name": "p", "key": "pk", "createdAt": "2026-01-01"});
index
.put_from_space(&space, "did:c", nsid::REPO, "p", &private)
.unwrap();
let taken = json!({"did": "did:a"});
index
.take_down("did:web:me", "did:a", &taken, true)
.unwrap();
index
.take_down("did:web:me", "did:a", &taken, false)
.unwrap();
index.report(json!({"subject": taken})).unwrap();
let shown = || index.repos("", 10).unwrap().len();
assert_eq!(shown(), 2);
// Deactivated, it is hidden, and back when it is active again.
index.set_status("did:a", "deactivated").unwrap();
assert_eq!(
index.status("did:a").unwrap().as_deref(),
Some("deactivated")
);
assert!(index.down(&["did:a"]).unwrap());
assert_eq!(shown(), 1);
index.set_status("did:a", "active").unwrap();
assert_eq!(index.status("did:a").unwrap(), None);
assert_eq!(shown(), 2);
// Deleted, all of it goes, but what others wrote and what this
// server holds about it.
index.set_status("did:a", "takendown").unwrap();
index.purge("did:a").unwrap();
assert_eq!(index.status("did:a").unwrap(), None);
assert_eq!(index.repos("", 10).unwrap().len(), 0);
let left = index.forge("did:a", "k").unwrap();
assert_eq!(
left.iter().map(|e| e.uri.as_str()).collect::<Vec<_>>(),
["at://did:b/dev.knotless.issue/2"]
);
assert_eq!(index.handle("did:a").unwrap(), None);
assert_eq!(index.seen("did:a").unwrap(), None);
assert!(index.spaces().unwrap().is_empty());
assert!(index.actor_stars("did:a").unwrap().is_empty());
let db = index.db.lock().unwrap();
let count = |table: &str| -> i64 {
db.query_row(
&format!("SELECT count(*) FROM {table} -- scans: {table}, a test's count"),
[],
|row| row.get(0),
)
.unwrap()
};
assert_eq!((count("push"), count("space_record")), (0, 0));
assert_eq!((count("emitted"), count("report")), (2, 1));
}
#[test]
fn organizations() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let (then, took) = ("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z");
let org = json!({"gateway": "did:web:gw", "createdAt": then});
index.put("did:o", nsid::ORG, "self", &org).unwrap();
let admin = json!({"member": "did:m", "roles": [{"role": "admin"}],
"invitedAt": then, "joinedAt": took});
index
.put("did:o", nsid::ORG_MEMBER, "did:m", &admin)
.unwrap();
let invited = json!({"member": "did:i", "invitedAt": took, "expiresAt": took});
index
.put("did:o", nsid::ORG_MEMBER, "did:i", &invited)
.unwrap();
// Keyed other than self, it is no organization's record; keyed
// other than by the DID it names, no member's.
index.put("did:x", nsid::ORG, "other", &org).unwrap();
assert!(!index.is_org("did:x").unwrap());
index
.put("did:o", nsid::ORG_MEMBER, "did:z", &admin)
.unwrap();
assert!(index.is_org("did:o").unwrap());
let named = |space: &str| -> Vec<(String, bool)> {
let roster = index.org_roster("did:o", space).unwrap();
roster.into_iter().map(|m| (m.did, m.joined)).collect()
};
let pair = |did: &str, joined| (did.to_string(), joined);
assert_eq!(named(""), [pair("did:m", false), pair("did:i", false)]);
let roster = index.org_roster("did:o", "").unwrap();
assert_eq!(roster[0].role.as_deref(), Some("admin"));
assert_eq!(roster[1].role.as_deref(), Some("member"));
assert_eq!(roster[1].expires_at.as_deref(), Some(took));
// Nor does one keyed other than self touch the one that is.
index.put("did:o", nsid::ORG, "other", &org).unwrap();
index.delete("did:o", nsid::ORG, "other").unwrap();
assert!(index.is_org("did:o").unwrap());
assert_eq!(
index.org_gateway("did:o").unwrap().as_deref(),
Some("did:web:gw")
);
// A record of joining counts keyed by the organization it names.
let joined = json!({"subject": "did:o", "createdAt": took});
index
.put("did:m", nsid::ORG_ACCEPTANCE, "did:z", &joined)
.unwrap();
assert!(index.orgs_of("did:m", false).unwrap().is_empty());
index
.put("did:m", nsid::ORG_ACCEPTANCE, "did:o", &joined)
.unwrap();
index
.put("did:i", nsid::ORG_ACCEPTANCE, "did:o", &joined)
.unwrap();
assert_eq!(named(""), [pair("did:m", true), pair("did:i", true)]);
// Not yet taken in, an invitee is no member.
assert_eq!(index.orgs_of("did:m", false).unwrap(), ["did:o"]);
assert!(index.orgs_of("did:i", false).unwrap().is_empty());
// A member's push is theirs; an invitee's is the owner's.
let repo = json!({"name": "r", "key": "k", "createdAt": then});
index.put("did:o", nsid::REPO, "r", &repo).unwrap();
for (rkey, pusher) in [("k.m.1", "did:m"), ("k.m.2", "did:i")] {
let claim = json!({"repo": "k", "name": "refs/heads/main", "to": "a1",
"createdAt": took, "pusher": pusher});
index.put("did:o", nsid::CLAIM, rkey, &claim).unwrap();
}
let pushes = index.pushes(10, None, None, None, false).unwrap();
let mut by: Vec<String> = pushes.into_iter().map(|p| p.by).collect();
by.sort();
assert_eq!(by, ["did:m", "did:o"]);
// Its space's records stand apart from its public ones.
let space = knotless::org::space_uri("did:o");
let hidden = json!({"member": "did:h", "invitedAt": then, "joinedAt": took});
index
.put_from_space(&space, "did:o", nsid::ORG_MEMBER, "did:h", &hidden)
.unwrap();
index
.put_from_space(&space, "did:h", nsid::ORG_ACCEPTANCE, "did:o", &joined)
.unwrap();
assert_eq!(named(&space), [pair("did:h", true)]);
assert_eq!(named("").len(), 2);
assert_eq!(index.orgs_of("did:h", true).unwrap(), ["did:o"]);
assert!(index.orgs_of("did:h", false).unwrap().is_empty());
assert!(index.orgs_of("did:m", true).unwrap().is_empty());
// Named there, an account may be a member of the space.
index.follow_space(&space, "did:o").unwrap();
assert_eq!(index.maybe_member("did:h").unwrap(), vec![space.clone()]);
index
.delete_from_space(&space, "did:o", nsid::ORG_MEMBER, "did:h")
.unwrap();
assert!(index.org_roster("did:o", &space).unwrap().is_empty());
assert_eq!(named("").len(), 2);
// Leaving ends it; an account deleted takes its rows with it.
index
.delete("did:m", nsid::ORG_ACCEPTANCE, "did:o")
.unwrap();
assert!(index.orgs_of("did:m", false).unwrap().is_empty());
index.purge("did:o").unwrap();
index.purge("did:i").unwrap();
assert!(!index.is_org("did:o").unwrap());
let db = index.db.lock().unwrap();
let count = |table: &str| -> i64 {
db.query_row(
&format!("SELECT count(*) FROM {table} -- scans: {table}, a test's count"),
[],
|row| row.get(0),
)
.unwrap()
};
assert_eq!(
(count("org"), count("org_member"), count("membership")),
(0, 0, 0)
);
}
#[test]
fn writes_tell_of_their_repositories() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
// With no one following, nothing is told.
index
.put("did:a", nsid::REPO, "r", &json!({"key": "k"}))
.unwrap();
let mut told = index.changes.subscribe();
let mut heard = || {
let mut all = vec![];
while let Ok(repo) = told.try_recv() {
all.push(repo.to_string());
}
all
};
let named = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
index
.put("did:a", nsid::REPO, "r", &json!({"key": "k"}))
.unwrap();
// A pull into it from a fork, whose pushes are heard of there too.
let head =
json!({"repo": {"uri": "at://did:b/dev.knotless.repo/f", "key": "fk"}, "branch": "b"});
let pull = json!({"repo": named, "head": head, "title": "p"});
index.put("did:b", nsid::PULL, "1", &pull).unwrap();
index
.put(
"did:b",
nsid::CLAIM,
"c",
&json!({"repo": "fk", "name": "refs/heads/b"}),
)
.unwrap();
index
.put("did:c", nsid::STAR, "s", &json!({"subject": named}))
.unwrap();
index
.put("did:r", nsid::CHECK, "x", &json!({"repo": named}))
.unwrap();
index
.put("did:c", nsid::FOLLOW, "f", &json!({"subject": "did:a"}))
.unwrap();
let release = json!({"repo": "k", "package": "p", "version": "1.0.0", "files": [],
"createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::RELEASE, "v", &release).unwrap();
assert_eq!(
heard(),
[
"did:a/k", "did:a/k", "did:b/fk", "did:a/k", "did:a/k", "did:a/k", "did:a/k"
]
);
// What goes is about what it was about.
index.delete("did:b", nsid::PULL, "1").unwrap();
index.delete("did:r", nsid::CHECK, "x").unwrap();
index.delete("did:c", nsid::FOLLOW, "f").unwrap();
index.delete("did:a", nsid::RELEASE, "v").unwrap();
assert_eq!(heard(), ["did:a/k", "did:a/k", "did:a/k"]);
// A yank is about the repository its release is of.
let yank = json!({"repo": "k", "createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::RELEASE_YANK, "v", &yank).unwrap();
assert_eq!(heard(), ["did:a/k"]);
}
#[test]
fn a_yank_marks_its_release_until_it_goes() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let release = json!({"repo": "k", "package": "p", "version": "1.0.0", "files": [],
"createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::RELEASE, "v", &release).unwrap();
let yanked = |did: &str| index.releases(did, "k").unwrap()[0].1.yanked;
assert!(!yanked("did:a"));
let yank = json!({"repo": "k", "createdAt": "2026-01-02T00:00:00Z"});
// Another account's record of the same key says nothing of it.
index.put("did:b", nsid::RELEASE_YANK, "v", &yank).unwrap();
assert!(!yanked("did:a"));
index.put("did:a", nsid::RELEASE_YANK, "v", &yank).unwrap();
assert!(yanked("did:a"));
index.delete("did:a", nsid::RELEASE_YANK, "v").unwrap();
assert!(!yanked("did:a"));
}
#[test]
fn records_in_and_out() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let repo =
json!({"name": "r", "key": "k", "description": "a thing", "createdAt": "2026-01-01"});
index.put("did:a", nsid::REPO, "r", &repo).unwrap();
let uri = "at://did:a/dev.knotless.repo/r";
let named = json!({"uri": uri, "key": "k"});
let fork = json!({"name": "r", "key": "k2", "forkOf": named, "createdAt": "2026-01-02"});
index.put("did:b", nsid::REPO, "r", &fork).unwrap();
index
.put(
"did:b",
nsid::ISSUE,
"1",
&json!({"repo": named, "title": "t"}),
)
.unwrap();
// About a repository since deleted, whose name was taken again.
let gone = json!({"uri": uri, "key": "k0"});
index
.put(
"did:b",
nsid::ISSUE,
"3",
&json!({"repo": gone, "title": "old"}),
)
.unwrap();
index
.put("did:b", nsid::ISSUE, "2", &json!({"title": "no repo"}))
.unwrap();
assert_eq!(index.repo(uri).unwrap(), Some(repo));
assert_eq!(index.repo("at://did:x/dev.knotless.repo/r").unwrap(), None);
assert_eq!(index.forge("did:a", "k").unwrap().len(), 1);
assert_eq!(index.name("did:a", "k").unwrap().as_deref(), Some("r"));
assert_eq!(index.name("did:a", "k0").unwrap(), None);
let all = index.repos("", 10).unwrap();
assert_eq!(
all.iter().map(|r| r.did.as_str()).collect::<Vec<_>>(),
["did:b", "did:a"]
);
assert_eq!(index.repos("THING", 10).unwrap().len(), 1);
let (page, cursor) = index.repos_page("", 1, None, Among::default()).unwrap();
assert_eq!(
(page[0].did.as_str(), cursor.as_deref()),
("did:b", Some("2026-01-02 at://did:b/dev.knotless.repo/r"))
);
let (page, cursor) = index
.repos_page("", 1, cursor.as_deref(), Among::default())
.unwrap();
assert_eq!(page[0].did, "did:a");
assert_eq!(
index
.repos_page("", 1, cursor.as_deref(), Among::default())
.unwrap(),
(vec![], None)
);
// Those naming a runner, a triager or a topic, for it to find them.
let ran = json!({"name": "s", "key": "k3", "runners": [{"did": "did:ci"}], "triagers": [{"did": "did:bot", "scopes": [{"name": "kind"}]}], "topics": [{"name": "rust"}], "createdAt": "2026-01-03"});
index.put("did:a", nsid::REPO, "s", &ran).unwrap();
let named_by = [
(Some("did:ci"), None, None),
(None, Some("did:bot"), None),
(None, None, Some("rust")),
];
for (runner, triager, topic) in named_by {
let (named, _) = index
.repos_page(
"",
10,
None,
Among {
runner,
triager,
topic,
..Among::default()
},
)
.unwrap();
assert_eq!(
named.iter().map(|r| r.name.as_str()).collect::<Vec<_>>(),
["s"]
);
}
assert!(
index
.repos_page(
"",
10,
None,
Among {
triager: Some("did:ci"),
..Among::default()
}
)
.unwrap()
.0
.is_empty()
);
assert!(
index
.repos_page(
"",
10,
None,
Among {
runner: Some("did:x"),
..Among::default()
}
)
.unwrap()
.0
.is_empty()
);
assert!(
index
.repos_page(
"",
10,
None,
Among {
topic: Some("go"),
..Among::default()
}
)
.unwrap()
.0
.is_empty()
);
index.delete("did:a", nsid::REPO, "s").unwrap();
let listed = serde_json::to_value(&page[0]).unwrap();
assert!(listed.get("forkOf").is_none() && listed["description"] == "a thing");
assert_eq!(index.forks("did:a", "k").unwrap()[0].did, "did:b");
let all = index.repos("", 10).unwrap();
assert_eq!(all[0].fork_of.as_deref(), Some(uri));
index.delete("did:b", nsid::ISSUE, "1").unwrap();
index.delete("did:a", nsid::REPO, "r").unwrap();
assert!(index.forge("did:a", "k").unwrap().is_empty());
assert_eq!(index.repos("", 10).unwrap().len(), 1);
// Checks: only those by the runners asked for, newest first.
let check = |workflow: &str, r: &str, sha: &str, at: &str| {
json!({"repo": {"uri": uri, "key": "k"}, "ref": r, "commit": sha,
"workflow": workflow, "status": "success", "startedAt": at})
};
let runners = ["did:ci".to_string()];
index
.put(
"did:ci",
nsid::CHECK,
"c1",
&check("ci", "refs/heads/main", "a", "2026-01-01"),
)
.unwrap();
index
.put(
"did:ci",
nsid::CHECK,
"c2",
&check("lint", "refs/heads/x", "b", "2026-01-02"),
)
.unwrap();
index
.put(
"did:other",
nsid::CHECK,
"c3",
&check("ci", "refs/heads/main", "a", "2026-01-03"),
)
.unwrap();
index
.put(
"did:ci",
nsid::CHECK,
"c4",
&json!({"ref": "refs/heads/main"}),
)
.unwrap();
let all = CheckFilter::default();
let (page, next) = index.checks("did:a", "k", &runners, all, 10, None).unwrap();
assert_eq!(
page.iter()
.map(|e| e.value["workflow"].as_str().unwrap())
.collect::<Vec<_>>(),
["lint", "ci"]
);
assert_eq!(next, None);
let rerun = json!({"repo": {"uri": uri, "key": "k"}, "ref": "refs/heads/main"});
index.put("did:a", nsid::RERUN, "k.1", &rerun).unwrap();
assert!(index.forge("did:a", "k").unwrap().is_empty());
let (page, next) = index.checks("did:a", "k", &runners, all, 1, None).unwrap();
assert_eq!(page[0].uri, "at://did:ci/dev.knotless.check/c2");
let (page, _) = index
.checks("did:a", "k", &runners, all, 1, next.as_deref())
.unwrap();
assert_eq!(page[0].uri, "at://did:ci/dev.knotless.check/c1");
let main = CheckFilter {
refname: Some("refs/heads/main"),
..CheckFilter::default()
};
assert_eq!(
index
.checks("did:a", "k", &runners, main, 10, None)
.unwrap()
.0
.len(),
1
);
let lint = CheckFilter {
workflow: Some("lint"),
..CheckFilter::default()
};
let (page, _) = index
.checks("did:a", "k", &runners, lint, 10, None)
.unwrap();
assert_eq!(page.len(), 1);
assert_eq!(page[0].uri, "at://did:ci/dev.knotless.check/c2");
let of_b = CheckFilter {
commit: Some("b"),
..CheckFilter::default()
};
assert_eq!(
index
.checks("did:a", "k", &runners, of_b, 10, None)
.unwrap()
.0
.len(),
1
);
let shas = ["a".to_string(), "b".to_string(), "z".to_string()];
let of_page = CheckFilter {
commits: Some(&shas),
..CheckFilter::default()
};
let (page, _) = index
.checks("did:a", "k", &runners, of_page, 10, None)
.unwrap();
assert_eq!(page.len(), 2);
let mut of_pull = check("ci", "refs/heads/topic", "c", "2026-01-04");
of_pull["pull"] = "at://did:b/dev.knotless.pull/p".into();
index.put("did:ci", nsid::CHECK, "c5", &of_pull).unwrap();
let pull = CheckFilter {
pull: Some("at://did:b/dev.knotless.pull/p"),
..CheckFilter::default()
};
let (page, _) = index
.checks("did:a", "k", &runners, pull, 10, None)
.unwrap();
assert_eq!(page[0].uri, "at://did:ci/dev.knotless.check/c5");
assert_eq!(page.len(), 1);
let mut of_batch = check("ci", "refs/heads/main", "d", "2026-01-05");
of_batch["batch"] = json!([{"pull": "at://did:b/dev.knotless.pull/p", "commit": "d"}]);
index.put("did:ci", nsid::CHECK, "c6", &of_batch).unwrap();
let batches = CheckFilter {
batch: true,
..main
};
let (page, _) = index
.checks("did:a", "k", &runners, batches, 10, None)
.unwrap();
assert_eq!(page[0].uri, "at://did:ci/dev.knotless.check/c6");
assert_eq!(page.len(), 1);
index.delete("did:ci", nsid::CHECK, "c6").unwrap();
index.delete("did:ci", nsid::CHECK, "c5").unwrap();
assert!(
index
.checks("did:a", "k", &[], all, 10, None)
.unwrap()
.0
.is_empty()
);
index.delete("did:ci", nsid::CHECK, "c2").unwrap();
assert_eq!(
index
.checks("did:a", "k", &runners, all, 10, None)
.unwrap()
.0
.len(),
1
);
// A second open finds the tables already there.
drop(index);
assert!(Index::open(&dir.path().join("i.db")).is_ok());
assert!(Index::open(dir.path()).is_err());
}
#[test]
fn sites_are_kept_by_their_repository() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let file =
|hash: &str| json!({"path": "index.html", "size": 1, "sha256": hash, "parts": []});
let hash = "0".repeat(64);
let site = json!({"files": [file(&hash)], "createdAt": "2026-11-03T00:00:00Z"});
index.put("did:a", nsid::SITE, "k", &site).unwrap();
let kept = index.site("did:a", "k").unwrap().unwrap();
assert_eq!(serde_json::from_str::<Value>(&kept).unwrap(), site);
// Not a site's record: passed by.
index
.put("did:a", nsid::SITE, "j", &json!({"files": 1}))
.unwrap();
assert_eq!(index.site("did:a", "j").unwrap(), None);
// One in a space is its private repository's, found there alone.
let space = "at://did:a/space/dev.knotless.space/s";
index
.put_from_space(space, "did:a", nsid::SITE, "i", &site)
.unwrap();
assert_eq!(index.site("did:a", "i").unwrap(), None);
let private = index.space_site(space, "did:a", "i").unwrap().unwrap();
assert_eq!(serde_json::from_str::<Value>(&private).unwrap(), site);
// A hash that would name a path outside where files are kept.
let out = format!("{:.<64}", "../../../../etc/passwd");
let bad = json!({"files": [file(&out)], "createdAt": "2026-11-03T00:00:00Z"});
index.put("did:a", nsid::SITE, "h", &bad).unwrap();
assert_eq!(index.site("did:a", "h").unwrap(), None);
index.delete("did:a", nsid::SITE, "k").unwrap();
assert_eq!(index.site("did:a", "k").unwrap(), None);
}
}