chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
Searching repositories finds those named as searched first, and your own
crates/knotless-serve/src/appview.rs+72−7
| @@ -144,9 +144,13 @@ pub struct ListParams { | ||
| ||
/// Repositories, newest first, matching `q` by name or description and | ||
/// naming `runner` among their runners, `triager` among their triagers | ||
/// and `topic` among their topics, a page at a time. Asked as an account, | ||
/// the first page of a search with none of these starts with the private | ||
/// repositories it may read here that match too ([`readable`]). | ||
/// and `topic` among their topics, a page at a time. The first page of a | ||
/// search with none of these starts with the repositories named `q` | ||
/// ([`Index::named`](crate::index::Index::named)), originals before forks; | ||
/// asked as an account, its own of those first, and before them the | ||
/// private repositories it may read here that match ([`readable`]), and | ||
/// after them its other public ones that match. Each kind put first, when | ||
/// all of it fits on that page, no page lists again. | ||
pub async fn list_repos( | ||
State(app): State<Arc<App>>, | ||
axum::Extension(viewer): axum::Extension<crate::viewer::Viewer>, | ||
| @@ -154,12 +158,18 @@ pub async fn list_repos( | ||
) -> Result<Json<Value>, AppError> { | ||
let limit = p.limit.unwrap_or(50).clamp(1, 500); | ||
let q = p.q.as_deref().unwrap_or_default(); | ||
let (repos, cursor) = index(&app)? | ||
let named = p.runner.is_some() || p.triager.is_some() || p.topic.is_some(); | ||
let first = p.cursor.is_none() && !named; | ||
let index = index(&app)?; | ||
let (exact, own, not_named, not_owner) = first_of(index, q, limit, named, viewer.0.as_deref())?; | ||
let (repos, cursor) = index | ||
.repos_page( | ||
q, | ||
limit, | ||
p.cursor.as_deref(), | ||
crate::index::Among { | ||
not_named, | ||
not_owner, | ||
runner: p.runner.as_deref(), | ||
triager: p.triager.as_deref(), | ||
topic: p.topic.as_deref(), | ||
| @@ -168,14 +178,16 @@ pub async fn list_repos( | ||
) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
let named = p.runner.is_some() || p.triager.is_some() || p.topic.is_some(); | ||
let first = p.cursor.is_none() && !named; | ||
let private = match viewer.0.filter(|_| first).zip(app.spaces.as_deref()) { | ||
Some((viewer, spaces)) => readable(&app, spaces, &viewer, q, limit), | ||
None => Vec::new(), | ||
}; | ||
let marked = private.len(); | ||
let repos: Vec<_> = private.into_iter().chain(repos).collect(); | ||
let put = first.then(|| exact.into_iter().chain(own)); | ||
let mut seen = std::collections::HashSet::new(); | ||
let public = put.into_iter().flatten().chain(repos); | ||
let public = public.filter(|r| seen.insert(r.uri.clone())); | ||
let repos: Vec<_> = private.into_iter().chain(public).collect(); | ||
let owners = repos.iter().map(|r| crate::web::actor(&app, &r.did)); | ||
let owners = futures::future::join_all(owners).await; | ||
let repos: Vec<Value> = repos | ||
| @@ -198,6 +210,59 @@ pub async fn list_repos( | ||
Ok(Json(out)) | ||
} | ||
| ||
/// What a search puts first: the repositories named as searched, the | ||
/// viewer's own that match, then the name and the owner the rest leaves | ||
/// out. | ||
type First<'a> = ( | ||
Vec<crate::index::Listed>, | ||
Vec<crate::index::Listed>, | ||
Option<&'a str>, | ||
Option<&'a str>, | ||
); | ||
| ||
/// What a search for `q` puts first, as [`list_repos`] says, at most | ||
/// `limit` of each kind: the repositories named `q`, `viewer`'s first, | ||
/// and `viewer`'s other ones that match; then the name and the owner | ||
/// whose repositories the rest leaves out, each only when all of its kind | ||
/// fit. None for no search, or one `named` by runner, triager or topic. | ||
fn first_of<'a>( | ||
index: &crate::index::Index, | ||
q: &'a str, | ||
limit: usize, | ||
named: bool, | ||
viewer: Option<&'a str>, | ||
) -> Result<First<'a>, AppError> { | ||
if q.is_empty() || named { | ||
return Ok((Vec::new(), Vec::new(), None, None)); | ||
} | ||
let mut exact = index | ||
.named(q, limit + 1) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
let all_named = exact.len() <= limit; | ||
exact.truncate(limit); | ||
// Stable: the viewer's own first, each kind as it was. | ||
exact.sort_by_key(|r| Some(r.did.as_str()) != viewer); | ||
let mut own = match viewer { | ||
Some(viewer) => { | ||
let among = crate::index::Among { | ||
owner: Some(viewer), | ||
..Default::default() | ||
}; | ||
let (page, _) = index | ||
.repos_page(q, limit + 1, None, among) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
page | ||
} | ||
None => Vec::new(), | ||
}; | ||
let all_own = own.len() <= limit; | ||
own.truncate(limit); | ||
let not_named = all_named.then_some(q); | ||
Ok((exact, own, not_named, viewer.filter(|_| all_own))) | ||
} | ||
| ||
/// How many private repositories [`readable`] looks through for those its | ||
/// viewer may read. | ||
const READABLE: usize = 500; | ||
crates/knotless-serve/src/index.rs+27−4
| @@ -399,12 +399,15 @@ pub struct RepoPushes { | ||
} | ||
| ||
/// Which repositories [`Index::repos_page`] lists: each named, only those | ||
/// that `owner` owns, that name `runner` among their runners, `triager` | ||
/// that `owner` owns, that `not_owner` does not, not named `not_named` | ||
/// (ignoring case), that name `runner` among their runners, `triager` | ||
/// among their triagers or `topic` among their topics, or that | ||
/// `followed_by` or one it follows owns. | ||
#[derive(Debug, Clone, Copy, Default)] | ||
pub struct Among<'a> { | ||
pub owner: Option<&'a str>, | ||
pub not_owner: Option<&'a str>, | ||
pub not_named: Option<&'a str>, | ||
pub runner: Option<&'a str>, | ||
pub triager: Option<&'a str>, | ||
pub followed_by: Option<&'a str>, | ||
| @@ -570,6 +573,7 @@ pub(crate) fn connect(path: &Path) -> rusqlite::Result<(Connection, bool)> { | ||
CREATE INDEX IF NOT EXISTS repo_key ON repo (did, key); | ||
CREATE INDEX IF NOT EXISTS repo_fork ON repo (fork_did, fork_key); | ||
CREATE INDEX IF NOT EXISTS repo_created ON repo (created_at); | ||
CREATE INDEX IF NOT EXISTS repo_name ON repo (name COLLATE NOCASE); | ||
CREATE TABLE IF NOT EXISTS collaborator ( | ||
uri TEXT NOT NULL, repo_did TEXT NOT NULL, repo_key TEXT NOT NULL, | ||
did TEXT NOT NULL, PRIMARY KEY (uri, did)); | ||
| @@ -4109,10 +4113,14 @@ impl Index { | ||
AND (?8 IS NULL OR did = ?8 OR did IN (SELECT subject FROM follow WHERE did = ?8)) | ||
AND (?9 IS NULL OR EXISTS (SELECT 1 FROM json_each(repo.value, '$.topics') t | ||
WHERE json_extract(t.value, '$.name') = ?9)) | ||
AND (?10 IS NULL OR did != ?10) | ||
AND (?11 IS NULL OR name != ?11 COLLATE NOCASE) | ||
ORDER BY created_at DESC, uri DESC LIMIT ?4", | ||
)?; | ||
let Among { | ||
owner, | ||
not_owner, | ||
not_named, | ||
runner, | ||
triager, | ||
followed_by, | ||
| @@ -4127,7 +4135,9 @@ impl Index { | ||
runner, | ||
triager, | ||
followed_by, | ||
topic | ||
topic, | ||
not_owner, | ||
not_named | ||
]; | ||
let rows: Vec<Listed> = stmt | ||
.query_map(found, listed)? | ||
| @@ -4138,6 +4148,20 @@ impl Index { | ||
Ok((rows, next)) | ||
} | ||
| ||
/// At most `limit` repositories named `name`, ignoring case: the | ||
/// originals before the forks, each newest first. Those a search for | ||
/// the name means, before the rest that hold it. | ||
pub fn named(&self, name: &str, limit: usize) -> rusqlite::Result<Vec<Listed>> { | ||
let db = self.readers.take(&self.db); | ||
let mut stmt = db.prepare_cached( | ||
"SELECT did, name, description, fork_of, created_at, uri FROM repo | ||
WHERE name = ?1 COLLATE NOCASE AND did NOT IN down AND uri NOT IN down | ||
ORDER BY fork_of IS NOT NULL, created_at DESC, uri DESC LIMIT ?2", | ||
)?; | ||
stmt.query_map(params![name, limit as i64], listed)? | ||
.collect() | ||
} | ||
| ||
/// The handle `did` was last found to have, `None` for none that | ||
/// resolves back to it, and when it was looked up, in seconds since the | ||
/// epoch: if it ever was. | ||
| @@ -6370,7 +6394,7 @@ mod tests { | ||
.unwrap() | ||
.execute_batch(&format!( | ||
"PRAGMA user_version = {VERSION}; | ||
CREATE TABLE repo (did, key, fork_did, fork_key, created_at); | ||
CREATE TABLE repo (did, key, name, fork_did, fork_key, created_at); | ||
CREATE TABLE forge (author, repo_did, repo_key);" | ||
)) | ||
.unwrap(); | ||
| @@ -6381,7 +6405,6 @@ mod tests { | ||
assert!(index.repos("", 1).is_err()); | ||
assert!(index.forks("did:a", "k").is_err()); | ||
assert!(index.forge("did:a", "k").is_err()); | ||
assert!(index.name("did:a", "k").is_err()); | ||
assert!(index.recent(1, None, None, None, None, false).is_err()); | ||
assert!(index.pushes(1, None, None, None, false).is_err()); | ||
assert!(index.actor_days("did:a", None, "UTC").is_err()); | ||
crates/knotless-serve/tests/forge.rs+47−31
| @@ -554,40 +554,56 @@ fn forks_pulls_reviews_and_merges() { | ||
.is_err() | ||
); | ||
| ||
// The index knows the fork, and finds repositories by name. | ||
let found: Value = reqwest::blocking::get(format!( | ||
"{}/xrpc/dev.knotless.listRepos?q=forkable", | ||
av.base | ||
)) | ||
.unwrap() | ||
.json() | ||
.unwrap(); | ||
assert_eq!(found["repos"].as_array().unwrap().len(), 2); | ||
assert!(found.get("cursor").is_none()); | ||
// A page at a time, the same two. | ||
let list = |cursor: Option<&str>| -> Value { | ||
let mut query = vec![("q", "forkable"), ("limit", "1")]; | ||
// The index knows the fork, and finds repositories by name: those | ||
// named as asked first, the original before its newer fork. | ||
let dids = |page: &Value| -> Vec<String> { | ||
let repos = page["repos"].as_array().unwrap(); | ||
repos | ||
.iter() | ||
.map(|r| r["did"].as_str().unwrap().to_string()) | ||
.collect() | ||
}; | ||
let (theirs, mine) = (owner.account.did.as_str(), contrib.account.did.as_str()); | ||
let list = |cursor: Option<&str>, limit: &str, token: Option<&str>| -> Value { | ||
let mut query = vec![("q", "forkable"), ("limit", limit)]; | ||
query.extend(cursor.map(|c| ("cursor", c))); | ||
reqwest::blocking::Client::new() | ||
let mut req = reqwest::blocking::Client::new() | ||
.get(format!("{}/xrpc/dev.knotless.listRepos", av.base)) | ||
.query(&query) | ||
.send() | ||
.unwrap() | ||
.json() | ||
.unwrap() | ||
.query(&query); | ||
if let Some(token) = token { | ||
req = req.bearer_auth(token); | ||
} | ||
req.send().unwrap().json().unwrap() | ||
}; | ||
let first = list(None); | ||
let second = list(first["cursor"].as_str()); | ||
assert_eq!( | ||
[&first["repos"][0], &second["repos"][0]], | ||
[&found["repos"][0], &found["repos"][1]] | ||
); | ||
assert!( | ||
list(second["cursor"].as_str())["repos"] | ||
.as_array() | ||
.unwrap() | ||
.is_empty() | ||
); | ||
let found = list(None, "50", None); | ||
assert_eq!(dids(&found), [theirs, mine]); | ||
assert!(found.get("cursor").is_none()); | ||
// A page at a time, more named so than a page holds: the first still | ||
// starts with the original, and the rest come newest first, in full. | ||
let first = list(None, "1", None); | ||
assert_eq!(dids(&first), [theirs, mine]); | ||
let second = list(first["cursor"].as_str(), "1", None); | ||
assert_eq!(dids(&second), [theirs]); | ||
assert!(dids(&list(second["cursor"].as_str(), "1", None)).is_empty()); | ||
// Asked as an account, its own come first: the fork, to its owner. | ||
let token = |who: &Sandbox| { | ||
av.rt.block_on(async { | ||
let ctx = ctx(who); | ||
let me = knotless::session::me(&ctx).await.unwrap(); | ||
let lxm = "dev.knotless.listRepos"; | ||
knotless::auth::service_token(&ctx, &me, &av.did, lxm) | ||
.await | ||
.unwrap() | ||
}) | ||
}; | ||
let forker = token(&contrib); | ||
assert_eq!(dids(&list(None, "50", Some(&forker))), [mine, theirs]); | ||
// And all of its own fit on the first page, no page after lists them. | ||
let original = token(&owner); | ||
let first = list(None, "1", Some(&original)); | ||
assert_eq!(dids(&first), [theirs, mine]); | ||
let next = list(first["cursor"].as_str(), "1", Some(&original)); | ||
assert!(dids(&next).is_empty()); | ||
// And finds their owners by handle, ignoring case, with how many each owns. | ||
let owners = |query: &[(&str, &str)]| -> Value { | ||
reqwest::blocking::Client::new() | ||
docs/design/serve.md+2−1
| @@ -381,7 +381,8 @@ answers](../reference/api.md#appview)), and the web pages read the same answers: | ||
- `dev.knotless.getForge`: every record naming a repository, less what | ||
moderation removes. | ||
- `dev.knotless.listRepos`: repositories by name or description, newest first, | ||
or those naming a runner, for it to find them, or a topic. | ||
those named as searched and the asker's own put first, or those naming a | ||
runner, for it to find them, or a topic. | ||
- `dev.knotless.searchOwners`: the accounts owning those repositories, by | ||
handle, for search to find people as well as projects. | ||
- `dev.knotless.listChecks`: check runs by ref, commit or pull. Only the | ||
docs/reference/api.md+18−6
| @@ -153,12 +153,24 @@ as the index keeps of every account. | ||
| ||
Answers `repos` (`did`, `name`, `description?`, `forkOf?`, `createdAt`, | ||
`owner`), newest first, and `cursor` when the page was full. Taken-down | ||
owners and repositories are left out. Asked as an account, the first page | ||
with no `runner`, `triager` or `topic` starts with the private repositories | ||
matching `q` that it may read here, at most `limit` more, each `private`: | ||
its own, and those it was proven a member of, as it read one or as it | ||
was asked after in the background. The answer is then that account's | ||
alone. | ||
owners and repositories are left out. | ||
| ||
The first page of a search (a `q`, and no `runner`, `triager` or `topic`) | ||
puts some first, at most `limit` more of each kind, in this order: | ||
| ||
1. Asked as an account, the private repositories matching `q` that it may | ||
read here, each `private`: its own, and those it was proven a member | ||
of, as it read one or as it was asked after in the background. | ||
2. Those named `q`, ignoring case: the account's own first, then | ||
originals before forks, each newest first. Searching for a project's | ||
name finds it before the forks of it, and before the repositories that | ||
only mention it. | ||
3. Asked as an account, its other public repositories that match. | ||
| ||
A kind put first is left out of every page's newest-first rest when all of | ||
it fit on the first page; when more match than `limit`, the rest lists | ||
them all again in their place. Asked as an account, the answer is that | ||
account's alone. | ||
| ||
### `dev.knotless.searchCode` | ||
| ||
lexicons/dev/knotless/listRepos.json+1−1
| @@ -5,7 +5,7 @@ | ||
"defs": { | ||
"main": { | ||
"type": "query", | ||
"description": "Public repositories an appview has indexed, newest first. No authentication needed; asked as an account, with a service token or the pages' session cookie, the first page of a search with no runner, triager or topic starts with the private repositories that account may read on this appview that match too, each marked private, and the answer is that account's alone.", | ||
"description": "Public repositories an appview has indexed, newest first. The first page of a search with no runner, triager or topic starts with those named as searched, ignoring case, originals before forks. No authentication needed; asked as an account, with a service token or the pages' session cookie, the account's own of those come first, and before them the private repositories that account may read on this appview that match too, each marked private, and after them its own other public ones that match; the answer is that account's alone. What a first page puts first, when all of its kind fit there, no later page lists again.", | ||
"parameters": { | ||
"type": "params", | ||
"properties": { | ||