chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
A repository's settings no longer wait seconds for its storage
crates/knotless-serve/src/web/settings.rs+119−5
| @@ -16,6 +16,8 @@ use crate::viewer::Viewer; | ||
use axum::Extension; | ||
use knotless::env::Ctx; | ||
use knotless::manage; | ||
use knotless::storage::Storage; | ||
use std::time::Instant; | ||
| ||
#[derive(Deserialize)] | ||
pub struct SettingsParams { | ||
| @@ -59,8 +61,6 @@ pub struct Settings { | ||
gateway: Option<String>, | ||
#[serde(skip_serializing_if = "Option::is_none")] | ||
transfer: Option<RepoViewBasic>, | ||
#[serde(skip_serializing_if = "Option::is_none")] | ||
storage: Option<knotless::storage::Storage>, | ||
/// The gateway its owner delegated writes to from these pages, which | ||
/// sharing it for writing names: none until they have. | ||
#[serde(skip_serializing_if = "Option::is_none")] | ||
| @@ -126,8 +126,6 @@ pub async fn get_repo_settings( | ||
Some(to) => Some(basic(&app, to).await), | ||
None => None, | ||
}; | ||
// Counted from its pack records, which a sync no longer reads. | ||
let storage = knotless::storage::storage(&reader, &url).await.ok(); | ||
let mut readers = None; | ||
if repo.in_space.is_some() { | ||
// Only its owner may list them; anyone else is shown none. | ||
| @@ -149,13 +147,95 @@ pub async fn get_repo_settings( | ||
triagers, | ||
gateway: record.gateway, | ||
transfer, | ||
storage, | ||
delegated, | ||
readers, | ||
gates, | ||
})) | ||
} | ||
| ||
/// The most repositories whose storage [`Counts`] keeps: once full, it | ||
/// starts again, as counting one again costs only that page a wait. | ||
const COUNTS_KEPT: usize = 4096; | ||
| ||
/// Each public repository's storage as last counted, by its owner's DID | ||
/// and key, with the events of that repository seen then | ||
/// ([`App::stamp`]) and when it was counted: good until another such | ||
/// event, or, with no firehose watched, for as long as a mirror is. | ||
#[derive(Default)] | ||
pub(crate) struct Counts(Mutex<HashMap<(String, String), Counted>>); | ||
| ||
/// A repository's storage, with its events seen and the time it was counted. | ||
type Counted = (u64, Instant, Storage); | ||
| ||
impl Counts { | ||
/// What `did`'s repository `key` was counted at, if nothing of it has | ||
/// been seen since, `seen` being its events now, and it is not older | ||
/// than `ttl` when `watching` is unset. | ||
fn get( | ||
&self, | ||
did: &str, | ||
key: &str, | ||
seen: u64, | ||
watching: bool, | ||
ttl: Duration, | ||
) -> Option<Storage> { | ||
let kept = self.0.lock().unwrap(); | ||
let (then, at, storage) = kept.get(&(did.to_string(), key.to_string()))?; | ||
let fresh = *then == seen && (watching || at.elapsed() < ttl); | ||
fresh.then(|| storage.clone()) | ||
} | ||
| ||
fn put(&self, did: &str, key: &str, seen: u64, storage: Storage, most: usize) { | ||
let mut kept = self.0.lock().unwrap(); | ||
if kept.len() >= most { | ||
kept.clear(); | ||
} | ||
kept.insert((did.into(), key.into()), (seen, Instant::now(), storage)); | ||
} | ||
| ||
/// Has `did`'s repository `key` counted again, after a write here | ||
/// that the firehose has yet to tell of. | ||
pub(crate) fn forget(&self, did: &str, key: &str) { | ||
self.0 | ||
.lock() | ||
.unwrap() | ||
.remove(&(did.to_string(), key.to_string())); | ||
} | ||
} | ||
| ||
/// `dev.knotless.getRepoStorage`: counted from its pack records, which a | ||
/// sync no longer reads, so apart from its settings, and kept. | ||
pub async fn get_repo_storage( | ||
State(app): State<Arc<App>>, | ||
Query(p): Query<SettingsParams>, | ||
) -> Result<Response> { | ||
let repo = repo::found(&app, &p.repo).await?; | ||
let url = url_of(&repo); | ||
// A private one is read as the member asking, and counted each time: | ||
// no firehose tells of its writes. | ||
let private = crate::private::viewer().filter(|_| repo.in_space.is_some()); | ||
if let Some(viewer) = private { | ||
let reader = crate::login::ctx_for(&app, &viewer); | ||
let counted = knotless::storage::storage(&reader, &url).await; | ||
return Ok(answer(counted.map_err(AppError::upstream)?)); | ||
} | ||
// Its events taken before counting, so one seen meanwhile has it | ||
// counted again next time. | ||
let seen = app.stamp(&repo.did, Some(&repo.key)).1; | ||
let watching = app.watching.load(Ordering::Relaxed); | ||
let kept = app | ||
.counts | ||
.get(&repo.did, &repo.key, seen, watching, app.ttl); | ||
if let Some(storage) = kept { | ||
return Ok(answer(storage)); | ||
} | ||
let counted = knotless::storage::storage(&app.ctx, &url).await; | ||
let storage = counted.map_err(AppError::upstream)?; | ||
app.counts | ||
.put(&repo.did, &repo.key, seen, storage.clone(), COUNTS_KEPT); | ||
Ok(answer(storage)) | ||
} | ||
| ||
/// The repository `uri` names, for a change only its owner, logged in to | ||
/// the pages, may make, `act`; and a context acting as them. | ||
pub(super) async fn owner_only( | ||
| @@ -314,6 +394,7 @@ pub async fn compact_repo( | ||
let (repo, ctx) = owned(&app, viewer, &headers, &input.repo, "compact it").await?; | ||
let compacted = knotless::compact::compact_repo(&ctx, &url_of(&repo)).await; | ||
let c = compacted.map_err(refused)?; | ||
app.counts.forget(&repo.did, &repo.key); | ||
wrote(&app, &repo.did, &[]).await?; | ||
Ok(answer(Compacted { | ||
packs: c.before, | ||
| @@ -445,3 +526,36 @@ pub async fn delete_repo( | ||
crate::accounts::repo_deleted(&app, &repo.did, &repo.key); | ||
Ok(answer(Deleted { uri: repo.uri() })) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn counts_keep_until_the_repository_moves() { | ||
let counts = Counts::default(); | ||
let one = Storage { | ||
packs: 1, | ||
..Storage::default() | ||
}; | ||
let day = Duration::from_secs(86_400); | ||
assert_eq!(counts.get("d", "k", 0, true, day), None); | ||
counts.put("d", "k", 3, one.clone(), 2); | ||
assert_eq!(counts.get("d", "k", 3, true, day), Some(one.clone())); | ||
// Another event of it, or none watched for longer than a mirror | ||
// is trusted, and it is counted again. | ||
assert_eq!(counts.get("d", "k", 4, true, day), None); | ||
assert_eq!(counts.get("d", "k", 3, false, Duration::ZERO), None); | ||
assert_eq!(counts.get("d", "k", 3, false, day), Some(one.clone())); | ||
// Another repository's is its own. | ||
assert_eq!(counts.get("d", "j", 3, true, day), None); | ||
counts.forget("d", "k"); | ||
assert_eq!(counts.get("d", "k", 3, true, day), None); | ||
// Full, it starts again. | ||
counts.put("d", "a", 0, one.clone(), 2); | ||
counts.put("d", "b", 0, one.clone(), 2); | ||
counts.put("d", "c", 0, one.clone(), 2); | ||
assert_eq!(counts.get("d", "a", 0, true, day), None); | ||
assert_eq!(counts.get("d", "c", 0, true, day), Some(one)); | ||
} | ||
} | ||
crates/knotless-serve/src/lib.rs+4−0
| @@ -87,6 +87,9 @@ pub struct App { | ||
account_events: Mutex<HashMap<String, u64>>, | ||
/// And those that change one repository, by its owner's DID and key. | ||
repo_events: Mutex<HashMap<(String, String), u64>>, | ||
/// Each public repository's storage as last counted, kept until it | ||
/// moves (`dev.knotless.getRepoStorage`). | ||
pub(crate) counts: web::Counts, | ||
/// The last commit, by its rev, that wrote to each DID's repository | ||
/// collections ([`first_of`]). | ||
revs: Mutex<HashMap<String, String>>, | ||
| @@ -235,6 +238,7 @@ impl App { | ||
events: Mutex::default(), | ||
account_events: Mutex::default(), | ||
repo_events: Mutex::default(), | ||
counts: web::Counts::default(), | ||
revs: Mutex::default(), | ||
watching: AtomicBool::new(false), | ||
moves: Mutex::default(), | ||
crates/knotless-serve/src/web.rs+5−0
| @@ -68,6 +68,7 @@ mod repos; | ||
mod runs; | ||
mod search; | ||
mod settings; | ||
pub(crate) use settings::Counts; | ||
mod stars; | ||
mod steps; | ||
mod subscriptions; | ||
| @@ -115,6 +116,10 @@ pub fn queries() -> Router<Arc<App>> { | ||
"/xrpc/dev.knotless.getRepoSettings", | ||
get(settings::get_repo_settings), | ||
) | ||
.route( | ||
"/xrpc/dev.knotless.getRepoStorage", | ||
get(settings::get_repo_storage), | ||
) | ||
.route("/xrpc/dev.knotless.updateRepo", post(settings::update_repo)) | ||
.route("/xrpc/dev.knotless.renameRepo", post(settings::rename_repo)) | ||
.route( | ||
crates/knotless-serve/tests/pages.rs+21−2
| @@ -411,8 +411,19 @@ fn owners_change_their_settings_from_the_pages() { | ||
assert_eq!(set["collaborators"], json!([])); | ||
assert_eq!(set["triagers"][0]["actor"]["did"], theirs); | ||
assert_eq!(set["triagers"][0]["scopes"], json!([{"name": "kind"}])); | ||
assert!(set["storage"]["packs"].as_u64().unwrap() >= 1, "{set}"); | ||
assert_eq!(set.get("transfer"), None); | ||
// What it takes up, apart from its settings, which no longer count it; | ||
// and the same again from what was counted, as nothing of it moved. | ||
assert_eq!(set.get("storage"), None); | ||
let storage = |name: &str| { | ||
let url = format!("{}?repo={}", server.xrpc("getRepoStorage"), uri(name)); | ||
let (status, _, answer) = browse("GET", &url, None, None); | ||
assert_eq!(status, 200, "{answer}"); | ||
answer | ||
}; | ||
let counted = storage("kept"); | ||
assert!(counted["packs"].as_u64().unwrap() >= 1, "{counted}"); | ||
assert_eq!(storage("kept"), counted); | ||
| ||
// Its description and default branch, by the owner alone. | ||
let links = json!([{"url": "https://kept.example"}, {"url": "https://kept.example/docs", "title": "Docs"}]); | ||
| @@ -463,6 +474,11 @@ fn owners_change_their_settings_from_the_pages() { | ||
assert_eq!(status, 200, "{compacted}"); | ||
assert!(compacted["packs"].as_u64().unwrap() >= 1, "{compacted}"); | ||
assert_eq!(server.call("compactRepo", not_me, kept_uri.clone()).0, 403); | ||
// Counted again at once, before the firehose tells of it. | ||
let (before, folded) = (&compacted["packs"], &compacted["folded"]); | ||
let (before, folded) = (before.as_u64().unwrap(), folded.as_u64().unwrap()); | ||
let now = before - folded + u64::from(folded > 0); | ||
assert_eq!(storage("kept")["packs"], now, "{compacted}"); | ||
| ||
// Archived, which its page says, then unarchived; only by its owner, | ||
// and only once each way. | ||
| @@ -848,7 +864,10 @@ fn private_repositories_settings_and_forks() { | ||
let (status, set) = settings(me); | ||
assert_eq!(status, 200, "{set}"); | ||
assert_eq!(set.get("indexed"), None); | ||
assert_eq!(set["storage"]["packs"], 1); | ||
let counted = format!("{}?repo={at}", server.xrpc("getRepoStorage")); | ||
let (status, _, storage) = browse("GET", &counted, me, None); | ||
assert_eq!((status, &storage["packs"]), (200, &json!(1)), "{storage}"); | ||
assert_eq!(browse("GET", &counted, None, None).0, 404); | ||
assert_eq!(settings(None).0, 404); | ||
// Changed from the pages, they say so at once: read from this server's | ||
// index of its space, which the change is put in as it is made. | ||
docs/design/serve.md+9−0
| @@ -292,6 +292,15 @@ Details: | ||
turn. One still writing after ten minutes, as for a client that stalls, | ||
is cut short, so that slow clients cannot keep the rest waiting. One at | ||
a commit's hash never changes, so it is cached for good. | ||
- A repository's settings page shows what it takes up on its owner's PDS | ||
(`dev.knotless.getRepoStorage`), asked apart from its settings, as | ||
counting it lists every pack record from the PDS, which can take seconds. | ||
The page draws the rest at once and fills it in once it comes. The count | ||
is kept in memory by repository, up to 4,096 of them, until the firehose | ||
tells of a write to that repository's packs, claims or refs, or the pages | ||
compact it; with no firehose watched, for as long as a mirror is trusted. | ||
A private repository's is counted each time, as no firehose tells of its | ||
writes. | ||
- A repository has three Atom feeds, for feed readers: `commits.atom`, the | ||
newest 30 commits on its default branch; `tags.atom`, its newest 30 tags | ||
that name a commit; and `releases.atom`, its newest 30 releases, with | ||
docs/reference/api.md+2−1
| @@ -599,7 +599,8 @@ name it has now. | ||
| `dev.knotless.getAccountData` | what knotless data the account logged in to the pages holds, read from its PDS: `{did, collections, repos, spaces}`, each `dev.knotless.*` collection it writes in as `{collection, count}`, its repositories as `{name, private, collaborators, members?}` (`members`, for a private one, the accounts that read it but its runners), and the spaces of its private repositories; the pages' cookie only, 401 otherwise | | ||
| `dev.knotless.getEmails` | `{count}`, how many addresses the asker's `dev.knotless.emails` claims: only hashes are kept, so they cannot be listed. 401 with no asker | | ||
| `dev.knotless.listBlocks` | the asker's blocks, the latest first, from its own repo: each `uri`, `subject` (an actor), `repo` when it is from one repository only, and `createdAt`. Those naming a private repository live in its space and are not listed. 401 with no asker | | ||
| `dev.knotless.getRepoSettings?repo=` | what the owner sets: description, links, default branch, protected branches, labels, collaborators, triagers, gateway, an offer to move it (`transfer`), `storage`, as `knotless storage` counts it, the gateway the owner delegated to from these pages (`delegated`), for a private one, to its owner alone, the members who read but do not push (`readers`), and the accounts its rules name as [gates](../design/collaboration.md#gates), each once (`gates`) | | ||
| `dev.knotless.getRepoSettings?repo=` | what the owner sets: description, links, default branch, protected branches, labels, collaborators, triagers, gateway, an offer to move it (`transfer`), the gateway the owner delegated to from these pages (`delegated`), for a private one, to its owner alone, the members who read but do not push (`readers`), and the accounts its rules name as [gates](../design/collaboration.md#gates), each once (`gates`) | | ||
| `dev.knotless.getRepoStorage?repo=` | what it takes up on its owner's PDS, as `knotless storage` counts it: kept per repository, and counted again only once a write to its packs, claims or refs is seen, or the pages compact it | | ||
| `dev.knotless.getOrg?actor=` | an organization's members, as the index has its `dev.knotless.org` and `dev.knotless.orgMember` records and its members' `dev.knotless.orgAcceptance` ones: `{org, organization, inside, gateway?, members}`, `gateway` the one its public `dev.knotless.org` record names, each member `{actor, role, standing, invitedAt, expiresAt?, joinedAt?}`, the first invited first, `standing` one of `invited`, `lapsed`, `public` and `hidden`. To anyone, its members in public; to the organization and its members, logged in or with a service token (`inside`), its invitations and hidden members too, from its space, which this server keeps an index of; to an account it invited, its own entry. `organization: false`, with no members, for an account that is none; `ProfileNotFound` for an unknown account | | ||
| `dev.knotless.getActorStars?actor=` | the public repositories an account starred, newest first, each with its `language`, `starCount` and `forkCount`: 404 with no index, `ProfileNotFound` for an unknown account | | ||
| `dev.knotless.getMergeStatus?repo=&pull=` | whether an open pull can merge now, as `knotless merge --check` says ([below](#landing-and-checks)): 401 with no asker, 400 for a pull not open, 404 for none | | ||
harness/shots/fixtures.json+9−1
| @@ -273,7 +273,15 @@ | ||
"content-type": "application/json", | ||
"cache-control": "no-cache" | ||
}, | ||
"body": "{\"uri\":\"at://did:plc:abe24xewga4qdbzjcxmuzdt3/dev.knotless.repo/tally\",\"description\":\"Counts lines, words and bytes, with output that lines up\",\"links\":[{\"url\":\"https://tally.dev\"},{\"url\":\"https://docs.tally.dev\",\"title\":\"Docs\"}],\"defaultBranch\":\"main\",\"rules\":[{\"branch\":\"main\",\"history\":\"linear\"}],\"labels\":[{\"name\":\"kind/bug\",\"description\":\"Something is broken\",\"color\":\"#d73a4a\"},{\"name\":\"priority/high\",\"color\":\"#fbca04\",\"exclusive\":true},{\"name\":\"priority/low\",\"exclusive\":true}],\"collaborators\":[],\"triagers\":[],\"storage\":{\"packs\":2,\"objects\":35,\"bytes\":5031,\"blobBytes\":0}}" | ||
"body": "{\"uri\":\"at://did:plc:abe24xewga4qdbzjcxmuzdt3/dev.knotless.repo/tally\",\"description\":\"Counts lines, words and bytes, with output that lines up\",\"links\":[{\"url\":\"https://tally.dev\"},{\"url\":\"https://docs.tally.dev\",\"title\":\"Docs\"}],\"defaultBranch\":\"main\",\"rules\":[{\"branch\":\"main\",\"history\":\"linear\"}],\"labels\":[{\"name\":\"kind/bug\",\"description\":\"Something is broken\",\"color\":\"#d73a4a\"},{\"name\":\"priority/high\",\"color\":\"#fbca04\",\"exclusive\":true},{\"name\":\"priority/low\",\"exclusive\":true}],\"collaborators\":[],\"triagers\":[]}" | ||
}, | ||
"/xrpc/dev.knotless.getRepoStorage?repo=at%3A%2F%2Fdid%3Aplc%3Aabe24xewga4qdbzjcxmuzdt3%2Fdev.knotless.repo%2Ftally": { | ||
"status": 200, | ||
"headers": { | ||
"content-type": "application/json", | ||
"cache-control": "no-cache" | ||
}, | ||
"body": "{\"packs\":2,\"objects\":35,\"bytes\":5031,\"blobBytes\":0}" | ||
}, | ||
"/xrpc/dev.knotless.getStats": { | ||
"status": 200, | ||
lexicons/dev/knotless/getRepoSettings.json+2−2
| @@ -1,7 +1,7 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.getRepoSettings", | ||
"description": "What a repository's owner sets: its description, links, topics and default branch, protected branches, labels, collaborators and triagers, an offer to move it, and what it takes up.", | ||
"description": "What a repository's owner sets: its description, links, topics and default branch, protected branches, labels, collaborators and triagers, and an offer to move it.", | ||
"defs": { | ||
"main": { | ||
"type": "query", | ||
| @@ -30,7 +30,7 @@ | ||
"triagers": { "type": "array", "description": "The accounts whose labels of some scopes count.", "items": { "type": "ref", "ref": "#triagerView" } }, | ||
"gateway": { "type": "string", "format": "did", "description": "The gateway the owner delegated writes to, through which collaborators push: absent until the owner delegates." }, | ||
"transfer": { "type": "ref", "ref": "dev.knotless.defs#repoViewBasic", "description": "Where the owner has offered to move it, if they have: the account, and the name it would have there." }, | ||
"storage": { "type": "ref", "ref": "#storageView", "description": "What it takes up on its owner's PDS; absent if that could not be read." }, | ||
"storage": { "type": "ref", "ref": "#storageView", "description": "Deprecated, and no longer set: dev.knotless.getRepoStorage counts it, apart, as counting reads every pack record from the PDS. Was what it takes up on its owner's PDS." }, | ||
"indexed": { "type": "boolean", "description": "Deprecated, and no longer set: knotless.dev indexes every private repository its members read there. Was, for a private repository, whether its owner let this appview keep an index of it." }, | ||
"delegated": { "type": "string", "format": "did", "description": "The gateway its owner delegated writes to from this appview's pages (dev.knotless.delegateWrites), which sharing it for writing names as its gateway; absent until they have." }, | ||
"gates": { "type": "array", "description": "The accounts its protected branches' rules name as gates (dev.knotless.repo#reviews), each once: for the pages to show by handle. Absent when there are none.", "items": { "type": "ref", "ref": "dev.knotless.defs#actorView" } }, | ||
lexicons/dev/knotless/getRepoStorage.jsonnew+26−0
| @@ -0,0 +1,26 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.getRepoStorage", | ||
"description": "What a repository takes up on its owner's PDS.", | ||
"defs": { | ||
"main": { | ||
"type": "query", | ||
"description": "What a repository's packs take up on its owner's PDS, as `knotless storage` counts them: counted again only once a write to the repository's packs, claims or refs is seen, or compaction from these pages folds them. Apart from dev.knotless.getRepoSettings, as counting reads every pack record from the PDS.", | ||
"parameters": { | ||
"type": "params", | ||
"required": ["repo"], | ||
"properties": { | ||
"repo": { "type": "string", "format": "at-uri", "description": "The repository's dev.knotless.repo record, as dev.knotless.getRepo takes it." } | ||
} | ||
}, | ||
"output": { | ||
"encoding": "application/json", | ||
"schema": { "type": "ref", "ref": "dev.knotless.getRepoSettings#storageView" } | ||
}, | ||
"errors": [ | ||
{ "name": "NotFound", "description": "No such repository." }, | ||
{ "name": "UpstreamFailure", "description": "Its owner's PDS could not be read." } | ||
] | ||
} | ||
} | ||
} | ||
web/src/lib/organisms/SettingsStorage.svelte+12−8
| @@ -7,10 +7,10 @@ | ||
import { number, plural, size } from '#lib/format.ts'; | ||
/** | ||
* What the repository `uri` names takes up on its owner's PDS, | ||
* `storage`, as `knotless storage` counts it, and folding its packs | ||
* into one, as `knotless compact` does. | ||
* `storage`, as `knotless storage` counts it, once counted, and folding | ||
* its packs into one, as `knotless compact` does. | ||
*/ | ||
let { uri, storage }: { uri: AtUriString; storage?: StorageView } = $props(); | ||
let { uri, storage }: { uri: AtUriString; storage: Promise<StorageView | null> } = $props(); | ||
let done = $state(''); | ||
let problem = $state(''); | ||
let busy = $state(false); | ||
| @@ -30,11 +30,15 @@ | ||
| ||
<section class="setting"> | ||
<h2>Storage</h2> | ||
{#if storage} | ||
<p>{`${plural(storage.packs, 'pack', 'packs')}, ${number(storage.objects)} objects, ${size(storage.bytes)}, ${size(storage.blobBytes)} of it in blobs.`}</p> | ||
{:else} | ||
<p class="mute">What it takes up could not be read.</p> | ||
{/if} | ||
{#await storage} | ||
<p class="mute" data-loading>Counting its packs…</p> | ||
{:then counted} | ||
{#if counted} | ||
<p>{`${plural(counted.packs, 'pack', 'packs')}, ${number(counted.objects)} objects, ${size(counted.bytes)}, ${size(counted.blobBytes)} of it in blobs.`}</p> | ||
{:else} | ||
<p class="mute">What it takes up could not be read.</p> | ||
{/if} | ||
{/await} | ||
<p class="hint">Each push adds a pack. Compacting folds them into one, so clones read fewer records.</p> | ||
<div class="actions"><button class="btn" type="button" disabled={busy} aria-busy={busy} onclick={compact}>{busy ? 'Compacting…' : 'Compact'}</button>{#if done}<span class="mute" role="status">{done}</span>{/if}</div> | ||
{#if problem}<p class="problem" role="alert">{problem}</p>{/if} | ||