chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
Code search loads more files, 200 deep at most
.changeset/code-search-more.mdnew+7−0
| @@ -0,0 +1,7 @@ | ||
--- | ||
default: minor | ||
--- | ||
| ||
# Code search loads more results | ||
| ||
The code search page shows its first 20 files, then loads more below them, 200 files at most. `dev.knotless.searchCode` takes a `cursor` for the page after. | ||
crates/knotless-serve/src/web/repo.rs+6−6
| @@ -45,9 +45,9 @@ pub(super) async fn found(app: &App, uri: &str) -> Result<Repo> { | ||
} | ||
} | ||
| ||
/// How many of a list the pages before held, as `p`'s cursor says. | ||
fn skipped(p: &RepoParams) -> Result<usize> { | ||
match p.cursor.as_deref().map(str::parse::<usize>) { | ||
/// How many of a list the pages before held, as a cursor says. | ||
pub(super) fn skipped(cursor: Option<&str>) -> Result<usize> { | ||
match cursor.map(str::parse::<usize>) { | ||
None => Ok(0), | ||
Some(Ok(skip)) => Ok(skip), | ||
Some(Err(_)) => Err(invalid("cursor is no count".into())), | ||
| @@ -553,7 +553,7 @@ pub async fn list_commits( | ||
Query(p): Query<RepoParams>, | ||
) -> Result<Response> { | ||
let limit = p.limit.unwrap_or(COMMITS_PER_PAGE).clamp(1, COMMITS_MAX); | ||
let skip = skipped(&p)?; | ||
let skip = skipped(p.cursor.as_deref())?; | ||
let repo = found(&app, &p.repo).await?; | ||
if unpushed(&repo, &p)? { | ||
return Ok(answer(History { | ||
| @@ -641,7 +641,7 @@ pub async fn list_branches( | ||
Query(p): Query<RepoParams>, | ||
) -> Result<Response> { | ||
let limit = p.limit.unwrap_or(BRANCHES_PER_PAGE).clamp(1, BRANCHES_MAX); | ||
let skip = skipped(&p)?; | ||
let skip = skipped(p.cursor.as_deref())?; | ||
let repo = found(&app, &p.repo).await?; | ||
let all = repo.branches()?; | ||
let cursor = (all.len() > skip + limit).then(|| (skip + limit).to_string()); | ||
| @@ -676,7 +676,7 @@ pub async fn list_tags( | ||
Query(p): Query<RepoParams>, | ||
) -> Result<Response> { | ||
let limit = p.limit.unwrap_or(BRANCHES_PER_PAGE).clamp(1, BRANCHES_MAX); | ||
let skip = skipped(&p)?; | ||
let skip = skipped(p.cursor.as_deref())?; | ||
let repo = found(&app, &p.repo).await?; | ||
let all = repo.objects()?.tags().map_err(AppError::internal)?; | ||
let cursor = (all.len() > skip + limit).then(|| (skip + limit).to_string()); | ||
crates/knotless-serve/src/web/search.rs+25−7
| @@ -7,12 +7,16 @@ use serde_json::{Value, json}; | ||
/// How many files a search shows, if not asked, and at most. | ||
const FILES: usize = 20; | ||
const FILES_MAX: usize = 50; | ||
/// How deep the pages of a search go: Zoekt answers the best files from | ||
/// the first, so each page asks again for those before it. | ||
const DEEPEST: usize = 200; | ||
| ||
#[derive(Deserialize)] | ||
pub struct SearchParams { | ||
q: String, | ||
repo: Option<String>, | ||
limit: Option<usize>, | ||
cursor: Option<String>, | ||
} | ||
| ||
/// A file found, in the repository the index lists it in. | ||
| @@ -60,13 +64,23 @@ pub async fn search_code( | ||
.as_ref() | ||
.map(|(did, key)| (did.as_str(), key.as_str())); | ||
let limit = p.limit.unwrap_or(FILES).clamp(1, FILES_MAX); | ||
let (found, faceted) = | ||
tokio::join!(code.search(&p.q, within, limit), code.faceted(&p.q, within)); | ||
let skip = repo::skipped(p.cursor.as_deref())?; | ||
let upto = (skip + limit).min(DEEPEST); | ||
// The languages are secondary to what was found: a query Zoekt cannot | ||
// count them by lists none, rather than failing the search. | ||
let (found, faceted) = (found?, faceted.unwrap_or_default()); | ||
// count them by lists none, rather than failing the search. The first | ||
// page alone lists them, as the pages after add only files. | ||
let faceted = async { | ||
match skip { | ||
0 => code.faceted(&p.q, within).await.unwrap_or_default(), | ||
_ => Vec::new(), | ||
} | ||
}; | ||
let (found, faceted) = tokio::join!(code.search(&p.q, within, upto), faceted); | ||
let found = found?; | ||
let more = upto < DEEPEST && found.file_count > upto as u64; | ||
let listed = |h: crate::code::Hit| Some((index.listed(&h.did, &h.key).ok().flatten()?, h)); | ||
let shown: Vec<_> = found.hits.into_iter().filter_map(listed).collect(); | ||
let pages_before = found.hits.into_iter().skip(skip); | ||
let shown: Vec<_> = pages_before.filter_map(listed).collect(); | ||
let owners = shown.iter().map(|(r, _)| actor(&app, &r.did)); | ||
let owners = futures::future::join_all(owners).await; | ||
let files: Vec<FileView> = shown | ||
| @@ -109,10 +123,14 @@ pub async fn search_code( | ||
files, | ||
}) | ||
.collect(); | ||
Ok(answer(json!({ | ||
let mut answered = json!({ | ||
"files": files, | ||
"fileCount": found.file_count, | ||
"matchCount": found.match_count, | ||
"languages": languages, | ||
}))) | ||
}); | ||
if more { | ||
answered["cursor"] = json!(upto.to_string()); | ||
} | ||
Ok(answer(answered)) | ||
} | ||
crates/knotless-serve/tests/web.rs+16−0
| @@ -3518,6 +3518,22 @@ fn searching_code() { | ||
let painted = chunk["highlights"].as_str().unwrap(); | ||
assert!(painted.starts_with("comment:0,"), "{painted}"); | ||
| ||
// A page at a time, the best first, and the next after the cursor. | ||
let first = search("needle_&limit=1"); | ||
assert_eq!(first["files"].as_array().unwrap().len(), 1, "{first}"); | ||
assert_eq!(first["cursor"], "1", "{first}"); | ||
let next = search("needle_&limit=1&cursor=1"); | ||
assert_eq!(next["files"].as_array().unwrap().len(), 1, "{next}"); | ||
assert_ne!(next["files"][0]["repo"], first["files"][0]["repo"]); | ||
assert!(next.get("cursor").is_none(), "{next}"); | ||
// Its languages were listed with the first. | ||
assert_eq!(next["languages"], serde_json::json!([]), "{next}"); | ||
// No deeper than the pages go. | ||
let deep = search("needle_&cursor=1000"); | ||
assert_eq!(deep["files"].as_array().unwrap().len(), 0, "{deep}"); | ||
let bad = site.status("/xrpc/dev.knotless.searchCode?q=x&cursor=no"); | ||
assert_eq!(bad.0, 400, "{}", bad.1); | ||
| ||
// In one repository, by its record's URI. | ||
let within = |name: &str| { | ||
let repo = format!("at://{handle}/dev.knotless.repo/{name}"); | ||
docs/design/serve.md+7−0
| @@ -430,6 +430,13 @@ query, run at the same time: the same one with its `lang:` filters taken | ||
out, asking only for files (`type:file`), up to 1000 of them. Picking a | ||
language then leaves the others listed. | ||
| ||
Zoekt ranks the files it finds but has no cursor, so a page after the | ||
first asks it for the best files up to that page's end, and leaves out | ||
those the pages before held. Each page costs a little more than the one | ||
before, so the pages stop 200 files deep: a search that finds more is | ||
narrowed rather than read further. Only the first page asks for the | ||
languages. | ||
| ||
### Shared steps | ||
| ||
The index keeps a directory of what repositories publish for others to | ||
docs/reference/api.md+3−1
| @@ -162,6 +162,7 @@ alone. | ||
| `q` | | required: Zoekt's query language, as words, `"quoted text"`, `/a regex/`, `file:`, `lang:`, `case:yes` | | ||
| `repo` | | a repository's `at://` URI, its authority a DID or a handle: only its code | | ||
| `limit` | 20 | the most files answered, clamped to 1 to 50 | | ||
| `cursor` | | from the page before: how many files it held | | ||
| ||
Answers `files`, the best first, each with its `repo` (as `listRepos` | ||
answers one), `path`, `language?`, `commit?` (the commit of the default | ||
| @@ -171,7 +172,8 @@ by `line`, `byteStart` and `byteEnd` in UTF-8 bytes, and its | ||
`highlights?`, encoded as `getFile`'s `ranges` are. `languages` counts | ||
the files found in each language, most first, its `name`, `color?` and | ||
`files`: its `lang:` filters left out, so each can be picked, over the | ||
first 1000 files at most. `fileCount` and | ||
first 1000 files at most, on the first page alone. `cursor` is absent | ||
on the last page, and once the pages reach 200 files. `fileCount` and | ||
`matchCount` count all that was found. Only default branches are | ||
searched, and taken-down owners and repositories are left out. A query | ||
Zoekt cannot read answers 400, and a server without `--code-index` 501. | ||
lexicons/dev/knotless/searchCode.json+4−2
| @@ -12,7 +12,8 @@ | ||
"properties": { | ||
"q": { "type": "string", "maxLength": 1000, "description": "What to search for." }, | ||
"repo": { "type": "string", "format": "at-uri", "description": "Only this repository's dev.knotless.repo record, its authority a DID or a handle." }, | ||
"limit": { "type": "integer", "minimum": 1, "maximum": 50, "default": 20, "description": "The most files to answer with." } | ||
"limit": { "type": "integer", "minimum": 1, "maximum": 50, "default": 20, "description": "The most files to answer with." }, | ||
"cursor": { "type": "string", "maxLength": 32, "description": "How many files the pages before held. The pages go 200 files deep at most." } | ||
} | ||
}, | ||
"output": { | ||
| @@ -24,7 +25,8 @@ | ||
"files": { "type": "array", "items": { "type": "ref", "ref": "#fileView" } }, | ||
"fileCount": { "type": "integer", "minimum": 0, "description": "How many files the query found in all, of which files are the best." }, | ||
"matchCount": { "type": "integer", "minimum": 0, "description": "How many matches the query found in all." }, | ||
"languages": { "type": "array", "items": { "type": "ref", "ref": "#languageCount" }, "description": "The languages of the files found, counting those its lang: filters leave out, most files first: of the first 1000 files at most." } | ||
"cursor": { "type": "string", "description": "Absent on the last page, and once the pages are 200 files deep." }, | ||
"languages": { "type": "array", "items": { "type": "ref", "ref": "#languageCount" }, "description": "The languages of the files found, counting those its lang: filters leave out, most files first: of the first 1000 files at most. The first page alone lists them." } | ||
} | ||
} | ||
} | ||
web/src/routes/search/+page.svelte+13−1
| @@ -6,6 +6,8 @@ | ||
import Icon from '#lib/atoms/Icon.svelte'; | ||
import Count from '#lib/atoms/Count.svelte'; | ||
import Empty from '#lib/molecules/Empty.svelte'; | ||
import More from '#lib/atoms/More.svelte'; | ||
import { Pages } from '#lib/pages.svelte.ts'; | ||
import { plural, repoHref } from '#lib/format.ts'; | ||
let { data } = $props(); | ||
const q = $derived(data.query); | ||
| @@ -37,6 +39,14 @@ | ||
// as far as its share of the first's, which fills its own. | ||
const languages = $derived(code.kind === 'found' ? (code.answer.languages ?? []) : []); | ||
const most = $derived(languages[0]?.files ?? 1); | ||
// The files found, a page at a time, each added below those before. | ||
const files = new Pages( | ||
() => data, | ||
(d) => (d.code.kind === 'found' ? { items: d.code.answer.files, cursor: d.code.answer.cursor } : { items: [] }), | ||
(f) => `${f.repo.did}/${f.repo.name}/${f.path}` | ||
); | ||
// How deep the pages go, as searchCode stops them. | ||
const DEEPEST = 200; | ||
const narrowed = (name: string) => (name.toLowerCase() === picked ? of('code', unpicked) : of('code', `${unpicked} lang:${name.includes(' ') ? `"${name}"` : name}`)); | ||
</script> | ||
| ||
| @@ -61,7 +71,9 @@ | ||
{#if data.type === 'code'} | ||
{#if code.kind === 'found' && code.answer.files.length} | ||
<p class="search-count mute">{plural(code.answer.fileCount, 'file', 'files')}{#if within}{' in '}<a href={repoHref(within[1], within[2])}>{within[1]}/{within[2]}</a>{' · '}<a href={everywhere}>Search all of knotless</a>{/if}</p> | ||
<CodeResults files={code.answer.files} /> | ||
<CodeResults files={files.items} /> | ||
<More pages={files} label="More files" /> | ||
{#if !files.href && code.answer.fileCount > DEEPEST}<p class="search-deepest mute">These are the best {DEEPEST}. Add lang:, file: or repo: to find the rest.</p>{/if} | ||
{:else} | ||
<div class="box"> | ||
{#if code.kind === 'off'}<Empty title="Code search is off here">This server searches no code.</Empty> | ||
web/src/routes/search/+page.ts+6−3
| @@ -35,7 +35,8 @@ export const load: PageLoad = async (event) => { | ||
const words = type === 'code' ? q.replace(/(^|\s)-?lang:("[^"]*"|\S+)/g, ' ').trim() : q; | ||
const listed = (e: typeof event) => (words || type === 'repositories' ? xrpc<typeof listRepos.main>(e, 'dev.knotless.listRepos', { q: words, topic }, true) : null); | ||
if (type === 'code') { | ||
const [found, repos] = await Promise.all([code(event, q), REPO.test(q) ? null : listed(quiet)]); | ||
const cursor = event.url.searchParams.get('cursor') ?? undefined; | ||
const [found, repos] = await Promise.all([code(event, q, undefined, cursor), REPO.test(q) ? null : listed(quiet)]); | ||
const counts = { code: found.kind === 'found' ? found.answer.fileCount : undefined, ...reposCounted(repos) }; | ||
return { query: q, topic, type, code: found, counts, repos: [], owners: [], indexed: true }; | ||
} | ||
| @@ -57,17 +58,19 @@ function reposCounted(found: Answer<typeof listRepos.main> | null) { | ||
} | ||
| ||
/** | ||
* The code `q` finds: asked here rather than by `xrpc`, as a query Zoekt | ||
* The code `q` finds, after the files `cursor` says the pages before | ||
* held: asked here rather than by `xrpc`, as a query Zoekt | ||
* cannot read, or a server that searches no code, is said on the page | ||
* rather than failing it. | ||
*/ | ||
async function code(event: Parameters<PageLoad>[0], q: string, limit?: number): Promise<CodeFound> { | ||
async function code(event: Parameters<PageLoad>[0], q: string, limit?: number, cursor?: string): Promise<CodeFound> { | ||
const named = REPO.exec(q); | ||
const rest = named ? q.replace(named[0], ' ').trim() : q; | ||
if (!rest) return { kind: 'none' }; | ||
const params = new URLSearchParams({ q: rest }); | ||
if (named) params.set('repo', `at://${named[1]}/dev.knotless.repo/${named[2]}`); | ||
if (limit) params.set('limit', String(limit)); | ||
if (cursor) params.set('cursor', cursor); | ||
const res = await event.fetch(`/xrpc/dev.knotless.searchCode?${params}`, { cache: 'no-cache' }); | ||
if (res.ok) return { kind: 'found', answer: await res.json() }; | ||
if (res.status === 501) return { kind: 'off' }; | ||
web/src/app.css+1−0
| @@ -1228,6 +1228,7 @@ textarea.field{resize:vertical;font-family:var(--font-mono);font-size:var(--text | ||
.search-lang-bar{position:absolute;inset:0 auto 0 0;border-radius:inherit;background:color-mix(in srgb,var(--lang) calc(var(--fill) + var(--more)),var(--under,var(--bg)));transition:background var(--quick)} | ||
.search-nav-head{padding:var(--space-1-5) var(--space-2);font-size:var(--text-sm);font-weight:var(--weight-semibold);color:var(--mute)} | ||
.search-count{margin-bottom:var(--space-3);font-size:var(--text-sm)} | ||
.search-deepest{margin-top:var(--space-5);text-align:center;font-size:var(--text-sm)} | ||
.code-result-name{flex:1;min-width:0;overflow-wrap:break-word} | ||
.code-result-path{padding:var(--space-2) var(--pad-x);font-size:var(--text-sm)} | ||
.code-results .faint{color:var(--faint)} | ||