chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
knotless-serve searches public repositories' code with Zoekt
.knotless/workflows/test.yml+18−0
| @@ -27,6 +27,10 @@ env: | ||
ATPROTO_COMMIT: 11c5d98cd50261c5e1201fedc5c6c05a7b8d1ccf | ||
# Bumped when the steps that build the alpha change, to build it again. | ||
ALPHA_BUILD: "2" | ||
# Zoekt, which code search runs, as the Dockerfile builds it, and the Go | ||
# that builds it. | ||
ZOEKT_COMMIT: e1fafb15177040d792bd7c8c41a24a0fb39aada4 | ||
GO_VERSION: 1.27.3 | ||
steps: | ||
# Node into /opt/node, the harness's packages when their lockfile | ||
# changes, and cargo-llvm-cov into /opt/tools, each kept in the cache. | ||
| @@ -53,6 +57,20 @@ steps: | ||
# The PDS serves the OAuth sign-in pages from this bundle. | ||
pnpm --filter @atproto/oauth-provider-ui run build:ui | ||
echo "$ATPROTO_COMMIT $ALPHA_BUILD" > /opt/atproto/.built | ||
- name: zoekt | ||
run: | | ||
# Its indexer and webserver into /opt/tools, kept in the cache, built | ||
# again only when the commit changes. | ||
[ "$(cat /opt/tools/.zoekt 2>/dev/null)" = "$ZOEKT_COMMIT" ] && exit 0 | ||
mkdir -p /opt/tools | ||
go=$(mktemp -d) | ||
curl -fsSL "https://go.dev/dl/go$GO_VERSION.linux-$(dpkg --print-architecture).tar.gz" | tar -xz -C "$go" | ||
CGO_ENABLED=0 GOBIN=/opt/tools GOPATH="$go/path" GOCACHE="$go/cache" GOTOOLCHAIN=local \ | ||
"$go/go/bin/go" install \ | ||
"github.com/sourcegraph/zoekt/cmd/zoekt-git-index@$ZOEKT_COMMIT" \ | ||
"github.com/sourcegraph/zoekt/cmd/zoekt-webserver@$ZOEKT_COMMIT" | ||
rm -rf "$go" | ||
echo "$ZOEKT_COMMIT" > /opt/tools/.zoekt | ||
- name: sqlite | ||
run: | | ||
# The one install script the PDS needs, which the install above | ||
crates/knotless-serve/src/web/search.rsnew+75−0
| @@ -0,0 +1,75 @@ | ||
//! `dev.knotless.searchCode`: the code of public repositories, as Zoekt | ||
//! finds it ([`crate::code`]). | ||
| ||
use super::*; | ||
use serde_json::{Value, json}; | ||
| ||
/// How many files a search shows, if not asked, and at most. | ||
const FILES: usize = 20; | ||
const FILES_MAX: usize = 50; | ||
| ||
#[derive(Deserialize)] | ||
pub struct SearchParams { | ||
q: String, | ||
repo: Option<String>, | ||
limit: Option<usize>, | ||
} | ||
| ||
/// A file found, in the repository the index lists it in. | ||
#[derive(Serialize)] | ||
struct FileView { | ||
repo: Value, | ||
path: String, | ||
#[serde(skip_serializing_if = "String::is_empty")] | ||
language: String, | ||
chunks: Vec<crate::code::Chunk>, | ||
} | ||
| ||
/// `dev.knotless.searchCode`: what `q` finds in the default branches of | ||
/// the public repositories this server indexes, or of the one `repo` | ||
/// names; none of a repository the index no longer lists, or has taken | ||
/// down. | ||
pub async fn search_code( | ||
State(app): State<Arc<App>>, | ||
Query(p): Query<SearchParams>, | ||
) -> Result<Response> { | ||
let (Some(code), Some(index)) = (app.code.as_deref(), app.index.as_deref()) else { | ||
return Err(Shown( | ||
StatusCode::NOT_IMPLEMENTED, | ||
"this server searches no code".into(), | ||
)); | ||
}; | ||
let mut within = None; | ||
if let Some(uri) = &p.repo { | ||
let repo = repo::found(&app, uri).await?; | ||
within = Some((repo.did, repo.key)); | ||
} | ||
let within = within | ||
.as_ref() | ||
.map(|(did, key)| (did.as_str(), key.as_str())); | ||
let limit = p.limit.unwrap_or(FILES).clamp(1, FILES_MAX); | ||
let found = code.search(&p.q, within, limit).await?; | ||
let listed = |h: crate::code::Hit| Some((index.listed(&h.did, &h.key).ok().flatten()?, h)); | ||
let shown: Vec<_> = found.hits.into_iter().filter_map(listed).collect(); | ||
let owners = shown.iter().map(|(r, _)| actor(&app, &r.did)); | ||
let owners = futures::future::join_all(owners).await; | ||
let files: Vec<FileView> = shown | ||
.into_iter() | ||
.zip(owners) | ||
.map(|((r, hit), owner)| { | ||
let mut repo = json!(r); | ||
repo["owner"] = json!(owner); | ||
FileView { | ||
repo, | ||
path: hit.path, | ||
language: hit.language, | ||
chunks: hit.chunks, | ||
} | ||
}) | ||
.collect(); | ||
Ok(answer(json!({ | ||
"files": files, | ||
"fileCount": found.file_count, | ||
"matchCount": found.match_count, | ||
}))) | ||
} | ||
crates/knotless-serve/src/code.rsnew+448−0
448 lines changed: not loaded until asked for.
crates/knotless-serve/src/index.rs+63−0
| @@ -170,6 +170,12 @@ pub struct Index { | ||
recount: mpsc::UnboundedSender<(String, String)>, | ||
/// What counts them takes this, once ([`crate::languages::follow`]). | ||
counting: Mutex<Option<mpsc::UnboundedReceiver<(String, String)>>>, | ||
/// The public repositories asked to have their code indexed again, by | ||
/// owner and key, until it is ([`Index::reindex`]). | ||
coded: Mutex<HashSet<(String, String)>>, | ||
recode: mpsc::UnboundedSender<(String, String)>, | ||
/// What indexes their code takes this, once ([`crate::code::follow`]). | ||
coding: Mutex<Option<mpsc::UnboundedReceiver<(String, String)>>>, | ||
/// The releases of things shared whose files are to be read, by URI; | ||
/// what reads them takes the other end, once ([`crate::shared::follow`]). | ||
read: mpsc::UnboundedSender<String>, | ||
| @@ -929,6 +935,7 @@ impl Index { | ||
crate::telemetry::watch(&db); | ||
let (recount, counting) = mpsc::unbounded_channel(); | ||
let (read, reading) = mpsc::unbounded_channel(); | ||
let (recode, coding) = mpsc::unbounded_channel(); | ||
let index = Index { | ||
db: Mutex::new(db), | ||
readers: Readers::new(path), | ||
| @@ -937,6 +944,9 @@ impl Index { | ||
asked: Mutex::default(), | ||
recount, | ||
counting: Mutex::new(Some(counting)), | ||
coded: Mutex::default(), | ||
recode, | ||
coding: Mutex::new(Some(coding)), | ||
read, | ||
reading: Mutex::new(Some(reading)), | ||
stamps: Stamps::new(STAMPS_MAX), | ||
| @@ -1379,6 +1389,10 @@ impl Index { | ||
nsid::REPO => Some(key), | ||
_ => None, | ||
}; | ||
// And a public one's code is indexed again, if it is searched. | ||
if let Some(key) = moved.filter(|_| space.is_empty()) { | ||
self.reindex(did, key); | ||
} | ||
let counted = "SELECT 1 FROM language WHERE did = ?1 AND key = ?2"; | ||
let kept = |k: &&str| db.query_row(counted, [did, k], |_| Ok(())).is_ok(); | ||
if let Some(key) = moved.filter(kept) { | ||
| @@ -1397,6 +1411,29 @@ impl Index { | ||
} | ||
} | ||
| ||
/// Asks for the code of `did`'s public repository with `key` to be | ||
/// indexed again, or forgotten if it is gone, once however often it is | ||
/// asked before it is. | ||
pub fn reindex(&self, did: &str, key: &str) { | ||
let repo = (did.to_string(), key.to_string()); | ||
if self.coded.lock().unwrap().insert(repo.clone()) { | ||
let _ = self.recode.send(repo); | ||
} | ||
} | ||
| ||
/// What [`Index::reindex`] asks for, for the one that indexes code: | ||
/// none after the first take. | ||
pub fn indexing(&self) -> Option<mpsc::UnboundedReceiver<(String, String)>> { | ||
self.coding.lock().unwrap().take() | ||
} | ||
| ||
/// Takes back the ask for `did`'s repository with `key`, as it is | ||
/// indexed: one asked after is indexed again. | ||
pub fn reindexed(&self, did: &str, key: &str) { | ||
let repo = (did.to_string(), key.to_string()); | ||
self.coded.lock().unwrap().remove(&repo); | ||
} | ||
| ||
/// What [`Index::recount`] asks for, for the one that counts: none | ||
/// after the first take. | ||
pub fn counting(&self) -> Option<mpsc::UnboundedReceiver<(String, String)>> { | ||
| @@ -1459,6 +1496,11 @@ impl Index { | ||
let scope = |sql: &str| db.query_row(sql, [&uri], |row| row.get(0)).optional(); | ||
let repo: Option<String> = | ||
scope("DELETE FROM repo WHERE uri = ?1 RETURNING did || '/' || key")?; | ||
// A public repository's code is forgotten. | ||
let gone = repo.as_deref().and_then(|r| r.split_once('/')); | ||
if let Some((owner, key)) = gone.filter(|_| space.is_empty()) { | ||
self.reindex(owner, key); | ||
} | ||
db.execute("DELETE FROM collaborator WHERE uri = ?1", [&uri])?; | ||
let said: Option<String> = | ||
scope("DELETE FROM forge WHERE uri = ?1 RETURNING repo_did || '/' || repo_key")?; | ||
| @@ -3534,6 +3576,27 @@ impl Index { | ||
.optional() | ||
} | ||
| ||
/// `did`'s repository `key`, unless it or its owner is taken down. | ||
pub fn listed(&self, did: &str, key: &str) -> rusqlite::Result<Option<Listed>> { | ||
let db = self.readers.take(&self.db); | ||
let mut stmt = db.prepare_cached( | ||
"SELECT did, name, description, fork_of, created_at, uri FROM repo | ||
WHERE did = ?1 AND key = ?2 AND did NOT IN down AND uri NOT IN down", | ||
)?; | ||
stmt.query_row([did, key], listed).optional() | ||
} | ||
| ||
/// Every public repository, by owner and key, but those taken down. | ||
pub fn code_repos(&self) -> rusqlite::Result<Vec<(String, String)>> { | ||
let db = self.readers.take(&self.db); | ||
let mut stmt = db.prepare_cached( | ||
"-- scans: repo, as code search reads it once, as it starts | ||
SELECT did, key FROM repo WHERE did NOT IN down AND uri NOT IN down", | ||
)?; | ||
let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?; | ||
rows.collect() | ||
} | ||
| ||
/// The record of `did`'s repository `key`. | ||
pub fn repo_of(&self, did: &str, key: &str) -> rusqlite::Result<Option<Value>> { | ||
let db = self.readers.take(&self.db); | ||
crates/knotless-serve/src/lib.rs+16−1
| @@ -20,6 +20,7 @@ mod api; | ||
mod appview; | ||
pub mod archive; | ||
pub mod card; | ||
pub mod code; | ||
pub mod diff; | ||
pub mod firehose; | ||
pub mod gateway; | ||
| @@ -126,6 +127,9 @@ pub struct App { | ||
/// When this server last asked each space's authority to tell it of | ||
/// writes there. | ||
registered: tokio::sync::Mutex<HashMap<String, Instant>>, | ||
/// Set when this server searches the code of public repositories | ||
/// ([`code`]). | ||
pub code: Option<Arc<code::Code>>, | ||
/// Set when this server counts how it is used, for its operator. | ||
pub usage: Option<Arc<usage::Usage>>, | ||
/// Set when this server keeps the lines it logs, for its operator. | ||
| @@ -247,6 +251,7 @@ impl App { | ||
due: Mutex::default(), | ||
told: tokio::sync::Notify::new(), | ||
registered: tokio::sync::Mutex::default(), | ||
code: None, | ||
usage: None, | ||
logs: None, | ||
labels: tokio::sync::broadcast::channel(256).0, | ||
| @@ -1273,6 +1278,8 @@ pub struct Options { | ||
/// Where the PDS sends a gateway's OAuth logins back to, when not | ||
/// `--public-url`: an https front for it, as the PDS asks of a client. | ||
pub oauth_url: Option<String>, | ||
/// Where code search keeps its Zoekt shards, if it is on. | ||
pub code_index: Option<std::path::PathBuf>, | ||
} | ||
| ||
/// What `--private`, which is gone, says instead. | ||
| @@ -1304,6 +1311,7 @@ pub fn parse_args(args: &[String]) -> Result<Options, String> { | ||
bots: None, | ||
sign_up_pds: gateway::SIGN_UP_PDS.into(), | ||
oauth_url: None, | ||
code_index: None, | ||
}; | ||
let mut it = args.iter().skip(1); | ||
while let Some(flag) = it.next() { | ||
| @@ -1372,9 +1380,10 @@ pub fn parse_args(args: &[String]) -> Result<Options, String> { | ||
"--bots" => opts.bots = Some(value()?.into()), | ||
"--sign-up-pds" => opts.sign_up_pds = value()?, | ||
"--oauth-url" => opts.oauth_url = Some(value()?), | ||
"--code-index" => opts.code_index = Some(value()?.into()), | ||
other => { | ||
return Err(format!( | ||
"unexpected {other}\nusage: knotless-serve [--listen <addr>] [--ttl <secs>] [--firehose <wss://relay or pds>] [--jetstream <wss://host>] [--public-url <https://host>] [--oauth-url <https://host>] [--index <file>] [--backfill <https://pds or relay>] [--relay <https://relay>] [--labeler <did>]... [--default-runner <did>] [--client-minimum <version> [--client-minimum-reason <text>]] [--web <http://app>] [--releases <dir or knotless://repo>] [--stale <secs>] [--usage <file>] [--logs <file> [--logs-keep <lines>]] [--check-index <file>] [--metrics <addr>] [--bots <dir>] [--sign-up-pds <https://pds>]" | ||
"unexpected {other}\nusage: knotless-serve [--listen <addr>] [--ttl <secs>] [--firehose <wss://relay or pds>] [--jetstream <wss://host>] [--public-url <https://host>] [--oauth-url <https://host>] [--index <file>] [--backfill <https://pds or relay>] [--relay <https://relay>] [--labeler <did>]... [--default-runner <did>] [--client-minimum <version> [--client-minimum-reason <text>]] [--web <http://app>] [--releases <dir or knotless://repo>] [--stale <secs>] [--usage <file>] [--logs <file> [--logs-keep <lines>]] [--check-index <file>] [--metrics <addr>] [--bots <dir>] [--sign-up-pds <https://pds>] [--code-index <dir>]" | ||
)); | ||
} | ||
} | ||
| @@ -1443,6 +1452,10 @@ pub async fn run(opts: Options, ctx: Ctx) -> Result<()> { | ||
// MCP over HTTP, for any account not taken down, made once | ||
// the address it reads this server's index at is known. | ||
let mcp_ctx = ctx.clone(); | ||
let code = match &opts.code_index { | ||
Some(dir) => Some(Arc::new(code::Code::start(dir).await?)), | ||
None => None, | ||
}; | ||
let mut app = App::build(ctx, opts.ttl, gateway, index); | ||
let reason = opts.client_minimum_reason.clone(); | ||
let client_minimum = opts.client_minimum.clone().map(|m| (m, reason)); | ||
| @@ -1453,6 +1466,7 @@ pub async fn run(opts: Options, ctx: Ctx) -> Result<()> { | ||
app.logs = logs.clone(); | ||
app.default_runner = opts.default_runner.clone(); | ||
app.client_minimum = client_minimum; | ||
app.code = code; | ||
} | ||
// An index built anew is filled from the relay, unless asked otherwise. | ||
let rebuilt = app.index.as_ref().is_some_and(|i| i.rebuilt); | ||
| @@ -1479,6 +1493,7 @@ pub async fn run(opts: Options, ctx: Ctx) -> Result<()> { | ||
} | ||
tokio::spawn(spaces::follow(app.clone(), SPACES)); | ||
tokio::spawn(languages::follow(app.clone())); | ||
tokio::spawn(code::follow(app.clone())); | ||
tokio::spawn(shared::follow(app.clone())); | ||
tokio::spawn(telemetry::summaries(app.clone(), SUMMARY)); | ||
let listener = tokio::net::TcpListener::bind(&opts.listen).await?; | ||
crates/knotless-serve/src/web.rs+2−0
| @@ -66,6 +66,7 @@ mod releases; | ||
mod repo; | ||
mod repos; | ||
mod runs; | ||
mod search; | ||
mod settings; | ||
mod stars; | ||
mod steps; | ||
| @@ -96,6 +97,7 @@ pub use themes::{cookie as palette_cookie, palette}; | ||
pub fn queries() -> Router<Arc<App>> { | ||
Router::new() | ||
.route("/xrpc/dev.knotless.getRepo", get(get_repo)) | ||
.route("/xrpc/dev.knotless.searchCode", get(search::search_code)) | ||
.route( | ||
"/xrpc/dev.knotless.listReleases", | ||
get(releases::list_releases), | ||
crates/knotless-serve/tests/serve.rs+2−0
| @@ -330,6 +330,8 @@ fn binary_lifecycle() { | ||
&s.path("logs.db").to_string_lossy(), | ||
"--logs-keep", | ||
"100000", | ||
"--code-index", | ||
&s.path("code").to_string_lossy(), | ||
"--client-minimum", | ||
"0.1.0", | ||
"--client-minimum-reason", | ||
crates/knotless-serve/tests/web.rs+104−1
| @@ -6,6 +6,7 @@ | ||
mod common; | ||
use common::*; | ||
| ||
use knotless_serve::code::Code; | ||
use knotless_serve::index::Index; | ||
use knotless_serve::{App, router}; | ||
use reqwest::blocking::{Client, Response}; | ||
| @@ -26,13 +27,23 @@ fn serve(indexed: bool) -> Site { | ||
| ||
/// A server reading as `s`. | ||
fn serve_as(s: Sandbox, indexed: bool) -> Site { | ||
serve_with(s, indexed, false) | ||
} | ||
| ||
/// A server reading as `s`, searching code too if `code`, though not | ||
/// yet indexing it ([`knotless_serve::code::follow`]). | ||
fn serve_with(s: Sandbox, indexed: bool, code: bool) -> Site { | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let base = format!("http://{}", listener.local_addr().unwrap()); | ||
let index = indexed.then(|| Index::open(&s.path("index.db")).unwrap()); | ||
let app: Arc<App> = App::build(ctx(&s), Duration::ZERO, None, index); | ||
let mut app: Arc<App> = App::build(ctx(&s), Duration::ZERO, None, index); | ||
if code { | ||
let started = rt.block_on(Code::start(&s.path("code"))).unwrap(); | ||
Arc::get_mut(&mut app).unwrap().code = Some(Arc::new(started)); | ||
} | ||
let router = router(app.clone()); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
if indexed { | ||
| @@ -3439,3 +3450,95 @@ fn the_steps_shared() { | ||
._rt | ||
.block_on(knotless_serve::spaces::follow(plain.app.clone(), every)); | ||
} | ||
| ||
/// Code search, over Zoekt: what the default branch of each public | ||
/// repository holds, found in all of them or in one, as it is pushed; and | ||
/// a repository deleted found no more. | ||
#[test] | ||
fn searching_code() { | ||
// A server that searches no code says so. | ||
let plain = serve(false); | ||
let asked = plain.status("/xrpc/dev.knotless.searchCode?q=x"); | ||
assert_eq!(asked.0, 501, "{}", asked.1); | ||
| ||
let site = serve_with(Sandbox::new(), true, true); | ||
let s = person(account(), &site); | ||
let handle = s.account.handle.clone(); | ||
// Pushed before code is indexed: what has no shards yet is indexed as | ||
// indexing starts. | ||
for (name, n) in [("needle-one", 1), ("needle-two", 2)] { | ||
let work = s.work(name, 1); | ||
let lib = format!("// the haystack\nfn needle_{n}() {{}}\nfn other() {{}}\n"); | ||
write(&work, "src/lib.rs", lib.as_bytes()); | ||
s.git(&work, &["add", "-A"]); | ||
s.git(&work, &["commit", "-q", "-m", "Hide a needle"]); | ||
s.git(&work, &["push", "-q", &s.url(name), "main"]); | ||
} | ||
wait_until(|| names(&site.search("needle")["repos"]).len() == 2); | ||
site._rt | ||
.spawn(knotless_serve::code::follow(site.app.clone())); | ||
let search = |q: &str| site.json(&format!("/xrpc/dev.knotless.searchCode?q={q}")); | ||
let found = |q: &str| search(q)["files"].as_array().unwrap().len(); | ||
wait_until(|| found("needle_") == 2); | ||
| ||
let one = search("needle_1"); | ||
assert_eq!( | ||
(one["fileCount"].as_u64(), one["matchCount"].as_u64()), | ||
(Some(1), Some(1)) | ||
); | ||
let file = &one["files"][0]; | ||
assert_eq!(file["repo"]["name"], "needle-one"); | ||
assert_eq!(file["repo"]["owner"]["handle"], handle.as_str()); | ||
assert_eq!(file["path"], "src/lib.rs"); | ||
assert_eq!(file["language"], "Rust"); | ||
let chunk = &file["chunks"][0]; | ||
assert_eq!(chunk["startLine"], 1, "{chunk}"); | ||
assert_eq!( | ||
chunk["content"], | ||
"// the haystack\nfn needle_1() {}\nfn other() {}" | ||
); | ||
assert_eq!( | ||
chunk["ranges"], | ||
serde_json::json!([{"line": 2, "byteStart": 3, "byteEnd": 11}]) | ||
); | ||
| ||
// In one repository, by its record's URI. | ||
let within = |name: &str| { | ||
let repo = format!("at://{handle}/dev.knotless.repo/{name}"); | ||
site.json(&format!( | ||
"/xrpc/dev.knotless.searchCode?q=needle_&repo={repo}&limit=5" | ||
)) | ||
}; | ||
let two = within("needle-two"); | ||
assert_eq!(two["files"].as_array().unwrap().len(), 1, "{two}"); | ||
assert_eq!(two["files"][0]["repo"]["name"], "needle-two"); | ||
| ||
// A push is found once it is indexed again. | ||
let work = s.path("needle-two"); | ||
write(&work, "src/more.rs", b"fn needle_3() {}\n"); | ||
s.git(&work, &["add", "-A"]); | ||
s.git(&work, &["commit", "-q", "-m", "Hide another"]); | ||
s.git(&work, &["push", "-q", &s.url("needle-two"), "main"]); | ||
wait_until(|| found("needle_3") == 1); | ||
| ||
// A query Zoekt cannot read is the asker's to fix. | ||
let bad = site.status("/xrpc/dev.knotless.searchCode?q=("); | ||
assert_eq!(bad.0, 400, "{}", bad.1); | ||
| ||
// A repository deleted is found no more. | ||
// This account's, as others' repositories on the PDS are indexed too: | ||
// each named for its owner's DID, `:` written `_3a`. | ||
let theirs = s.account.did.replace(':', "_3a"); | ||
let shards = || { | ||
let dir = std::fs::read_dir(site._sandbox.path("code")).unwrap(); | ||
let names = dir.map(|e| e.unwrap().file_name().into_string().unwrap()); | ||
let mine = |n: &String| n.starts_with(&theirs) && n.ends_with(".zoekt"); | ||
names.filter(mine).count() | ||
}; | ||
assert_eq!(shards(), 2); | ||
ok(&s, &["delete", &s.url("needle-one"), "--yes"]); | ||
wait_until(|| found("needle_1") == 0); | ||
// And its shards are deleted. | ||
wait_until(|| shards() == 1); | ||
assert_eq!(found("needle_"), 2); | ||
} | ||
deploy/docker-compose.yml+4−0
| @@ -81,6 +81,10 @@ services: | ||
# /oauth/bot: run and triage read them from there. | ||
- --bots | ||
- /data/bots | ||
# Code search: the Zoekt shards of every public repository's default | ||
# branch, written from its mirror as it is pushed. | ||
- --code-index | ||
- /data/code | ||
# The runner a new public repository names unless its owner names | ||
# others: the runner host's account (runner-host/README.md). Empty, | ||
# none. | ||
docs/design/serve.md+24−0
| @@ -397,6 +397,30 @@ sees, is counted from its mirror the first time its list is read; after | ||
that, what was counted shows at once, and its mirror is synced and counted | ||
again in the background, as a sync waits on its owner's PDS. | ||
| ||
### Code search | ||
| ||
With `--code-index`, the server searches the code of public repositories. | ||
It runs [Zoekt](https://github.com/sourcegraph/zoekt), a trigram index | ||
built for searching code, as two programs beside it: | ||
| ||
- `zoekt-webserver` serves searches from the index's shards on disk, | ||
on a loopback port of its own, never reached from outside. It loads | ||
shards as they are written, so a repository written again is searched | ||
at once. | ||
- `zoekt-git-index` writes one repository's shards from its mirror, of | ||
its default branch. A commit already indexed is left as it is. | ||
| ||
A single task in the background indexes one repository at a time. As | ||
the server starts it indexes each public repository the index lists | ||
that has no shards yet. After that it indexes a repository again | ||
whenever a feed brings a change to one of its refs or its record. A | ||
repository deleted has its shards deleted when its turn comes. | ||
| ||
`dev.knotless.searchCode` asks Zoekt and answers only files of | ||
repositories the index still lists and has not taken down. So a takedown | ||
hides a repository's code at once, before its shards are gone. Private | ||
repositories are never indexed, as the index never sees them. | ||
| ||
### Shared steps | ||
| ||
The index keeps a directory of what repositories publish for others to | ||
docs/reference/api.md+16−0
| @@ -155,6 +155,22 @@ its own, and those it was proven a member of, as it read one or as it | ||
was asked after in the background. The answer is then that account's | ||
alone. | ||
| ||
### `dev.knotless.searchCode` | ||
| ||
| Parameter | Default | | | ||
| --- | --- | --- | | ||
| `q` | | required: Zoekt's query language, as words, `"quoted text"`, `/a regex/`, `file:`, `lang:`, `case:yes` | | ||
| `repo` | | a repository's `at://` URI, its authority a DID or a handle: only its code | | ||
| `limit` | 20 | the most files answered, clamped to 1 to 50 | | ||
| ||
Answers `files`, the best first, each with its `repo` (as `listRepos` | ||
answers one), `path`, `language?` and `chunks`: runs of lines holding | ||
matches, each its `startLine`, its `content` and its `ranges`, a match | ||
by `line`, `byteStart` and `byteEnd` in UTF-8 bytes. `fileCount` and | ||
`matchCount` count all that was found. Only default branches are | ||
searched, and taken-down owners and repositories are left out. A query | ||
Zoekt cannot read answers 400, and a server without `--code-index` 501. | ||
| ||
### `dev.knotless.searchOwners` | ||
| ||
| Parameter | Default | | | ||