~ knotless Search/
↑↓ move↵ open⇧↵ all results

chadtmiller.com/knotless
Switch repository
Loading…

git that lives in your Atmosphere account

knotless.dev
1 collaborator

knotless 0.13.0 Latest 2 packages, 1 image

1
10

Repositories have websites, served on knotless.page

Merged chadtmiller.com asked to merge into main from chadtmiller.com/knotless:feature/pages
.changeset/sites.mdnew+7−0
View file
@@ -0,0 +1,7 @@
---
default: minor
---
# Deploy a repository's website
`knotless site <url> <dir>` deploys a directory as the repository's website, served at `https://<owner>.knotless.page/<repo>/`, uploading only the files the live site lacks. `--fallback` serves a single-page app, `--not-found` a 404 page, and `--down` takes the site down.
.knotless/workflows/deploy.yml+4−0
View file
@@ -42,6 +42,10 @@ secrets: [DEPLOY_SSH_KEY, RELEASE_PASSWORD, RUNNER_HOST_SSH_KEY]
env:
KNOTLESS_HOST: root@knotless.dev
KNOTLESS_DOMAIN: knotless.dev
# Repositories' sites (docs/design/pages.md), whose certificate the
# proxy holds once deploy/setup.sh has run with this and the zone's
# token.
KNOTLESS_PAGES_DOMAIN: knotless.page
KNOTLESS_DATA: /mnt/data/knotless
KNOTLESS_ADMIN_DIDS: did:plc:bcgltzqazw5tb6k2g3ttenbj
# The runner's account (ci.knotless.dev), the triager's
crates/knotless/src/cli.rs+49−0
View file
@@ -396,6 +396,13 @@ const USAGE: &str = "usage: knotless <command>
--oci adds an image's, from an OCI
image layout, for knotless-serve's /v2/
releases <url> [<package>] the versions published, highest first
site <url> [<dir>] [--fallback <path>] [--not-found <path>] [--commit <sha>]
[--check <at-uri>] [--down]
deploy a directory as the repository's
website, as the owner, uploading only
what the live site lacks, and print
where it is served; with no <dir>, the
live site; --down takes it down
uses update [-C <dir>] [<uses>...] pin the shared steps the workflows use
in .knotless/uses.lock: each to the
highest release its @ takes (those
@@ -1618,6 +1625,48 @@ async fn command(
let uri = crate::release::publish(ctx, &url, spec).await?;
writeln!(out, "{uri}")?;
}
"site" => {
let url = args.url()?;
let mut spec = crate::site::Deploy {
commit: ctx.env.run_sha.clone(),
..Default::default()
};
let (mut dir, mut down) = (None, false);
while let Some(arg) = args.rest.next() {
match arg.as_str() {
"--fallback" => spec.fallback = Some(args.next("<path>")?.into()),
"--not-found" => spec.not_found = Some(args.next("<path>")?.into()),
"--commit" => spec.commit = Some(args.next("<sha>")?.into()),
"--check" => spec.check = Some(args.next("<at-uri>")?.into()),
"--down" => down = true,
flag if flag.starts_with("--") => bail!("unexpected {flag}"),
path => dir = Some(PathBuf::from(path)),
}
}
match (dir, down) {
(Some(_), true) => bail!("--down takes no <dir>"),
(None, true) => {
crate::site::take_down(ctx, &url).await?;
writeln!(out, "taken down")?;
}
(None, false) => {
let live = crate::site::read(ctx, &url).await?;
let Some((site, address)) = live else {
bail!("{url} has no site");
};
let commit = site.commit.as_deref().unwrap_or("-");
let (files, at) = (site.files.len(), site.created_at);
writeln!(out, "{address}\t{files} files\t{commit}\t{at}")?;
}
(Some(dir), false) => {
spec.dir = dir;
let done = crate::site::deploy(ctx, &url, spec).await?;
let (uploaded, files) = (done.uploaded, done.files);
writeln!(out, "uploaded {uploaded} of {files} files")?;
writeln!(out, "{}", done.address)?;
}
}
}
"releases" => {
let url = args.url()?;
let package = args.rest.next().map(String::as_str);
crates/knotless/src/jetstream.rs+4−3
View file
@@ -58,9 +58,9 @@ pub struct Op {
/// Whether `collection` is one knotless writes, the forge reads, or a
/// runner reports in, counts minutes in, or is asked in; a star, a
/// follow, a release, a theme for the pages, or an organization's
/// membership, on either side; a runner's key; or one an account's profile
/// is made of.
/// follow, a release, a site, a theme for the pages, or an
/// organization's membership, on either side; a runner's key; or one an
/// account's profile is made of.
pub fn knotless(collection: &str) -> bool {
nsid::ALL.contains(&collection)
|| nsid::FORGE.contains(&collection)
@@ -72,6 +72,7 @@ pub fn knotless(collection: &str) -> bool {
nsid::FOLLOW,
nsid::REACTION,
nsid::RELEASE,
nsid::SITE,
nsid::THEME,
nsid::ORG,
nsid::ORG_MEMBER,
crates/knotless/src/lib.rs+1−0
View file
@@ -59,6 +59,7 @@ pub mod reviews;
pub mod sealed;
pub mod secrets;
pub mod session;
pub mod site;
pub mod social;
pub mod space;
pub mod space_sync;
crates/knotless/src/records.rs+36−0
View file
@@ -74,6 +74,9 @@ pub mod nsid {
pub const RELEASE: &str = "dev.knotless.release";
/// A theme for the pages, in the account that made it.
pub const THEME: &str = "dev.knotless.theme";
/// A repository's website, in its owner's account, keyed by the
/// repository's key and replaced on each deploy.
pub const SITE: &str = "dev.knotless.site";
}
pub const PACK_MIME: &str = "application/x-git-packed-objects";
@@ -968,6 +971,39 @@ pub struct ReleaseFile {
pub parts: Vec<ReleasePart>,
}
/// A repository's site, keyed by the repository's key
/// ([`crate::site`]). Replaced on each deploy.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct SiteRecord {
pub files: Vec<SiteFile>,
/// The page served, with 404, for a path no file has.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub not_found: Option<String>,
/// The page served, with 200, for a page no file has.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fallback: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub commit: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub check: Option<String>,
pub created_at: String,
}
/// A site's file, over blobs in order: none, when it is empty.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct SiteFile {
pub path: String,
pub size: u64,
/// Lowercase hex.
pub sha256: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content_type: Option<String>,
#[serde(default)]
pub parts: Vec<ReleasePart>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub struct ReleasePart {
pub blob: BlobRef,
crates/knotless/src/site.rsnew+375−0
View file
@@ -0,0 +1,375 @@
//! Sites: a repository's static website, its files over blobs in its
//! owner's account under one record keyed by the repository's key,
//! replaced on each deploy, and served at `<owner>.knotless.page/<repo>/`
//! (see docs/design/pages.md).
use crate::env::Ctx;
use crate::push;
use crate::records::{self, RELEASE_MIME, ReleasePart, SiteFile, SiteRecord, nsid};
use crate::release::sha256_file;
use crate::remote::{self, Remote};
use crate::session;
use crate::url::RepoUrl;
use anyhow::{Context, Result, bail};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
/// The domain sites are served under, each owner at a label of it.
pub const DOMAIN: &str = "knotless.page";
/// The most files a site holds.
pub const MAX_FILES: usize = 2000;
/// The longest path a site's file has, in bytes.
pub const MAX_PATH: usize = 512;
/// The longest DNS label.
const MAX_LABEL: usize = 63;
/// The `Content-Type` a file is served with, by its extension.
const TYPES: &[(&str, &str)] = &[
("html", "text/html; charset=utf-8"),
("htm", "text/html; charset=utf-8"),
("css", "text/css; charset=utf-8"),
("js", "text/javascript; charset=utf-8"),
("mjs", "text/javascript; charset=utf-8"),
("json", "application/json"),
("map", "application/json"),
("webmanifest", "application/manifest+json"),
("txt", "text/plain; charset=utf-8"),
("md", "text/markdown; charset=utf-8"),
("xml", "application/xml"),
("rss", "application/rss+xml"),
("atom", "application/atom+xml"),
("svg", "image/svg+xml"),
("png", "image/png"),
("jpg", "image/jpeg"),
("jpeg", "image/jpeg"),
("gif", "image/gif"),
("webp", "image/webp"),
("avif", "image/avif"),
("ico", "image/x-icon"),
("woff", "font/woff"),
("woff2", "font/woff2"),
("ttf", "font/ttf"),
("otf", "font/otf"),
("wasm", "application/wasm"),
("pdf", "application/pdf"),
("mp4", "video/mp4"),
("webm", "video/webm"),
("mp3", "audio/mpeg"),
("ogg", "audio/ogg"),
("wav", "audio/wav"),
];
/// The type `path` is served with, by its extension, if knotless knows
/// it; else a reader sends it as bytes.
pub fn content_type(path: &str) -> Option<&'static str> {
let name = path.rsplit('/').next().unwrap_or(path);
let (_, ext) = name.rsplit_once('.')?;
let ext = ext.to_ascii_lowercase();
TYPES.iter().find(|(e, _)| *e == ext).map(|(_, t)| *t)
}
/// `handle` as one DNS label: each `-` doubled, then each `.` a `-`, so no
/// two handles give the same one. None for a handle whose label would be
/// longer than a label can be, or would begin `xn--`, which browsers read
/// as an internationalized name.
pub fn label(handle: &str) -> Option<String> {
let label = handle
.to_ascii_lowercase()
.replace('-', "--")
.replace('.', "-");
let fits = label.len() <= MAX_LABEL && !label.starts_with("xn--");
fits.then_some(label)
}
/// The handle `label` names, read back as [`label`] wrote it: a pair of
/// `-` a `-`, a single one a `.`. None for a label with no `-`, which is a
/// DID's ([`did_label`]), and for one no handle gives.
pub fn handle(label: &str) -> Option<String> {
let mut handle = String::new();
let mut chars = label.chars().peekable();
while let Some(c) = chars.next() {
match c {
'-' if chars.peek() == Some(&'-') => {
chars.next();
handle.push('-');
}
'-' => handle.push('.'),
c => handle.push(c),
}
}
let fits = |s: &str| !s.is_empty() && !s.starts_with('-') && !s.ends_with('-');
let parts = handle.contains('.') && handle.split('.').all(fits);
let back = self::label(&handle);
(parts && back.as_deref() == Some(label)).then_some(handle)
}
/// The label a `did:plc` account is served at whatever its handle: its
/// plc id, which has no `-`, so it is never a handle's.
pub fn did_label(did: &str) -> Option<&str> {
did.strip_prefix("did:plc:")
}
/// The address of `name`'s site, its owner at `label`.
pub fn address(label: &str, name: &str) -> String {
format!("https://{label}.{DOMAIN}/{name}/")
}
/// Whether `path` is a site's file's: `/`-separated, with no leading `/`,
/// no empty, `.` or `..` segment, and at most [`MAX_PATH`] bytes.
pub fn valid_path(path: &str) -> bool {
!path.is_empty()
&& path.len() <= MAX_PATH
&& path
.split('/')
.all(|s| !s.is_empty() && s != "." && s != "..")
}
/// Every file under `dir`, by its path from it, in order. A link is
/// refused rather than followed, so a site holds what the directory does,
/// and so is a name that is not UTF-8, as two such could read as one path.
pub fn walk(dir: &Path) -> Result<Vec<(String, PathBuf)>> {
let mut found = Vec::new();
let mut dirs = vec![(String::new(), dir.to_path_buf())];
while let Some((prefix, at)) = dirs.pop() {
let reading = || format!("reading {}", at.display());
for entry in std::fs::read_dir(&at).with_context(reading)? {
let entry = entry.with_context(reading)?;
let path = entry.path();
let utf8 = format!("{}: a site's file is named in UTF-8", path.display());
let name = entry.file_name().into_string().ok().context(utf8)?;
let kind = entry.file_type().with_context(reading)?;
let named = format!("{prefix}{name}");
if kind.is_dir() {
dirs.push((format!("{named}/"), path));
} else if kind.is_file() {
found.push((named, path));
} else {
bail!(
"{}: a site holds files and directories, not links",
path.display()
);
}
}
}
found.sort();
Ok(found)
}
/// What `knotless site` deploys.
#[derive(Debug, Default)]
pub struct Deploy {
pub dir: PathBuf,
pub fallback: Option<String>,
pub not_found: Option<String>,
pub commit: Option<String>,
pub check: Option<String>,
}
/// What a deploy did: the files it uploaded, of all it holds, and where it
/// is served.
#[derive(Debug)]
pub struct Deployed {
pub uploaded: usize,
pub files: usize,
pub address: String,
}
/// `url`'s repository, public and found, with its key.
async fn open(ctx: &Ctx, url: &RepoUrl) -> Result<(Remote, String)> {
let remote = Remote::open(ctx, url).await?;
if remote.space.is_some() {
bail!("{url}: a private repository's site is not published yet");
}
let Some(repo) = remote.repo().await? else {
bail!("{url}: no such repository");
};
Ok((remote, repo.value.key))
}
/// Where `url`'s site is served: at its owner's handle, as `url` names
/// them, or else at the DID of the account `remote` found.
fn served_at(url: &RepoUrl, remote: &Remote) -> Result<String> {
let owner = Some(url.authority.as_str()).filter(|a| !a.starts_with("did:"));
let named = owner.and_then(label);
let label = named.or_else(|| did_label(remote.did()).map(String::from));
let served = label.map(|label| address(&label, &url.name));
served.context(format!("{url}: its owner has no name a site is served at"))
}
/// Deploys `spec`'s directory as `url`'s site, as its owner, logged in:
/// each file whose bytes the live site does not hold uploaded, and the
/// record replaced. Everything is checked before anything is uploaded.
pub async fn deploy(ctx: &Ctx, url: &RepoUrl, spec: Deploy) -> Result<Deployed> {
let found = walk(&spec.dir)?;
if found.is_empty() || found.len() > MAX_FILES {
bail!("a site holds 1 to {MAX_FILES} files");
}
let mut measured = Vec::new();
for (path, at) in found {
if !valid_path(&path) {
bail!(
"{path:?} cannot be a site's path: at most {MAX_PATH} bytes, with no empty, . or .. part"
);
}
let (size, sha256) = sha256_file(&at)?;
measured.push((path, at, size, sha256));
}
for page in [&spec.fallback, &spec.not_found].into_iter().flatten() {
if !measured.iter().any(|(path, ..)| path == page) {
bail!("{page} is no file of the site");
}
}
let (remote, key) = open(ctx, url).await?;
let address = served_at(url, &remote)?;
let live = remote.get_record::<SiteRecord>(nsid::SITE, &key).await?;
let auth = session::ensure(ctx, &remote.identity).await?.auth();
let mut held: HashMap<String, Vec<ReleasePart>> = HashMap::new();
for file in live.iter().flat_map(|l| &l.value.files) {
held.insert(file.sha256.clone(), file.parts.clone());
}
let (count, mut uploaded, mut files) = (measured.len(), 0, Vec::new());
for (path, at, size, sha256) in measured {
let parts = match held.get(&sha256) {
Some(parts) => parts.clone(),
None if size == 0 => Vec::new(),
None => {
uploaded += 1;
let blobs = push::upload_file(ctx, &remote, &auth, &at, size, RELEASE_MIME);
let parts: Vec<ReleasePart> = blobs
.await?
.into_iter()
.map(|blob| ReleasePart { blob })
.collect();
held.insert(sha256.clone(), parts.clone());
parts
}
};
let content_type = content_type(&path).map(String::from);
files.push(SiteFile {
path,
size,
sha256,
content_type,
parts,
});
}
let record = SiteRecord {
files,
not_found: spec.not_found,
fallback: spec.fallback,
commit: spec.commit,
check: spec.check,
created_at: records::now(),
};
let typed = records::typed(nsid::SITE, &record);
let write = match live {
Some(_) => remote::update(nsid::SITE, &key, typed),
None => remote::create(nsid::SITE, &key, typed),
};
remote.apply(vec![write], &auth).await?;
Ok(Deployed {
uploaded,
files: count,
address,
})
}
/// Takes `url`'s site down, as its owner, logged in: its record deleted,
/// and with it the blobs only it held.
pub async fn take_down(ctx: &Ctx, url: &RepoUrl) -> Result<()> {
let (remote, key) = open(ctx, url).await?;
if remote
.get_record::<SiteRecord>(nsid::SITE, &key)
.await?
.is_none()
{
bail!("{url} has no site");
}
let auth = session::ensure(ctx, &remote.identity).await?.auth();
remote
.apply(vec![remote::delete(nsid::SITE, &key)], &auth)
.await
}
/// `url`'s live site, and where it is served, if it has one.
pub async fn read(ctx: &Ctx, url: &RepoUrl) -> Result<Option<(SiteRecord, String)>> {
let (remote, key) = open(ctx, url).await?;
let address = served_at(url, &remote)?;
let live = remote.get_record::<SiteRecord>(nsid::SITE, &key).await?;
Ok(live.map(|l| (l.value, address)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn handles_are_labels_and_back() {
let named = [
("chadtmiller.com", "chadtmiller-com"),
("alice.bsky.social", "alice-bsky-social"),
("a-b.c", "a--b-c"),
("a.b-c", "a-b--c"),
("Mixed.Case", "mixed-case"),
];
for (h, l) in named {
assert_eq!(label(h).as_deref(), Some(l), "{h}");
assert_eq!(
handle(l).as_deref(),
Some(h.to_ascii_lowercase().as_str()),
"{l}"
);
}
assert_eq!(label(&format!("{}.com", "a".repeat(60))), None);
assert_eq!(label("xn--bcher-kva.example"), None);
// A DID's label, and labels no handle gives.
for l in [
"abc123",
"-a-b",
"a-b-",
"a---b",
&format!("{}-c", "a".repeat(62)),
] {
assert_eq!(handle(l), None, "{l}");
}
assert_eq!(did_label("did:plc:abc"), Some("abc"));
assert_eq!(did_label("did:web:example.com"), None);
assert_eq!(address("a-b", "site"), "https://a-b.knotless.page/site/");
}
#[test]
fn paths_and_types() {
for ok in ["index.html", "a/b.css", ".well-known/x"] {
assert!(valid_path(ok), "{ok}");
}
for bad in ["", "/a", "a//b", "a/./b", "../a", "a/", &"a".repeat(513)] {
assert!(!valid_path(bad), "{bad}");
}
assert_eq!(
content_type("a/INDEX.HTML"),
Some("text/html; charset=utf-8")
);
assert_eq!(
content_type("app.js"),
Some("text/javascript; charset=utf-8")
);
assert_eq!(content_type("data.bin"), None);
assert_eq!(content_type("dir.d/README"), None);
}
// Two names that are not UTF-8 could read as one path, so neither is
// taken. Linux alone lets a file be named so.
#[cfg(target_os = "linux")]
#[test]
fn a_name_not_in_utf8_is_refused() {
use std::os::unix::ffi::OsStrExt;
let dir = tempfile::tempdir().unwrap();
let name = std::ffi::OsStr::from_bytes(b"a\xff.html");
std::fs::write(dir.path().join(name), b"").unwrap();
let refused = walk(dir.path()).unwrap_err().to_string();
assert!(refused.contains("named in UTF-8"), "{refused}");
}
}
crates/knotless/tests/cli.rs+167−0
View file
@@ -1343,6 +1343,173 @@ fn releases() {
}
}
#[test]
fn sites() {
let mut s = Sandbox::new();
let handle = s.account.handle.clone();
s.set("KNOTLESS_IDENTIFIER", &handle);
s.set("KNOTLESS_CHUNK_SIZE", "1000");
let url = s.url("site");
let work = s.work("w", 1);
s.git(&work, &["push", "-q", &url, "main"]);
let dist = s.path("dist");
let write = |path: &str, bytes: &[u8]| {
let at = dist.join(path);
std::fs::create_dir_all(at.parent().unwrap()).unwrap();
std::fs::write(at, bytes).unwrap();
};
let big: Vec<u8> = (0..2500u32).map(|i| (i * 7 % 251) as u8).collect();
write("index.html", b"<h1>home</h1>");
write("404.html", b"gone");
write("assets/app.js", &big);
write("assets/same.js", &big);
write(".nojekyll", b"");
let dir = dist.to_string_lossy().into_owned();
let label = handle.replace('-', "--").replace('.', "-");
let address = format!("https://{label}.knotless.page/site/");
// No site yet.
let (ok, _, err) = s.knotless(&["site", &url], b"");
assert!(!ok && err.contains("has no site"), "{err}");
// Deployed as the owner, at the run's commit: the same bytes once.
s.set("KNOTLESS_RUN_SHA", &"c".repeat(40));
let deploy = ["site", &url, &dir, "--not-found", "404.html"];
let (code, out) = s.cli(&deploy, b"");
assert_eq!(code, 0, "{out}");
assert_eq!(out, format!("uploaded 3 of 5 files\n{address}"));
let records = s.records("dev.knotless.site");
assert_eq!(records.len(), 1);
let (rkey, r) = &records[0];
let paths: Vec<&str> = r["files"]
.as_array()
.unwrap()
.iter()
.map(|f| f["path"].as_str().unwrap())
.collect();
assert_eq!(
paths,
[
".nojekyll",
"404.html",
"assets/app.js",
"assets/same.js",
"index.html"
]
);
assert_eq!(r["files"][0]["parts"].as_array().unwrap().len(), 0);
assert_eq!(r["files"][2]["parts"].as_array().unwrap().len(), 3);
assert_eq!(r["files"][2]["parts"], r["files"][3]["parts"]);
assert_eq!(
r["files"][2]["contentType"],
"text/javascript; charset=utf-8"
);
assert_eq!(r["files"][0].get("contentType"), None);
assert_eq!(r["notFound"], "404.html");
assert_eq!(r["commit"], "c".repeat(40));
let (_, live) = s.cli(&["site", &url], b"");
assert!(
live.starts_with(&format!("{address}\t5 files\t{}\t", "c".repeat(40))),
"{live}"
);
// Deployed again: only what changed is uploaded, and the record replaced.
write("index.html", b"<h1>home, again</h1>");
std::fs::remove_file(dist.join("404.html")).unwrap();
let check = format!("at://{}/dev.knotless.check/x", s.account.did);
let again = [
"site",
&url,
&dir,
"--fallback",
"index.html",
"--commit",
&"d".repeat(40),
"--check",
&check,
];
let (code, out) = s.cli(&again, b"");
assert_eq!(code, 0, "{out}");
assert!(out.starts_with("uploaded 1 of 4 files\n"), "{out}");
let records = s.records("dev.knotless.site");
assert_eq!(records.len(), 1);
assert_eq!(&records[0].0, rkey);
let r = &records[0].1;
assert_eq!(
(r["fallback"].as_str(), r.get("notFound")),
(Some("index.html"), None)
);
assert_eq!(
(r["commit"].as_str(), r["check"].as_str()),
(
Some("dddddddddddddddddddddddddddddddddddddddd"),
Some(check.as_str())
)
);
// Named by its owner's DID, it is served at the DID's label.
let by_did = format!("knotless://{}/site", s.account.did);
let (_, live) = s.cli(&["site", &by_did], b"");
let plc = s.account.did.strip_prefix("did:plc:").unwrap();
assert!(
live.starts_with(&format!("https://{plc}.knotless.page/site/\t")),
"{live}"
);
// Taken down, once.
assert_eq!(
s.cli(&["site", &url, "--down"], b""),
(0, "taken down".into())
);
assert!(s.records("dev.knotless.site").is_empty());
let (ok, _, err) = s.knotless(&["site", &url, "--down"], b"");
assert!(!ok && err.contains("has no site"), "{err}");
// What is refused before anything is uploaded.
let empty = s.path("empty");
std::fs::create_dir_all(&empty).unwrap();
let deep = s.path("deep");
let long = format!("{}/x", vec!["d".repeat(100); 6].join("/"));
std::fs::create_dir_all(deep.join(&long).parent().unwrap()).unwrap();
std::fs::write(deep.join(&long), b"x").unwrap();
let linked = s.path("linked");
std::fs::create_dir_all(&linked).unwrap();
std::os::unix::fs::symlink(dist.join("index.html"), linked.join("index.html")).unwrap();
let (empty, deep, linked) = (
empty.to_string_lossy().into_owned(),
deep.to_string_lossy().into_owned(),
linked.to_string_lossy().into_owned(),
);
let missing = s.path("missing").to_string_lossy().into_owned();
let refused = [
(vec![empty.as_str()], "a site holds 1 to 2000 files"),
(vec![deep.as_str()], "cannot be a site's path"),
(vec![linked.as_str()], "not links"),
(vec![missing.as_str()], "reading"),
(
vec![dir.as_str(), "--fallback", "nope.html"],
"nope.html is no file of the site",
),
(vec![dir.as_str(), "--down"], "--down takes no <dir>"),
(vec![dir.as_str(), "--bogus"], "unexpected --bogus"),
];
for (args, said) in refused {
let mut argv = vec!["site", url.as_str()];
argv.extend(args);
let (ok, _, err) = s.knotless(&argv, b"");
assert!(!ok && err.contains(said), "{argv:?}: {err}");
}
let gone = s.url("nothing-here");
let (ok, _, err) = s.knotless(&["site", &gone, &dir], b"");
assert!(!ok && err.contains("no such repository"), "{err}");
if net().spaces {
let hidden = s.url("hidden");
assert_eq!(s.cli(&["create", &hidden, "--private"], b"").0, 0);
let (ok, _, err) = s.knotless(&["site", &hidden, &dir], b"");
assert!(!ok && err.contains("private repository's site"), "{err}");
}
}
#[test]
fn shared_steps_are_pinned() {
let mut s = Sandbox::new();
crates/knotless-serve/src/index.rs+61−2
View file
@@ -13,8 +13,9 @@ use jiff::{Timestamp, tz::TimeZone};
use knotless::aturi::{AtUri, SpaceRef};
use knotless::forge::{Entry, Item};
use knotless::records::{
FailedCheck, OrgMemberRecord, OrgRecord, ReleaseRecord, RepoRecord, RepoRef, nsid,
FailedCheck, OrgMemberRecord, OrgRecord, ReleaseRecord, RepoRecord, RepoRef, SiteRecord, nsid,
};
use knotless::release::valid_sha256;
use knotless::secrets::Secret;
use knotless::usage::{Day, Network};
use rusqlite::functions::{Context, FunctionFlags};
@@ -594,6 +595,8 @@ pub(crate) fn connect(path: &Path) -> rusqlite::Result<(Connection, bool)> {
uri TEXT PRIMARY KEY, did TEXT NOT NULL, repo_key TEXT NOT NULL,
created_at TEXT NOT NULL, value TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS release_repo ON release (did, repo_key, created_at);
CREATE TABLE IF NOT EXISTS site (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, repo_key TEXT NOT NULL, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS theme (
uri TEXT PRIMARY KEY, did TEXT NOT NULL, created_at TEXT NOT NULL,
name TEXT NOT NULL, value TEXT NOT NULL);
@@ -773,7 +776,7 @@ const CACHE: &str = "DROP TABLE IF EXISTS repo; DROP TABLE IF EXISTS forge;
DROP TABLE IF EXISTS profile_record; DROP TABLE IF EXISTS handle;
DROP TABLE IF EXISTS home;
DROP TABLE IF EXISTS star; DROP TABLE IF EXISTS follow; DROP TABLE IF EXISTS comment;
DROP TABLE IF EXISTS reaction; DROP TABLE IF EXISTS release; DROP TABLE IF EXISTS mention;
DROP TABLE IF EXISTS reaction; DROP TABLE IF EXISTS release; DROP TABLE IF EXISTS site; DROP TABLE IF EXISTS mention;
DROP TABLE IF EXISTS reviewer; DROP TABLE IF EXISTS assigned;
DROP TABLE IF EXISTS org; DROP TABLE IF EXISTS org_member; DROP TABLE IF EXISTS membership;
DROP TABLE IF EXISTS collaborator; DROP TABLE IF EXISTS shared;
@@ -848,6 +851,7 @@ const FORGET_REPO: &[&str] = &[
"DELETE FROM forge WHERE repo_did = ?1 AND repo_key = ?2",
"DELETE FROM check_run WHERE repo_did = ?1 AND repo_key = ?2",
"DELETE FROM release WHERE did = ?1 AND repo_key = ?2",
"DELETE FROM site WHERE uri = 'at://' || ?1 || '/dev.knotless.site/' || ?2",
"DELETE FROM shared WHERE did = ?1 AND repo_key = ?2",
"DELETE FROM shared_use WHERE did = ?1 AND key = ?2",
"DELETE FROM star WHERE repo_did = ?1 AND repo_key = ?2",
@@ -894,6 +898,7 @@ const PURGE: &[&str] = &[
"DELETE FROM follow WHERE did = ?1",
"DELETE FROM reaction WHERE did = ?1 -- scans: reaction, once, as an account is forgotten",
"DELETE FROM release WHERE did = ?1",
"DELETE FROM site WHERE uri >= ?2 AND uri < ?3",
"DELETE FROM shared WHERE did = ?1",
"DELETE FROM shared_use WHERE did = ?1",
"DELETE FROM theme WHERE did = ?1",
@@ -1249,6 +1254,18 @@ impl Index {
text
],
)
} else if collection == nsid::SITE {
// A public repository's site, keyed by the repository's key:
// one that is not a site's record is passed by, as is one in a
// space, which has no site yet, and one with a file whose hash
// is not one, as the hash names the file it is kept in.
let site = serde_json::from_value::<SiteRecord>(value.clone());
let hashed = |s: SiteRecord| s.files.iter().all(|f| valid_sha256(&f.sha256));
let kept = site.is_ok_and(hashed) && space.is_empty();
db.execute(
"INSERT OR REPLACE INTO site SELECT ?1, ?2, ?3, ?4 WHERE ?5",
params![uri, did, rkey, text, kept],
)
} else if collection == nsid::RELEASE {
// Kept by the owner and the key of the repository it is of, as
// its record names it: one that is not a release, by its names,
@@ -1518,6 +1535,7 @@ impl Index {
db.execute("DELETE FROM minutes WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM runner_key WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM release WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM site WHERE uri = ?1", [&uri])?;
db.execute("DELETE FROM shared WHERE uri = ?1", [&uri])?;
db.execute(
"DELETE FROM profile_record WHERE did = ?1 AND collection = ?2 AND rkey = ?3",
@@ -1603,6 +1621,17 @@ impl Index {
Ok(rows.filter_map(Result::ok).flatten().collect())
}
/// The site of `did`'s repository `key`, as its record was last
/// indexed: none when it has none, or it is taken down.
pub fn site(&self, did: &str, key: &str) -> rusqlite::Result<Option<String>> {
let db = self.readers.take(&self.db);
let uri = format!("at://{did}/{}/{key}", nsid::SITE);
let mut stmt = db.prepare_cached(
"SELECT value FROM site WHERE uri = ?1 AND uri NOT IN down AND did NOT IN down",
)?;
stmt.query_row([&uri], |row| row.get(0)).optional()
}
/// How many accounts starred `did`'s repository with `key`, leaving
/// out those taken down.
pub fn star_count(&self, did: &str, key: &str) -> rusqlite::Result<u64> {
@@ -6966,4 +6995,34 @@ mod tests {
assert!(Index::open(&dir.path().join("i.db")).is_ok());
assert!(Index::open(dir.path()).is_err());
}
#[test]
fn sites_are_kept_by_their_repository() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let file =
|hash: &str| json!({"path": "index.html", "size": 1, "sha256": hash, "parts": []});
let hash = "0".repeat(64);
let site = json!({"files": [file(&hash)], "createdAt": "2026-11-03T00:00:00Z"});
index.put("did:a", nsid::SITE, "k", &site).unwrap();
let kept = index.site("did:a", "k").unwrap().unwrap();
assert_eq!(serde_json::from_str::<Value>(&kept).unwrap(), site);
// Not a site's record, or one in a space: passed by.
index
.put("did:a", nsid::SITE, "j", &json!({"files": 1}))
.unwrap();
let space = "ats://did:a/dev.knotless.space/s";
index
.put_from_space(space, "did:a", nsid::SITE, "i", &site)
.unwrap();
assert_eq!(index.site("did:a", "j").unwrap(), None);
// A hash that would name a path outside where files are kept.
let out = format!("{:.<64}", "../../../../etc/passwd");
let bad = json!({"files": [file(&out)], "createdAt": "2026-11-03T00:00:00Z"});
index.put("did:a", nsid::SITE, "h", &bad).unwrap();
assert_eq!(index.site("did:a", "h").unwrap(), None);
assert_eq!(index.site("did:a", "i").unwrap(), None);
index.delete("did:a", nsid::SITE, "k").unwrap();
assert_eq!(index.site("did:a", "k").unwrap(), None);
}
}
crates/knotless-serve/src/lib.rs+30−1
View file
@@ -41,6 +41,7 @@ pub mod private;
pub mod releases;
mod secrets;
pub mod shared;
pub mod sites;
mod smart;
pub mod spaces;
pub mod status;
@@ -860,6 +861,7 @@ fn indexed_collections() -> impl Iterator<Item = &'static str> {
knotless::records::nsid::FOLLOW,
knotless::records::nsid::REACTION,
knotless::records::nsid::RELEASE,
knotless::records::nsid::SITE,
knotless::records::nsid::THEME,
knotless::records::nsid::ORG,
knotless::records::nsid::ORG_MEMBER,
@@ -1305,6 +1307,9 @@ pub struct Options {
pub oauth_url: Option<String>,
/// Where code search keeps its Zoekt shards, if it is on.
pub code_index: Option<std::path::PathBuf>,
/// The domain repositories' sites are answered on, each owner at a
/// label of it, as `knotless.page`: none answers none.
pub pages_domain: Option<String>,
}
/// What `--private`, which is gone, says instead.
@@ -1337,6 +1342,7 @@ pub fn parse_args(args: &[String]) -> Result<Options, String> {
sign_up_pds: gateway::SIGN_UP_PDS.into(),
oauth_url: None,
code_index: None,
pages_domain: None,
};
let mut it = args.iter().skip(1);
while let Some(flag) = it.next() {
@@ -1355,6 +1361,7 @@ pub fn parse_args(args: &[String]) -> Result<Options, String> {
"--firehose" => opts.firehose = Some(value()?).filter(|v| !v.is_empty()),
"--jetstream" => opts.jetstream = Some(value()?).filter(|v| !v.is_empty()),
"--public-url" => opts.public_url = Some(value()?),
"--pages-domain" => opts.pages_domain = Some(value()?).filter(|v| !v.is_empty()),
"--index" => opts.index = Some(value()?.into()),
"--backfill" => opts.backfill = Some(value()?).filter(|v| !v.is_empty()),
"--relay" => opts.relay = Some(value()?),
@@ -1408,7 +1415,7 @@ pub fn parse_args(args: &[String]) -> Result<Options, String> {
"--code-index" => opts.code_index = Some(value()?.into()),
other => {
return Err(format!(
"unexpected {other}\nusage: knotless-serve [--listen <addr>] [--ttl <secs>] [--firehose <wss://relay or pds>] [--jetstream <wss://host>] [--public-url <https://host>] [--oauth-url <https://host>] [--index <file>] [--backfill <https://pds or relay>] [--relay <https://relay>] [--labeler <did>]... [--default-runner <did>] [--client-minimum <version> [--client-minimum-reason <text>]] [--web <http://app>] [--releases <dir or knotless://repo>] [--stale <secs>] [--usage <file>] [--logs <file> [--logs-keep <lines>]] [--check-index <file>] [--metrics <addr>] [--bots <dir>] [--sign-up-pds <https://pds>] [--code-index <dir>]"
"unexpected {other}\nusage: knotless-serve [--listen <addr>] [--ttl <secs>] [--firehose <wss://relay or pds>] [--jetstream <wss://host>] [--public-url <https://host>] [--oauth-url <https://host>] [--index <file>] [--backfill <https://pds or relay>] [--relay <https://relay>] [--labeler <did>]... [--default-runner <did>] [--client-minimum <version> [--client-minimum-reason <text>]] [--web <http://app>] [--releases <dir or knotless://repo>] [--stale <secs>] [--usage <file>] [--logs <file> [--logs-keep <lines>]] [--check-index <file>] [--metrics <addr>] [--bots <dir>] [--sign-up-pds <https://pds>] [--code-index <dir>] [--pages-domain <domain>]"
));
}
}
@@ -1442,6 +1449,11 @@ pub async fn run(opts: Options, ctx: Ctx) -> Result<()> {
if opts.public_url.is_none() && opts.bots.is_some() {
anyhow::bail!("--bots takes --public-url, where bots log in");
}
if opts.public_url.is_none() && opts.pages_domain.is_some() {
anyhow::bail!(
"--pages-domain takes --public-url, where a site's owner is sent from its root"
);
}
let gateway = match &opts.public_url {
Some(url) => {
let mut gw = gateway::Gateway::new(url).map_err(anyhow::Error::msg)?;
@@ -1540,13 +1552,20 @@ pub async fn run(opts: Options, ctx: Ctx) -> Result<()> {
.map(|url| mcp_routes(mcp_ctx, url, index, addr, usage.clone()));
let mcp = mcp.transpose()?.unwrap_or_default();
let registry = oci::routes(app.ctx.clone(), opts.public_url.clone());
let home = opts.public_url.clone().unwrap_or_default();
let releases = releases::routes(opts.releases, opts.public_url, app.ctx.clone());
let stopping = app.live.stopping.clone();
let sites = opts
.pages_domain
.map(|d| sites::Sites::new(app.clone(), d, home));
let mut routes = router(app)
.merge(releases)
.merge(registry)
.merge(mcp)
.merge(pages);
if let Some(sites) = sites {
routes = sites::hosted(routes, sites);
}
if let Some(usage) = usage.clone() {
tokio::spawn(usage::flushing(usage.clone(), usage::FLUSH));
routes = usage::counted(routes, usage);
@@ -1708,4 +1727,14 @@ mod tests {
let down = AppError::upstream(anyhow::anyhow!("resolver down"));
assert_eq!(down.0, StatusCode::BAD_GATEWAY);
}
// A collection a backfill reads and the firehose passes by is indexed
// once and never again: each list names it, or the index goes stale.
#[test]
fn the_firehose_keeps_every_indexed_collection() {
for c in indexed_collections() {
let kept = knotless::jetstream::knotless(c);
assert!(kept, "{c} is backfilled but not kept live");
}
}
}
crates/knotless-serve/src/plans.rs+2−1
View file
crates/knotless-serve/src/sites.rsnew+476−0
View file
476 lines changed: not loaded until asked for.
crates/knotless-serve/tests/serve.rs+13−0
View file
crates/knotless-serve/tests/sites.rsnew+168−0
View file
deploy/deploy.sh+1−0
View file
deploy/docker-compose.yml+5−0
View file
deploy/local.env.example+5−0
View file
deploy/README.md+30−0
View file
deploy/setup.sh+59−4
View file
docs/design/pages.mdnew+330−0
View file
docs/design/README.md+4−2
View file
docs/design/serve.md+9−0
View file
docs/reference/cli.md+2−0
View file
docs-site/astro.config.mjs+1−0
View file
lexicons/dev/knotless/site.jsonnew+44−0
View file