chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
Crawlers skip list filters, and a page read retries a lost connection
crates/knotless-serve/src/web/pages.rs+21−3
| @@ -6,11 +6,17 @@ use axum::body::Body; | ||
use axum::extract::Request; | ||
use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; | ||
use axum::response::Response; | ||
use knotless::telemetry::send; | ||
use std::sync::LazyLock; | ||
use std::time::Duration; | ||
| ||
/// The largest request body handed on: the pages take none to speak of. | ||
const BODY_LIMIT: usize = 1 << 20; | ||
| ||
/// How long an idle connection to the app is kept: less than the 5 seconds | ||
/// node keeps one, so that it is not used just as node closes it. | ||
const IDLE: Duration = Duration::from_secs(4); | ||
| ||
/// Headers about one hop of a connection, which are not handed on. | ||
const HOP: [header::HeaderName; 6] = [ | ||
header::CONNECTION, | ||
| @@ -24,6 +30,7 @@ const HOP: [header::HeaderName; 6] = [ | ||
static CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| { | ||
reqwest::Client::builder() | ||
.redirect(reqwest::redirect::Policy::none()) | ||
.pool_idle_timeout(IDLE) | ||
.build() | ||
.expect("a client builds") | ||
}); | ||
| @@ -45,7 +52,9 @@ async fn page(web: &str, req: Request) -> Response { | ||
} | ||
| ||
/// `req`, as the app at `web` answers it. The app sees the host it was | ||
/// asked for, as `x-forwarded-host`. | ||
/// asked for, as `x-forwarded-host`. A read that finds the connection gone, | ||
/// as when the app is being replaced, is asked once more; a write may have | ||
/// been done, so is not. | ||
async fn handed(web: &str, req: Request) -> anyhow::Result<Response> { | ||
let (parts, body) = req.into_parts(); | ||
let at = parts.uri.path_and_query().map_or("/", |p| p.as_str()); | ||
| @@ -57,8 +66,17 @@ async fn handed(web: &str, req: Request) -> anyhow::Result<Response> { | ||
.entry("x-forwarded-proto") | ||
.or_insert(HeaderValue::from_static("http")); | ||
let body = axum::body::to_bytes(body, BODY_LIMIT).await?; | ||
let asked = CLIENT.request(parts.method, format!("{web}{at}")); | ||
let answer = knotless::telemetry::send(asked.headers(headers).body(body), 0).await?; | ||
let url = format!("{web}{at}"); | ||
let ask = || { | ||
CLIENT | ||
.request(parts.method.clone(), &url) | ||
.headers(headers.clone()) | ||
.body(body.clone()) | ||
}; | ||
let answer = match send(ask(), 0).await { | ||
Err(_) if parts.method.is_safe() => send(ask(), 1).await?, | ||
answer => answer?, | ||
}; | ||
let mut res = Response::builder().status(answer.status()); | ||
let mut headers = kept(answer.headers()); | ||
for (name, value) in [ | ||
crates/knotless-serve/tests/web.rs+19−1
| @@ -2648,7 +2648,7 @@ fn the_feed() { | ||
| ||
/// The pages: anything knotless-serve does not answer itself is handed to | ||
/// the app at `--web`, as asked, with the host it was asked of; and a | ||
/// failure to reach it is said. | ||
/// failure to reach it is asked over if a read, and said. | ||
#[test] | ||
fn pages_drawn_elsewhere() { | ||
use axum::http::HeaderMap; | ||
| @@ -2682,6 +2682,16 @@ fn pages_drawn_elsewhere() { | ||
res | ||
}, | ||
); | ||
// An app that drops the first connection it takes, then answers. | ||
let dropping = |echo: axum::Router| { | ||
let (app, at) = listen(); | ||
rt.spawn(async move { | ||
drop(app.accept().await.unwrap()); | ||
axum::serve(app, echo).await.unwrap() | ||
}); | ||
at | ||
}; | ||
let (reread, rewritten) = (dropping(echo.clone()), dropping(echo.clone())); | ||
rt.spawn(async move { axum::serve(app, echo).await.unwrap() }); | ||
| ||
let s = Sandbox::new(); | ||
| @@ -2736,6 +2746,14 @@ fn pages_drawn_elsewhere() { | ||
Ok(res) => assert_eq!(res.status().as_u16(), 502), | ||
Err(e) => assert!(e.is_request(), "{e}"), | ||
} | ||
// A connection lost is asked over on a read, and not on a write. | ||
let res = client.get(format!("{}/x", front(reread))).send().unwrap(); | ||
assert_eq!(res.status().as_u16(), 200); | ||
let written = front(rewritten); | ||
let res = client.post(format!("{written}/x")).send().unwrap(); | ||
assert_eq!(res.status().as_u16(), 502); | ||
let res = client.post(format!("{written}/x")).send().unwrap(); | ||
assert_eq!(res.status().as_u16(), 200); | ||
let (gone, nowhere) = listen(); | ||
drop(gone); | ||
let res = client.get(front(nowhere)).send().unwrap(); | ||
docs/design/serve.md+5−3
| @@ -741,9 +741,11 @@ given to another repository. | ||
### Crawlers | ||
| ||
Each file at each branch and commit, and each file's history and blame, | ||
is a page of its own, so a crawler that follows every link never runs | ||
out. `web/static/robots.txt` keeps crawlers to a repository's code on | ||
`main`, its pulls and issues, and the pages around them. A crawler that | ||
is a page of its own, and each mix of a list's labels and authors is a | ||
list of its own, so a crawler that follows every link never runs out. | ||
`web/static/robots.txt` keeps crawlers to a repository's code on `main`, | ||
its open and closed pulls and issues, and the pages around them, and a | ||
filter's links are `rel="nofollow"`. A crawler that | ||
calls itself a browser may not read it, and each request's log line names | ||
its client (`user_agent.original`), so they can be told apart. | ||
| ||
web/src/lib/atoms/Label.svelte+2−1
| @@ -5,6 +5,7 @@ | ||
const color = $derived(/^#[0-9a-fA-F]{6}$/.test(label.color ?? '') ? label.color : undefined); | ||
</script> | ||
| ||
{#if href}<a class="label" {href} style:--label={color} title={label.description}>{label.name}</a | ||
{#if href}<a class="label" {href} rel="nofollow" style:--label={color} title={label.description} | ||
>{label.name}</a | ||
>{:else}<span class="label" style:--label={color} title={label.description}>{label.name}</span | ||
>{/if} | ||
web/src/lib/molecules/FilterMenu.svelte+1−1
| @@ -18,7 +18,7 @@ | ||
<div class="menu"> | ||
<div class="menu-head">{head}</div> | ||
{#each options as o (o.name)} | ||
<a href={o.href} aria-current={o.on ? 'true' : undefined} | ||
<a href={o.href} rel="nofollow" aria-current={o.on ? 'true' : undefined} | ||
><span class="tick" | ||
>{#if o.on}<Icon name="check" />{/if}</span | ||
>{#if o.color !== undefined}<span class="dot" style:--label={hex(o.color)}></span>{/if}<span | ||
web/static/robots.txt+11−1
| @@ -1,8 +1,18 @@ | ||
# A repository's code on main is read; its every branch, commit, history | ||
# and blame are pages without end, for each file at each of them. | ||
# and blame are pages without end, for each file at each of them. So are | ||
# a list's filters, one for each mix of its labels and authors: its open | ||
# and closed lists are read, and no other. | ||
User-agent: * | ||
Allow: /*/*/tree/main | ||
Allow: /*/*/blob/main | ||
Allow: /*/*/issues?q=state%3Aclosed$ | ||
Allow: /*/*/issues?q=state%3Aclosed&cursor= | ||
Allow: /*/*/pulls?q=state%3Aclosed$ | ||
Allow: /*/*/pulls?q=state%3Aclosed&cursor= | ||
Disallow: /*/*/issues?q= | ||
Disallow: /*/*/pulls?q= | ||
Disallow: /*/*/issues/__data.json | ||
Disallow: /*/*/pulls/__data.json | ||
Disallow: /*/*/tree/ | ||
Disallow: /*/*/blob/ | ||
Disallow: /*/*/blame/ | ||