chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
serve holds crawlers to robots.txt
crates/knotless-serve/src/web/pages.rs+78−1
| @@ -5,7 +5,7 @@ use axum::Router; | ||
use axum::body::Body; | ||
use axum::extract::Request; | ||
use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; | ||
use axum::response::Response; | ||
use axum::response::{IntoResponse, Response}; | ||
use knotless::telemetry::send; | ||
use std::sync::LazyLock; | ||
use std::time::Duration; | ||
| @@ -45,7 +45,84 @@ pub fn pages(web: String) -> Router { | ||
.layer(tower_http::compression::CompressionLayer::new()) | ||
} | ||
| ||
/// The pages' `robots.txt`, as served: what it keeps crawlers off is held | ||
/// here too, for those that do not read it again for a day or more, or | ||
/// at all. | ||
static ROBOTS: LazyLock<Vec<(bool, String)>> = | ||
LazyLock::new(|| rules(include_str!("../../../../web/static/robots.txt"))); | ||
| ||
/// `robots`'s rules, each whether it allows and its path, escapes | ||
/// decoded. It has one group, for every crawler. | ||
fn rules(robots: &str) -> Vec<(bool, String)> { | ||
let rule = |line: &str| { | ||
let (field, path) = line.split_once(':')?; | ||
let allows = match field.trim().to_ascii_lowercase().as_str() { | ||
"allow" => true, | ||
"disallow" => false, | ||
_ => return None, | ||
}; | ||
Some((allows, super::unescaped(path.trim()))) | ||
}; | ||
robots.lines().filter_map(rule).collect() | ||
} | ||
| ||
/// Whether `path`, a robots.txt rule's, with `*` for any run and a last | ||
/// `$` for the end, matches the start of `at`, or all of it. | ||
fn matches(path: &str, at: &str) -> bool { | ||
let (path, whole) = match path.strip_suffix('$') { | ||
Some(path) => (path, true), | ||
None => (path, false), | ||
}; | ||
let mut parts = path.split('*'); | ||
let first = parts.next().unwrap_or_default(); | ||
let Some(mut left) = at.strip_prefix(first) else { | ||
return false; | ||
}; | ||
let mut parts = parts.peekable(); | ||
while let Some(part) = parts.next() { | ||
if whole && parts.peek().is_none() { | ||
return left.ends_with(part); | ||
} | ||
let Some(found) = left.find(part) else { | ||
return false; | ||
}; | ||
left = &left[found + part.len()..]; | ||
} | ||
!whole || left.is_empty() | ||
} | ||
| ||
/// Whether robots.txt lets crawlers read `at`, a path and query: by the | ||
/// longest rule that matches it, an allowing one if two are as long. | ||
fn allowed(at: &str) -> bool { | ||
let at = super::unescaped(at); | ||
let matching = ROBOTS.iter().filter(|(_, path)| matches(path, &at)); | ||
let rule = matching.max_by_key(|(allows, path)| (path.len(), *allows)); | ||
rule.is_none_or(|(allows, _)| *allows) | ||
} | ||
| ||
/// Whether `agent` is a crawler, which robots.txt is for: one that says | ||
/// it is a bot, a crawler or a spider, as they do, but not one drawing a | ||
/// preview of a link someone shared, which that person asked for. | ||
fn crawler(agent: &str) -> bool { | ||
const CRAWLERS: [&str; 5] = ["bot", "crawl", "spider", "slurp", "googleother"]; | ||
const PREVIEWS: [&str; 7] = [ | ||
"slack", "discord", "twitter", "telegram", "whatsapp", "linkedin", "embed", | ||
]; | ||
let agent = agent.to_ascii_lowercase(); | ||
let says = |words: &[&str]| words.iter().any(|w| agent.contains(w)); | ||
says(&CRAWLERS) && !says(&PREVIEWS) | ||
} | ||
| ||
async fn page(web: &str, req: Request) -> Response { | ||
// A crawler that reads what robots.txt keeps it off is turned away | ||
// here, rather than the pages drawing what it should not have asked. | ||
let agent = req.headers().get(header::USER_AGENT); | ||
let agent = agent.and_then(|a| a.to_str().ok()).unwrap_or_default(); | ||
let at = req.uri().path_and_query().map_or("/", |p| p.as_str()); | ||
if crawler(agent) && !allowed(at) { | ||
let why = "robots.txt keeps crawlers off this page."; | ||
return (StatusCode::FORBIDDEN, why).into_response(); | ||
} | ||
match handed(web, req).await { | ||
Ok(res) => res, | ||
Err(e) => crate::telemetry::failed( | ||
crates/knotless-serve/tests/web.rs+46−0
| @@ -2754,6 +2754,52 @@ fn pages_drawn_elsewhere() { | ||
.send() | ||
.unwrap(); | ||
assert!(ours.text().unwrap().contains("keeps no index")); | ||
// A crawler is kept off what robots.txt keeps it off, whether it read | ||
// it or not: by the longest rule, with its escapes or without. A | ||
// browser, or a preview of a link shared, is not. | ||
const GPT: &str = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4)"; | ||
const SLACK: &str = "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"; | ||
const BROWSER: &str = "Mozilla/5.0 (Macintosh) AppleWebKit/605.1.15 Safari/605.1.15"; | ||
for (agent, at, status) in [ | ||
( | ||
GPT, | ||
"/alice.test/tally/issues?q=label:area/serve%20author:bob", | ||
403, | ||
), | ||
( | ||
GPT, | ||
"/alice.test/tally/pulls/__data.json?x-sveltekit-invalidated=01", | ||
403, | ||
), | ||
(GPT, "/alice.test/tally/tree/dev/src", 403), | ||
(GPT, "/search?q=x", 403), | ||
(GPT, "/alice.test/tally/issues?q=state:closed", 200), | ||
( | ||
GPT, | ||
"/alice.test/tally/issues?q=state%3Aclosed&cursor=2", | ||
200, | ||
), | ||
( | ||
GPT, | ||
"/alice.test/tally/issues?q=state%3Aclosed%20label:x", | ||
403, | ||
), | ||
(GPT, "/alice.test/tally/issues", 200), | ||
(GPT, "/alice.test/tally/tree/main/src", 200), | ||
(GPT, "/alice.test", 200), | ||
(SLACK, "/alice.test/tally/commit/abc", 200), | ||
(BROWSER, "/alice.test/tally/issues?q=label:x", 200), | ||
] { | ||
let res = client | ||
.get(format!("{base}{at}")) | ||
.header("user-agent", agent) | ||
.send() | ||
.unwrap(); | ||
assert_eq!(res.status().as_u16(), status, "{agent} at {at}"); | ||
if status == 403 { | ||
assert!(res.text().unwrap().contains("robots.txt")); | ||
} | ||
} | ||
// Too much to hand on, or no app to hand it to. | ||
// The answer can come before all of it is sent, which closes the | ||
// connection under the rest. | ||
docs/design/serve.md+14−3
| @@ -754,9 +754,20 @@ is a page of its own, and each mix of a list's labels and authors is a | ||
list of its own, so a crawler that follows every link never runs out. | ||
`web/static/robots.txt` keeps crawlers to a repository's code on `main`, | ||
its open and closed pulls and issues, and the pages around them, and a | ||
filter's links are `rel="nofollow"`. A crawler that | ||
calls itself a browser may not read it, and each request's log line names | ||
its client (`user_agent.original`), so they can be told apart. | ||
filter's links are `rel="nofollow"`. | ||
| ||
Some crawlers do not read it, or read it again only days later, so the | ||
server holds them to it too. A page request whose `User-Agent` says it is | ||
a bot, a crawler or a spider, and whose path and query the file's rules | ||
disallow, gets a 403 before the page is drawn. The server reads the same | ||
file, built into it, and matches the rules as crawlers do: `*` for any | ||
run, `$` for the end, escapes decoded, and the longest matching rule | ||
deciding, an `Allow` winning a tie. A link preview, such as Slack's or | ||
Discord's, is not held to it, as someone shared that link. | ||
| ||
A crawler that calls itself a browser is not caught, and each request's | ||
log line names its client (`user_agent.original`), so they can be told | ||
apart. | ||
| ||
### Who asks | ||
| ||
web/static/robots.txt+2−1
| @@ -1,7 +1,8 @@ | ||
# A repository's code on main is read; its every branch, commit, history | ||
# and blame are pages without end, for each file at each of them. So are | ||
# a list's filters, one for each mix of its labels and authors: its open | ||
# and closed lists are read, and no other. | ||
# and closed lists are read, and no other. knotless-serve reads these | ||
# rules too, and turns away a crawler that asks for a page they disallow. | ||
User-agent: * | ||
Allow: /*/*/tree/main | ||
Allow: /*/*/blob/main | ||