johnandersen777.bsky.social/knotless

Switch repository

Loading…

//! The pages: every request this server does not answer itself, handed to
//! the SvelteKit app that draws them, which reads back its XRPC queries.

use axum::Router;
use axum::body::Body;
use axum::extract::Request;
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
use axum::response::{IntoResponse, Response};
use knotless::telemetry::send;
use std::sync::LazyLock;
use std::time::Duration;

/// The largest request body handed on: the pages take none to speak of.
const BODY_LIMIT: usize = 1 << 20;

/// How long an idle connection to the app is kept: less than the 5 seconds
/// node keeps one, so that it is not used just as node closes it.
const IDLE: Duration = Duration::from_secs(4);

/// Headers about one hop of a connection, which are not handed on.
const HOP: [header::HeaderName; 6] = [
    header::CONNECTION,
    header::HOST,
    header::TRANSFER_ENCODING,
    header::CONTENT_LENGTH,
    header::UPGRADE,
    header::TE,
];

static CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
    reqwest::Client::builder()
        .redirect(reqwest::redirect::Policy::none())
        .pool_idle_timeout(IDLE)
        .build()
        .expect("a client builds")
});

/// A router that hands every request to the app at `web`: what it answers
/// compressed as this server's own answers are, brotli to a browser that
/// takes it, but for what the app compressed itself, as its scripts. But
/// `/favicon.ico`, which crawlers and old browsers ask for whatever a page
/// names, goes to the pages' icon: handed on, it would be drawn as the
/// profile of a handle `favicon.ico`, looked up at every handle resolver.
pub fn pages(web: String) -> Router {
    let web: &'static str = web.trim_end_matches('/').to_string().leak();
    Router::new()
        .route(
            "/favicon.ico",
            axum::routing::get(|| async { axum::response::Redirect::permanent("/favicon.svg") }),
        )
        .fallback(move |req: Request| page(web, req))
        .layer(tower_http::compression::CompressionLayer::new())
}

/// The pages' `robots.txt`, as served: what it keeps crawlers off is held
/// here too, for those that do not read it again for a day or more, or
/// at all.
static ROBOTS: LazyLock<Vec<(bool, String)>> =
    LazyLock::new(|| rules(include_str!("../../../../web/static/robots.txt")));

/// `robots`'s rules, each whether it allows and its path, escapes
/// decoded. It has one group, for every crawler.
fn rules(robots: &str) -> Vec<(bool, String)> {
    let rule = |line: &str| {
        let (field, path) = line.split_once(':')?;
        let allows = match field.trim().to_ascii_lowercase().as_str() {
            "allow" => true,
            "disallow" => false,
            _ => return None,
        };
        Some((allows, super::unescaped(path.trim())))
    };
    robots.lines().filter_map(rule).collect()
}

/// Whether `path`, a robots.txt rule's, with `*` for any run and a last
/// `$` for the end, matches the start of `at`, or all of it.
fn matches(path: &str, at: &str) -> bool {
    let (path, whole) = match path.strip_suffix('$') {
        Some(path) => (path, true),
        None => (path, false),
    };
    let mut parts = path.split('*');
    let first = parts.next().unwrap_or_default();
    let Some(mut left) = at.strip_prefix(first) else {
        return false;
    };
    let mut parts = parts.peekable();
    while let Some(part) = parts.next() {
        if whole && parts.peek().is_none() {
            return left.ends_with(part);
        }
        let Some(found) = left.find(part) else {
            return false;
        };
        left = &left[found + part.len()..];
    }
    !whole || left.is_empty()
}

/// Whether robots.txt lets crawlers read `at`, a path and query: by the
/// longest rule that matches it, an allowing one if two are as long.
fn allowed(at: &str) -> bool {
    let at = super::unescaped(at);
    let matching = ROBOTS.iter().filter(|(_, path)| matches(path, &at));
    let rule = matching.max_by_key(|(allows, path)| (path.len(), *allows));
    rule.is_none_or(|(allows, _)| *allows)
}

/// Whether `agent` is a crawler, which robots.txt is for: one that says
/// it is a bot, a crawler or a spider, as they do, but not one drawing a
/// preview of a link someone shared, which that person asked for.
fn crawler(agent: &str) -> bool {
    const CRAWLERS: [&str; 5] = ["bot", "crawl", "spider", "slurp", "googleother"];
    const PREVIEWS: [&str; 7] = [
        "slack", "discord", "twitter", "telegram", "whatsapp", "linkedin", "embed",
    ];
    let agent = agent.to_ascii_lowercase();
    let says = |words: &[&str]| words.iter().any(|w| agent.contains(w));
    says(&CRAWLERS) && !says(&PREVIEWS)
}

async fn page(web: &str, req: Request) -> Response {
    // A crawler that reads what robots.txt keeps it off is turned away
    // here, rather than the pages drawing what it should not have asked.
    let agent = req.headers().get(header::USER_AGENT);
    let agent = agent.and_then(|a| a.to_str().ok()).unwrap_or_default();
    let at = req.uri().path_and_query().map_or("/", |p| p.as_str());
    if crawler(agent) && !allowed(at) {
        let why = "robots.txt keeps crawlers off this page.";
        return (StatusCode::FORBIDDEN, why).into_response();
    }
    match handed(web, req).await {
        Ok(res) => res,
        Err(e) => crate::telemetry::failed(
            StatusCode::BAD_GATEWAY,
            format!("The pages are not being drawn: {e:#}"),
        ),
    }
}

/// `req`, as the app at `web` answers it, its body sent on as it comes.
/// The app sees the host it was
/// asked for, as `x-forwarded-host`. A read that finds the connection gone,
/// as when the app is being replaced, is asked once more; a write may have
/// been done, so is not.
async fn handed(web: &str, req: Request) -> anyhow::Result<Response> {
    let (parts, body) = req.into_parts();
    let at = parts.uri.path_and_query().map_or("/", |p| p.as_str());
    let mut headers = kept(&parts.headers);
    if let Some(host) = parts.headers.get(header::HOST) {
        headers.insert("x-forwarded-host", host.clone());
    }
    headers
        .entry("x-forwarded-proto")
        .or_insert(HeaderValue::from_static("http"));
    let body = axum::body::to_bytes(body, BODY_LIMIT).await?;
    let url = format!("{web}{at}");
    let ask = || {
        CLIENT
            .request(parts.method.clone(), &url)
            .headers(headers.clone())
            .body(body.clone())
    };
    let answer = match send(ask(), 0).await {
        Err(_) if parts.method.is_safe() => send(ask(), 1).await?,
        answer => answer?,
    };
    let mut res = Response::builder().status(answer.status());
    let mut headers = kept(answer.headers());
    for (name, value) in [
        (header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
        (header::REFERRER_POLICY, "same-origin"),
    ] {
        headers
            .entry(name)
            .or_insert(HeaderValue::from_static(value));
    }
    *res.headers_mut().unwrap() = headers;
    Ok(res.body(Body::from_stream(streamed(answer)))?)
}

/// `answer`'s body as it comes, each part sent on as it is read: what a
/// page streams, as a directory's last commits, reaches the browser as the
/// app sends it, not once it has sent the last of it.
fn streamed(
    answer: reqwest::Response,
) -> impl futures::Stream<Item = reqwest::Result<axum::body::Bytes>> {
    futures::stream::try_unfold(answer, |mut answer| async move {
        let part = answer.chunk().await?;
        Ok(part.map(|part| (part, answer)))
    })
}

/// `headers`, but for those about one hop.
fn kept(headers: &HeaderMap) -> HeaderMap {
    let mut out = headers.clone();
    for name in HOP {
        out.remove(name);
    }
    out
}