chadtmiller.com/knotless

Switch repository

Loading…

#!/usr/bin/env bash
# Makes a fresh Ubuntu 24.04 host a runner host: a box that runs other
# people's workflows and holds nothing else of knotless's.
#
#   deploy/runner-host/setup.sh <ssh destination>
#   KNOTLESS_RUNNER_HOST=root@host deploy/runner-host/setup.sh
#
# or with that in deploy/local.env. Safe to run again: each step leaves
# what is already done as it is, and the egress rules are made afresh.
#
# What it sets up (deploy/runner-host/README.md says why):
#
# - docker, from Docker's own apt repository;
# - gVisor's runsc, registered with the daemon as the runtime `runsc`,
#   which runs' containers start with (knotless-run --runtime runsc);
# - the network runs join, knotless-runs, a bridge named kl-runs whose
#   containers cannot reach each other;
# - egress rules for that bridge, made again at each boot after docker: no
#   private, link-local (the cloud's metadata service) or host addresses,
#   and no mail ports; the rest of the internet is open;
# - a firewall letting in SSH alone;
# - unattended security upgrades, and a daily prune of images unused for
#   a week;
# - /etc/knotless, with an empty pause list and secrets file, and the
#   resolv.conf runs see: docker's resolver on its own networks is reached
#   through NAT rules gVisor does not apply, so runs name public ones.
#
# It ends by checking from inside a gVisor container on that network that
# the internet answers and the metadata service and private ranges do not.

set -euo pipefail
cd "$(dirname "$0")/../.."
[ -f deploy/local.env ] && { set -a; . deploy/local.env; set +a; }
DEST="${1:-${KNOTLESS_RUNNER_HOST:-}}"
[ -n "$DEST" ] || { sed -n '5,6p' "$0" >&2; exit 2; }

ssh -o StrictHostKeyChecking=accept-new "$DEST" bash -s <<'HOST'
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive

echo "==> packages"
apt-get update -q
apt-get install -yq ca-certificates curl gnupg ufw unattended-upgrades >/dev/null
install -d -m 0755 /etc/apt/keyrings

echo "==> docker"
if ! command -v docker >/dev/null; then
  curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
  . /etc/os-release
  echo "deb [signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" \
    >/etc/apt/sources.list.d/docker.list
  apt-get update -q
  apt-get install -yq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin >/dev/null
fi

echo "==> gVisor"
if ! command -v runsc >/dev/null; then
  curl -fsSL https://gvisor.dev/archive.key | gpg --dearmor -o /etc/apt/keyrings/gvisor.gpg
  echo "deb [signed-by=/etc/apt/keyrings/gvisor.gpg] https://storage.googleapis.com/gvisor/releases release main" \
    >/etc/apt/sources.list.d/gvisor.list
  apt-get update -q
  apt-get install -yq runsc >/dev/null
fi
# Adds the runtime `runsc` to /etc/docker/daemon.json, keeping the rest.
runsc install
systemctl restart docker
docker info --format '{{json .Runtimes}}' | grep -q '"runsc"' || { echo "docker does not know runsc" >&2; exit 1; }

echo "==> the runs' network"
docker network inspect knotless-runs >/dev/null 2>&1 ||
  docker network create --driver bridge \
    --opt com.docker.network.bridge.enable_icc=false \
    --opt com.docker.network.bridge.name=kl-runs \
    knotless-runs >/dev/null

echo "==> egress rules"
cat >/usr/local/sbin/knotless-egress <<'EOF'
#!/bin/sh
# What a run on kl-runs may not reach: the host, the private and
# link-local ranges (the cloud's metadata service among them), and mail.
# Made afresh each time, in a chain of its own that docker's DOCKER-USER
# jumps to for traffic from the bridge.
set -eu
iptables -N KNOTLESS-RUNS 2>/dev/null || iptables -F KNOTLESS-RUNS
iptables -A KNOTLESS-RUNS -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN
for net in 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 \
  172.16.0.0/12 192.168.0.0/16 224.0.0.0/4 240.0.0.0/4; do
  iptables -A KNOTLESS-RUNS -d "$net" -j DROP
done
for ip in $(hostname -I); do
  case "$ip" in *:*) ;; *) iptables -A KNOTLESS-RUNS -d "$ip" -j DROP ;; esac
done
iptables -A KNOTLESS-RUNS -p tcp -m multiport --dports 25,465,587 -j DROP
iptables -A KNOTLESS-RUNS -j RETURN
iptables -N DOCKER-USER 2>/dev/null || true
iptables -C DOCKER-USER -i kl-runs -j KNOTLESS-RUNS 2>/dev/null ||
  iptables -I DOCKER-USER -i kl-runs -j KNOTLESS-RUNS
# A run reaching the host itself does not pass through FORWARD.
iptables -C INPUT -i kl-runs -j DROP 2>/dev/null || iptables -I INPUT -i kl-runs -j DROP
EOF
chmod 0755 /usr/local/sbin/knotless-egress
cat >/etc/systemd/system/knotless-egress.service <<'EOF'
[Unit]
Description=What knotless runs may not reach
After=docker.service
PartOf=docker.service

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/sbin/knotless-egress

[Install]
WantedBy=docker.service
EOF
systemctl daemon-reload
systemctl enable --now knotless-egress.service >/dev/null
systemctl restart knotless-egress.service

echo "==> firewall"
ufw default deny incoming >/dev/null
ufw default allow outgoing >/dev/null
ufw allow OpenSSH >/dev/null
# Caddy, for the runner's bot login (docker-compose.yml's `bots` profile).
ufw allow 80,443/tcp >/dev/null
ufw --force enable >/dev/null

echo "==> logs kept 30 days, as the privacy policy says"
install -d /etc/systemd/journald.conf.d
cat >/etc/systemd/journald.conf.d/knotless.conf <<'EOF'
[Journal]
MaxRetentionSec=30day
EOF
systemctl restart systemd-journald

echo "==> upgrades and pruning"
dpkg-reconfigure -f noninteractive unattended-upgrades
cat >/etc/systemd/system/knotless-prune.service <<'EOF'
[Unit]
Description=Removes images no container has used for a week

[Service]
Type=oneshot
ExecStart=/usr/bin/docker image prune -af --filter until=168h
EOF
cat >/etc/systemd/system/knotless-prune.timer <<'EOF'
[Unit]
Description=Daily image prune

[Timer]
OnCalendar=daily
RandomizedDelaySec=1h
Persistent=true

[Install]
WantedBy=timers.target
EOF
systemctl daemon-reload
systemctl enable --now knotless-prune.timer >/dev/null

echo "==> /etc/knotless"
install -d -m 0700 /etc/knotless /var/lib/knotless-run
[ -f /etc/knotless/paused.json ] || echo '{}' >/etc/knotless/paused.json
[ -f /etc/knotless/run-secrets.json ] || (umask 077; echo '{}' >/etc/knotless/run-secrets.json)
cat >/etc/knotless/resolv.conf <<'EOF'
nameserver 1.1.1.1
nameserver 9.9.9.9
nameserver 2606:4700:4700::1111
options edns0
EOF
chmod 0644 /etc/knotless/resolv.conf

echo "==> check"
docker run --rm --runtime runsc --network knotless-runs \
  -v /etc/knotless/resolv.conf:/etc/resolv.conf:ro alpine:3 sh -c '
  wget -q -T 10 -O /dev/null https://example.com && echo "internet: answers" || echo "internet: NO ANSWER"
  wget -q -T 3 -O /dev/null http://169.254.169.254/ && echo "metadata: REACHABLE" || echo "metadata: blocked"
  wget -q -T 3 -O /dev/null http://10.0.0.1/ && echo "private: REACHABLE" || echo "private: blocked"
  nc -z -w 3 smtp.gmail.com 25 && echo "mail: REACHABLE" || echo "mail: blocked"
  dmesg 2>/dev/null | head -1 | grep -q gVisor && echo "kernel: gVisor" || echo "kernel: NOT gVisor"
'
HOST