~ knotless Search/
↑↓ move↵ open⇧↵ all results

chadtmiller.com/knotless
Switch repository
Loading…

git that lives in your Atmosphere account

knotless.dev
1 collaborator

knotless 0.13.0 Latest 2 packages, 1 image

1
10

A deploy keeps the runner unless its build changed

Merged chadtmiller.com asked to merge into main from chadtmiller.com/knotless:deploy/run-fingerprint
crates/knotless-agent/src/main.rs+7−2
View file
@@ -9,8 +9,13 @@ fn main() -> ExitCode {
return ExitCode::from(2);
}
};
let (version, commit) = (env!("CARGO_PKG_VERSION"), option_env!("KNOTLESS_COMMIT"));
knotless::telemetry::init("knotless-agent", version, commit);
// KNOTLESS_COMMIT is the commit the image was built from, set as it
// runs (Dockerfile), so the binary is the same at every commit.
let (version, commit) = (
env!("CARGO_PKG_VERSION"),
std::env::var("KNOTLESS_COMMIT").ok(),
);
knotless::telemetry::init("knotless-agent", version, commit.as_deref());
let ctx = knotless::env::Ctx::new(knotless::env::Env::from_process());
let runtime = tokio::runtime::Runtime::new().expect("a runtime builds");
match runtime.block_on(knotless_agent::run(opts, ctx)) {
crates/knotless-run/src/main.rs+7−3
View file
@@ -7,9 +7,13 @@ fn main() {
std::process::exit(2);
}
};
// KNOTLESS_COMMIT is the commit the image was built from (Dockerfile).
let (version, commit) = (env!("CARGO_PKG_VERSION"), option_env!("KNOTLESS_COMMIT"));
knotless::telemetry::init("knotless-run", version, commit);
// KNOTLESS_COMMIT is the commit the image was built from, set as it
// runs (Dockerfile), so the binary is the same at every commit.
let (version, commit) = (
env!("CARGO_PKG_VERSION"),
std::env::var("KNOTLESS_COMMIT").ok(),
);
knotless::telemetry::init("knotless-run", version, commit.as_deref());
let ctx = knotless::env::Ctx::new(knotless::env::Env::from_process());
let runtime = tokio::runtime::Runtime::new().expect("a runtime builds");
if let Err(err) = runtime.block_on(knotless_run::run(opts, ctx)) {
crates/knotless-serve/src/main.rs+7−3
View file
@@ -7,9 +7,13 @@ fn main() {
std::process::exit(2);
}
};
// KNOTLESS_COMMIT is the commit the image was built from (Dockerfile).
let (version, commit) = (env!("CARGO_PKG_VERSION"), option_env!("KNOTLESS_COMMIT"));
knotless::telemetry::init("knotless-serve", version, commit);
// KNOTLESS_COMMIT is the commit the image was built from, set as it
// runs (Dockerfile), so the binary is the same at every commit.
let (version, commit) = (
env!("CARGO_PKG_VERSION"),
std::env::var("KNOTLESS_COMMIT").ok(),
);
knotless::telemetry::init("knotless-serve", version, commit.as_deref());
let ctx = knotless::env::Ctx::new(knotless::env::Env::from_process());
let runtime = tokio::runtime::Runtime::new().expect("a runtime builds");
if let Err(err) = runtime.block_on(knotless_serve::run(opts, ctx)) {
crates/knotless-triage/src/main.rs+7−2
View file
@@ -10,8 +10,13 @@ fn main() -> ExitCode {
return ExitCode::from(2);
}
};
let (version, commit) = (env!("CARGO_PKG_VERSION"), option_env!("KNOTLESS_COMMIT"));
knotless::telemetry::init("knotless-triage", version, commit);
// KNOTLESS_COMMIT is the commit the image was built from, set as it
// runs (Dockerfile), so the binary is the same at every commit.
let (version, commit) = (
env!("CARGO_PKG_VERSION"),
std::env::var("KNOTLESS_COMMIT").ok(),
);
knotless::telemetry::init("knotless-triage", version, commit.as_deref());
let ctx = knotless::env::Ctx::new(knotless::env::Env::from_process());
let runtime = tokio::runtime::Runtime::new().expect("a runtime builds");
match runtime.block_on(knotless_triage::run(opts, ctx)) {
deploy/deploy.sh+23−18
View file
@@ -49,9 +49,9 @@
# KNOTLESS_RUN_DATA (/var/lib/knotless-run), and the secrets it grants in
# /etc/knotless/run-secrets.json, which starts empty. KNOTLESS_RUN_ARGS
# adds to its command line: `--pulls`, say, to run open pulls' heads too.
# A deploy restarts it only when what it runs changed: the sources it is
# built from (deploy/run-unchanged.sh) or its settings; KNOTLESS_RUN_RESTART=1
# restarts it anyway. The deploy log says which, and why.
# A deploy restarts it only when what it runs changed: its build, as the
# image's /etc/knotless-run.fingerprint says (Dockerfile), or its settings;
# KNOTLESS_RUN_RESTART=1 restarts it anyway. The deploy log says which, and why.
#
# knotless-triage too, if KNOTLESS_TRIAGE_REPOS names repositories to
# triage, once /etc/knotless/triage.env is on the host: a deploy with
@@ -137,15 +137,10 @@ echo /system.slice/knotless-build
EOF
)
fi
# The commit the runner is built from, for the next deploy to compare its
# own with (up): left empty, as unknown, when this checkout's runner
# sources are not that commit's, uncommitted changes and all.
local run_commit=$COMMIT
deploy/run-unchanged.sh "$COMMIT" >/dev/null || run_commit=
say "building $CANDIDATE on $DEST${cgroup:+, in $cgroup}"
local began=$SECONDS
DOCKER_HOST="ssh://$DEST" docker build ${cgroup:+--cgroup-parent "$cgroup"} \
--build-arg "KNOTLESS_COMMIT=$COMMIT" --label "dev.knotless.run-commit=$run_commit" \
--build-arg "KNOTLESS_COMMIT=$COMMIT" \
-t "$CANDIDATE" .
say "built $CANDIDATE in $((SECONDS - began))s"
# buildkit keeps every layer it ever built, with no cap of its own: it
@@ -267,9 +262,12 @@ scp -q -o StrictHostKeyChecking=accept-new deploy/docker-compose.yml deploy/roll
# "A stop"). What it runs is two things, each of which must be known to be
# unchanged, or it is restarted as before:
#
# - its build: the image's dev.knotless.run-commit label (build), read
# from the image the running container was made from, against this
# checkout (deploy/run-unchanged.sh);
# - its build: the fingerprint the image keeps of what the runner runs
# (Dockerfile), read from the running container, against the one in
# knotless:current, the image just built. The commit is set as the
# binaries run rather than built in, so a commit that changes nothing
# the runner is made of leaves the same fingerprint, whatever else in
# the image or the repository it changes;
# - its settings: a hash of the `run` service as compose resolves it from
# docker-compose.yml and the .env just written, and of run.env, against
# /etc/knotless/run-deployed, "<container id> <hash>", which each restart
@@ -288,8 +286,8 @@ STATUS=$($SSH sh -s <<'EOF'
id=$(docker compose ps -q --status running run 2>/dev/null | head -n 1)
[ -n "$id" ] || exit 0
echo "container $id"
image=$(docker inspect -f '{{.Image}}' "$id") || exit 0
echo "commit $(docker image inspect -f '{{index .Config.Labels "dev.knotless.run-commit"}}' "$image" 2>/dev/null)"
echo "build $(docker exec "$id" cat /etc/knotless-run.fingerprint 2>/dev/null)"
echo "candidate $(docker run --rm --entrypoint cat knotless:current /etc/knotless-run.fingerprint 2>/dev/null)"
[ ! -f run-deployed ] || echo "record $(head -n 1 run-deployed)"
} </dev/null
EOF
@@ -297,7 +295,10 @@ EOF
field() { printf '%s\n' "$STATUS" | awk -v k="$1" '$1 == k { $1 = ""; sub(/^ /, ""); print; exit }'; }
SETTINGS=$(field settings)
RUNNING=$(field container)
RUN_COMMIT=$(field commit)
RUN_BUILD=$(field build)
NEW_BUILD=$(field candidate)
[[ $RUN_BUILD =~ ^[0-9a-f]{64}$ ]] || RUN_BUILD=""
[[ $NEW_BUILD =~ ^[0-9a-f]{64}$ ]] || NEW_BUILD=""
[[ $SETTINGS =~ ^[0-9a-f]{64}$ ]] || SETTINGS=""
RESTART_RUN=""
if [ -z "$RUN_REPOS" ]; then
@@ -310,14 +311,18 @@ elif [ -z "$RUNNING" ]; then
RESTART_RUN="it is not running"
elif [ "$(field record)" != "$RUNNING $SETTINGS" ]; then
RESTART_RUN="its settings changed, or were not recorded for the container running"
elif ! why=$(deploy/run-unchanged.sh "$RUN_COMMIT"); then
RESTART_RUN=${why:-its build may have changed}
elif [ -z "$RUN_BUILD" ]; then
RESTART_RUN="the one running keeps no fingerprint of its build"
elif [ -z "$NEW_BUILD" ]; then
RESTART_RUN="the new image's fingerprint could not be read"
elif [ "$RUN_BUILD" != "$NEW_BUILD" ]; then
RESTART_RUN="its build changed, ${RUN_BUILD:0:12} to ${NEW_BUILD:0:12}"
fi
if [ -n "$RESTART_RUN" ]; then
[ -z "$RUN_REPOS" ] || say "run: restarting: $RESTART_RUN"
RUN_MODE=restart
else
say "run: unchanged since ${RUN_COMMIT:0:12}, not restarted"
say "run: its build, ${RUN_BUILD:0:12}, and settings are the same, not restarted"
RUN_MODE=keep
fi
deploy/README.md+21−23
View file
@@ -8,7 +8,6 @@ One Docker host, a reverse proxy in front of it for TLS, and a domain.
| `docker-compose.yml` | `/etc/knotless/docker-compose.yml`: `serve` (knotless-serve, on :2590), `pages` (the SvelteKit app, on :2591) and, if asked, `run` (knotless-run), `triage` (knotless-triage) and `review` (knotless-agent review) |
| `deploy.sh` | builds the image on the host over SSH, writes the compose file and its `.env`, and restarts what changed; or one of those steps, `build`, `up` or `drop` |
| `rollout.sh` | run on the host by `deploy.sh`: moves `pages` and `serve` to a new container each, with no gap |
| `run-unchanged.sh` | run by `deploy.sh`: whether the runner built from this checkout is the one built from a given commit, so a deploy can leave `run` running |
| `rollout-test.sh` | tries `rollout.sh` on a local docker behind a Caddy, counting failed requests |
| `release.sh` | builds the client for macOS and Linux here and puts the builds on the host, for `/install.sh` |
| `setup.sh` | makes a fresh Ubuntu 24.04 host ready for `deploy.sh`: docker, a Caddy of its own for the domain, a firewall, swap, upgrades and pruning |
@@ -520,21 +519,19 @@ three times in all. It reads every repository again every `--sweep`
A restart costs the runs still waiting for a job, and one that cuts off
a merge-queue batch still being checked makes the batch be checked again, so a
deploy restarts `run` only when what it runs changed. The image cannot
say so: it is rebuilt at every commit, with the commit in it. So
`deploy.sh up` compares two things, and leaves `run` running only when
it knows both are the same:
- **Its build.** `deploy.sh build` labels the image
`dev.knotless.run-commit=<commit>`, or leaves the label empty when the
checkout's runner sources are not that commit's (uncommitted changes,
or no git). `up` reads the label from the image the running `run`
container was made from, which is the build it runs whatever
`knotless:current` now is, and `run-unchanged.sh` compares that commit
with this checkout: `crates/knotless` and `crates/knotless-run` (less
their `tests/` and `examples/`), `Cargo.toml`, `Cargo.lock`, the
`Dockerfile`, `.dockerignore` and a pinned toolchain, committed or not,
untracked files included.
deploy restarts `run` only when what it runs changed. `deploy.sh up`
compares two things, and leaves `run` running only when it knows both
are the same:
- **Its build.** The image keeps a fingerprint of what the runner is
made of, in `/etc/knotless-run.fingerprint`: the `knotless-run`
binary, the docker client its steps run through, and the packages
installed (`/var/lib/dpkg/status`: git and the libraries). `up` reads it
from the running container and from `knotless:current`, the image just
built. The commit the binaries log is set as they run (`ENV
KNOTLESS_COMMIT`, the image's last layer) rather than built in, so a
commit that changes only serve, the pages, another binary or the docs
leaves the runner's binary, and so its fingerprint, as it was.
- **Its settings.** A hash of the `run` service as `docker compose config
run` resolves it from the compose file and the `.env` this deploy just
wrote (its command line, `KNOTLESS_RUN_ARGS` and `KNOTLESS_RUN_REPOS`
@@ -544,23 +541,24 @@ it knows both are the same:
hash, or whose running container, is not the one recorded restarts it.
`run-secrets.json` is not in it: the runner reads that at each run.
Anything it cannot tell, a container not running, no label, a commit the
checkout lacks, a record missing, restarts `run` as every deploy once did;
Anything it cannot tell, a container not running, no fingerprint, a
record missing, restarts `run` as every deploy once did;
so the first deploy with this does, and so does one while a restart is
still under way. Either way the deploy log says which, and why:
```text
2026-10-27T04:02:11Z 1a2b3c4d5e6f → run: unchanged since c7252c6fe7d9, not restarted
2026-10-27T05:13:40Z 7d8e9f0a1b2c → run: restarting: 2 file(s) it is built from changed since c7252c6fe7d9: crates/knotless-run/src/queue.rs crates/knotless/src/queue.rs
2026-10-27T04:02:11Z 1a2b3c4d5e6f → run: its build, 5d41402abc4b, and settings are the same, not restarted
2026-10-27T05:13:40Z 7d8e9f0a1b2c → run: restarting: its build changed, 5d41402abc4b to 7d793037a076
```
A runner left running keeps logging the commit it was built from at its
start, as it should. To restart it anyway, deploy with
`KNOTLESS_RUN_RESTART=1`, or on the host `cd /etc/knotless && docker
compose up -d --force-recreate run`, after which the next deploy, finding
no record for that container, restarts it once more. `deploy/run-unchanged.sh
<commit>` asks the first question by hand: it exits 0 when the runner built
from the checkout is that commit's, and otherwise says why not.
no record for that container, restarts it once more. To ask the first
question by hand, on the host: `docker exec <container> cat
/etc/knotless-run.fingerprint` beside `docker run --rm --entrypoint cat
knotless:current /etc/knotless-run.fingerprint`.
## knotless-triage
deploy/run-unchanged.shdeleted+0−45
@@ -1,45 +0,0 @@
#!/usr/bin/env bash
# Whether knotless-run built from this checkout would be the one built
# from <commit>: whether a deploy can leave `run` running.
#
# deploy/run-unchanged.sh <commit>
#
# Exits 0, saying nothing, when no file the runner is built from differs
# between <commit> and this checkout as it is now (HEAD and its working
# tree, which the image is built from, uncommitted changes included).
# Anything else, a change or anything it cannot tell, exits 1 and says why
# on stdout: deploy.sh then restarts the runner, as every deploy once did.
#
# What the runner is built from: its crate and the knotless crate it
# depends on, less their tests and examples; the workspace's Cargo.toml
# and Cargo.lock; the Dockerfile and .dockerignore; a pinned toolchain.
# Its settings (the `run` service in docker-compose.yml, the .env and
# run.env it reads) are compared on the host by deploy.sh, as resolved.
set -uo pipefail
cd "$(dirname "$0")/.." || { echo "no checkout"; exit 1; }
PATHS=(
crates/knotless crates/knotless-run
':(exclude)crates/knotless/tests' ':(exclude)crates/knotless/examples'
':(exclude)crates/knotless-run/tests'
Cargo.toml Cargo.lock Dockerfile .dockerignore rust-toolchain rust-toolchain.toml
)
# The deploy workflow's checkout is root's, in a container whose user may
# not be: git would refuse it as another's.
git() { command git -c safe.directory='*' "$@"; }
old="${1:-}"
[[ $old =~ ^[0-9a-f]{40}$ ]] || { echo "the running runner's commit is unknown (${old:-none})"; exit 1; }
git cat-file -e "$old^{commit}" 2>/dev/null || { echo "${old:0:12}, which it runs, is not in this checkout"; exit 1; }
# Between the commit and the working tree, staged or not; then files git
# does not know yet, which the build copies in too.
changed=$(git diff --name-only "$old" -- "${PATHS[@]}") || { echo "git diff failed"; exit 1; }
untracked=$(git ls-files --others --exclude-standard -- "${PATHS[@]}") || { echo "git ls-files failed"; exit 1; }
changed=$(printf '%s\n%s\n' "$changed" "$untracked" | sed '/^$/d')
if [ -n "$changed" ]; then
count=$(printf '%s\n' "$changed" | wc -l | tr -d ' ')
echo "$count file(s) it is built from changed since ${old:0:12}: $(printf '%s\n' "$changed" | head -n 5 | paste -sd ' ' -)"
exit 1
fi
exit 0
docs/design/runner.md+5−3
View file
@@ -492,9 +492,11 @@ A stop never waits forever on a hung step. One did once, when a deploy's
Since a stop costs the runs still waiting, and can cut off a merge-queue
batch still being checked, which is then checked again, knotless.dev's
deploys stop the runner only when what it runs changed: the sources it
is built from, or its settings. Each deploy rebuilds the image, so the
image alone cannot tell; when unsure, it restarts the runner
deploys stop the runner only when what it runs changed: its binary and
the tools it runs, or its settings. The image keeps a fingerprint of the
first, and the commit is given to the binaries as they run rather than
built in, so a commit that changes nothing the runner is made of leaves
the fingerprint as it was. When unsure, it restarts the runner
([When a deploy restarts the runner](../../deploy/README.md#when-a-deploy-restarts-the-runner)).
A run that has not started its steps when the stop comes is dropped: its
Dockerfile+11−3
View file
@@ -13,9 +13,6 @@ COPY . .
# The cache mounts keep the registry and the build between builds on the
# same machine, so a change rebuilds the crates it touches and not the rest.
# The deploy profile (Cargo.toml) is release's, built for the build's time.
# KNOTLESS_COMMIT, from deploy.sh, is the commit serve and the runner log
# that they run: only their main.rs reads it, so it rebuilds nothing else.
ARG KNOTLESS_COMMIT
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/src/target \
cargo build --profile deploy --locked \
@@ -80,6 +77,17 @@ COPY crates/knotless-agent/claude-review /usr/local/bin/knotless-claude-review
COPY --from=docker:28-cli /usr/local/bin/docker /usr/local/bin/docker
COPY --from=pages /web/build /app/build
COPY --from=pages /web/package.json /app/package.json
# What knotless-run is, for a deploy to leave a runner running whose is the
# same (deploy/deploy.sh): its binary, the docker client its steps run
# through, and the packages installed, git and the libraries among them.
RUN sha256sum /usr/local/bin/knotless-run /usr/local/bin/docker /var/lib/dpkg/status \
| sha256sum | cut -d ' ' -f 1 > /etc/knotless-run.fingerprint
USER node
# The commit, from deploy.sh, that the binaries log they run. Set as they
# run rather than built in, and last, it changes no binary and no layer
# before it: a runner built at another commit from the same code is the
# same runner.
ARG KNOTLESS_COMMIT
ENV KNOTLESS_COMMIT=$KNOTLESS_COMMIT
ENTRYPOINT ["tini", "--"]
CMD ["knotless-serve", "--listen", "0.0.0.0:2590"]