0
0
// A staging network's knotless (dev up --with-knotless) as a state machine: seed the runner's composition and
// bindings in the account's repo (the gateway owner, the runner's account), then each step a script against the
// staging knotless (knotless CLI, git, the network's env), each failing on what it checks. Entries:
// push push a workflow to a fresh repo: the runner rents a network bidder's VM, the check lands success
// after build keeps an artifact, test (after: [build]) reads it with $KNOTLESS_ARTIFACTS_URL/TOKEN in its VM,
// `knotless run artifacts --out` downloads it
// executor two workflows on two bindings (executor: small | big): each VM has its binding's vCPUs
// pull a sealed secret: the push to main sees it, a pull's run gets none
// fallback one workflow per network bidder, each VM sized so a bidder holds one (its offering's maxCpus / 2 + 1):
// all succeed, and in one of up to 3 rounds a bidder declined an accept (409 CapacityExceeded) and the
// rent took the next bid (the atlibs requester module's accept fallback; acceptance 0014's atlibs path)
// cancel every bidder held by a `hold-<n>` workflow (a VM each, on the `fill` binding with wait_capacity_s), then a `waiter`
// rent that has to wait for capacity is cancelled (`knotless run cancel`) and ends cancelled at once (not after
// the wait), and a hold cancelled with its VM held gives it back: the waiter's next rent gets that bidder and succeeds
// terminated a step powers off its own VM: its bidder's vm.terminated reaches the cicd workload as a market event
// and the step fails at once with the reason (not after the read retries)
// Result {pass, error?, phase (the step), readings (seconds per step)}. The host deletes the repo and the seeded
// records however it ends; the runner releases each VM.
use abi::{Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{MIN, S, host};
struct Step {
id: &'static str,
script: String,
}
const CI: &str = "on:\n push:\n refs: [refs/heads/main]\nsteps:\n - name: where\n run: uname -a && nproc && git log --oneline -1\n - name: test\n run: test \"$(cat answer.txt)\" = 42\n";
const BUILD: &str = "on:\n push:\n refs: [refs/heads/main]\nartifacts: [out/hello.txt]\nsteps:\n - name: build\n run: mkdir -p out && echo \"hello $KNOTLESS_RUN_SHA\" > out/hello.txt\n";
const TEST: &str = "on:\n push:\n refs: [refs/heads/main]\nafter: [build]\nsteps:\n - name: fetch\n run: |\n curl -sSfo got.txt -H \"Authorization: Bearer $KNOTLESS_ARTIFACTS_TOKEN\" \"$KNOTLESS_ARTIFACTS_URL?workflow=build&name=out/hello.txt\" \n test \"$(cat got.txt)\" = \"hello $KNOTLESS_RUN_SHA\"\n";
const SMALL: &str = "on:\n push:\n refs: [refs/heads/main]\nexecutor: small\nsteps:\n - name: size\n run: nproc && test \"$(nproc)\" = 1\n";
const BIG: &str = "on:\n push:\n refs: [refs/heads/main]\nexecutor: big\nsteps:\n - name: size\n run: nproc && test \"$(nproc)\" = 2\n";
const POWEROFF: &str = "on:\n push:\n refs: [refs/heads/main]\nsteps:\n - name: poweroff\n run: |\n id -u; cat /proc/sys/kernel/sysrq; ls -d /sys/firmware/acpi || true\n if [ -d /sys/firmware/acpi ]; then t=o; else t=b; fi\n echo \"sysrq $t in 20 s\"\n setsid sh -c \"sleep 20; echo 1 > /proc/sys/kernel/sysrq; echo $t > /proc/sysrq-trigger\" > /dev/null 2>&1 < /dev/null &\n sleep 1200\n";
const SECRET: &str = "on:\n push:\n refs: [refs/heads/main]\n pull:\n base: [refs/heads/main]\nsecrets: [STAGING_SECRET]\nsteps:\n - name: secret\n run: if [ -n \"${KNOTLESS_RUN_PULL:-}\" ]; then test -z \"${STAGING_SECRET:-}\" && echo pull-has-no-secret; else test \"$STAGING_SECRET\" = staging-secret-value && echo push-has-the-secret; fi\n";
/// Repo `r`, then the wait until the runner watches it: a runner acts on no ref, pull or rerun it found already there.
// ponytail: the appview listing it, then 3 of the runner's 10 s discovery sweeps; a runner-side "watching" read when
// one exists.
fn create(r: &str) -> String {
format!(
"knotless repo create knotless://$KNOTLESS_IDENTIFIER/{r} --description 'swc-test knotless'\nfor i in $(seq 60); do curl -sf \"$KNOTLESS_APPVIEW/xrpc/dev.knotless.listRepos?runner=$KNOTLESS_IDENTIFIER\" | grep -q '\"name\":\"{r}\"' && break; sleep 2; done\ncurl -sf \"$KNOTLESS_APPVIEW/xrpc/dev.knotless.listRepos?runner=$KNOTLESS_IDENTIFIER\" | grep -q '\"name\":\"{r}\"'\nsleep 30\n"
)
}
/// Repo `r` with `answer.txt` and the workflows, pushed to main; its commit in `<r>.sha`.
fn push_script(r: &str, workflows: &[(&str, &str)]) -> String {
let mut s = format!(
"U=knotless://$KNOTLESS_IDENTIFIER/{r}\nrm -rf {r} && mkdir -p {r}/.knotless/workflows && cd {r}\ngit init -q -b main\necho 42 > answer.txt\n"
);
for (name, body) in workflows {
s.push_str(&format!(
"cat > .knotless/workflows/{name}.yml <<'EOF'\n{body}EOF\n"
));
}
s.push_str("git add . && git commit -q -m 'swc-test knotless'\ngit remote add origin \"$U\"\ngit push -q -u origin main\ngit rev-parse HEAD > ../");
s.push_str(&format!("{r}.sha\ncat ../{r}.sha\n"));
s
}
/// Waits for every check of the commit in `<r>.sha` (or the pull in `<r>.pull`), then each named workflow's must be
/// success; their logs on a failure.
fn checks(r: &str, of: &str, workflows: &[&str]) -> String {
let sel = match of {
"pull" => format!("--pull \"$(cat {r}.pull)\""),
_ => format!("--commit \"$(cat {r}.sha)\""),
};
let want: Vec<String> = workflows
.iter()
.map(|w| format!("grep -q \"^success\t{w}\t\" {r}.{of}.checks"))
.collect();
format!(
"U=knotless://$KNOTLESS_IDENTIFIER/{r}\nknotless run list \"$U\" {sel} --wait 1500 > {r}.{of}.checks\ncat {r}.{of}.checks\n{{ {}; }} || {{ knotless run list \"$U\" {sel} --log; exit 1; }}\n",
want.join(" && ")
)
}
/// The workflows of `fallback`: one per bidder, each on the `fill` binding.
const FALLBACK_FILES: &str = "for i in $(seq \"$n\"); do printf 'on:\n push:\n refs: [refs/heads/main]\nexecutor: fill\nsteps:\n - name: size\n run: nproc\n' > .knotless/workflows/fill-$i.yml; done";
/// The `fill` binding (the bidders' tightest maxCpus / 2 + 1 vCPUs, 1 GiB) beside the seeded ones, then one workflow
/// per bidder on it (`fill-<n>`), pushed; the bidders are the network relay's repos with a compute offering.
fn fill_script(r: &str) -> String {
fill_with(r, "", FALLBACK_FILES)
}
/// `fill_script`, the binding's inputs given `inputs` more (`, wait_capacity_s: 900`), and `files` the shell that writes the workflows
/// (it runs in the new repo's directory, `$n` bidders known).
fn fill_with(r: &str, inputs: &str, files: &str) -> String {
format!(
r##"S=$(jq -n --arg i "$KNOTLESS_IDENTIFIER" --arg p "$KNOTLESS_PASSWORD" '{{identifier: $i, password: $p}}' | curl -sf -H 'content-type: application/json' -d @- "$KNOTLESS_PDS/xrpc/com.atproto.server.createSession")
J=$(printf %s "$S" | jq -r .accessJwt); D=$(printf %s "$S" | jq -r .did)
: > {r}.max
O=com.publicdomainrelay.temp.market.offering
for b in $(curl -sf --cacert "$KNOTLESS_CA_CERTS" "$KNOTLESS_RELAY/xrpc/com.atproto.sync.listReposByCollection?collection=$O" | jq -r '.repos[].did' | sort -u); do
p=$(curl -sf "$KNOTLESS_PLC_URL/$b" | jq -r '.service[] | select(.id == "#atproto_pds") | .serviceEndpoint') || continue
m=$(curl -sf --cacert "$KNOTLESS_CA_CERTS" "$p/xrpc/com.atproto.repo.getRecord?repo=$b&collection=$O&rkey=self" | jq -r '.value.maxCpus // empty') || continue
[ -n "$m" ] && echo "$b $m" | tee -a {r}.max
done
n=$(wc -l < {r}.max); test "$n" -ge 2
cpus=$(( $(sort -n -k2 {r}.max | head -1 | cut -d' ' -f2) / 2 + 1 ))
echo "$n bidders, $cpus vCPUs a VM: one VM fills a bidder"
comp=$(curl -sf "$KNOTLESS_PDS/xrpc/com.atproto.repo.getRecord?repo=$D&collection=com.fedfork.exec.binding&rkey=self" | jq -c .value.composition)
jq -n --arg d "$D" --argjson c "$comp" --argjson cpus "$cpus" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '{{repo: $d, collection: "com.fedfork.exec.binding", rkey: "fill", record: {{"$type": "com.fedfork.exec.binding", composition: $c, inputs: {{cpus: $cpus, mem: 1{inputs}}}, createdAt: $at}}}}' |
curl -sf -H "authorization: Bearer $J" -H 'content-type: application/json' -d @- "$KNOTLESS_PDS/xrpc/com.atproto.repo.putRecord" > /dev/null
U=knotless://$KNOTLESS_IDENTIFIER/{r}
rm -rf {r} && mkdir -p {r}/.knotless/workflows && cd {r}
git init -q -b main
{files}
echo 1 > round
git add . && git commit -q -m 'swc-test knotless fallback'
git remote add origin "$U"
git push -q -u origin main
"##,
inputs = inputs,
files = files
)
}
/// The workflows of `cancel`: a `hold-<n>` per bidder (a VM each, held 15 min), and a `waiter` only an ask runs.
const CANCEL_FILES: &str = "for i in $(seq \"$n\"); do printf 'on:\\n push:\\n refs: [refs/heads/main]\\nexecutor: fill\\nsteps:\\n - name: hold\\n run: sleep 900\\n' > .knotless/workflows/hold-$i.yml; done\nprintf 'on:\\n asked:\\n refs: [refs/heads/main]\\nexecutor: fill\\nsteps:\\n - name: size\\n run: nproc\\n' > .knotless/workflows/waiter.yml";
/// With every bidder held: a waiter's rent, waiting on capacity, is cancelled and its check says so within 90 s; a hold cancelled
/// while it holds its VM gives it back, so the waiter's next rent succeeds.
fn cancel_script(r: &str) -> String {
format!(
r#"U=knotless://$KNOTLESS_IDENTIFIER/{r}
n=$(ls {r}/.knotless/workflows | grep -c '^hold-')
line() {{ knotless run list "$U" --ref refs/heads/main --workflow "$1" 2>/dev/null | head -1; }}
show() {{ echo "$1: $(line "$1")"; knotless run list "$U" --ref refs/heads/main --workflow "$1" --log | tail -30; }}
until_line() {{ for i in $(seq "$3"); do line "$1" | grep -q "$2" && return 0; sleep 1; done; show "$1"; echo "wanted /$2/ in $3 s"; return 1; }}
for i in $(seq "$n"); do until_line hold-$i '^running.*step hold' 900 || exit 1; done
echo "$n bidders held"
knotless run rerun "$U" --ref refs/heads/main --workflow waiter
until_line waiter '^running' 120 || exit 1
sleep 30
line waiter | grep -q '^running' || {{ show waiter; exit 1; }}
t=$(date +%s)
knotless run cancel "$U" --ref refs/heads/main --workflow waiter
until_line waiter '^cancelled' 90 || exit 1
echo "the rent waiting on capacity was cancelled in $(( $(date +%s) - t )) s"
knotless run list "$U" --ref refs/heads/main --workflow waiter --log | grep -q 'cancelled while renting its machine' || {{ show waiter; exit 1; }}
knotless run cancel "$U" --ref refs/heads/main --workflow hold-1
until_line hold-1 '^cancelled' 180 || exit 1
knotless run rerun "$U" --ref refs/heads/main --workflow waiter
until_line waiter '^success' 400 || exit 1
echo "a hold cancelled with its VM gave it back: the waiter got a bidder"
knotless run cancel "$U" --ref refs/heads/main
"#
)
}
/// Up to 3 rounds: every fill-* check of the round's commit succeeds; a round whose logs show a declined accept
/// (the rent's notes) ends it, else the next round is a new commit.
fn rounds_script(r: &str) -> String {
format!(
r#"U=knotless://$KNOTLESS_IDENTIFIER/{r}
cd {r}
for round in 1 2 3; do
C=$(git rev-parse HEAD)
knotless run list "$U" --commit "$C" --wait 1500 > ../{r}.checks
cat ../{r}.checks
knotless run list "$U" --commit "$C" --log > ../{r}.log
if grep -v "^success fill-" ../{r}.checks | grep -q " fill-"; then cat ../{r}.log; exit 1; fi
if grep 'declined the accept' ../{r}.log; then echo "round $round: a declined accept was replaced"; exit 0; fi
echo "round $round: every first pick had room; the next once the bidders hold no reservation (90 s)"
sleep 90
echo $((round + 1)) > round && git commit -q -am "round $((round + 1))" && git push -q origin main
done
echo "no accept was declined in 3 rounds"; exit 1
"#
)
}
fn plan(entry: &str, r: &str) -> (Value, Vec<Step>) {
let one = json!({ "cpus": 1, "mem": 1 });
match entry {
"after" => (
json!({ "self": one }),
vec![
Step {
id: "create",
script: create(r),
},
Step {
id: "push",
script: push_script(r, &[("build", BUILD), ("test", TEST)]),
},
Step {
id: "checks",
script: checks(r, "push", &["build", "test"]),
},
Step {
id: "download",
script: format!(
"U=knotless://$KNOTLESS_IDENTIFIER/{r}\nrm -rf {r}.got\nknotless run artifacts \"$U\" --workflow build --commit \"$(cat {r}.sha)\"\nknotless run artifacts \"$U\" --workflow build --commit \"$(cat {r}.sha)\" --out {r}.got\nfind {r}.got -type f\ntest \"$(cat \"$(find {r}.got -name hello.txt)\")\" = \"hello $(cat {r}.sha)\"\n"
),
},
],
),
"executor" => (
json!({ "self": one, "small": one, "big": { "cpus": 2, "mem": 2 } }),
vec![
Step {
id: "create",
script: create(r),
},
Step {
id: "push",
script: push_script(r, &[("small", SMALL), ("big", BIG)]),
},
Step {
id: "checks",
script: checks(r, "push", &["small", "big"]),
},
],
),
"fallback" => (
json!({ "self": one }),
vec![
Step {
id: "create",
script: create(r),
},
Step {
id: "fill",
script: fill_script(r),
},
Step {
id: "rounds",
script: rounds_script(r),
},
],
),
"terminated" => (
json!({ "self": one }),
vec![
Step {
id: "create",
script: create(r),
},
Step {
id: "push",
script: push_script(r, &[("terminated", POWEROFF)]),
},
Step {
id: "check",
script: format!(
"U=knotless://$KNOTLESS_IDENTIFIER/{r}\nC=\"$(cat {r}.sha)\"\nknotless run list \"$U\" --commit \"$C\" --wait 1500 > {r}.checks\ncat {r}.checks\nknotless run list \"$U\" --commit \"$C\" --log > {r}.log\ngrep -q \"^failure\tterminated\t\" {r}.checks || {{ cat {r}.log; exit 1; }}\ngrep 'the VM was terminated by its bidder: ' {r}.log || {{ cat {r}.log; exit 1; }}\nms=$(knotless run list \"$U\" --commit \"$C\" --json | jq '[.[] | select(.workflow == \"terminated\")][0].steps[0].durationMs')\necho \"step $ms ms\"\ntest \"$ms\" -lt 300000\n"
),
},
],
),
"cancel" => (
json!({ "self": one }),
vec![
Step {
id: "create",
script: create(r),
},
Step {
id: "fill",
script: fill_with(r, ", wait_capacity_s: 900", CANCEL_FILES),
},
Step {
id: "cancel",
script: cancel_script(r),
},
],
),
"pull" => (
json!({ "self": one }),
vec![
Step {
id: "create",
script: create(r),
},
Step {
id: "secret",
script: format!(
"printf staging-secret-value | knotless secret set knotless://$KNOTLESS_IDENTIFIER/{r} STAGING_SECRET\nknotless secret list knotless://$KNOTLESS_IDENTIFIER/{r}\n"
),
},
Step {
id: "push",
script: push_script(r, &[("secret", SECRET)]),
},
Step {
id: "checks",
script: checks(r, "push", &["secret"]),
},
Step {
id: "open-pull",
script: format!(
"U=knotless://$KNOTLESS_IDENTIFIER/{r}\ncd {r}\ngit checkout -q -b change\necho 43 > answer.txt\ngit commit -q -am 'a pull'\ngit push -q origin change\nknotless pull create \"$U\" main \"$U\" change 'swc-test pull' -m 'a pull gets no secrets' | tee ../{r}.pull.out\ngrep -o 'at://[^ ]*' ../{r}.pull.out | head -1 > ../{r}.pull\ntest -s ../{r}.pull\n"
),
},
Step {
id: "pull-checks",
script: checks(r, "pull", &["secret"]),
},
Step {
id: "pull-log",
script: format!(
"knotless run list knotless://$KNOTLESS_IDENTIFIER/{r} --pull \"$(cat {r}.pull)\" --log | tee {r}.pull.log\ngrep -q pull-has-no-secret {r}.pull.log\n"
),
},
],
),
_ => (
json!({ "self": one }),
vec![
Step {
id: "create",
script: create(r),
},
Step {
id: "push",
script: push_script(r, &[("ci", CI)]),
},
Step {
id: "checks",
script: checks(r, "push", &["ci"]),
},
],
),
}
}
fn run_entry(entry: &'static str, run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let repo = format!(
"{}-{entry}",
args["name"].as_str().ok_or("name is required")?
);
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let (bindings, steps) = plan(entry, &repo);
let mut h = Handle::new();
let step = |s: &mut S, h: Handle, i: usize| -> Handle {
s.st["i"] = json!(i);
s.set(steps[i].id);
s.mark("step");
h.call(host(
&format!("step-{i}"),
"knotless.sh",
json!({ "script": steps[i].script, "timeoutSecs": 1800 }),
))
};
match run.event {
Event::Start => {
s.set("seed");
s.mark("step");
let mut cleanup = format!(
"knotless repo delete knotless://$KNOTLESS_IDENTIFIER/{repo} --yes || true"
);
if entry == "fallback" || entry == "cancel" {
cleanup.push_str("\nS=$(jq -n --arg i \"$KNOTLESS_IDENTIFIER\" --arg p \"$KNOTLESS_PASSWORD\" '{identifier: $i, password: $p}' | curl -sf -H 'content-type: application/json' -d @- \"$KNOTLESS_PDS/xrpc/com.atproto.server.createSession\") && jq -n --arg d \"$(printf %s \"$S\" | jq -r .did)\" '{repo: $d, collection: \"com.fedfork.exec.binding\", rkey: \"fill\"}' | curl -sf -H \"authorization: Bearer $(printf %s \"$S\" | jq -r .accessJwt)\" -H 'content-type: application/json' -d @- \"$KNOTLESS_PDS/xrpc/com.atproto.repo.deleteRecord\" > /dev/null || true");
}
h = h
.call(host(
"cleanup",
"knotless.sh",
json!({ "cleanup": cleanup }),
))
.call(host(
"seed",
"knotless.seed",
json!({ "bindings": bindings }),
))
.timer("deadline", 60 * MIN);
}
Event::Timer { id } if id == "deadline" => {
return Ok(s.fail(format!("not done in 60 min (step {})", s.phase())));
}
Event::Result { id, result, .. } => {
let o = match steps::out(&s, &id, result.output()) {
Ok(o) => o,
Err(f) => return Ok(f),
};
if id == "cleanup" {
return Ok(s.keep(h));
}
let t = s.since("step");
if id == "seed" {
s.reading("seed", json!({ "seconds": t, "composition": o["composition"], "modules": o["modules"] }));
h = step(&mut s, h, 0);
return Ok(s.keep(h));
}
let i = s.st["i"].as_u64().unwrap_or_default() as usize;
if o["exit"] != 0 {
let out = o["stdout"].as_str().unwrap_or_default();
let tail: String = out
.chars()
.rev()
.take(1500)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect();
return Ok(s.fail(format!(
"step {}: exit {}: {} {}",
steps[i].id, o["exit"], tail, o["stderr"]
)));
}
s.reading(steps[i].id, t);
if i + 1 == steps.len() {
return Ok(s.pass());
}
h = step(&mut s, h, i + 1);
}
_ => {}
}
Ok(s.keep(h))
}
fn push_entry(r: Run<Value>) -> Result<Handle, String> {
run_entry("push", r)
}
fn after_entry(r: Run<Value>) -> Result<Handle, String> {
run_entry("after", r)
}
fn executor_entry(r: Run<Value>) -> Result<Handle, String> {
run_entry("executor", r)
}
fn pull_entry(r: Run<Value>) -> Result<Handle, String> {
run_entry("pull", r)
}
fn fallback_entry(r: Run<Value>) -> Result<Handle, String> {
run_entry("fallback", r)
}
fn terminated_entry(r: Run<Value>) -> Result<Handle, String> {
run_entry("terminated", r)
}
fn cancel_entry(r: Run<Value>) -> Result<Handle, String> {
run_entry("cancel", r)
}
entries! { push => push_entry, after => after_entry, executor => executor_entry, pull => pull_entry, fallback => fallback_entry, terminated => terminated_entry, cancel => cancel_entry }