// A staging network's knotless (dev up --with-knotless) as a state machine: seed the runner's composition and // bindings in the account's repo (the gateway owner, the runner's account), then each step a script against the // staging knotless (knotless CLI, git, the network's env), each failing on what it checks. Entries: // push push a workflow to a fresh repo: the runner rents a network bidder's VM, the check lands success // after build keeps an artifact, test (after: [build]) reads it with $KNOTLESS_ARTIFACTS_URL/TOKEN in its VM, // `knotless run artifacts --out` downloads it // executor two workflows on two bindings (executor: small | big): each VM has its binding's vCPUs // pull a sealed secret: the push to main sees it, a pull's run gets none // fallback one workflow per network bidder, each VM sized so a bidder holds one (its offering's maxCpus / 2 + 1): // all succeed, and in one of up to 3 rounds a bidder declined an accept (409 CapacityExceeded) and the // rent took the next bid (the atlibs requester module's accept fallback; acceptance 0014's atlibs path) // cancel every bidder held by a `hold-` workflow (a VM each, on the `fill` binding with wait_capacity_s), then a `waiter` // rent that has to wait for capacity is cancelled (`knotless run cancel`) and ends cancelled at once (not after // the wait), and a hold cancelled with its VM held gives it back: the waiter's next rent gets that bidder and succeeds // terminated a step powers off its own VM: its bidder's vm.terminated reaches the cicd workload as a market event // and the step fails at once with the reason (not after the read retries) // Result {pass, error?, phase (the step), readings (seconds per step)}. The host deletes the repo and the seeded // records however it ends; the runner releases each VM. use abi::{Event, Handle, Run, entries}; use serde_json::{Value, json}; use steps::{MIN, S, host}; struct Step { id: &'static str, script: String, } const CI: &str = "on:\n push:\n refs: [refs/heads/main]\nsteps:\n - name: where\n run: uname -a && nproc && git log --oneline -1\n - name: test\n run: test \"$(cat answer.txt)\" = 42\n"; const BUILD: &str = "on:\n push:\n refs: [refs/heads/main]\nartifacts: [out/hello.txt]\nsteps:\n - name: build\n run: mkdir -p out && echo \"hello $KNOTLESS_RUN_SHA\" > out/hello.txt\n"; const TEST: &str = "on:\n push:\n refs: [refs/heads/main]\nafter: [build]\nsteps:\n - name: fetch\n run: |\n curl -sSfo got.txt -H \"Authorization: Bearer $KNOTLESS_ARTIFACTS_TOKEN\" \"$KNOTLESS_ARTIFACTS_URL?workflow=build&name=out/hello.txt\" \n test \"$(cat got.txt)\" = \"hello $KNOTLESS_RUN_SHA\"\n"; const SMALL: &str = "on:\n push:\n refs: [refs/heads/main]\nexecutor: small\nsteps:\n - name: size\n run: nproc && test \"$(nproc)\" = 1\n"; const BIG: &str = "on:\n push:\n refs: [refs/heads/main]\nexecutor: big\nsteps:\n - name: size\n run: nproc && test \"$(nproc)\" = 2\n"; const POWEROFF: &str = "on:\n push:\n refs: [refs/heads/main]\nsteps:\n - name: poweroff\n run: |\n id -u; cat /proc/sys/kernel/sysrq; ls -d /sys/firmware/acpi || true\n if [ -d /sys/firmware/acpi ]; then t=o; else t=b; fi\n echo \"sysrq $t in 20 s\"\n setsid sh -c \"sleep 20; echo 1 > /proc/sys/kernel/sysrq; echo $t > /proc/sysrq-trigger\" > /dev/null 2>&1 < /dev/null &\n sleep 1200\n"; const SECRET: &str = "on:\n push:\n refs: [refs/heads/main]\n pull:\n base: [refs/heads/main]\nsecrets: [STAGING_SECRET]\nsteps:\n - name: secret\n run: if [ -n \"${KNOTLESS_RUN_PULL:-}\" ]; then test -z \"${STAGING_SECRET:-}\" && echo pull-has-no-secret; else test \"$STAGING_SECRET\" = staging-secret-value && echo push-has-the-secret; fi\n"; /// Repo `r`, then the wait until the runner watches it: a runner acts on no ref, pull or rerun it found already there. // ponytail: the appview listing it, then 3 of the runner's 10 s discovery sweeps; a runner-side "watching" read when // one exists. fn create(r: &str) -> String { format!( "knotless repo create knotless://$KNOTLESS_IDENTIFIER/{r} --description 'swc-test knotless'\nfor i in $(seq 60); do curl -sf \"$KNOTLESS_APPVIEW/xrpc/dev.knotless.listRepos?runner=$KNOTLESS_IDENTIFIER\" | grep -q '\"name\":\"{r}\"' && break; sleep 2; done\ncurl -sf \"$KNOTLESS_APPVIEW/xrpc/dev.knotless.listRepos?runner=$KNOTLESS_IDENTIFIER\" | grep -q '\"name\":\"{r}\"'\nsleep 30\n" ) } /// Repo `r` with `answer.txt` and the workflows, pushed to main; its commit in `.sha`. fn push_script(r: &str, workflows: &[(&str, &str)]) -> String { let mut s = format!( "U=knotless://$KNOTLESS_IDENTIFIER/{r}\nrm -rf {r} && mkdir -p {r}/.knotless/workflows && cd {r}\ngit init -q -b main\necho 42 > answer.txt\n" ); for (name, body) in workflows { s.push_str(&format!( "cat > .knotless/workflows/{name}.yml <<'EOF'\n{body}EOF\n" )); } s.push_str("git add . && git commit -q -m 'swc-test knotless'\ngit remote add origin \"$U\"\ngit push -q -u origin main\ngit rev-parse HEAD > ../"); s.push_str(&format!("{r}.sha\ncat ../{r}.sha\n")); s } /// Waits for every check of the commit in `.sha` (or the pull in `.pull`), then each named workflow's must be /// success; their logs on a failure. fn checks(r: &str, of: &str, workflows: &[&str]) -> String { let sel = match of { "pull" => format!("--pull \"$(cat {r}.pull)\""), _ => format!("--commit \"$(cat {r}.sha)\""), }; let want: Vec = workflows .iter() .map(|w| format!("grep -q \"^success\t{w}\t\" {r}.{of}.checks")) .collect(); format!( "U=knotless://$KNOTLESS_IDENTIFIER/{r}\nknotless run list \"$U\" {sel} --wait 1500 > {r}.{of}.checks\ncat {r}.{of}.checks\n{{ {}; }} || {{ knotless run list \"$U\" {sel} --log; exit 1; }}\n", want.join(" && ") ) } /// The workflows of `fallback`: one per bidder, each on the `fill` binding. const FALLBACK_FILES: &str = "for i in $(seq \"$n\"); do printf 'on:\n push:\n refs: [refs/heads/main]\nexecutor: fill\nsteps:\n - name: size\n run: nproc\n' > .knotless/workflows/fill-$i.yml; done"; /// The `fill` binding (the bidders' tightest maxCpus / 2 + 1 vCPUs, 1 GiB) beside the seeded ones, then one workflow /// per bidder on it (`fill-`), pushed; the bidders are the network relay's repos with a compute offering. fn fill_script(r: &str) -> String { fill_with(r, "", FALLBACK_FILES) } /// `fill_script`, the binding's inputs given `inputs` more (`, wait_capacity_s: 900`), and `files` the shell that writes the workflows /// (it runs in the new repo's directory, `$n` bidders known). fn fill_with(r: &str, inputs: &str, files: &str) -> String { format!( r##"S=$(jq -n --arg i "$KNOTLESS_IDENTIFIER" --arg p "$KNOTLESS_PASSWORD" '{{identifier: $i, password: $p}}' | curl -sf -H 'content-type: application/json' -d @- "$KNOTLESS_PDS/xrpc/com.atproto.server.createSession") J=$(printf %s "$S" | jq -r .accessJwt); D=$(printf %s "$S" | jq -r .did) : > {r}.max O=com.publicdomainrelay.temp.market.offering for b in $(curl -sf --cacert "$KNOTLESS_CA_CERTS" "$KNOTLESS_RELAY/xrpc/com.atproto.sync.listReposByCollection?collection=$O" | jq -r '.repos[].did' | sort -u); do p=$(curl -sf "$KNOTLESS_PLC_URL/$b" | jq -r '.service[] | select(.id == "#atproto_pds") | .serviceEndpoint') || continue m=$(curl -sf --cacert "$KNOTLESS_CA_CERTS" "$p/xrpc/com.atproto.repo.getRecord?repo=$b&collection=$O&rkey=self" | jq -r '.value.maxCpus // empty') || continue [ -n "$m" ] && echo "$b $m" | tee -a {r}.max done n=$(wc -l < {r}.max); test "$n" -ge 2 cpus=$(( $(sort -n -k2 {r}.max | head -1 | cut -d' ' -f2) / 2 + 1 )) echo "$n bidders, $cpus vCPUs a VM: one VM fills a bidder" comp=$(curl -sf "$KNOTLESS_PDS/xrpc/com.atproto.repo.getRecord?repo=$D&collection=com.fedfork.exec.binding&rkey=self" | jq -c .value.composition) jq -n --arg d "$D" --argjson c "$comp" --argjson cpus "$cpus" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '{{repo: $d, collection: "com.fedfork.exec.binding", rkey: "fill", record: {{"$type": "com.fedfork.exec.binding", composition: $c, inputs: {{cpus: $cpus, mem: 1{inputs}}}, createdAt: $at}}}}' | curl -sf -H "authorization: Bearer $J" -H 'content-type: application/json' -d @- "$KNOTLESS_PDS/xrpc/com.atproto.repo.putRecord" > /dev/null U=knotless://$KNOTLESS_IDENTIFIER/{r} rm -rf {r} && mkdir -p {r}/.knotless/workflows && cd {r} git init -q -b main {files} echo 1 > round git add . && git commit -q -m 'swc-test knotless fallback' git remote add origin "$U" git push -q -u origin main "##, inputs = inputs, files = files ) } /// The workflows of `cancel`: a `hold-` per bidder (a VM each, held 15 min), and a `waiter` only an ask runs. const CANCEL_FILES: &str = "for i in $(seq \"$n\"); do printf 'on:\\n push:\\n refs: [refs/heads/main]\\nexecutor: fill\\nsteps:\\n - name: hold\\n run: sleep 900\\n' > .knotless/workflows/hold-$i.yml; done\nprintf 'on:\\n asked:\\n refs: [refs/heads/main]\\nexecutor: fill\\nsteps:\\n - name: size\\n run: nproc\\n' > .knotless/workflows/waiter.yml"; /// With every bidder held: a waiter's rent, waiting on capacity, is cancelled and its check says so within 90 s; a hold cancelled /// while it holds its VM gives it back, so the waiter's next rent succeeds. fn cancel_script(r: &str) -> String { format!( r#"U=knotless://$KNOTLESS_IDENTIFIER/{r} n=$(ls {r}/.knotless/workflows | grep -c '^hold-') line() {{ knotless run list "$U" --ref refs/heads/main --workflow "$1" 2>/dev/null | head -1; }} show() {{ echo "$1: $(line "$1")"; knotless run list "$U" --ref refs/heads/main --workflow "$1" --log | tail -30; }} until_line() {{ for i in $(seq "$3"); do line "$1" | grep -q "$2" && return 0; sleep 1; done; show "$1"; echo "wanted /$2/ in $3 s"; return 1; }} for i in $(seq "$n"); do until_line hold-$i '^running.*step hold' 900 || exit 1; done echo "$n bidders held" knotless run rerun "$U" --ref refs/heads/main --workflow waiter until_line waiter '^running' 120 || exit 1 sleep 30 line waiter | grep -q '^running' || {{ show waiter; exit 1; }} t=$(date +%s) knotless run cancel "$U" --ref refs/heads/main --workflow waiter until_line waiter '^cancelled' 90 || exit 1 echo "the rent waiting on capacity was cancelled in $(( $(date +%s) - t )) s" knotless run list "$U" --ref refs/heads/main --workflow waiter --log | grep -q 'cancelled while renting its machine' || {{ show waiter; exit 1; }} knotless run cancel "$U" --ref refs/heads/main --workflow hold-1 until_line hold-1 '^cancelled' 180 || exit 1 knotless run rerun "$U" --ref refs/heads/main --workflow waiter until_line waiter '^success' 400 || exit 1 echo "a hold cancelled with its VM gave it back: the waiter got a bidder" knotless run cancel "$U" --ref refs/heads/main "# ) } /// Up to 3 rounds: every fill-* check of the round's commit succeeds; a round whose logs show a declined accept /// (the rent's notes) ends it, else the next round is a new commit. fn rounds_script(r: &str) -> String { format!( r#"U=knotless://$KNOTLESS_IDENTIFIER/{r} cd {r} for round in 1 2 3; do C=$(git rev-parse HEAD) knotless run list "$U" --commit "$C" --wait 1500 > ../{r}.checks cat ../{r}.checks knotless run list "$U" --commit "$C" --log > ../{r}.log if grep -v "^success fill-" ../{r}.checks | grep -q " fill-"; then cat ../{r}.log; exit 1; fi if grep 'declined the accept' ../{r}.log; then echo "round $round: a declined accept was replaced"; exit 0; fi echo "round $round: every first pick had room; the next once the bidders hold no reservation (90 s)" sleep 90 echo $((round + 1)) > round && git commit -q -am "round $((round + 1))" && git push -q origin main done echo "no accept was declined in 3 rounds"; exit 1 "# ) } fn plan(entry: &str, r: &str) -> (Value, Vec) { let one = json!({ "cpus": 1, "mem": 1 }); match entry { "after" => ( json!({ "self": one }), vec![ Step { id: "create", script: create(r), }, Step { id: "push", script: push_script(r, &[("build", BUILD), ("test", TEST)]), }, Step { id: "checks", script: checks(r, "push", &["build", "test"]), }, Step { id: "download", script: format!( "U=knotless://$KNOTLESS_IDENTIFIER/{r}\nrm -rf {r}.got\nknotless run artifacts \"$U\" --workflow build --commit \"$(cat {r}.sha)\"\nknotless run artifacts \"$U\" --workflow build --commit \"$(cat {r}.sha)\" --out {r}.got\nfind {r}.got -type f\ntest \"$(cat \"$(find {r}.got -name hello.txt)\")\" = \"hello $(cat {r}.sha)\"\n" ), }, ], ), "executor" => ( json!({ "self": one, "small": one, "big": { "cpus": 2, "mem": 2 } }), vec![ Step { id: "create", script: create(r), }, Step { id: "push", script: push_script(r, &[("small", SMALL), ("big", BIG)]), }, Step { id: "checks", script: checks(r, "push", &["small", "big"]), }, ], ), "fallback" => ( json!({ "self": one }), vec![ Step { id: "create", script: create(r), }, Step { id: "fill", script: fill_script(r), }, Step { id: "rounds", script: rounds_script(r), }, ], ), "terminated" => ( json!({ "self": one }), vec![ Step { id: "create", script: create(r), }, Step { id: "push", script: push_script(r, &[("terminated", POWEROFF)]), }, Step { id: "check", script: format!( "U=knotless://$KNOTLESS_IDENTIFIER/{r}\nC=\"$(cat {r}.sha)\"\nknotless run list \"$U\" --commit \"$C\" --wait 1500 > {r}.checks\ncat {r}.checks\nknotless run list \"$U\" --commit \"$C\" --log > {r}.log\ngrep -q \"^failure\tterminated\t\" {r}.checks || {{ cat {r}.log; exit 1; }}\ngrep 'the VM was terminated by its bidder: ' {r}.log || {{ cat {r}.log; exit 1; }}\nms=$(knotless run list \"$U\" --commit \"$C\" --json | jq '[.[] | select(.workflow == \"terminated\")][0].steps[0].durationMs')\necho \"step $ms ms\"\ntest \"$ms\" -lt 300000\n" ), }, ], ), "cancel" => ( json!({ "self": one }), vec![ Step { id: "create", script: create(r), }, Step { id: "fill", script: fill_with(r, ", wait_capacity_s: 900", CANCEL_FILES), }, Step { id: "cancel", script: cancel_script(r), }, ], ), "pull" => ( json!({ "self": one }), vec![ Step { id: "create", script: create(r), }, Step { id: "secret", script: format!( "printf staging-secret-value | knotless secret set knotless://$KNOTLESS_IDENTIFIER/{r} STAGING_SECRET\nknotless secret list knotless://$KNOTLESS_IDENTIFIER/{r}\n" ), }, Step { id: "push", script: push_script(r, &[("secret", SECRET)]), }, Step { id: "checks", script: checks(r, "push", &["secret"]), }, Step { id: "open-pull", script: format!( "U=knotless://$KNOTLESS_IDENTIFIER/{r}\ncd {r}\ngit checkout -q -b change\necho 43 > answer.txt\ngit commit -q -am 'a pull'\ngit push -q origin change\nknotless pull create \"$U\" main \"$U\" change 'swc-test pull' -m 'a pull gets no secrets' | tee ../{r}.pull.out\ngrep -o 'at://[^ ]*' ../{r}.pull.out | head -1 > ../{r}.pull\ntest -s ../{r}.pull\n" ), }, Step { id: "pull-checks", script: checks(r, "pull", &["secret"]), }, Step { id: "pull-log", script: format!( "knotless run list knotless://$KNOTLESS_IDENTIFIER/{r} --pull \"$(cat {r}.pull)\" --log | tee {r}.pull.log\ngrep -q pull-has-no-secret {r}.pull.log\n" ), }, ], ), _ => ( json!({ "self": one }), vec![ Step { id: "create", script: create(r), }, Step { id: "push", script: push_script(r, &[("ci", CI)]), }, Step { id: "checks", script: checks(r, "push", &["ci"]), }, ], ), } } fn run_entry(entry: &'static str, run: Run) -> Result { let args = &run.args; let repo = format!( "{}-{entry}", args["name"].as_str().ok_or("name is required")? ); let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let (bindings, steps) = plan(entry, &repo); let mut h = Handle::new(); let step = |s: &mut S, h: Handle, i: usize| -> Handle { s.st["i"] = json!(i); s.set(steps[i].id); s.mark("step"); h.call(host( &format!("step-{i}"), "knotless.sh", json!({ "script": steps[i].script, "timeoutSecs": 1800 }), )) }; match run.event { Event::Start => { s.set("seed"); s.mark("step"); let mut cleanup = format!( "knotless repo delete knotless://$KNOTLESS_IDENTIFIER/{repo} --yes || true" ); if entry == "fallback" || entry == "cancel" { cleanup.push_str("\nS=$(jq -n --arg i \"$KNOTLESS_IDENTIFIER\" --arg p \"$KNOTLESS_PASSWORD\" '{identifier: $i, password: $p}' | curl -sf -H 'content-type: application/json' -d @- \"$KNOTLESS_PDS/xrpc/com.atproto.server.createSession\") && jq -n --arg d \"$(printf %s \"$S\" | jq -r .did)\" '{repo: $d, collection: \"com.fedfork.exec.binding\", rkey: \"fill\"}' | curl -sf -H \"authorization: Bearer $(printf %s \"$S\" | jq -r .accessJwt)\" -H 'content-type: application/json' -d @- \"$KNOTLESS_PDS/xrpc/com.atproto.repo.deleteRecord\" > /dev/null || true"); } h = h .call(host( "cleanup", "knotless.sh", json!({ "cleanup": cleanup }), )) .call(host( "seed", "knotless.seed", json!({ "bindings": bindings }), )) .timer("deadline", 60 * MIN); } Event::Timer { id } if id == "deadline" => { return Ok(s.fail(format!("not done in 60 min (step {})", s.phase()))); } Event::Result { id, result, .. } => { let o = match steps::out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; if id == "cleanup" { return Ok(s.keep(h)); } let t = s.since("step"); if id == "seed" { s.reading("seed", json!({ "seconds": t, "composition": o["composition"], "modules": o["modules"] })); h = step(&mut s, h, 0); return Ok(s.keep(h)); } let i = s.st["i"].as_u64().unwrap_or_default() as usize; if o["exit"] != 0 { let out = o["stdout"].as_str().unwrap_or_default(); let tail: String = out .chars() .rev() .take(1500) .collect::>() .into_iter() .rev() .collect(); return Ok(s.fail(format!( "step {}: exit {}: {} {}", steps[i].id, o["exit"], tail, o["stderr"] ))); } s.reading(steps[i].id, t); if i + 1 == steps.len() { return Ok(s.pass()); } h = step(&mut s, h, i + 1); } _ => {} } Ok(s.keep(h)) } fn push_entry(r: Run) -> Result { run_entry("push", r) } fn after_entry(r: Run) -> Result { run_entry("after", r) } fn executor_entry(r: Run) -> Result { run_entry("executor", r) } fn pull_entry(r: Run) -> Result { run_entry("pull", r) } fn fallback_entry(r: Run) -> Result { run_entry("fallback", r) } fn terminated_entry(r: Run) -> Result { run_entry("terminated", r) } fn cancel_entry(r: Run) -> Result { run_entry("cancel", r) } entries! { push => push_entry, after => after_entry, executor => executor_entry, pull => pull_entry, fallback => fallback_entry, terminated => terminated_entry, cancel => cancel_entry }