~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
446 lines · 21.8 KB Raw History
// A staging network's knotless (dev up --with-knotless) as a state machine: seed the runner's composition and
// bindings in the account's repo (the gateway owner, the runner's account), then each step a script against the
// staging knotless (knotless CLI, git, the network's env), each failing on what it checks. Entries:
//   push      push a workflow to a fresh repo: the runner rents a network bidder's VM, the check lands success
//   after     build keeps an artifact, test (after: [build]) reads it with $KNOTLESS_ARTIFACTS_URL/TOKEN in its VM,
//             `knotless run artifacts --out` downloads it
//   executor  two workflows on two bindings (executor: small | big): each VM has its binding's vCPUs
//   pull      a sealed secret: the push to main sees it, a pull's run gets none
//   fallback  one workflow per network bidder, each VM sized so a bidder holds one (its offering's maxCpus / 2 + 1):
//             all succeed, and in one of up to 3 rounds a bidder declined an accept (409 CapacityExceeded) and the
//             rent took the next bid (the atlibs requester module's accept fallback; acceptance 0014's atlibs path)
//   cancel    every bidder held by a `hold-<n>` workflow (a VM each, on the `fill` binding with wait_capacity_s), then a `waiter`
//             rent that has to wait for capacity is cancelled (`knotless run cancel`) and ends cancelled at once (not after
//             the wait), and a hold cancelled with its VM held gives it back: the waiter's next rent gets that bidder and succeeds
//   terminated a step powers off its own VM: its bidder's vm.terminated reaches the cicd workload as a market event
//             and the step fails at once with the reason (not after the read retries)
// Result {pass, error?, phase (the step), readings (seconds per step)}. The host deletes the repo and the seeded
// records however it ends; the runner releases each VM.
use abi::{Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{MIN, S, host};

struct Step {
    id: &'static str,
    script: String,
}

const CI: &str = "on:\n  push:\n    refs: [refs/heads/main]\nsteps:\n  - name: where\n    run: uname -a && nproc && git log --oneline -1\n  - name: test\n    run: test \"$(cat answer.txt)\" = 42\n";
const BUILD: &str = "on:\n  push:\n    refs: [refs/heads/main]\nartifacts: [out/hello.txt]\nsteps:\n  - name: build\n    run: mkdir -p out && echo \"hello $KNOTLESS_RUN_SHA\" > out/hello.txt\n";
const TEST: &str = "on:\n  push:\n    refs: [refs/heads/main]\nafter: [build]\nsteps:\n  - name: fetch\n    run: |\n      curl -sSfo got.txt -H \"Authorization: Bearer $KNOTLESS_ARTIFACTS_TOKEN\" \"$KNOTLESS_ARTIFACTS_URL?workflow=build&name=out/hello.txt\" \n      test \"$(cat got.txt)\" = \"hello $KNOTLESS_RUN_SHA\"\n";
const SMALL: &str = "on:\n  push:\n    refs: [refs/heads/main]\nexecutor: small\nsteps:\n  - name: size\n    run: nproc && test \"$(nproc)\" = 1\n";
const BIG: &str = "on:\n  push:\n    refs: [refs/heads/main]\nexecutor: big\nsteps:\n  - name: size\n    run: nproc && test \"$(nproc)\" = 2\n";
const POWEROFF: &str = "on:\n  push:\n    refs: [refs/heads/main]\nsteps:\n  - name: poweroff\n    run: |\n      id -u; cat /proc/sys/kernel/sysrq; ls -d /sys/firmware/acpi || true\n      if [ -d /sys/firmware/acpi ]; then t=o; else t=b; fi\n      echo \"sysrq $t in 20 s\"\n      setsid sh -c \"sleep 20; echo 1 > /proc/sys/kernel/sysrq; echo $t > /proc/sysrq-trigger\" > /dev/null 2>&1 < /dev/null &\n      sleep 1200\n";
const SECRET: &str = "on:\n  push:\n    refs: [refs/heads/main]\n  pull:\n    base: [refs/heads/main]\nsecrets: [STAGING_SECRET]\nsteps:\n  - name: secret\n    run: if [ -n \"${KNOTLESS_RUN_PULL:-}\" ]; then test -z \"${STAGING_SECRET:-}\" && echo pull-has-no-secret; else test \"$STAGING_SECRET\" = staging-secret-value && echo push-has-the-secret; fi\n";

/// Repo `r`, then the wait until the runner watches it: a runner acts on no ref, pull or rerun it found already there.
// ponytail: the appview listing it, then 3 of the runner's 10 s discovery sweeps; a runner-side "watching" read when
// one exists.
fn create(r: &str) -> String {
    format!(
        "knotless repo create knotless://$KNOTLESS_IDENTIFIER/{r} --description 'swc-test knotless'\nfor i in $(seq 60); do curl -sf \"$KNOTLESS_APPVIEW/xrpc/dev.knotless.listRepos?runner=$KNOTLESS_IDENTIFIER\" | grep -q '\"name\":\"{r}\"' && break; sleep 2; done\ncurl -sf \"$KNOTLESS_APPVIEW/xrpc/dev.knotless.listRepos?runner=$KNOTLESS_IDENTIFIER\" | grep -q '\"name\":\"{r}\"'\nsleep 30\n"
    )
}

/// Repo `r` with `answer.txt` and the workflows, pushed to main; its commit in `<r>.sha`.
fn push_script(r: &str, workflows: &[(&str, &str)]) -> String {
    let mut s = format!(
        "U=knotless://$KNOTLESS_IDENTIFIER/{r}\nrm -rf {r} && mkdir -p {r}/.knotless/workflows && cd {r}\ngit init -q -b main\necho 42 > answer.txt\n"
    );
    for (name, body) in workflows {
        s.push_str(&format!(
            "cat > .knotless/workflows/{name}.yml <<'EOF'\n{body}EOF\n"
        ));
    }
    s.push_str("git add . && git commit -q -m 'swc-test knotless'\ngit remote add origin \"$U\"\ngit push -q -u origin main\ngit rev-parse HEAD > ../");
    s.push_str(&format!("{r}.sha\ncat ../{r}.sha\n"));
    s
}

/// Waits for every check of the commit in `<r>.sha` (or the pull in `<r>.pull`), then each named workflow's must be
/// success; their logs on a failure.
fn checks(r: &str, of: &str, workflows: &[&str]) -> String {
    let sel = match of {
        "pull" => format!("--pull \"$(cat {r}.pull)\""),
        _ => format!("--commit \"$(cat {r}.sha)\""),
    };
    let want: Vec<String> = workflows
        .iter()
        .map(|w| format!("grep -q \"^success\t{w}\t\" {r}.{of}.checks"))
        .collect();
    format!(
        "U=knotless://$KNOTLESS_IDENTIFIER/{r}\nknotless run list \"$U\" {sel} --wait 1500 > {r}.{of}.checks\ncat {r}.{of}.checks\n{{ {}; }} || {{ knotless run list \"$U\" {sel} --log; exit 1; }}\n",
        want.join(" && ")
    )
}

/// The workflows of `fallback`: one per bidder, each on the `fill` binding.
const FALLBACK_FILES: &str = "for i in $(seq \"$n\"); do printf 'on:\n  push:\n    refs: [refs/heads/main]\nexecutor: fill\nsteps:\n  - name: size\n    run: nproc\n' > .knotless/workflows/fill-$i.yml; done";

/// The `fill` binding (the bidders' tightest maxCpus / 2 + 1 vCPUs, 1 GiB) beside the seeded ones, then one workflow
/// per bidder on it (`fill-<n>`), pushed; the bidders are the network relay's repos with a compute offering.
fn fill_script(r: &str) -> String {
    fill_with(r, "", FALLBACK_FILES)
}

/// `fill_script`, the binding's inputs given `inputs` more (`, wait_capacity_s: 900`), and `files` the shell that writes the workflows
/// (it runs in the new repo's directory, `$n` bidders known).
fn fill_with(r: &str, inputs: &str, files: &str) -> String {
    format!(
        r##"S=$(jq -n --arg i "$KNOTLESS_IDENTIFIER" --arg p "$KNOTLESS_PASSWORD" '{{identifier: $i, password: $p}}' | curl -sf -H 'content-type: application/json' -d @- "$KNOTLESS_PDS/xrpc/com.atproto.server.createSession")
J=$(printf %s "$S" | jq -r .accessJwt); D=$(printf %s "$S" | jq -r .did)
: > {r}.max
O=com.publicdomainrelay.temp.market.offering
for b in $(curl -sf --cacert "$KNOTLESS_CA_CERTS" "$KNOTLESS_RELAY/xrpc/com.atproto.sync.listReposByCollection?collection=$O" | jq -r '.repos[].did' | sort -u); do
  p=$(curl -sf "$KNOTLESS_PLC_URL/$b" | jq -r '.service[] | select(.id == "#atproto_pds") | .serviceEndpoint') || continue
  m=$(curl -sf --cacert "$KNOTLESS_CA_CERTS" "$p/xrpc/com.atproto.repo.getRecord?repo=$b&collection=$O&rkey=self" | jq -r '.value.maxCpus // empty') || continue
  [ -n "$m" ] && echo "$b $m" | tee -a {r}.max
done
n=$(wc -l < {r}.max); test "$n" -ge 2
cpus=$(( $(sort -n -k2 {r}.max | head -1 | cut -d' ' -f2) / 2 + 1 ))
echo "$n bidders, $cpus vCPUs a VM: one VM fills a bidder"
comp=$(curl -sf "$KNOTLESS_PDS/xrpc/com.atproto.repo.getRecord?repo=$D&collection=com.fedfork.exec.binding&rkey=self" | jq -c .value.composition)
jq -n --arg d "$D" --argjson c "$comp" --argjson cpus "$cpus" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)"   '{{repo: $d, collection: "com.fedfork.exec.binding", rkey: "fill", record: {{"$type": "com.fedfork.exec.binding", composition: $c, inputs: {{cpus: $cpus, mem: 1{inputs}}}, createdAt: $at}}}}' |
  curl -sf -H "authorization: Bearer $J" -H 'content-type: application/json' -d @- "$KNOTLESS_PDS/xrpc/com.atproto.repo.putRecord" > /dev/null
U=knotless://$KNOTLESS_IDENTIFIER/{r}
rm -rf {r} && mkdir -p {r}/.knotless/workflows && cd {r}
git init -q -b main
{files}
echo 1 > round
git add . && git commit -q -m 'swc-test knotless fallback'
git remote add origin "$U"
git push -q -u origin main
"##,
        inputs = inputs,
        files = files
    )
}

/// The workflows of `cancel`: a `hold-<n>` per bidder (a VM each, held 15 min), and a `waiter` only an ask runs.
const CANCEL_FILES: &str = "for i in $(seq \"$n\"); do printf 'on:\\n  push:\\n    refs: [refs/heads/main]\\nexecutor: fill\\nsteps:\\n  - name: hold\\n    run: sleep 900\\n' > .knotless/workflows/hold-$i.yml; done\nprintf 'on:\\n  asked:\\n    refs: [refs/heads/main]\\nexecutor: fill\\nsteps:\\n  - name: size\\n    run: nproc\\n' > .knotless/workflows/waiter.yml";

/// With every bidder held: a waiter's rent, waiting on capacity, is cancelled and its check says so within 90 s; a hold cancelled
/// while it holds its VM gives it back, so the waiter's next rent succeeds.
fn cancel_script(r: &str) -> String {
    format!(
        r#"U=knotless://$KNOTLESS_IDENTIFIER/{r}
n=$(ls {r}/.knotless/workflows | grep -c '^hold-')
line() {{ knotless run list "$U" --ref refs/heads/main --workflow "$1" 2>/dev/null | head -1; }}
show() {{ echo "$1: $(line "$1")"; knotless run list "$U" --ref refs/heads/main --workflow "$1" --log | tail -30; }}
until_line() {{ for i in $(seq "$3"); do line "$1" | grep -q "$2" && return 0; sleep 1; done; show "$1"; echo "wanted /$2/ in $3 s"; return 1; }}
for i in $(seq "$n"); do until_line hold-$i '^running.*step hold' 900 || exit 1; done
echo "$n bidders held"
knotless run rerun "$U" --ref refs/heads/main --workflow waiter
until_line waiter '^running' 120 || exit 1
sleep 30
line waiter | grep -q '^running' || {{ show waiter; exit 1; }}
t=$(date +%s)
knotless run cancel "$U" --ref refs/heads/main --workflow waiter
until_line waiter '^cancelled' 90 || exit 1
echo "the rent waiting on capacity was cancelled in $(( $(date +%s) - t )) s"
knotless run list "$U" --ref refs/heads/main --workflow waiter --log | grep -q 'cancelled while renting its machine' || {{ show waiter; exit 1; }}
knotless run cancel "$U" --ref refs/heads/main --workflow hold-1
until_line hold-1 '^cancelled' 180 || exit 1
knotless run rerun "$U" --ref refs/heads/main --workflow waiter
until_line waiter '^success' 400 || exit 1
echo "a hold cancelled with its VM gave it back: the waiter got a bidder"
knotless run cancel "$U" --ref refs/heads/main
"#
    )
}

/// Up to 3 rounds: every fill-* check of the round's commit succeeds; a round whose logs show a declined accept
/// (the rent's notes) ends it, else the next round is a new commit.
fn rounds_script(r: &str) -> String {
    format!(
        r#"U=knotless://$KNOTLESS_IDENTIFIER/{r}
cd {r}
for round in 1 2 3; do
  C=$(git rev-parse HEAD)
  knotless run list "$U" --commit "$C" --wait 1500 > ../{r}.checks
  cat ../{r}.checks
  knotless run list "$U" --commit "$C" --log > ../{r}.log
  if grep -v "^success	fill-" ../{r}.checks | grep -q "	fill-"; then cat ../{r}.log; exit 1; fi
  if grep 'declined the accept' ../{r}.log; then echo "round $round: a declined accept was replaced"; exit 0; fi
  echo "round $round: every first pick had room; the next once the bidders hold no reservation (90 s)"
  sleep 90
  echo $((round + 1)) > round && git commit -q -am "round $((round + 1))" && git push -q origin main
done
echo "no accept was declined in 3 rounds"; exit 1
"#
    )
}

fn plan(entry: &str, r: &str) -> (Value, Vec<Step>) {
    let one = json!({ "cpus": 1, "mem": 1 });
    match entry {
        "after" => (
            json!({ "self": one }),
            vec![
                Step {
                    id: "create",
                    script: create(r),
                },
                Step {
                    id: "push",
                    script: push_script(r, &[("build", BUILD), ("test", TEST)]),
                },
                Step {
                    id: "checks",
                    script: checks(r, "push", &["build", "test"]),
                },
                Step {
                    id: "download",
                    script: format!(
                        "U=knotless://$KNOTLESS_IDENTIFIER/{r}\nrm -rf {r}.got\nknotless run artifacts \"$U\" --workflow build --commit \"$(cat {r}.sha)\"\nknotless run artifacts \"$U\" --workflow build --commit \"$(cat {r}.sha)\" --out {r}.got\nfind {r}.got -type f\ntest \"$(cat \"$(find {r}.got -name hello.txt)\")\" = \"hello $(cat {r}.sha)\"\n"
                    ),
                },
            ],
        ),
        "executor" => (
            json!({ "self": one, "small": one, "big": { "cpus": 2, "mem": 2 } }),
            vec![
                Step {
                    id: "create",
                    script: create(r),
                },
                Step {
                    id: "push",
                    script: push_script(r, &[("small", SMALL), ("big", BIG)]),
                },
                Step {
                    id: "checks",
                    script: checks(r, "push", &["small", "big"]),
                },
            ],
        ),
        "fallback" => (
            json!({ "self": one }),
            vec![
                Step {
                    id: "create",
                    script: create(r),
                },
                Step {
                    id: "fill",
                    script: fill_script(r),
                },
                Step {
                    id: "rounds",
                    script: rounds_script(r),
                },
            ],
        ),
        "terminated" => (
            json!({ "self": one }),
            vec![
                Step {
                    id: "create",
                    script: create(r),
                },
                Step {
                    id: "push",
                    script: push_script(r, &[("terminated", POWEROFF)]),
                },
                Step {
                    id: "check",
                    script: format!(
                        "U=knotless://$KNOTLESS_IDENTIFIER/{r}\nC=\"$(cat {r}.sha)\"\nknotless run list \"$U\" --commit \"$C\" --wait 1500 > {r}.checks\ncat {r}.checks\nknotless run list \"$U\" --commit \"$C\" --log > {r}.log\ngrep -q \"^failure\tterminated\t\" {r}.checks || {{ cat {r}.log; exit 1; }}\ngrep 'the VM was terminated by its bidder: ' {r}.log || {{ cat {r}.log; exit 1; }}\nms=$(knotless run list \"$U\" --commit \"$C\" --json | jq '[.[] | select(.workflow == \"terminated\")][0].steps[0].durationMs')\necho \"step $ms ms\"\ntest \"$ms\" -lt 300000\n"
                    ),
                },
            ],
        ),
        "cancel" => (
            json!({ "self": one }),
            vec![
                Step {
                    id: "create",
                    script: create(r),
                },
                Step {
                    id: "fill",
                    script: fill_with(r, ", wait_capacity_s: 900", CANCEL_FILES),
                },
                Step {
                    id: "cancel",
                    script: cancel_script(r),
                },
            ],
        ),
        "pull" => (
            json!({ "self": one }),
            vec![
                Step {
                    id: "create",
                    script: create(r),
                },
                Step {
                    id: "secret",
                    script: format!(
                        "printf staging-secret-value | knotless secret set knotless://$KNOTLESS_IDENTIFIER/{r} STAGING_SECRET\nknotless secret list knotless://$KNOTLESS_IDENTIFIER/{r}\n"
                    ),
                },
                Step {
                    id: "push",
                    script: push_script(r, &[("secret", SECRET)]),
                },
                Step {
                    id: "checks",
                    script: checks(r, "push", &["secret"]),
                },
                Step {
                    id: "open-pull",
                    script: format!(
                        "U=knotless://$KNOTLESS_IDENTIFIER/{r}\ncd {r}\ngit checkout -q -b change\necho 43 > answer.txt\ngit commit -q -am 'a pull'\ngit push -q origin change\nknotless pull create \"$U\" main \"$U\" change 'swc-test pull' -m 'a pull gets no secrets' | tee ../{r}.pull.out\ngrep -o 'at://[^ ]*' ../{r}.pull.out | head -1 > ../{r}.pull\ntest -s ../{r}.pull\n"
                    ),
                },
                Step {
                    id: "pull-checks",
                    script: checks(r, "pull", &["secret"]),
                },
                Step {
                    id: "pull-log",
                    script: format!(
                        "knotless run list knotless://$KNOTLESS_IDENTIFIER/{r} --pull \"$(cat {r}.pull)\" --log | tee {r}.pull.log\ngrep -q pull-has-no-secret {r}.pull.log\n"
                    ),
                },
            ],
        ),
        _ => (
            json!({ "self": one }),
            vec![
                Step {
                    id: "create",
                    script: create(r),
                },
                Step {
                    id: "push",
                    script: push_script(r, &[("ci", CI)]),
                },
                Step {
                    id: "checks",
                    script: checks(r, "push", &["ci"]),
                },
            ],
        ),
    }
}

fn run_entry(entry: &'static str, run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let repo = format!(
        "{}-{entry}",
        args["name"].as_str().ok_or("name is required")?
    );
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let (bindings, steps) = plan(entry, &repo);
    let mut h = Handle::new();
    let step = |s: &mut S, h: Handle, i: usize| -> Handle {
        s.st["i"] = json!(i);
        s.set(steps[i].id);
        s.mark("step");
        h.call(host(
            &format!("step-{i}"),
            "knotless.sh",
            json!({ "script": steps[i].script, "timeoutSecs": 1800 }),
        ))
    };
    match run.event {
        Event::Start => {
            s.set("seed");
            s.mark("step");
            let mut cleanup = format!(
                "knotless repo delete knotless://$KNOTLESS_IDENTIFIER/{repo} --yes || true"
            );
            if entry == "fallback" || entry == "cancel" {
                cleanup.push_str("\nS=$(jq -n --arg i \"$KNOTLESS_IDENTIFIER\" --arg p \"$KNOTLESS_PASSWORD\" '{identifier: $i, password: $p}' | curl -sf -H 'content-type: application/json' -d @- \"$KNOTLESS_PDS/xrpc/com.atproto.server.createSession\") && jq -n --arg d \"$(printf %s \"$S\" | jq -r .did)\" '{repo: $d, collection: \"com.fedfork.exec.binding\", rkey: \"fill\"}' | curl -sf -H \"authorization: Bearer $(printf %s \"$S\" | jq -r .accessJwt)\" -H 'content-type: application/json' -d @- \"$KNOTLESS_PDS/xrpc/com.atproto.repo.deleteRecord\" > /dev/null || true");
            }
            h = h
                .call(host(
                    "cleanup",
                    "knotless.sh",
                    json!({ "cleanup": cleanup }),
                ))
                .call(host(
                    "seed",
                    "knotless.seed",
                    json!({ "bindings": bindings }),
                ))
                .timer("deadline", 60 * MIN);
        }
        Event::Timer { id } if id == "deadline" => {
            return Ok(s.fail(format!("not done in 60 min (step {})", s.phase())));
        }
        Event::Result { id, result, .. } => {
            let o = match steps::out(&s, &id, result.output()) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            if id == "cleanup" {
                return Ok(s.keep(h));
            }
            let t = s.since("step");
            if id == "seed" {
                s.reading("seed", json!({ "seconds": t, "composition": o["composition"], "modules": o["modules"] }));
                h = step(&mut s, h, 0);
                return Ok(s.keep(h));
            }
            let i = s.st["i"].as_u64().unwrap_or_default() as usize;
            if o["exit"] != 0 {
                let out = o["stdout"].as_str().unwrap_or_default();
                let tail: String = out
                    .chars()
                    .rev()
                    .take(1500)
                    .collect::<Vec<_>>()
                    .into_iter()
                    .rev()
                    .collect();
                return Ok(s.fail(format!(
                    "step {}: exit {}: {} {}",
                    steps[i].id, o["exit"], tail, o["stderr"]
                )));
            }
            s.reading(steps[i].id, t);
            if i + 1 == steps.len() {
                return Ok(s.pass());
            }
            h = step(&mut s, h, i + 1);
        }
        _ => {}
    }
    Ok(s.keep(h))
}

fn push_entry(r: Run<Value>) -> Result<Handle, String> {
    run_entry("push", r)
}

fn after_entry(r: Run<Value>) -> Result<Handle, String> {
    run_entry("after", r)
}

fn executor_entry(r: Run<Value>) -> Result<Handle, String> {
    run_entry("executor", r)
}

fn pull_entry(r: Run<Value>) -> Result<Handle, String> {
    run_entry("pull", r)
}

fn fallback_entry(r: Run<Value>) -> Result<Handle, String> {
    run_entry("fallback", r)
}

fn terminated_entry(r: Run<Value>) -> Result<Handle, String> {
    run_entry("terminated", r)
}

fn cancel_entry(r: Run<Value>) -> Result<Handle, String> {
    run_entry("cancel", r)
}

entries! { push => push_entry, after => after_entry, executor => executor_entry, pull => pull_entry, fallback => fallback_entry, terminated => terminated_entry, cancel => cancel_entry }