~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
198 lines · 12.4 KB Raw History
// docs/acceptance/0020-bidder-follows-release.md (plan 0040 built 4): an installed bidder follows the software.release channel.
// On a staging root host: install a bidder (its own network handle, --release-repo the account's DID), write the account's
// project, manifest, version and channel `socialweb-computer:rc` for build A, wait until the bidder's heartbeat names A, move the
// channel to build B, wait for B, then delete the records and uninstall. A is the installed build's own binary
// (the bucket's main/<commit> after `release` on main, else local/<commit> of a hand build; neither: the entry fails), B input `b` (default DEFAULT_B); each is fetched on
// the host for its sha256, size and version. A staging entry (tests/entries.txt): the prod gate does not run it.
use abi::{Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{MIN, S, host, out};

const PROJECT: &str = "socialweb-computer";
const HB: &str = "com.publicdomainrelay.temp.market.heartbeat";
const ST: &str = "/var/lib/socialweb-computer/swc-bidder";
const NET: &str = "/var/tmp/swc-0020-net.json";
const SPEC: &str = "/var/tmp/swc-0020-spec.json";
/// The installed bidder's agent DID (its offering's repo; the handle names the bidder's account, not the agent that heartbeats),
/// read from its own unit's lines: a uid is reused, so the uid's journal also holds the offerings of accounts that had it before.
fn agent(handle: &str) -> String {
    format!("journalctl _UID=$(id -u swc-bidder) _SYSTEMD_USER_UNIT=socialweb-computer-bidder@{handle}.service --no-pager -o cat | grep -o '\"offering\":\"at://did:plc:[a-z0-9]*' | tail -1 | sed 's|.*at://||'")
}
const BUCKET: &str = "https://socialweb-computer-kcp.sfo3.digitaloceanspaces.com";
const DEFAULT_B: &str = "main/bce9465bade6221988f5fce9d14301948b957143";

/// On the host: the first of `urls` that downloads, as `<sha256> <size> <version> <url>`; exit 7 when none does.
fn measure(urls: &[String], n: usize) -> String {
    format!("f=/root/swc-0020-{n}; u=; for c in {}; do if curl -fsSL -o $f $c; then u=$c; break; fi; done; [ -n \"$u\" ] || exit 7; chmod 0755 $f; printf '%s %s %s %s\\n' $(sha256sum $f | cut -d' ' -f1) $(stat -c %s $f) $($f version | sed -n 1p | cut -d' ' -f2) $u; rm -f $f", urls.join(" "))
}

fn cid_of(sha256: &str) -> Option<String> {
    let mut bytes = vec![0x01u8, 0x55, 0x12, 0x20];
    for i in (0..sha256.len()).step_by(2) {
        bytes.push(u8::from_str_radix(sha256.get(i..i + 2)?, 16).ok()?);
    }
    if bytes.len() != 36 {
        return None;
    }
    const A: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567";
    let (mut s, mut buf, mut bits) = (String::from("b"), 0u32, 0u32);
    for b in bytes {
        buf = (buf << 8) | b as u32;
        bits += 8;
        while bits >= 5 {
            bits -= 5;
            s.push(A[((buf >> bits) & 31) as usize] as char);
        }
    }
    if bits > 0 {
        s.push(A[((buf << (5 - bits)) & 31) as usize] as char);
    }
    Some(s)
}

fn manifest(project: &str, b: &Value, now: &str) -> Option<Value> {
    Some(json!({ "$type": "software.release.manifest", "project": project, "createdAt": now, "artifacts": [{
        "$type": "software.release.defs#httpArtifact", "name": "socialweb-computer-linux-amd64", "url": b["url"],
        "cid": cid_of(b["sha256"].as_str()?)?, "size": b["size"], "target": { "os": "linux", "arch": ["amd64"] } }] }))
}

fn run_0020(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let now = args["now"].as_str().unwrap_or_default().to_string();
    let mut s = S::new(run.state, &now);
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    if args["target"] == "prod" {
        return Ok(s.unmet("staging only: it installs a bidder as root on a staging host"));
    }
    let builds: Vec<Value> = s.st["builds"].as_array().cloned().unwrap_or_default();
    let did = args["did"].as_str().ok_or("did")?.to_string();
    let first = args["bidders"][0]["name"].as_str().unwrap_or_default();
    let Some(rest) = first.strip_prefix("bob2-") else {
        return Ok(s.unmet("0020 names its bidder after the target's bob2- bidder"));
    };
    let tag: String = s.st["tag"].as_str().map(String::from).unwrap_or_else(|| now.chars().filter(char::is_ascii_digit).skip(8).take(6).collect());
    s.st["tag"] = json!(tag);
    let handle = format!("bob{}t{tag}-{rest}", args["bidders"].as_array().map_or(0, |b| b.len()) + 2);
    let project = format!("at://{did}/software.release.project/{PROJECT}");
    let root = |id: &str, cmd: String| host(id, "root.exec", json!({ "cmd": cmd, "stdinPassword": false }));
    let bin = "/root/swc-st-socialweb-computer";
    let mut h = Handle::new();
    match run.event {
        Event::Start => h = h.call(host("put", "root.putBinary", json!({}))).timer("deadline", 40 * MIN),
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 40 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "poll" => match s.st["agent"].as_str() {
            Some(agent) => h = h.call(steps::get_record("hb", &format!("at://{agent}/{HB}/self"))),
            None => h = h.call(root("agent", agent(&handle))),
        },
        Event::Result { id, result, .. } => {
            let o = match out(&s, &id, result.output()) {
                Ok(o) => o,
                Err(_) if id == "hb" => Value::Null,
                Err(f) => return Ok(f),
            };
            if o.get("exit").is_some_and(|e| e != 0) {
                return Ok(s.fail(format!("{id}: exit {} {}", o["exit"], o["stderr"])));
            }
            match id.as_str() {
                "put" => h = h.call(root("install", format!("install -m 0644 /home/stg/stg/net.json {NET} && printf '{{\"options\":[{{\"name\":\"release-repo\",\"value\":\"{did}\"}}]}}\\n' > {SPEC} && chmod 0644 {SPEC} && {bin} install bidder --handle {handle} --market-network-file {NET} --bidder-spec-file {SPEC} --network-timeout 20m"))),
                "install" => {
                    s.set("up");
                    s.mark("wait");
                    h = h.timer("poll", 10_000);
                }
                "agent" => {
                    if let Some(did) = o["stdout"].as_str().map(str::trim).filter(|d| d.starts_with("did:")) {
                        s.st["agent"] = json!(did);
                        s.reading("agent", json!(did));
                    } else if s.since("wait") > 600.0 {
                        return Ok(s.fail("the installed bidder wrote no offering in 10 min"));
                    }
                    h = h.timer("poll", 5_000);
                }
                "hb" => {
                    let running = o["value"]["version"]["version"].as_str().unwrap_or_default().to_string();
                    let want = match s.phase().as_str() {
                        "up" => "",
                        "a" => builds[0]["version"].as_str().unwrap_or_default(),
                        _ => builds[1]["version"].as_str().unwrap_or_default(),
                    };
                    if s.is("up") && !running.is_empty() {
                        let commit = o["value"]["version"]["commit"].as_str().unwrap_or_default().to_string();
                        let b = args["b"].as_str().unwrap_or(DEFAULT_B).trim_matches('/').to_string();
                        s.reading("installed_version", json!(running));
                        s.set("measure");
                        let name = "binaries/socialweb-computer-linux-amd64";
                        let a = [format!("{BUCKET}/main/{commit}/{name}"), format!("{BUCKET}/local/{commit}/{name}")];
                        h = h.call(root("ma", measure(&a, 0))).call(root("mb", measure(&[format!("{BUCKET}/{b}/{name}")], 1)));
                        return Ok(s.keep(h));
                    }
                    if running.is_empty() || (!want.is_empty() && running != want) {
                        if s.since("wait") > 600.0 {
                            let at = format!("at://{}/{HB}/self", s.st["agent"].as_str().unwrap_or_default());
                            let got = if o.is_null() { "is not there".to_string() } else if running.is_empty() { "names no version".to_string() } else { format!("names {running:?}") };
                            let want = if want.is_empty() { "any version".to_string() } else { format!("{want:?}") };
                            return Ok(s.fail(format!("phase {}: the heartbeat {at} {got}, want {want}", s.phase())));
                        }
                        h = h.timer("poll", 10_000);
                    } else if s.is("a") {
                        s.reading("followed_a_s", json!(s.since("wait")));
                        let m = manifest(&project, &builds[1], &now).ok_or("builds[1] sha256")?;
                        h = h.call(host("mb2", "release.put", json!({ "records": [{ "collection": "software.release.manifest", "rkey": "m-b", "record": m }] })));
                    } else {
                        s.reading("followed_b_s", json!(s.since("wait")));
                        s.set("cleanup");
                        let uris: Vec<String> = ["software.release.channel/socialweb-computer:rc", "software.release.version/socialweb-computer:a",
                            "software.release.version/socialweb-computer:b", "software.release.manifest/m-a", "software.release.manifest/m-b",
                            "software.release.project/socialweb-computer"].iter().map(|r| format!("at://{did}/{r}")).collect();
                        h = h.call(host("gone", "release.delete", json!({ "uris": uris })));
                    }
                }
                "ma" | "mb" => {
                    let k = if id == "ma" { 0 } else { 1 };
                    let f: Vec<String> = o["stdout"].as_str().unwrap_or_default().split_whitespace().map(String::from).collect();
                    if f.len() < 4 {
                        return Ok(s.fail(format!("{id}: no sha256/size/version/url: {}", o["stdout"])));
                    }
                    s.st[format!("b{k}")] = json!({ "url": f[3], "sha256": f[0], "size": f[1].parse::<u64>().unwrap_or(0), "version": f[2] });
                    if s.st["b0"].is_object() && s.st["b1"].is_object() {
                        s.st["builds"] = json!([s.st["b0"].clone(), s.st["b1"].clone()]);
                        s.reading("builds", s.st["builds"].clone());
                        let m = manifest(&project, &s.st["b0"], &now).ok_or("build A sha256")?;
                        h = h.call(host("ab", "release.put", json!({ "records": [
                            { "collection": "software.release.project", "rkey": PROJECT, "record": { "$type": "software.release.project", "name": PROJECT, "createdAt": now } },
                            { "collection": "software.release.manifest", "rkey": "m-a", "record": m } ] })));
                    }
                }
                "ab" | "mb2" => {
                    let recs = o["records"].as_array().cloned().unwrap_or_default();
                    let m = recs.last().cloned().unwrap_or_default();
                    let (b, key) = if id == "ab" { (&builds[0], "a") } else { (&builds[1], "b") };
                    s.st["manifest"] = m.clone();
                    s.st["vkey"] = json!(key);
                    h = h.call(host("v", "release.put", json!({ "records": [{ "collection": "software.release.version", "rkey": format!("{PROJECT}:{key}"),
                        "record": { "$type": "software.release.version", "project": project, "version": b["version"], "manifest": { "uri": m["uri"], "cid": m["cid"] }, "createdAt": now } }] })));
                }
                "v" => {
                    let v = o["records"][0].clone();
                    h = h.call(host("ch", "release.put", json!({ "records": [{ "collection": "software.release.channel", "rkey": format!("{PROJECT}:rc"),
                        "record": { "$type": "software.release.channel", "project": project, "name": "rc", "version": { "uri": v["uri"], "cid": v["cid"] }, "updatedAt": now } }] })));
                }
                "ch" => {
                    let next = if s.st["vkey"] == "a" { "a" } else { "b" };
                    s.set(next);
                    s.mark("wait");
                    h = h.timer("poll", 10_000);
                }
                "gone" => h = h.call(root("uninstall", format!("{ST}/bin/current/socialweb-computer uninstall bidder --account swc-bidder --purge"))),
                "uninstall" => return Ok(s.pass()),
                _ => {}
            }
        }
        _ => {}
    }
    Ok(s.keep(h))
}

entries! { run => run_0020 }