~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
369 lines · 22.9 KB Raw History
// docs/acceptance/0015-vm-memory.md, entry `pressure`: the bidder's memory events fire exactly once per crossing and re-arm.
// This run's own VM holds ~55% of its RAM in the guest; the test host lowers THIS VM unit's MemoryHigh (a runtime property of
// the unit in the network bidder's user manager: fault injection on the test's own VM only) to half its usage while the guest
// touches its memory, so memory.events `high` rises again and again for WINDOW_S. The bidder polls memory.events every 15 s and
// publishes one compute.events.vm.memoryPressure {kind high} per crossing, at most one per kind per MemoryTypeGap (10 min), and
// only after MemoryRearmQuiet (5 min without a rise, usage under 90% of the limit) re-arms it. So:
//   crossing 1 (t0): exactly one record, though `high` rose many times
//   crossing 2 (crossing 1's restore + 5 min + 4 polls, re-armed but inside the gap): still exactly one record
//   crossing 3 (record 1 + 10.75 min, past the gap): exactly two
// and never a `max` record or a vm.oomKilled; the VM stays Held and Running, the guest's boot id never changes; then the release.
// Dev / staging network whose bidder runs VM units under systemd-user (cgroup limits); SKIPs on prod and on a proc bidder.
use abi::{Call, Event, Handle, Run};
use serde_json::{Value, json};
use steps::{MIN, S, VM_TERMINATED, get_record, host, kube, out, secs, termination_rkey};

const PRESSURE: &str = "com.publicdomainrelay.temp.compute.events.vm.memoryPressure";
const OOM_KILLED: &str = "com.publicdomainrelay.temp.compute.events.vm.oomKilled";

/// internal/market/bidder/memory.go: MemoryRearmQuiet 5 min, MemoryTypeGap 10 min, MemoryPoll 15 s.
const REARM_S: f64 = 300.0;
const GAP_S: f64 = 600.0;
const POLL_S: f64 = 15.0;
const WINDOW_S: f64 = 60.0;

const HOLD_PY: &str = r#"import os, sys, time
n = int(sys.argv[1])
hold = [bytearray(os.urandom(64 << 20)) for _ in range(n // 64)]
open('/var/tmp/memhold.ready', 'w').write(str(len(hold) * 64))
while True:
    if os.path.exists('/var/tmp/memhold.go'):
        for c in hold:
            c[::4096]
    else:
        time.sleep(0.2)
"#;

fn ssh(id: &str, name: &str, exec: &str) -> Call {
    host(id, "request.ssh", json!({ "name": name, "exec": exec }))
}

fn vm_memory(id: &str, s: &S, set_high: Option<Value>) -> Call {
    let mut i = json!({ "receipt": s.st["receipt"], "cluster": s.st["cluster"], "vm": s.st["vm"] });
    if let Some(v) = set_high {
        i["setHigh"] = v;
    }
    host(id, "vm.memory", i)
}

fn list(id: &str, repo: &str, collection: &str) -> Call {
    Call::new(id, "com.atproto.repo.listRecords").params(json!({ "repo": repo, "collection": collection, "limit": 100 })).service(repo)
}

/// This receipt's records of `nsid` of `kinds` (rkey ends -payload-<key_hash("<nsid>:<kind>-<n>:<receipt>")>, n 1..10).
fn mine(o: &Value, nsid: &str, kinds: &[&str], receipt: &str) -> Vec<Value> {
    let tails: Vec<String> = kinds.iter().flat_map(|k| (1..=10).map(move |n| format!("{k}-{n}"))).map(|t| format!("-payload-{}", steps::key_hash(&format!("{nsid}:{t}:{receipt}")))).collect();
    o["records"].as_array().into_iter().flatten().filter(|r| r["uri"].as_str().is_some_and(|u| tails.iter().any(|t| u.ends_with(t.as_str())))).cloned().collect()
}

/// The crossing n starts: MemoryHigh to half the VM's usage (256 MiB at least), the guest touches its memory.
fn cross(s: &mut S, h: Handle, name: &str, n: u64) -> Handle {
    let cur = s.st["m"]["current"].as_u64().unwrap_or(0);
    let lowered = (cur / 2).max(256 << 20);
    s.st["x"] = json!({ "n": n, "lowered": lowered, "cur": cur, "highBefore": s.st["m"]["events"]["high"], "startS": s.now, "until": s.now + WINDOW_S });
    s.set("cross");
    s.mark("cross");
    h.call(vm_memory("mx", s, Some(json!(lowered)))).call(ssh("go", name, "touch /var/tmp/memhold.go; echo ok")).timer("tick", 5_000)
}

pub fn run(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
    let receipt = s.st["receipt"].as_str().unwrap_or_default().to_string();
    let winner = receipt.trim_start_matches("at://").split('/').next().unwrap_or_default().to_string();
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            if args["target"] == "prod" {
                return Ok(s.unmet("staging only: it lowers its own VM's MemoryHigh through the network bidder's user manager (fault injection; prod bidders are shared)"));
            }
            // a bidder publishes its offering once its pinned guest image verified: a network that just came up is minutes
            // ahead of it, so the rent waits for the first bidder's offering (10 s retries, 10 min at most)
            s.set("offering");
            h = h.timer("deadline", 40 * MIN);
            h = match bidders.first() {
                Some(b) => h.call(get_record("off", &format!("at://{}/com.publicdomainrelay.temp.market.offering/self", b["did"].as_str().unwrap_or_default()))),
                None => h.timer("offered", 1),
            };
        }
        Event::Timer { id } if id == "offretry" => {
            let b = bidders.first().cloned().unwrap_or_default();
            h = h.call(get_record("off", &format!("at://{}/com.publicdomainrelay.temp.market.offering/self", b["did"].as_str().unwrap_or_default())));
        }
        Event::Timer { id } if id == "offered" && s.is("offering") => {
            s.set("held");
            let a = json!(["--hold", "--cpus", "2", "--mem", "2G", "--expires-in", "1h"]);
            h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a })));
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 40 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "ready" => h = h.call(ssh("ready", &name, "cat /var/tmp/memhold.ready 2>/dev/null; echo; echo BOOT $(cat /proc/sys/kernel/random/boot_id)")),
        // the crossing's watch: memory.events each 5 s; at its end the limit is restored and the guest stops touching
        Event::Timer { id } if id == "tick" && s.is("cross") => {
            // a reading still out (vm.memory runs on the winner's host over ssh) is not asked again
            if s.st["mtOut"] == true {
                h = h.timer("tick", 5_000);
            } else {
                s.st["mtOut"] = json!(true);
                h = h.call(vm_memory("mt", &s, None)).timer("tick", 5_000);
            }
        }
        Event::Timer { id } if id == "lst" && (s.is("after") || s.is("verify")) => {
            h = h.call(list("ls", &winner, PRESSURE)).timer("lst", 15_000);
        }
        Event::Timer { id } if id == "next" && s.is("idle") => {
            let n = s.st["x"]["n"].as_u64().unwrap_or(0) + 1;
            h = cross(&mut s, h, &name, n);
        }
        Event::Message { data, .. } => {
            if s.st["receipt"].is_null() && data["receipt"].is_string() {
                s.st["receipt"] = data["receipt"].clone();
            }
            match (data["msg"].as_str().unwrap_or_default(), data["phase"].as_str().unwrap_or_default()) {
                ("contract", "Held") => s.flag("held"),
                ("contract", p) if s.has("held") && !s.has("releasing") && !p.is_empty() => {
                    return Ok(s.fail(format!("the contract left Held: {p} {}", data["message"])));
                }
                ("contract", "Failed") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                ("exited", _) if !s.has("releasing") => return Ok(s.fail(format!("run request exited before the release: {data}"))),
                _ => {}
            }
            if s.is("held") && s.has("held") && s.st["receipt"].is_string() {
                let w = s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default().to_string();
                s.reading("winner", w.clone());
                let Some(b) = bidders.iter().find(|b| b["did"] == w.as_str()).cloned() else {
                    return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target")));
                };
                let Some(c) = b["cluster"].as_str() else {
                    return Ok(s.unmet(&format!("the target names no kcp for the winner {w}")));
                };
                s.st["cluster"] = json!(c);
                s.set("find");
                h = h.call(kube("vms", "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })));
            }
        }
        Event::Result { id, result, .. } => {
            let r = result.output();
            if id == "off" {
                if r.as_ref().is_err_and(|e| e.contains("RecordNotFound")) {
                    let n = s.st["offTries"].as_u64().unwrap_or(0) + 1;
                    s.st["offTries"] = json!(n);
                    if n > 60 {
                        return Ok(s.fail("the bidder's offering is not published after 10 min"));
                    }
                    return Ok(s.keep(h.timer("offretry", 10_000)));
                }
                if let Err(e) = r {
                    return Ok(s.fail(format!("off: {e}")));
                }
                s.set("held");
                let a = json!(["--hold", "--cpus", "2", "--mem", "2G", "--expires-in", "1h"]);
                return Ok(s.keep(h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a })))));
            }
            // the VM's memory cgroup is the dev network's: a bidder without VM cgroups (proc backend) skips the entry
            if (id == "m0" || id == "mx") && r.as_ref().is_err_and(|e| e.contains("no VM unit") || e.contains("no readable memory cgroup")) {
                let why = format!("the winner's VM has no memory cgroup under a user manager of this account: {}", r.err().unwrap_or_default());
                s.flag("releasing");
                s.set("release");
                s.reading("unmet", why);
                return Ok(s.keep(h.call(host("release", "request.release", json!({ "name": name })))));
            }
            // no vm.terminated for the receipt is the pass: its absence reads RecordNotFound
            let o = if id == "term" && r.as_ref().is_err_and(|e| e.contains("RecordNotFound")) {
                json!({ "none": true })
            } else {
                match out(&s, &id, r) {
                    Ok(o) => o,
                    Err(f) => return Ok(f),
                }
            };
            match id.as_str() {
                "vms" => {
                    let c: Vec<&Value> = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).collect();
                    let [v] = c.as_slice() else {
                        return Ok(s.fail(format!("{} ComputeVMs of this run's receipt on the winner", c.len())));
                    };
                    s.st["vm"] = v["metadata"]["name"].clone();
                    s.set("deploy");
                    let start = format!(
                        "cat > /var/tmp/memhold.py <<'PY_EOF'\n{HOLD_PY}PY_EOF\ntotal=$(awk '/^MemTotal:/{{print $2}}' /proc/meminfo); rm -f /var/tmp/memhold.go /var/tmp/memhold.ready\nsetsid nohup python3 /var/tmp/memhold.py $((total * 55 / 100 / 1024)) > /var/tmp/memhold.log 2>&1 < /dev/null & echo started"
                    );
                    h = h.call(ssh("deploy", &name, &start));
                }
                "deploy" => {
                    s.mark("deploy");
                    h = h.timer("ready", 5_000);
                }
                "ready" => {
                    let text = o["output"].as_str().unwrap_or_default();
                    let held: Option<u64> = text.lines().next().and_then(|l| l.trim().parse().ok());
                    let boot = text.lines().find_map(|l| l.strip_prefix("BOOT ")).map(|b| b.trim().to_string());
                    match (held, boot) {
                        (Some(m), Some(b)) => {
                            s.st["boot0"] = json!(b);
                            s.reading("guest_held_mib", m);
                            s.set("base");
                            h = h.call(vm_memory("m0", &s, None));
                        }
                        _ if s.since("deploy") > 180.0 => return Ok(s.fail(format!("the guest did not hold its memory in 3 min: {text}"))),
                        _ => h = h.timer("ready", 5_000),
                    }
                }
                "m0" => {
                    s.st["m"] = o.clone();
                    s.st["high0"] = o["high"].clone();
                    s.reading("limits", json!({ "unit": o["unit"], "high": o["high"], "max": o["max"], "swapMax": o["swapMax"], "currentBeforeMiB": o["current"].as_u64().unwrap_or(0) >> 20, "events": o["events"] }));
                    h = cross(&mut s, h, &name, 1);
                }
                "mx" => {
                    s.st["x"]["setHigh"] = o["high"].clone();
                }
                "mt" => {
                    s.st["mtOut"] = json!(false);
                    s.st["m"] = o.clone();
                    if s.now >= s.st["x"]["until"].as_f64().unwrap_or(0.0) && s.is("cross") {
                        let restore = match s.st["high0"].as_str() {
                            Some("max") | None => json!("max"),
                            Some(v) => v.parse::<u64>().map(Value::from).unwrap_or(json!("max")),
                        };
                        s.set("restore");
                        h = h.call(vm_memory("mr", &s, Some(restore))).call(ssh("stop", &name, "rm -f /var/tmp/memhold.go; echo ok"));
                    }
                }
                "mr" => {
                    let n = s.st["x"]["n"].as_u64().unwrap_or(0);
                    let rose = o["events"]["high"].as_u64().unwrap_or(0).saturating_sub(s.st["x"]["highBefore"].as_u64().unwrap_or(0));
                    let x = json!({ "n": n, "lowered": s.st["x"]["lowered"], "currentMiB": s.st["x"]["cur"].as_u64().unwrap_or(0) >> 20, "highEventsRose": rose, "restoredHigh": o["high"], "oomKill": o["events"]["oom_kill"], "max": o["events"]["max"] });
                    s.st["xs"].as_array_mut().map(|a| a.push(x.clone())).unwrap_or_else(|| s.st["xs"] = json!([x.clone()]));
                    s.reading("crossings", s.st["xs"].clone());
                    if rose < 3 {
                        return Ok(s.fail(format!("crossing {n}: memory.events high rose only {rose} times in {WINDOW_S} s (limit lowered to {} B)", s.st["x"]["lowered"])));
                    }
                    s.st["m"] = o.clone();
                    s.st["x"]["endS"] = json!(s.now);
                    s.set("after");
                    s.mark("after");
                    h = h.timer("lst", 1_000);
                }
                "ls" => {
                    let n = s.st["x"]["n"].as_u64().unwrap_or(0);
                    let recs = mine(&o, PRESSURE, &["high"], &receipt);
                    let maxes = mine(&o, PRESSURE, &["max"], &receipt);
                    if !maxes.is_empty() {
                        return Ok(s.fail(format!("a kind max record: {}", json!(maxes))));
                    }
                    let want_total = if n == 3 { 2 } else { 1 };
                    s.st["recs"] = json!(recs.iter().map(|r| json!({ "uri": r["uri"], "kind": r["value"]["kind"], "threshold": r["value"]["threshold"], "observed": r["value"]["observed"], "observedAt": r["value"]["observedAt"] })).collect::<Vec<_>>());
                    if recs.len() > want_total {
                        return Ok(s.fail(format!("crossing {n}: {} memoryPressure records, exactly {want_total} expected: {}", recs.len(), s.st["recs"])));
                    }
                    // the crossing's records are in once the expected count appeared; then 45 s more must add none
                    if recs.len() == want_total && !s.has(&format!("seen{n}")) {
                        s.flag(&format!("seen{n}"));
                        s.mark(&format!("seen{n}"));
                        if n == 1 {
                            s.st["t1"] = json!(secs(recs[0]["value"]["observedAt"].as_str().unwrap_or_default()));
                        }
                    }
                    let settled = s.has(&format!("seen{n}")) && s.since(&format!("seen{n}")) >= 45.0;
                    // crossing 2 must add none: 75 s after its window
                    let quiet2 = n == 2 && recs.len() == 1 && s.since("after") >= 75.0;
                    if s.is("after") && (settled || quiet2) {
                        let ok1 = recs[0]["value"]["kind"] == "high" && recs[0]["value"]["observed"].as_i64().unwrap_or(0) > 0;
                        if !ok1 {
                            return Ok(s.fail(format!("the first record is not a high record with an observed size: {}", recs[0]["value"])));
                        }
                        let t1 = s.st["t1"].as_f64().unwrap_or(s.now);
                        match n {
                            1 => {
                                s.reading("record1", s.st["recs"][0].clone());
                                // the bidder's quiet starts at its first poll after the restore (endS), its re-arm comes at a poll
                                // 5 min after that one or the next: crossing 2 starts after both, and must end inside the gap
                                let end1 = s.st["x"]["endS"].as_f64().unwrap_or(s.now);
                                let at = end1 + REARM_S + 4.0 * POLL_S;
                                if at + WINDOW_S + 2.0 * POLL_S >= t1 + GAP_S {
                                    return Ok(s.fail(format!("crossing 1's limit was restored {:.0} s after record 1: crossing 2 cannot start after the re-arm and end inside the gap", end1 - t1)));
                                }
                                s.reading("crossing2AfterRecord1S", ((at - t1) * 10.0).round() / 10.0);
                                s.set("idle");
                                h = h.timer("next", ((at - s.now).max(1.0) * 1000.0) as u64);
                            }
                            2 => {
                                s.reading("afterGap2", json!({ "records": recs.len() }));
                                s.set("idle");
                                h = h.timer("next", ((t1 + GAP_S + 45.0 - s.now).max(1.0) * 1000.0) as u64);
                            }
                            _ => {
                                s.reading("records", s.st["recs"].clone());
                                s.set("verify");
                                s.st["need"] = json!(["vf", "boot1", "oom", "term"]);
                                h = h
                                    .call(kube("vf", "list", &s.st["cluster"].as_str().unwrap_or_default().to_string(), json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })))
                                    .call(ssh("boot1", &name, "echo BOOT $(cat /proc/sys/kernel/random/boot_id)"))
                                    .call(list("oom", &winner, OOM_KILLED))
                                    .call(get_record("term", &format!("at://{winner}/{VM_TERMINATED}/{}", termination_rkey(&receipt))));
                            }
                        }
                    } else if s.is("after") && s.since("after") > 240.0 && recs.len() < want_total {
                        return Ok(s.fail(format!("crossing {n}: {} memoryPressure records 4 min after the window, {want_total} expected: {} (the bidder polls VM cgroups only when its core runs under a systemd user manager, user@<uid>.service: a dev network started from a login session has its core in a session scope, and none are published; start `dev up` inside `systemd-run --user --scope`)", recs.len(), s.st["recs"])));
                    }
                }
                "vf" | "boot1" | "oom" | "term" => {
                    s.st["got"][id.as_str()] = if id == "term" { json!({ "terminated": o["none"] != true, "value": o["value"] }) } else { o.clone() };
                    let all = ["vf", "boot1", "oom", "term"].iter().all(|k| !s.st["got"][*k].is_null());
                    if all {
                        let g = s.st["got"].clone();
                        let mut p = vec![];
                        let v = g["vf"]["items"].as_array().into_iter().flatten().find(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]);
                        if v.is_none_or(|v| v["status"]["phase"] != "Running" || v["status"]["lostReason"].is_string()) {
                            p.push(format!("the ComputeVM is not Running: {}", v.map(|v| v["status"].to_string()).unwrap_or("gone".into())));
                        }
                        let b1 = g["boot1"]["output"].as_str().unwrap_or_default().lines().find_map(|l| l.strip_prefix("BOOT ")).map(|b| b.trim().to_string());
                        if b1.as_deref() != s.st["boot0"].as_str() {
                            p.push(format!("the guest rebooted: {} -> {:?}", s.st["boot0"], b1));
                        }
                        if g["term"]["terminated"] == true {
                            p.push(format!("vm.terminated: {}", g["term"]["value"]));
                        }
                        let ooms = mine(&g["oom"], OOM_KILLED, &["oom"], &receipt);
                        s.reading("oomKilledRecords", ooms.len());
                        if !ooms.is_empty() {
                            p.push(format!("vm.oomKilled records: {}", json!(ooms)));
                        }
                        if !p.is_empty() {
                            return Ok(s.fail(p.join("; ")));
                        }
                        s.flag("releasing");
                        s.set("release");
                        h = h.call(host("release", "request.release", json!({ "name": name })));
                    }
                }
                "release" => {
                    s.reading("release_s", o["seconds"].clone());
                    if s.has("skip") || s.st["readings"]["unmet"].is_string() {
                        return Ok(s.unmet(s.st["readings"]["unmet"].as_str().unwrap_or_default()));
                    }
                    h = h.call(host("records", "request.records", json!({ "name": name })));
                }
                "records" => {
                    s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
                    h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
                }
                "gone" => {
                    if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                        return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
                    }
                    return Ok(s.pass());
                }
                _ => {}
            }
        }
        Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
        _ => {}
    }
    Ok(s.keep(h))
}