~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
299 lines · 14.8 KB Raw History
// docs/acceptance/0004-vm-atirc-agent.md as a state machine: an agent VM rented by `run request --hold --agent`
// (discovery), its ident.link chain, the owner's mesh node reaches it and it answers, then a fast release that deletes
// the renter's records and tombstones both DIDs. Input {did (the account; the renter is its requester, from requester_association_written), handle, name}. Result {pass, error?, phase,
// readings}. The host cleans up whatever this run started (requesters, invocations, kube objects), however it ends.
use abi::{Call, Event, Handle, Run, entries};
use serde_json::{Value, json};

const T: &str = "computer.socialweb.test.";
const MIN: u64 = 60_000;

/// Seconds of an ISO time (UTC, `Z`), for elapsed readings.
fn secs(iso: &str) -> f64 {
    let n = |a: usize, b: usize| {
        iso.get(a..b)
            .and_then(|s| s.parse::<i64>().ok())
            .unwrap_or(0)
    };
    let (y, m, d) = (n(0, 4), n(5, 7), n(8, 10));
    let (y2, m2) = if m <= 2 { (y - 1, m + 9) } else { (y, m - 3) };
    let days = 365 * y2 + y2 / 4 - y2 / 100 + y2 / 400 + (153 * m2 + 2) / 5 + d;
    let frac = iso
        .get(19..)
        .and_then(|f| f.trim_end_matches('Z').parse::<f64>().ok())
        .unwrap_or(0.0);
    (days * 86400 + n(11, 13) * 3600 + n(14, 16) * 60 + n(17, 19)) as f64 + frac
}

fn fail(st: &Value, error: String) -> Handle {
    Handle::new().result(json!({ "pass": false, "error": error, "phase": st["phase"], "readings": st["readings"] })).done()
}

fn call(id: &str, what: &str, input: Value) -> Call {
    Call::new(id, format!("{T}{what}")).input(input)
}

fn run_0004(run: Run<Value>) -> Result<Handle, String> {
    let now = secs(run.args["now"].as_str().unwrap_or_default());
    let name = run.args["name"]
        .as_str()
        .ok_or("name is required")?
        .to_string();
    let account = run.args["did"]
        .as_str()
        .ok_or("did is required")?
        .to_string();
    let st = if run.state.is_null() {
        json!({ "phase": "start", "t0": now, "readings": {}, "f": {} })
    } else {
        run.state
    };
    let renter = st["renter"].as_str().unwrap_or_default().to_string();
    let mut s = steps::S { st, now };
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let mut st = s.st;
    let t = |st: &Value| ((now - st["t0"].as_f64().unwrap_or(now)) * 10.0).round() / 10.0;
    let set = |st: &mut Value, phase: &str| st["phase"] = json!(phase);
    let phase = st["phase"].as_str().unwrap_or_default().to_string();
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            h = h
                .call(call("owner", "requester.start", json!({ "name": "owner" })))
                .timer("deadline", 45 * MIN);
        }
        Event::Timer { id } if id == "deadline" => {
            return Ok(fail(&st, format!("not done in 45 min (phase {phase})")));
        }
        // the agent posts `<nick> online` by itself (plan 0040 decision 24c): none 60 s after the owner node subscribed
        // is a FAIL
        Event::Timer { id } if id == "first" && phase == "online" => {
            st["readings"]["online_after_agent_start_s"] = json!("not seen");
            return Ok(fail(&st, "the agent never posted `<nick> online`: no line with \"online\" from it 60 s after the owner node subscribed".into()));
        }
        // the prompt again every 30 s until it answers (5 min)
        Event::Timer { id } if id == "reprompt" && phase == "answer" => {
            if now - st["t_prompt1"].as_f64().unwrap_or(now) > 300.0 {
                return Ok(fail(&st, "no answer 5 min after the first prompt".into()));
            }
            st["readings"]["prompts"] = json!(st["readings"]["prompts"].as_u64().unwrap_or(0) + 1);
            st["t_prompt"] = json!(now);
            let nick = st["identity"].as_str().unwrap_or_default().split('.').next().unwrap_or_default().to_string();
            h = h.call(call("prompt", "atirc.meshPub", json!({ "cluster": "requester:owner", "node": "st-owner", "text": format!("{nick}: uname -m") }))).timer("reprompt", 30_000);
        }
        Event::Result { id, result, .. } => {
            let out = match result.output() {
                Ok(o) => o.clone(),
                Err(_) if id == "summary" => return Ok(h.state(st)),
                Err(e) => return Ok(fail(&st, format!("{id}: {e}"))),
            };
            match id.as_str() {
                "owner" => {
                    set(&mut st, "held");
                    let args = json!([
                        "--hold",
                        "--agent-config",
                        "@agentConfig",
                        "--agent=/etc/atirc/atirc.env"
                    ]);
                    h = h.call(
                        steps::host_sub("req", "request.start", json!({ "name": name, "args": args })),
                    );
                }
                "unit" => {
                    // a second ssh beside the --hold one can be refused (0003's pitfall): then this check is a reading
                    let o = out["output"].as_str().unwrap_or_default();
                    if out["ok"] != true {
                        return Ok(fail(&st, format!("unmet: the unit check before agentStart was not read: the guest ssh beside the hold process failed: {}", out["output"])));
                    } else if !o.contains("inactive") || o.matches("root 600").count() != 3 {
                        return Ok(fail(&st, format!("before agentStart: {o}")));
                    }
                    st["t_start"] = json!(now);
                    set(&mut st, "agent");
                    h = h.call(call("agent", "request.agent", json!({ "name": name })));
                }
                "agent" => {}
                // the assert module asks again for up to 3 min (R2 writes its side seconds after it reports)
                "chain" => {
                    if out["ok"] != true {
                        return Ok(fail(&st, format!("chain (after 3 min): {}", out["problems"])));
                    }
                    st["readings"]["chain_after_association_s"] = out["seconds"].clone();
                    // the owner's mesh node: `atirc mesh run --peer <identity>@<agent DID>` as the renter (the hand run's)
                    h = h.call(call("node", "atirc.meshRun", json!({ "name": "st-owner", "peer": st["peer"] })));
                }
                "node" => {
                    set(&mut st, "online");
                    h = h.call(
                        steps::host_sub("mesh", "atirc.meshSub", json!({ "cluster": "requester:owner", "node": "st-owner" })),
                    ).timer("first", 60_000);
                }
                "prompt" => {}
                "summary" => {
                    st["receipt"] = out["contract"]["receiptUri"].clone();
                    st["readings"]["winner"] = out["contract"]["winnerDid"].clone();
                }
                "release" => {
                    let s = out["seconds"].as_f64().unwrap_or(f64::MAX);
                    st["readings"]["release_s"] = json!(s);
                    if s >= 60.0 {
                        return Ok(fail(
                            &st,
                            format!("release took {s} s (fast release: ~15 s)"),
                        ));
                    }
                    set(&mut st, "released");
                    h = h.call(call("summary", "request.summary", json!({ "name": name })));
                    // the log may have reported both before the release returned: no message comes after it
                    if st["f"]["r2released"] == true && st["f"]["guestTomb"] == true {
                        set(&mut st, "records");
                        h = h.call(call("records", "request.records", json!({ "name": name })));
                    }
                }
                "records" => {
                    let mut uris = out["uris"].as_array().cloned().unwrap_or_default();
                    for c in [
                        "link.ident.endorsement",
                        "com.publicdomainrelay.temp.market.association",
                    ] {
                        uris.push(json!(format!(
                            "at://{renter}/{c}/{}",
                            st["r2"].as_str().unwrap_or_default()
                        )));
                    }
                    st["readings"]["records_checked"] = json!(uris.len());
                    h = h.call(call("gone", "assert.recordsGone", json!({ "uris": uris })));
                }
                "gone" => {
                    if out["left"].as_array().is_none_or(|l| !l.is_empty())
                        || out["errors"].as_array().is_some_and(|e| !e.is_empty())
                    {
                        return Ok(fail(
                            &st,
                            format!(
                                "records after release: left {} errors {}",
                                out["left"], out["errors"]
                            ),
                        ));
                    }
                    h = h.call(call(
                        "tomb",
                        "assert.tombstoned",
                        json!({ "dids": [st["r2"], st["agent"]] }),
                    ));
                }
                "tomb" => {
                    if out["alive"].as_array().is_none_or(|a| !a.is_empty()) {
                        return Ok(fail(&st, format!("not tombstoned: {}", out["alive"])));
                    }
                    st["readings"]["seconds"] = json!(t(&st));
                    return Ok(Handle::new()
                        .result(json!({ "pass": true, "readings": st["readings"] }))
                        .done());
                }
                _ => {}
            }
        }
        Event::Message { id, data, .. } if id == "req" => {
            let msg = data["msg"].as_str().unwrap_or_default();
            match msg {
                "contract" if data["phase"] == "Failed" && st["f"]["held"].is_null() => {
                    return Ok(fail(&st, format!("the contract failed before Held: {}", data["message"])));
                }
                "exited" if st["f"]["held"].is_null() => return Ok(fail(&st, "run request exited before the VM was Held".into())),
                "contract" if data["phase"] == "Held" && st["f"]["held"].is_null() => {
                    st["f"]["held"] = json!(true);
                    st["readings"]["held_s"] = json!(t(&st));
                }
                "requester_agent_mesh_key_written" => st["f"]["key"] = json!(true),
                "exited" => st["f"]["exited"] = json!(true),
                "requester_association_written" => {
                    st["r2"] = data["subject"].clone();
                    st["renter"] = data["did"].clone();
                }
                "requester_agent_identity" => {
                    let peer = data["peer"].as_str().unwrap_or_default();
                    let (identity, agent) = peer.split_once('@').unwrap_or_default();
                    st["peer"] = json!(peer);
                    st["identity"] = json!(identity);
                    st["agent"] = json!(agent);
                }
                "requester_guest_requester_released" => st["f"]["r2released"] = json!(true),
                "requester_guest_dids_tombstoned" => st["f"]["guestTomb"] = json!(true),
                "requester_guest_requester_release_timeout" => {
                    return Ok(fail(
                        &st,
                        "the release waited for the guest's timeout".into(),
                    ));
                }
                _ => {}
            }
            let f = &st["f"];
            // the --hold process stays (its requester core handles R2's report and the release): no wait for its exit
            if phase == "held" && f["held"] == true && f["key"] == true {
                set(&mut st, "unit");
                let exec = "systemctl is-active atirc-agent.service; stat -c '%U %a' /etc/atirc/iroh_key.hex /etc/atirc/atirc.env /etc/atirc/config.toml";
                h = h.call(call(
                    "unit",
                    "request.ssh",
                    json!({ "name": name, "exec": exec }),
                ));
            }
            if phase == "agent" && st["r2"].is_string() && st["agent"].is_string() {
                st["readings"]["association_after_agent_start_s"] =
                    json!(now - st["t_start"].as_f64().unwrap_or(now));
                st["readings"]["r2"] = st["r2"].clone();
                st["readings"]["agent"] = st["agent"].clone();
                set(&mut st, "chain");
                h = h.call(call("chain", "assert.agentChain", json!({ "renter": renter, "account": account, "r2": st["r2"], "agent": st["agent"], "identity": st["identity"] })));
            }
            if phase == "released" && st["f"]["r2released"] == true && st["f"]["guestTomb"] == true
            {
                set(&mut st, "records");
                h = h.call(call("records", "request.records", json!({ "name": name })));
            }
        }
        Event::Message { id, data, .. } if id == "mesh" => {
            let from_agent = data["identity"] == st["identity"];
            let text = data["text"].as_str().unwrap_or_default();
            if phase == "online" && from_agent && text.contains("online") {
                st["readings"]["online_after_agent_start_s"] =
                    json!(now - st["t_start"].as_f64().unwrap_or(now));
                st["t_prompt"] = json!(now);
                st["t_prompt1"] = json!(now);
                st["readings"]["prompts"] = json!(1);
                set(&mut st, "answer");
                h = h.cancel("first").timer("reprompt", 30_000);
                let nick = st["identity"]
                    .as_str()
                    .unwrap_or_default()
                    .split('.')
                    .next()
                    .unwrap_or_default()
                    .to_string();
                h = h.call(call("prompt", "atirc.meshPub", json!({ "cluster": "requester:owner", "node": "st-owner", "text": format!("{nick}: uname -m") })));
            } else if phase == "answer"
                && from_agent
                && (text.contains("x86_64") || text.contains("aarch64"))
            {
                st["readings"]["answer_s"] = json!(now - st["t_prompt"].as_f64().unwrap_or(now));
                set(&mut st, "release");
                // the summary is written when the --hold process ends: release first, read it after
                st["t_rel"] = json!(now);
                h = h.cancel("mesh").cancel("reprompt").call(call("release", "request.release", json!({ "name": name })));
            }
        }
        Event::Closed { id, error } if id == "req" || (id == "mesh" && phase != "release") => {
            return Ok(fail(
                &st,
                format!("{id} closed: {}", error.unwrap_or_default()),
            ));
        }
        _ => {}
    }
    Ok(h.state(st))
}

entries! { run => run_0004 }