// docs/acceptance/0014-accept-fallback.md: rentals that outnumber the bidders' room, started together, on bidders this // run sizes from their offerings (maxCpus, less the live guests on their kcp). A bidder whose committed capacity the // accept would exceed answers 409 CapacityExceeded; the requester deletes that accept, denies the bidder and takes the // next bid on the same RFP. run: 2 bidders, one VM each, 2 rentals (a refusal is a reading: both rentals may pick the same bidder first, and the one refused falls back to the other; up to 2 rounds): the rentals end on different bidders, scaling across them. concurrent: // 2 bidders, two VMs each, 4 rentals; every Held rental's accept is then submitted 5 times more at once (each answered 202, one // ComputeVM per receipt); after the release the same round runs again, so the room the release returned must be there (a refusal // that leaves a rental waiting fails it). exhausted: 2 bidders, one VM each, 3 rentals: the third is refused for capacity by both (a // 409 to its accept or `declined: capacity` to its RFP, before it waits), after a 409 its new RFP (the refusers excluded) has no bid: it waits. All through the kcp CLI requester (`run request`); the atlibs requester // module's fallback is tests/compositions/knotless entry `fallback`. Both: every rental Held, never on a bidder that refused it, no bidder over its // vCPUs, its memory or its guest quota, the refused accepts kept, then all released and every record gone. Staging only (it fills bidders). // A bidder commits cores x its cpu-overcommit (read from its Bidder spec.cpuOvercommit, 0: the default 2; an input `overcommit` // must equal it) in vCPUs and its maxMem in guest memory plus each VM's overhead (contract/host.VMCommitMiB), and one VM is at // most its cores: the room of a bidder is the least of maxCpus x overcommit, maxMem and its guest quota less its live guests, and // a filler is at most maxCpus (a bidder's room above that is filled by several pinned fillers). Entry `overcommit`: see overcommit_0014. use abi::{Event, Handle, Run, entries}; use serde_json::{Value, json}; use steps::{MIN, S, get_record, host, kube, out}; const OFFERING: &str = "com.publicdomainrelay.temp.market.offering"; const LIVE: [&str; 5] = ["", "Pending", "Preparing", "Booting", "Running"]; struct Shape { bidders: usize, rentals: usize, fit: i64, need_refusal: bool, rounds: u64, /// rentals beyond the bidders' room: each refused by every bidder, then its new RFP finds no bid: the requester /// WAITS for capacity (requester_waiting_for_capacity) instead of failing fails: usize, /// 0: the accept-fallback runs; 1 giveup: every bidder filled, one rental with a short --expires-in must fail at its /// deadline "gave up waiting for capacity"; 2 wait: every bidder filled, a rental waits, a filler is released and it is Held mode: u8, /// the --expires-in of the rental of mode 1 and 2 expires: &'static str, /// vCPUs of each contending guest when a bidder has more room than that and the input `cpus` is unset (0: sized from the room) small: i64, /// every Held rental's accept is submitted again RESENDS times at once to its winner: each answered 202, no second VM resend: bool, /// after the release the same round runs again (up to `rounds`): the released room must be there for it again: bool, } const RESENDS: usize = 5; fn mib(s: &str) -> i64 { let t = s.trim().trim_end_matches(['B', 'b']).trim_end_matches(['i', 'I']); let (n, unit) = t.split_at(t.find(|c: char| !c.is_ascii_digit()).unwrap_or(t.len())); let n: i64 = n.parse().unwrap_or(0); match unit.to_ascii_uppercase().as_str() { "K" => ((n + 512) / 1024).max(1), "G" => n * 1024, "T" => n * 1024 * 1024, _ => n, } } /// a guest's memory as the bidder commits it (contract/host.VMCommitMiB): the guest's plus its VMM overhead fn commit_mib(m: i64) -> i64 { m + (m / 10).max(512) } /// a guest's committed memory as the bidder counts it: a deno sandbox runs on the host and holds none fn mem_of(v: &Value) -> i64 { if !v["spec"]["sandbox"].is_null() { return 0; } commit_mib(mib(v["spec"]["vm"]["mem"].as_str().unwrap_or_default())) } fn vms(id: &str, cluster: &str) -> abi::Call { kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })) } fn live_vms(list: &Value) -> Vec { list["items"] .as_array() .into_iter() .flatten() .filter(|v| v["metadata"]["deletionTimestamp"].is_null() && LIVE.contains(&v["status"]["phase"].as_str().unwrap_or_default())) .cloned() .collect() } fn cpus_of(v: &Value) -> i64 { v["spec"]["vm"]["cpus"].as_i64().unwrap_or(0) } fn did_of(uri: &str) -> String { uri.trim_start_matches("at://").split('/').next().unwrap_or_default().to_string() } /// One more answer of the step's fan-out; true when it was the last. fn landed(s: &mut S) -> bool { let left = s.st["wait"].as_u64().unwrap_or(1).saturating_sub(1); s.st["wait"] = json!(left); left == 0 } fn chosen(s: &S) -> Vec { s.st["chosen"].as_array().cloned().unwrap_or_default() } fn rentals(s: &S) -> Vec { s.st["rentals"].as_array().cloned().unwrap_or_default() } /// The rentals and this round's fillers: what the release ends and the record check reads. fn everyone(s: &S) -> Vec { let mut all = rentals(s); all.extend(s.st["fillers"].as_array().cloned().unwrap_or_default()); all } /// A round: the fillers first (each pinned to its bidder, leaving it room for `fit` guests of `cpus`), then the rentals. fn start_round(s: &mut S, mut h: Handle, name: &str, shape: &Shape) -> Handle { let plan = s.st["fill"].as_object().cloned().unwrap_or_default(); if plan.is_empty() { s.st["fillers"] = json!([]); return launch(s, h, name, shape); } let round = s.st["round"].as_u64().unwrap_or(0) + 1; s.set("fillers"); s.mark("fillers"); let mut fs = vec![]; let mut j = 0; for (did, total) in plan.iter() { // one VM is at most the bidder's cores: its room is filled by pinned fillers of at most that let cores = s.st["cores"][did].as_i64().unwrap_or(i64::MAX).max(1); let mut left = total.as_i64().unwrap_or(0); while left > 0 { let cpus = left.min(cores); left -= cpus; let n = format!("{name}-{round}-f{j}"); let a = json!(["--hold", "--cpus", cpus.to_string(), "--mem", s.st["mem"], "--expires-in", "1h"]); h = h.call(steps::host_sub(&format!("fill:{j}"), "request.start", json!({ "name": n, "args": a, "bidderDids": [did] }))); fs.push(json!({ "name": n, "bidder": did, "cpus": cpus })); j += 1; } } s.st["fillers"] = json!(fs); h.timer(&format!("fillers:{round}"), 15 * MIN) } fn launch(s: &mut S, mut h: Handle, name: &str, shape: &Shape) -> Handle { let round = s.st["round"].as_u64().unwrap_or(0) + 1; s.st["round"] = json!(round); s.set("held"); s.mark("round"); let dids: Vec = chosen(s).iter().map(|b| b["did"].clone()).collect(); let mut rs = vec![]; for i in 0..shape.rentals { let n = format!("{name}-{round}-{i}"); let exp = if shape.mode > 0 { shape.expires } else { "1h" }; let a = json!(["--hold", "--cpus", s.st["cpus"].to_string(), "--mem", s.st["mem"], "--expires-in", exp]); h = h.call(steps::host_sub(&format!("req:{i}"), "request.start", json!({ "name": n, "args": a, "bidderDids": dids }))); rs.push(json!({ "name": n, "declined": [], "rfpDeclined": [], "replaced": 0, "newRfp": 0, "waits": 0 })); } s.st["rentals"] = json!(rs); h.timer(&format!("round:{round}"), 20 * MIN) } /// How each bidder refused a rental for capacity before its first wait (round 0), once per bidder: `accept409` (it bid, then /// refused the accept 409 CapacityExceeded) or `rfpCapacity` (it declined the RFP itself, `declined: capacity`: full when the /// RFP came). Which of the two a full bidder answers is the order the RFP and another rental's admission reach it /// (vmsize.go capacityRefused, both). A waiting round's RFP asks every bidder again: its answers do not explain the wait. fn capacity_refusals(r: &Value) -> serde_json::Map { let mut forms = serde_json::Map::new(); for d in r["declined"].as_array().into_iter().flatten().filter(|d| d["round"] == 0) { if let Some(b) = d["bidder"].as_str().filter(|_| d["status"] == 409 && d["body"]["error"] == "CapacityExceeded") { forms.entry(b.to_string()).or_insert(json!("accept409")); } } for d in r["rfpDeclined"].as_array().into_iter().flatten().filter(|d| d["round"] == 0) { if let Some(b) = d["bidder"].as_str().filter(|_| d["kind"] == "capacity") { forms.entry(b.to_string()).or_insert(json!("rfpCapacity")); } } forms } /// Every rental Held: where each landed, against what refused it and each bidder's room. fn check(s: &mut S, mut h: Handle, shape: &Shape) -> Result { let free = s.st["free"].clone(); let mut per: serde_json::Map = Default::default(); let mut p = vec![]; let mut declined_uris = vec![]; let mut failed = vec![]; for r in rentals(s) { if r["held"] != true && r["failed"] != true && shape.mode == 0 && shape.fails > 0 && r["waits"].as_u64().unwrap_or(0) > 0 { // the documented end: every bidder refused it for capacity before it waited, and after a 409 its new RFP (the refusers // excluded) found no bid (with no bid at all there is no accept to refuse and nothing to ask again): it waits for // capacity, a new RFP asking everyone after each backoff, until its deadline for d in r["declined"].as_array().into_iter().flatten() { declined_uris.push(d["accept"].clone()); } let forms = capacity_refusals(&r); let every = chosen(s).iter().all(|b| b["did"].as_str().is_some_and(|d| forms.contains_key(d))); let refused409 = forms.values().any(|f| f == "accept409"); if !every || (refused409 && r["newRfp"].as_u64().unwrap_or(0) == 0) { p.push(format!("{} waits for capacity, not after every bidder refused it and its new RFP found none: {} refusals, newRfp {}, refused by {} (409s {}, RFP declines {})", r["name"], forms.len(), r["newRfp"], Value::Object(forms), r["declined"], r["rfpDeclined"])); } failed.push(r["name"].clone()); continue; } if r["failed"] == true { for d in r["declined"].as_array().into_iter().flatten() { declined_uris.push(d["accept"].clone()); } let msg = r["failMsg"].as_str().unwrap_or_default(); let waits = r["waits"].as_u64().unwrap_or(0); if shape.mode == 1 { // the round that gives up logs no waiting line: "after N rounds" has N-1 of them if !msg.contains("gave up waiting for capacity after") || waits < 1 { p.push(format!("{} failed, not by giving up waiting for capacity after waiting at least once ({waits} waiting lines): {msg}", r["name"])); } } else { p.push(format!("{} failed instead of waiting for capacity: {msg}", r["name"])); } failed.push(r["name"].clone()); continue; } let w = did_of(r["receipt"].as_str().unwrap_or_default()); if !chosen(s).iter().any(|b| b["did"] == w.as_str()) { p.push(format!("{} Held on {w}, not one of the run's bidders", r["name"])); } let waits = r["waits"].as_u64().unwrap_or(0); if waits > 0 && shape.mode == 0 && shape.fails == 0 { // the rentals fit the bidders' room (fit each, sized from their live guests): a refusal is a race the next bid // settles; waiting means every bidder refused it with room left, room held by no guest of this run (a reservation // not backed by a guest: docs/acceptance/0014 pitfalls) p.push(format!("{} waited {waits} rounds for capacity though the run's rentals fit the bidders' room ({} each), refusals {}", r["name"], shape.fit, r["declined"])); } for d in r["declined"].as_array().into_iter().flatten() { // a refuser is excluded until the next waiting round, whose RFP asks every bidder again (runner.go waitForCapacity) if d["bidder"] == w.as_str() && d["round"].as_u64().unwrap_or(0) == waits { p.push(format!("{} Held on {w}, which refused it in the same round ({} waiting rounds, refusals {})", r["name"], waits, r["declined"])); } if d["status"] != 409 || d["body"]["error"] != "CapacityExceeded" { p.push(format!("{} declined {} {}: not a 409 CapacityExceeded", r["name"], d["status"], d["body"])); } declined_uris.push(d["accept"].clone()); } let c = per.get(&w).and_then(Value::as_i64).unwrap_or(0) + s.st["cpus"].as_i64().unwrap_or(0); per.insert(w, json!(c)); } for (w, c) in per.iter().filter(|_| shape.mode == 0) { if c.as_i64().unwrap_or(0) > free[w].as_i64().unwrap_or(0) { p.push(format!("bidder {w} holds {c} vCPUs of this run, its room was {}", free[w])); } let held = c.as_i64().unwrap_or(0) / s.st["cpus"].as_i64().unwrap_or(1).max(1); let m = held * s.st["vmMiB"].as_i64().unwrap_or(0); if m > s.st["freeMiB"][w].as_i64().unwrap_or(i64::MAX) { p.push(format!("bidder {w} holds {m}M of this run's guests, its memory room was {}M", s.st["freeMiB"][w])); } } if failed.len() != shape.fails { p.push(format!("{} rentals unresolved ({}), expected {}", failed.len(), json!(failed), shape.fails)); } if !p.is_empty() { if s.st["preLive"].is_object() { p.push(format!("sized around guests live before this run (free {}): {}", s.st["free"], s.st["preLive"])); } return Err(s.fail(p.join("; "))); } let round = s.st["round"].clone(); let rs = rentals(s); let refusals: usize = rs.iter().map(|r| r["declined"].as_array().map_or(0, Vec::len)).sum(); let summary = json!({ "round": round, "heldS": s.since("round"), "perBidder": per, "refusals": refusals, "failed": failed, "maxRefusalsOneRental": rs.iter().map(|r| r["declined"].as_array().map_or(0, Vec::len)).max().unwrap_or(0), "rentals": rs.iter().map(|r| json!({ "winner": did_of(r["receipt"].as_str().unwrap_or_default()), "failed": r["failed"], "refusedBy": r["declined"].as_array().into_iter().flatten().map(|d| d["bidder"].clone()).collect::>(), "rfpDeclinedBy": r["rfpDeclined"].as_array().into_iter().flatten().map(|d| json!({ "bidder": d["bidder"], "kind": d["kind"] })).collect::>(), "capacityRefusals": Value::Object(capacity_refusals(r)), "replaced": r["replaced"], "newRfp": r["newRfp"] })).collect::>(), }); // serde_json's index panics past an array's end: rounds grow by push let k = round.as_u64().unwrap_or(1) as usize - 1; if !s.st["readings"]["rounds"].is_array() { s.st["readings"]["rounds"] = json!([]); } let rounds = s.st["readings"]["rounds"].as_array_mut().unwrap(); rounds.truncate(k); rounds.push(summary); // a refused accept is kept (a bidder that has it may still act on it; a missing record would be retried until the bidder gives up) s.reading(&format!("refusedAcceptsKept_{round}"), declined_uris.len()); let mut kept = s.st["kept"].as_array().cloned().unwrap_or_default(); kept.extend(declined_uris.iter().cloned()); s.st["kept"] = json!(kept); if shape.resend { s.set("resend"); let rs = rentals(s); let mut n = 0; for (i, r) in rs.iter().enumerate() { if let Some(u) = r["receipt"].as_str() { h = h.call(get_record(&format!("acc:{i}"), u)); n += 1; } } if n > 0 { s.st["wait"] = json!(n); return Ok(h); } } Ok(verify_vms(s, h)) } /// Every chosen bidder's ComputeVMs after the round (and after the re-sends): no bidder over its room, one VM per receipt. fn verify_vms(s: &mut S, mut h: Handle) -> Handle { s.set("verify"); let mut wait = 0; for (j, b) in chosen(s).iter().enumerate() { if let Some(c) = b["cluster"].as_str() { h = h.call(vms(&format!("vms:{j}"), c)); wait += 1; } } if wait == 0 { return release(s, h); } s.st["wait"] = json!(wait); h } /// Offerings and Bidder specs read: each bidder commits cores x its overcommit in vCPUs; then the live guests before the fill. fn pre(s: &mut S, mut h: Handle) -> Result { for b in chosen(s) { let d = b["did"].as_str().unwrap_or_default(); s.st["max"][d] = json!(s.st["cores"][d].as_i64().unwrap_or(0) * s.st["overcommit"][d].as_i64().unwrap_or(2)); } s.reading("overcommit", s.st["overcommit"].clone()); s.reading("quota", s.st["quota"].clone()); s.set("pre"); let with: Vec<(usize, String)> = chosen(s).iter().enumerate().filter_map(|(j, b)| b["cluster"].as_str().map(|c| (j, c.to_string()))).collect(); s.st["wait"] = json!(with.len()); for (j, c) in &with { h = h.call(vms(&format!("pre:{j}"), c)); } if with.is_empty() { return Err(s.unmet("the target names no bidder kcp: the live guests before the fill are not read")); } Ok(h) } fn release(s: &mut S, mut h: Handle) -> Handle { s.set("release"); s.mark("release"); let rs = everyone(s); s.st["wait"] = json!(rs.len()); for (i, r) in rs.iter().enumerate() { h = h.call(host(&format!("rel:{i}"), "request.release", json!({ "name": r["name"] }))); } // the round's invocations are being released: their subscriptions end here, so a next round can open its own for i in 0..rentals(s).len() { h = h.cancel(&format!("req:{i}")); } for j in 0..s.st["fillers"].as_array().map_or(0, Vec::len) { h = h.cancel(&format!("fill:{j}")); } h } fn after_release(s: &mut S, mut h: Handle, name: &str, shape: &Shape) -> Handle { let rounds: Vec = s.st["readings"]["rounds"].as_array().cloned().unwrap_or_default(); let refusals: u64 = rounds.iter().map(|r| r["refusals"].as_u64().unwrap_or(0)).sum(); s.reading("refusals", refusals); if shape.again && s.st["round"].as_u64().unwrap_or(0) < shape.rounds { // a late re-send of every released accept (a requester's retry that lands after the release) holds nothing: the next // round, the same shape, must fit as the first did let late: Vec<(usize, Value)> = rentals(s).into_iter().enumerate().filter(|(_, r)| r["accept"].is_object()).collect(); if shape.resend && !late.is_empty() && !s.is("late") { s.set("late"); s.st["wait"] = json!(late.len() * RESENDS); for (i, r) in late { for k in 0..RESENDS { h = h.call(host(&format!("late:{i}:{k}"), "bidder.submitAccept", json!({ "name": r["accept"]["bidder"], "rkey": r["accept"]["rkey"], "cid": r["accept"]["cid"] }))); } } return h; } return start_round(s, h, name, shape); } if shape.need_refusal && refusals == 0 { if s.st["round"].as_u64().unwrap_or(0) < shape.rounds { return start_round(s, h, name, shape); } return s.fail(format!("no accept refused in {} rounds: every rental's first pick had room (a race; run again)", shape.rounds)); } s.pass() } fn drive(run: Run, shape: Shape) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let mut h = Handle::new(); match run.event { Event::Start => { if args["target"] == "prod" { return Ok(s.unmet("staging only: it fills bidders to their committed capacity (prod bidders are shared)")); } let bidders: Vec = args["bidders"].as_array().cloned().unwrap_or_default(); if bidders.iter().any(|b| !b["cluster"].is_string()) { return Ok(s.unmet("the target names no kcp for one of its bidders: every filled bidder's VMs are read")); } if bidders.len() < shape.bidders { return Ok(s.unmet(&format!("needs {} bidders; the target has {}", shape.bidders, bidders.len()))); } let picked: Vec = bidders.into_iter().take(shape.bidders).collect(); s.st["chosen"] = json!(picked); s.st["mem"] = json!(args["mem"].as_str().unwrap_or("1G")); s.st["vmMiB"] = json!(commit_mib(mib(s.st["mem"].as_str().unwrap_or_default()))); // each bidder's offering (maxCpus, maxMem) and its own Bidder spec (cpuOvercommit, maxConcurrentGuests): its room as it counts it s.st["wait"] = json!(2 * shape.bidders); s.set("offerings"); for (j, b) in picked.iter().enumerate() { h = h .call(get_record(&format!("off:{j}"), &format!("at://{}/{OFFERING}/self", b["did"].as_str().unwrap_or_default()))) .call(kube(&format!("bd:{j}"), "get", b["cluster"].as_str().unwrap_or_default(), steps::sw("Bidder", "bidder"))); } h = h.timer("deadline", (25 * shape.rounds + 15) * MIN); } Event::Timer { id } if id.starts_with("offretry:") => { let j: usize = id[9..].parse().unwrap_or(0); let b = chosen(&s).get(j).cloned().unwrap_or_default(); h = h.call(get_record(&format!("off:{j}"), &format!("at://{}/{OFFERING}/self", b["did"].as_str().unwrap_or_default()))); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in time (phase {})", s.phase()))), Event::Timer { id } if id.starts_with("round:") => { if (s.is("held") || s.is("settle")) && id == format!("round:{}", s.st["round"]) { return Ok(s.fail(format!("not every rental Held in 20 min: {}", s.st["rentals"]))); } } // each bidder's room: its offering's maxCpus less its live guests; sized so `fit` VMs fit the tightest Event::Timer { id } if id == "sized" => { let mut free = json!({}); for b in chosen(&s) { let d = b["did"].as_str().unwrap_or_default(); free[d] = json!(s.st["max"][d].as_i64().unwrap_or(0) - s.st["used"][d].as_i64().unwrap_or(0)); } if shape.mode > 0 { // every bidder filled to its last vCPU by one pinned filler each; then one rental of `cpus` that has no room let small = args["cpus"].as_i64().unwrap_or(2); let mut plan = json!({}); for (d, f) in free.as_object().cloned().unwrap_or_default() { if f.as_i64().unwrap_or(0) > 0 { plan[&d] = f.clone(); } free[&d] = json!(0); } s.reading("fillers", plan.clone()); s.reading("free", free.clone()); s.reading("cpus", small); s.reading("mem", s.st["mem"].clone()); s.st["fill"] = plan; s.st["free"] = free; s.st["cpus"] = json!(small); h = start_round(&mut s, h, &name, &shape); return Ok(s.keep(h)); } let tight = free.as_object().into_iter().flatten().filter_map(|(_, v)| v.as_i64()).min().unwrap_or(0); let mut cpus = if shape.fit == 1 { tight / 2 + 1 } else { tight / shape.fit }; // big bidders (a dev network's: the whole host's CPUs) get a filler each, so the contending guests stay // small (input `cpus`, default 2 over 8 free): many large guests booting at once miss the 2 min on-network let cores = chosen(&s).iter().filter_map(|b| s.st["cores"][b["did"].as_str().unwrap_or_default()].as_i64()).min().unwrap_or(0); let auto = if shape.small > 0 { shape.small } else if tight > 8 || cpus > cores { 2 } else { 0 }; let small = args["cpus"].as_i64().unwrap_or(auto); if small > 0 && small < cpus { let mut plan = json!({}); for (d, f) in free.as_object().cloned().unwrap_or_default() { let filler = f.as_i64().unwrap_or(0) - shape.fit * small; if filler > 0 { plan[&d] = json!(filler); } free[&d] = json!(shape.fit * small); } s.reading("fillers", plan.clone()); s.st["fill"] = plan; cpus = small; } // the room of a bidder in guests of this run: its vCPUs, its memory (maxMem less its live guests' and the fillers' // committed memory) and its guest quota (less its live guests and the fillers), whichever is least let vm_mib = s.st["vmMiB"].as_i64().unwrap_or(0).max(1); let mut free_mib = json!({}); let mut room = 0; for b in chosen(&s) { let d = b["did"].as_str().unwrap_or_default(); let cores = s.st["cores"][d].as_i64().unwrap_or(i64::MAX).max(1); let filler = s.st["fill"][d].as_i64().unwrap_or(0); let fillers = (filler + cores - 1) / cores; let m = s.st["maxMiB"][d].as_i64().unwrap_or(0) - s.st["usedMiB"][d].as_i64().unwrap_or(0) - fillers * vm_mib; free_mib[d] = json!(m); let mut n = (free[d].as_i64().unwrap_or(0) / cpus.max(1)).min(m / vm_mib); if let Some(q) = s.st["quota"][d].as_i64().filter(|q| *q > 0) { n = n.min(q - s.st["guests"][d].as_i64().unwrap_or(0) - fillers); } room += n.max(0); } s.reading("free", free.clone()); s.reading("freeMiB", free_mib.clone()); s.reading("cpus", cpus); s.reading("mem", s.st["mem"].clone()); s.st["free"] = free; s.st["freeMiB"] = free_mib; s.st["cpus"] = json!(cpus); if cpus < 1 || room < (shape.rentals - shape.fails) as i64 { return Ok(s.fail(format!("the bidders have room for {room} VMs of {cpus} vCPUs and {vm_mib}M, the run needs {}: {} {}", shape.rentals, s.st["free"], s.st["freeMiB"]))); } h = start_round(&mut s, h, &name, &shape); } Event::Timer { id } if id.starts_with("fillers:") => { if s.is("fillers") && id == format!("fillers:{}", s.st["round"].as_u64().unwrap_or(0) + 1) { return Ok(s.fail(format!("not every filler Held in 15 min: {}", s.st["fillers"]))); } } Event::Message { id, data, .. } if id.starts_with("fill:") => { let j: usize = id[5..].parse().unwrap_or(0); let f = &mut s.st["fillers"][j]; if f["receipt"].is_null() && data["receipt"].is_string() { f["receipt"] = data["receipt"].clone(); } match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Held" => f["held"] = json!(true), "contract" if data["phase"] == "Failed" && f["held"] != true => { return Ok(s.fail(format!("filler {} failed before Held: {}", s.st["fillers"][j]["name"], data["message"]))); } "exited" if f["held"] != true => return Ok(s.fail(format!("filler {} exited before Held: {data}", s.st["fillers"][j]["name"]))), _ => {} } let fs = s.st["fillers"].as_array().cloned().unwrap_or_default(); if s.is("fillers") && fs.iter().all(|f| f["held"] == true && f["receipt"].is_string()) { // each filler on its own bidder (it was pinned there) if let Some(f) = fs.iter().find(|f| did_of(f["receipt"].as_str().unwrap_or_default()) != f["bidder"].as_str().unwrap_or_default()) { return Ok(s.fail(format!("filler {} Held off its bidder: {}", f["name"], f["receipt"]))); } s.reading("fillersHeldS", s.since("fillers")); h = launch(&mut s, h, &name, &shape); } } Event::Timer { id } if id == "settle" => { h = match check(&mut s, h, &shape) { Ok(h) => h, Err(f) => return Ok(f), }; } Event::Timer { id } if id == "poll" => { s.st["wait"] = json!(0); for (j, b) in chosen(&s).iter().enumerate() { if let Some(c) = b["cluster"].as_str() { h = h.call(vms(&format!("post:{j}"), c)); s.st["wait"] = json!(s.st["wait"].as_u64().unwrap_or(0) + 1); } } s.st["left"] = json!([]); } Event::Message { id, data, .. } if id.starts_with("req:") => { let i: usize = id[4..].parse().unwrap_or(0); let (freed, held_phase, since_freed) = (s.has("freed"), s.is("held"), s.since("freed")); let after_settle = !s.is("held") && !s.is("settle"); let mut free_filler = false; let mut fail_now: Option = None; let mut held_readings: Option<(f64, Value, Value)> = None; let r = &mut s.st["rentals"][i]; // the receipt: from the requester's own lines (requester_*, the core log); a refused accept has none if r["receipt"].is_null() && data["receipt"].is_string() { r["receipt"] = data["receipt"].clone(); } match data["msg"].as_str().unwrap_or_default() { "requester_submit_accept_declined" => { // its round: the waiting lines before it (both requester_* lines of the core log, in order); a waiting // round clears the requester's refusers (runner.go waitForCapacity: the next RFP asks every bidder) let d = json!({ "bidder": data["bidder"], "accept": data["accept"], "status": data["status"], "body": data["body"], "round": r["waits"].as_u64().unwrap_or(0) }); r["declined"].as_array_mut().map(|a| a.push(d)); } "requester_bidder_declined" => { let round = r["waits"].as_u64().unwrap_or(0); let d = json!({ "bidder": data["bidder"], "kind": data["kind"], "reason": data["reason"], "round": round }); if !r["rfpDeclined"].is_array() { r["rfpDeclined"] = json!([]); } r["rfpDeclined"].as_array_mut().map(|a| a.push(d)); if shape.mode > 0 && held_phase && !freed && data["kind"] == "capacity" && r["bidsBy"].as_array().into_iter().flatten().any(|b| b["bidder"] == data["bidder"] && b["round"] == round) { fail_now = Some(format!("{} round {round}: {} declined its RFP for capacity and also bid on it", r["name"], data["bidder"])); } } // giveup and wait hold every bidder full from before the rental's first RFP until the end (giveup) or the // filler's release (wait), so a bidder that declined a round's RFP for capacity has no room in that round: // its bid in the same round, in either order, is a decline and a bid from one bidder (docs/protocol/market.md) "requester_bid_received" if shape.mode > 0 && held_phase && !freed => { let round = r["waits"].as_u64().unwrap_or(0); if !r["bidsBy"].is_array() { r["bidsBy"] = json!([]); } r["bidsBy"].as_array_mut().map(|a| a.push(json!({ "bidder": data["bidder"], "round": round }))); if r["rfpDeclined"].as_array().into_iter().flatten() .any(|d| d["bidder"] == data["bidder"] && d["kind"] == "capacity" && d["round"] == round) { fail_now = Some(format!("{} round {round}: {} declined its RFP for capacity and also bid on it", r["name"], data["bidder"])); } } "requester_accept_declined_replace" => r["replaced"] = json!(r["replaced"].as_u64().unwrap_or(0) + 1), "requester_bids_exhausted_new_rfp" => r["newRfp"] = json!(r["newRfp"].as_u64().unwrap_or(0) + 1), "requester_waiting_for_capacity" => { r["waits"] = json!(r["waits"].as_u64().unwrap_or(0) + 1); r["lastSlot"] = data["slot"].clone(); r["lastRound"] = data["round"].clone(); if shape.mode == 2 && held_phase && !freed && r["waits"].as_u64().unwrap_or(0) >= 2 { // two rounds shown: free the first filler's bidder free_filler = true; } } "contract" if data["phase"] == "Held" => { if shape.mode == 2 { if !freed { fail_now = Some("the rental was Held before any filler was released: the bidders were not full".into()); } else if since_freed > 300.0 { fail_now = Some(format!("the waiting rental was Held {since_freed} s after the filler's release (more than one backoff and a boot: 300 s)")); } held_readings = Some((since_freed, r["waits"].clone(), r["lastSlot"].clone())); } r["held"] = json!(true); if r["receipt"].is_null() { r["receipt"] = data["receipt"].clone(); } } "contract" if data["phase"] == "Failed" && r["held"] != true && (shape.fails > 0 || shape.mode > 0) => { r["failed"] = json!(true); r["failMsg"] = data["message"].clone(); } "contract" if data["phase"] == "Failed" && r["held"] != true => { return Ok(s.fail(format!("{} failed before Held: {}", s.st["rentals"][i]["name"], data["message"]))); } "exited" if r["held"] != true && r["failed"] != true && !(after_settle && r["waits"].as_u64().unwrap_or(0) > 0) => return Ok(s.fail(format!("{} exited before Held: {}", s.st["rentals"][i]["name"], data))), _ => {} } if let Some(m) = fail_now { return Ok(s.fail(m)); } if let Some((t, rounds, slot)) = held_readings { s.reading("heldAfterFreeS", t); s.reading("waitRounds", rounds); s.reading("waitSlot", slot); } if free_filler { s.flag("freed"); s.mark("freed"); s.st["fillers"][0]["failed"] = json!(true); let n = s.st["fillers"][0]["name"].clone(); h = h.call(host("freeing", "request.release", json!({ "name": n }))); } // every rental settled: the request log (Held, Failed) and the core log (requester_*) are two files, so // their lines settle 5 s before the check if s.is("held") && rentals(&s).iter().all(|r| (r["held"] == true && r["receipt"].is_string()) || r["failed"] == true || (shape.mode == 0 && shape.fails > 0 && r["waits"].as_u64().unwrap_or(0) > 0)) { s.set("settle"); h = h.timer("settle", 5_000); } } Event::Result { id, result, .. } => { // a bidder publishes its offering once its pinned guest image verified (a network that just came up may be // seconds to minutes ahead of it): the read is asked again every 10 s, 10 min at most if id.starts_with("off:") && result.output().is_err_and(|e| e.contains("RecordNotFound")) { let j: usize = id[4..].parse().unwrap_or(0); let n = s.st["offTries"].as_u64().unwrap_or(0) + 1; s.st["offTries"] = json!(n); if n > 60 * chosen(&s).len() as u64 { return Ok(s.fail(format!("{id}: the bidder's offering is not published after 10 min"))); } return Ok(s.keep(h.timer(&format!("offretry:{j}"), 10_000))); } // the release of a rental that Failed (no VM, its process already ending) may find the invocation's kcp // gone ("its kcp does not answer", "no invocation"): nothing is left to release, the records are checked after let tolerated = id.starts_with("rel:") && result.output().is_err() && id[4..].parse::().ok().and_then(|i| everyone(&s).get(i).map(|r| r["failed"] == true)) == Some(true); let o = if tolerated { json!({}) } else { match out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), } }; let j: usize = id.split(':').nth(1).and_then(|x| x.parse().ok()).unwrap_or(0); let b = chosen(&s).get(j).cloned().unwrap_or_default(); let did = b["did"].as_str().unwrap_or_default().to_string(); match id.split(':').next().unwrap_or_default() { "off" => { let Some(m) = o["value"]["maxCpus"].as_i64() else { return Ok(s.fail(format!("bidder {did}'s offering has no maxCpus: {}", o["value"]))); }; s.st["cores"][&did] = json!(m); s.st["maxMiB"][&did] = json!(mib(o["value"]["maxMem"].as_str().unwrap_or_default())); if landed(&mut s) { h = match pre(&mut s, h) { Ok(h) => h, Err(u) => return Ok(u), }; } } "bd" => { // the bidder's commitment per core (0: its default, DefaultCPUOvercommit 2) and its guest quota (0: none) let n = o["spec"]["cpuOvercommit"].as_i64().filter(|n| *n > 0).unwrap_or(2); if let Some(want) = args["overcommit"].as_i64() && want != n { return Ok(s.fail(format!("input overcommit {want}, but bidder {did} commits {n} vCPUs per core (its Bidder spec.cpuOvercommit)"))); } s.st["overcommit"][&did] = json!(n); s.st["quota"][&did] = json!(o["spec"]["maxConcurrentGuests"].as_i64().unwrap_or(0)); if landed(&mut s) { h = match pre(&mut s, h) { Ok(h) => h, Err(u) => return Ok(u), }; } } "acc" => { // the receipt names the accept: submitted again RESENDS times at once to the bidder that holds it let i: usize = id[4..].parse().unwrap_or(0); let (uri, cid) = (o["value"]["accept"]["uri"].as_str().unwrap_or_default(), o["value"]["accept"]["cid"].as_str().unwrap_or_default()); let w = did_of(s.st["rentals"][i]["receipt"].as_str().unwrap_or_default()); let Some(b) = chosen(&s).into_iter().find(|b| b["did"] == w.as_str()) else { return Ok(s.fail(format!("{}'s winner {w} is not one of the run's bidders", s.st["rentals"][i]["name"]))); }; if did_of(uri) != args["did"].as_str().unwrap_or_default() || cid.is_empty() { return Ok(s.fail(format!("{}'s receipt names the accept {uri} {cid}, not one of this account's", s.st["rentals"][i]["name"]))); } let rkey = uri.rsplit('/').next().unwrap_or_default().to_string(); s.st["rentals"][i]["accept"] = json!({ "rkey": rkey, "cid": cid, "bidder": b["name"] }); let n = s.st["wait"].as_u64().unwrap_or(1) - 1 + RESENDS as u64; s.st["wait"] = json!(n); for k in 0..RESENDS { h = h.call(host(&format!("rs:{i}:{k}"), "bidder.submitAccept", json!({ "name": b["name"], "rkey": rkey, "cid": cid }))); } } "late" => { if o["reply"]["ok"] != true { return Ok(s.fail(format!("{id}: the accept re-sent after its release was not answered ok: {o}"))); } if landed(&mut s) { s.reading(&format!("lateResent_{}", s.st["round"]), s.st["rentals"].as_array().map_or(0, Vec::len) * RESENDS); h = start_round(&mut s, h, &name, &shape); } } "rs" => { if o["reply"]["ok"] != true { return Ok(s.fail(format!("{id}: the re-sent accept was not answered ok: {o}"))); } let n = s.st["resent"].as_u64().unwrap_or(0) + 1; s.st["resent"] = json!(n); if landed(&mut s) { s.reading(&format!("resent_{}", s.st["round"]), n); s.st["resent"] = json!(0); h = verify_vms(&mut s, h); } } "pre" => { let live = live_vms(&o); let used: i64 = live.iter().map(cpus_of).sum(); s.st["used"][&did] = json!(used); s.st["usedMiB"][&did] = json!(live.iter().map(mem_of).sum::()); s.st["guests"][&did] = json!(live.len()); if !live.is_empty() { s.st["preLive"][&did] = json!(live.iter().map(|v| json!({ "name": v["metadata"]["name"], "phase": v["status"]["phase"], "cpus": v["spec"]["vm"]["cpus"], "mem": v["spec"]["vm"]["mem"], "sandbox": !v["spec"]["sandbox"].is_null(), "created": v["metadata"]["creationTimestamp"], "receipt": v["spec"]["contract"]["receipt"]["uri"] })).collect::>()); } if landed(&mut s) { if s.st["preLive"].is_object() { s.reading("preLive", s.st["preLive"].clone()); } return Ok(s.keep(h.timer("sized", 1))); } } "vms" => { let lv = live_vms(&o); let live: i64 = lv.iter().map(cpus_of).sum(); let live_mib: i64 = lv.iter().map(mem_of).sum(); let max = s.st["max"][&did].as_i64().unwrap_or(0); let max_mib = s.st["maxMiB"][&did].as_i64().unwrap_or(0); let quota = s.st["quota"][&did].as_i64().unwrap_or(0); let k = s.st["round"].as_u64().unwrap_or(1) as usize - 1; s.st["readings"]["rounds"][k]["liveCpus"][&did] = json!(live); s.st["readings"]["rounds"][k]["liveMiB"][&did] = json!(live_mib); if live > max { return Ok(s.fail(format!("bidder {did} runs {live} guest vCPUs of its {max}: overcommitted"))); } if max_mib > 0 && live_mib > max_mib { return Ok(s.fail(format!("bidder {did} runs {live_mib}M of guest memory of its {max_mib}M: overcommitted"))); } if quota > 0 && lv.len() as i64 > quota { return Ok(s.fail(format!("bidder {did} runs {} guests, its quota is {quota}", lv.len()))); } // one ComputeVM per receipt of this run, however often its accept was sent let ours: Vec = everyone(&s).iter().filter_map(|r| r["receipt"].as_str().map(String::from)).collect(); for u in &ours { let n = lv.iter().filter(|v| v["spec"]["contract"]["receipt"]["uri"].as_str() == Some(u.as_str())).count(); if n > 1 { return Ok(s.fail(format!("bidder {did} runs {n} ComputeVMs for the receipt {u}"))); } } if landed(&mut s) { h = release(&mut s, h); } } "gone" if id == "gone:declined" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("refused accepts not deleted: {} {}", o["left"], o["errors"]))); } if landed(&mut s) { h = release(&mut s, h); } } "rel" => { if landed(&mut s) { s.reading("releaseS", s.since("release")); let rs = everyone(&s); s.st["wait"] = json!(rs.len()); s.st["uris"] = json!([]); for (i, r) in rs.iter().enumerate() { h = h.call(host(&format!("rec:{i}"), "request.records", json!({ "name": r["name"] }))); } } } "rec" => { let kept = s.st["kept"].as_array().cloned().unwrap_or_default(); for u in o["uris"].as_array().into_iter().flatten().filter(|u| !kept.contains(u)) { s.st["uris"].as_array_mut().map(|a| a.push(u.clone())); } if landed(&mut s) { s.reading("recordsChecked", s.st["uris"].as_array().map_or(0, Vec::len)); h = h.call(host("gone:records", "assert.recordsGone", json!({ "uris": s.st["uris"] }))); } } "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left after the release: {} {}", o["left"], o["errors"]))); } if chosen(&s).iter().all(|b| b["cluster"].is_null()) { let h = after_release(&mut s, h, &name, &shape); return Ok(s.keep(h)); } s.set("orphans"); s.mark("orphans"); h = h.timer("poll", 1); } "post" => { let ours: Vec = everyone(&s).iter().filter_map(|r| r["receipt"].as_str().map(String::from)).collect(); for v in o["items"].as_array().into_iter().flatten() { if v["spec"]["contract"]["receipt"]["uri"].as_str().is_some_and(|u| ours.iter().any(|x| x == u)) { s.st["left"].as_array_mut().map(|a| a.push(v["metadata"]["name"].clone())); } } if landed(&mut s) { if s.st["left"].as_array().is_some_and(|l| !l.is_empty()) { if s.since("orphans") > 180.0 { return Ok(s.fail(format!("ComputeVMs of this run left 3 min after the release: {}", s.st["left"]))); } return Ok(s.keep(h.timer("poll", 5_000))); } let h = after_release(&mut s, h, &name, &shape); return Ok(s.keep(h)); } } _ => {} } } Event::Closed { id, error } if id.starts_with("req:") && s.is("held") && s.st["rentals"][id[4..].parse::().unwrap_or(0)]["failed"] != true => { return Ok(s.fail(format!("{id} closed: {}", error.unwrap_or_default()))); } _ => {} } Ok(s.keep(h)) } fn fallback(r: Run) -> Result { drive(r, Shape { bidders: 2, rentals: 2, fit: 1, need_refusal: false, rounds: 2, fails: 0, mode: 0, expires: "1h", small: 0, resend: false, again: false }) } fn concurrent_0014(r: Run) -> Result { drive(r, Shape { bidders: 2, rentals: 4, fit: 2, need_refusal: false, rounds: 2, fails: 0, mode: 0, expires: "1h", small: 0, resend: true, again: true }) } fn exhausted_0014(r: Run) -> Result { drive(r, Shape { bidders: 2, rentals: 3, fit: 1, need_refusal: true, rounds: 1, fails: 1, mode: 0, expires: "1h", small: 0, resend: false, again: false }) } /// giveup: both bidders full (one pinned filler each), one rental with --expires-in 150s: it waits for capacity in rounds /// (a fresh RFP after 30 s, 1 min, ...) and fails at its deadline with "gave up waiting for capacity after N rounds". fn giveup_0014(r: Run) -> Result { drive(r, Shape { bidders: 2, rentals: 1, fit: 0, need_refusal: false, rounds: 1, fails: 1, mode: 1, expires: "150s", small: 0, resend: false, again: false }) } /// wait: both bidders full, one rental with --expires-in 15m waits (rounds 1..k logged); after two rounds a filler is /// released: the rental is Held on the freed bidder within one backoff and a boot. fn wait_0014(r: Run) -> Result { drive(r, Shape { bidders: 2, rentals: 1, fit: 0, need_refusal: false, rounds: 1, fails: 0, mode: 2, expires: "15m", small: 0, resend: false, again: false }) } /// impossible: requests NO bidder can ever take fail fast with the reason instead of waiting for capacity: a VM of more vCPUs than /// the largest offering's maxCpus, and (when no offering sets nestedVirt) a nestedVirt request. Each must end Failed within 120 s /// naming why; nothing is Held and no requester_waiting_for_capacity line appears. fn impossible_0014(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let bidders: Vec = args["bidders"].as_array().cloned().unwrap_or_default(); let mut h = Handle::new(); // the cases still to run: "maxCpus", then "nested" when no offering sets it let launch = |s: &mut S, h: Handle, case: &str| -> Handle { s.set(case); s.mark("case"); s.st["cur"] = json!({ "case": case, "waits": 0 }); let a = match case { "maxCpus" => json!(["--hold", "--cpus", (s.st["maxMax"].as_i64().unwrap_or(0) + 1).to_string(), "--mem", "1G", "--expires-in", "1h"]), "nested" => json!(["--hold", "--cpus", "1", "--mem", "1G", "--nested-virt", "--expires-in", "1h"]), // a deadline that passes while the requester starts: the CLI fixes the label when it is invoked and creates the contract // only once its own kcp is Ready (seconds later: 9 s and more on the dev nets), so this deadline is always behind the // contract, and no RFP may go out _ => json!(["--hold", "--cpus", "1", "--mem", "1G", "--expires-in", "1s"]), }; let (id, tag) = match case { "maxCpus" => ("req", "mx"), "nested" => ("req2", "nv"), _ => ("req3", "ex"), }; h.call(steps::host_sub(id, "request.start", json!({ "name": format!("{name}-{tag}"), "args": a }))).timer(&format!("case:{case}"), 4 * MIN) }; match run.event { Event::Start => { if bidders.is_empty() { return Ok(s.unmet("the target names no bidder")); } s.st["wait"] = json!(bidders.len()); s.st["maxMax"] = json!(0); s.st["nestedAny"] = json!(false); s.set("offerings"); for (j, b) in bidders.iter().enumerate() { h = h.call(get_record(&format!("off:{j}"), &format!("at://{}/{OFFERING}/self", b["did"].as_str().unwrap_or_default()))); } h = h.timer("deadline", 15 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 15 min (phase {})", s.phase()))), Event::Timer { id } if id.starts_with("case:") => { if s.is(&id[5..]) { return Ok(s.fail(format!("the {} request did not fail within 4 min: it waits for something nobody can serve", &id[5..]))); } } Event::Result { id, result, .. } if id.starts_with("off:") => { let o = match steps::out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; let m = o["value"]["maxCpus"].as_i64().unwrap_or(0); if m > s.st["maxMax"].as_i64().unwrap_or(0) { s.st["maxMax"] = json!(m); } if o["value"]["nestedVirt"] == true { s.st["nestedAny"] = json!(true); } if landed(&mut s) { if s.st["maxMax"].as_i64().unwrap_or(0) <= 0 { return Ok(s.unmet("no offering names a maxCpus")); } s.reading("maxCpus", s.st["maxMax"].clone()); s.reading("nestedOffered", s.st["nestedAny"].clone()); h = launch(&mut s, h, "maxCpus"); } } Event::Closed { id, error } if (id == "req" && s.is("maxCpus")) || (id == "req2" && s.is("nested")) || (id == "req3" && s.is("expired")) => { return Ok(s.fail(format!("the request closed before it failed: {}", error.unwrap_or_default()))); } Event::Message { id, data, .. } if (id == "req" && s.is("maxCpus")) || (id == "req2" && s.is("nested")) || (id == "req3" && s.is("expired")) => { let case = s.st["cur"]["case"].as_str().unwrap_or_default().to_string(); match data["msg"].as_str().unwrap_or_default() { "requester_waiting_for_capacity" => return Ok(s.fail(format!("the {case} request waits for capacity: {data}"))), "contract" if data["phase"] == "Held" => return Ok(s.fail(format!("the {case} request was Held: {data}"))), "contract" if data["phase"] == "Failed" => { let msg = data["message"].as_str().unwrap_or_default().to_string(); let want = match case.as_str() { "maxCpus" => "maxCpus", "nested" => "nestedVirt", _ => "deadline", }; let t = s.since("case"); s.reading(&format!("{case}_failed_s"), t); s.reading(&format!("{case}_message"), msg.clone()); if !msg.contains(want) || (case != "expired" && !msg.contains("no bidder") && !msg.contains("requester_no_bidders")) { return Ok(s.fail(format!("the {case} request failed, but not naming why ({want}): {msg}"))); } if t > 120.0 { return Ok(s.fail(format!("the {case} request took {t} s to fail (more than 120 s): {msg}"))); } s.set(&format!("done-{case}")); s.flag(&format!("{case}Done")); if case == "maxCpus" && s.st["nestedAny"] != true { h = launch(&mut s, h, "nested"); } else if case != "expired" { h = launch(&mut s, h, "expired"); } else { return Ok(s.pass()); } } _ => {} } } _ => {} } Ok(s.keep(h)) } /// The CID of a record that does not exist (the example CID of the atproto docs). const NO_SUCH_CID: &str = "bafyreigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"; /// An accept whose record cannot be read is abandoned by the bidder within this many seconds of the submit (the bound: 5 failed tries /// and 3 min, the outbox's stall retry is the next attempt at about 5.5 min). const ABANDON_WITHIN_S: f64 = 540.0; /// After the abandon the bidder does not try again for this long. const ABANDON_QUIET_S: f64 = 120.0; /// abandon: a requester that submits an accept and has no such record (it deleted it before the bidder read it): the bidder retries /// it a bounded number of times and gives up (bidder_accept_abandoned, one line, with the tries and the age), after which it logs /// nothing more of that accept: no bidder_accept_unresolved, no "the provision of an accept failed". The first bidder of the target. fn abandon_0014(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let bidders: Vec = args["bidders"].as_array().cloned().unwrap_or_default(); let polls = |s: &mut S, h: Handle| -> Handle { let (name, uri) = (s.st["bidder"].clone(), s.st["uri"].as_str().unwrap_or_default().to_string()); s.st["wait"] = json!(3); let rows = |id: &str, text: &str| host(id, "bidder.logLines", json!({ "name": name, "texts": [[uri, text]], "keys": ["accept", "kind", "tries", "age", "reservationReleased", "err"] })); h.call(rows("ab", "bidder_accept_abandoned")).call(rows("un", "bidder_accept_unresolved")).call(rows("gf", "the provision of an accept failed")) }; let mut h = Handle::new(); match run.event { Event::Start => { let Some(b) = bidders.first() else { return Ok(s.unmet("the target names no bidder")) }; s.st["bidder"] = b["name"].clone(); s.set("submit"); let rkey = format!("abandon{}", (s.now * 1000.0) as u64); h = h.call(host("sub", "bidder.submitAccept", json!({ "name": b["name"], "rkey": rkey, "cid": NO_SUCH_CID }))).timer("deadline", 15 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 15 min (phase {})", s.phase()))), Event::Timer { id } if id == "poll" => { if s.is("watch") || s.is("settle") { h = polls(&mut s, h); } } Event::Result { id, result, .. } => { let o = match steps::out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; if id == "sub" { s.st["uri"] = o["acceptUri"].clone(); s.reading("submitted", json!({ "uri": o["acceptUri"], "reply": o["reply"] })); s.mark("submitted"); s.set("watch"); return Ok(s.keep(h.timer("poll", 20_000))); } let n = o["lines"].as_array().map_or(0, Vec::len); s.st[&id] = json!({ "n": n, "lines": o["lines"] }); if !landed(&mut s) { return Ok(s.keep(h)); } let (ab, un, gf) = (s.st["ab"]["n"].as_u64().unwrap_or(0), s.st["un"]["n"].as_u64().unwrap_or(0), s.st["gf"]["n"].as_u64().unwrap_or(0)); if gf > 0 { return Ok(s.fail(format!("the generic provision failure was logged {gf} times for an accept that is not there: {}", s.st["gf"]["lines"]))); } if s.is("watch") { let t = s.since("submitted"); if ab == 0 { if t > ABANDON_WITHIN_S { return Ok(s.fail(format!("the bidder did not abandon the accept within {ABANDON_WITHIN_S} s ({un} unresolved lines): {}", s.st["un"]["lines"]))); } return Ok(s.keep(h.timer("poll", 20_000))); } if ab != 1 { return Ok(s.fail(format!("{ab} bidder_accept_abandoned lines for one accept: {}", s.st["ab"]["lines"]))); } s.reading("abandoned_s", t); s.reading("abandoned", s.st["ab"]["lines"][0].clone()); s.reading("unresolved_before_abandon", un); s.st["un0"] = json!(un); s.mark("abandoned"); s.set("settle"); return Ok(s.keep(h.timer("poll", 20_000))); } if ab != 1 { return Ok(s.fail(format!("{ab} bidder_accept_abandoned lines after the abandon: {}", s.st["ab"]["lines"]))); } if un != s.st["un0"].as_u64().unwrap_or(0) { return Ok(s.fail(format!("the bidder logged {un} bidder_accept_unresolved lines, {} at the abandon: it retries an abandoned accept", s.st["un0"]))); } if s.since("abandoned") < ABANDON_QUIET_S { return Ok(s.keep(h.timer("poll", 20_000))); } s.reading("quiet_s", s.since("abandoned")); return Ok(s.pass()); } _ => {} } Ok(s.keep(h)) } /// The bidder log line of a VM its placer saw inactive and killed before the VMM answered (91103758, fixed f15dda15). const EARLY_KILL: &str = "before its VMM answered"; /// backtoback: `count` (6) small VMs one after another on ONE bidder (the target's first), each pinned to it: rent, /// Held, release, the next (input overlap: the next is requested while the release runs); a unit of the same slot was just reset each time. Every one Held, none ended before it /// came on network, the bidder's log gains no EARLY_KILL line. A rental that ends before Held is recorded and the run /// goes on, so the readings count them all. fn backtoback_0014(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let count = args["count"].as_u64().unwrap_or(6); let overlap = args["overlap"] == true; let mut h = Handle::new(); let i = s.st["i"].as_u64().unwrap_or(0); let rental = |i: u64| format!("{name}-b{i}"); let next = |s: &mut S, h: Handle, i: u64| -> Handle { s.st["i"] = json!(i); s.set("held"); s.mark("rent"); let a = json!(["--hold", "--cpus", s.st["cpus"].to_string(), "--mem", s.st["mem"], "--expires-in", "1h"]); h.call(steps::host_sub(&format!("req:{i}"), "request.start", json!({ "name": rental(i), "args": a, "bidderDids": [s.st["bidder"]["did"]] }))) .timer(&format!("held:{i}"), 6 * MIN) }; match run.event { Event::Start => { if args["target"] == "prod" { return Ok(s.unmet("staging only: it books one bidder for a run of VMs (prod bidders are shared)")); } let Some(b) = args["bidders"].as_array().and_then(|a| a.first()).cloned() else { return Ok(s.unmet("needs a bidder; the target has none")); }; s.st["bidder"] = b.clone(); s.st["cpus"] = json!(args["cpus"].as_i64().unwrap_or(1)); s.st["mem"] = json!(args["mem"].as_str().unwrap_or("1G")); s.st["results"] = json!([]); s.reading("bidder", b["did"].clone()); s.set("baseline"); // the bidder's offering first (it publishes it once its pinned guest image verified: minutes after dev up) h = h .call(host("log0", "bidder.logCount", json!({ "name": b["name"], "texts": [EARLY_KILL] }))) .call(get_record("off", &format!("at://{}/com.publicdomainrelay.temp.market.offering/self", b["did"].as_str().unwrap_or_default()))) .timer("deadline", (count * 7 + 10) * MIN + 10 * MIN); } Event::Timer { id } if id == "offretry" => { h = h.call(get_record("off", &format!("at://{}/com.publicdomainrelay.temp.market.offering/self", s.st["bidder"]["did"].as_str().unwrap_or_default()))); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in time (phase {}, VM {i})", s.phase()))), Event::Timer { id } if id == format!("held:{i}") && s.is("held") => { s.st["results"].as_array_mut().map(|a| a.push(json!({ "vm": rental(i), "held": false, "why": "not Held in 6 min" }))); s.set("release"); h = h.cancel(&format!("req:{i}")).call(host(&format!("rel:{i}"), "request.release", json!({ "name": rental(i) }))); } Event::Message { id, data, .. } if id == format!("req:{i}") && s.is("held") => { if data["receipt"].is_string() && s.st["receipt"].is_null() { s.st["receipt"] = data["receipt"].clone(); } if data["msg"] == "requester_guest_on_network" { s.flag(&format!("on:{i}")); } let ended = match (data["msg"].as_str().unwrap_or_default(), data["phase"].as_str().unwrap_or_default()) { ("contract", "Held") => Some(json!({ "vm": rental(i), "held": true, "heldS": s.since("rent"), "winner": data["receipt"].as_str().or(s.st["receipt"].as_str()).map(|r| r.trim_start_matches("at://").split('/').next().unwrap_or_default().to_string()) })), // the bidder ended it (vm.terminated) or the contract failed before the guest came up ("contract", p @ ("Released" | "Failed")) => Some(json!({ "vm": rental(i), "held": false, "phase": p, "onNetwork": s.has(&format!("on:{i}")), "why": data["message"] })), ("exited", _) => Some(json!({ "vm": rental(i), "held": false, "why": format!("run request exited: {data}") })), _ => None, }; if let Some(r) = ended { s.st["results"].as_array_mut().map(|a| a.push(r)); s.st["receipt"] = Value::Null; s.set("release"); h = h.cancel(&format!("req:{i}")).call(host(&format!("rel:{i}"), "request.release", json!({ "name": rental(i) }))); // overlap: the next VM is requested while this one's release runs (its unit is reset as the next is placed) if overlap && i + 1 < count { h = next(&mut s, h, i + 1); } } } Event::Result { id, result, .. } => { let r = result.output(); if id == "off" { if r.as_ref().is_err_and(|e| e.contains("RecordNotFound")) { let n = s.st["offTries"].as_u64().unwrap_or(0) + 1; s.st["offTries"] = json!(n); if n > 60 { return Ok(s.fail("the bidder's offering is not published after 10 min")); } return Ok(s.keep(h.timer("offretry", 10_000))); } if let Err(e) = r { return Ok(s.fail(format!("off: {e}"))); } s.flag("offered"); if s.has("logged") { h = next(&mut s, h, 0); } return Ok(s.keep(h)); } match id.as_str() { "log0" | "log1" => { let o = match out(&s, &id, r) { Ok(o) => o, Err(f) => return Ok(f), }; let n = o["counts"][EARLY_KILL].as_u64().unwrap_or(0); if id == "log0" { s.st["early0"] = json!(n); s.flag("logged"); if s.has("offered") { h = next(&mut s, h, 0); } } else { let early = n.saturating_sub(s.st["early0"].as_u64().unwrap_or(0)); s.reading("earlyKillLines", early); let rs = s.st["results"].as_array().cloned().unwrap_or_default(); let bad: Vec = rs.iter().filter(|r| r["held"] != true).cloned().collect(); s.reading("vms", json!(rs)); s.reading("held", rs.len() - bad.len()); if !bad.is_empty() || early > 0 { return Ok(s.fail(format!("{} of {} VMs never Held on one bidder back to back, {early} '{EARLY_KILL}' lines: {}", bad.len(), rs.len(), json!(bad)))); } s.set("records"); s.st["wait"] = json!(count); s.st["uris"] = json!([]); for k in 0..count { h = h.call(host(&format!("rec:{k}"), "request.records", json!({ "name": rental(k) }))); } } } x if x.starts_with("rel:") => { // a Failed invocation leaves nothing to release if let Err(e) = &r && !e.contains("no invocation") && !e.contains("kcp does not answer") { return Ok(s.fail(format!("{id}: {e}"))); } if overlap { let n = s.st["rels"].as_u64().unwrap_or(0) + 1; s.st["rels"] = json!(n); if n == count { s.set("log"); h = h.call(host("log1", "bidder.logCount", json!({ "name": s.st["bidder"]["name"], "texts": [EARLY_KILL] }))); } } else if i + 1 < count { h = next(&mut s, h, i + 1); } else { s.set("log"); h = h.call(host("log1", "bidder.logCount", json!({ "name": s.st["bidder"]["name"], "texts": [EARLY_KILL] }))); } } x if x.starts_with("rec:") => { let o = match out(&s, &id, r) { Ok(o) => o, Err(f) => return Ok(f), }; for u in o["uris"].as_array().into_iter().flatten() { s.st["uris"].as_array_mut().map(|a| a.push(u.clone())); } if landed(&mut s) { s.reading("recordsChecked", s.st["uris"].as_array().map_or(0, Vec::len)); h = h.call(host("gone", "assert.recordsGone", json!({ "uris": s.st["uris"] }))); } } "gone" => { let o = match out(&s, &id, r) { Ok(o) => o, Err(f) => return Ok(f), }; if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"]))); } return Ok(s.pass()); } _ => {} } } _ => {} } Ok(s.keep(h)) } /// CPU overcommit (the bidder default 2x, `--cpu-overcommit`): one bidder commits cores x `overcommit` vCPUs (input, default 2: the /// bidder's default; run it against a network brought up with `--cpu-overcommit N` and `--input '{"overcommit":N}'`). Pinned /// fillers leave exactly room for `overcommit` guests of 2 vCPUs (fillers are at most the bidder's cores each, so the bidder is /// filled well past its cores); then `overcommit` + 1 such guests are rented at once: `overcommit` are Held, and the last is /// refused 409 CapacityExceeded by the bidder, finds no bid, and waits for capacity. At 2: two accepted, the third declined; at 1, /// the second. A guest above the bidder's cores is declined `unservable` whatever its commitment: entry `impossible` (maxCpus + 1). fn overcommit_0014(r: Run) -> Result { let n = r.args["overcommit"].as_i64().unwrap_or(2).max(1); drive(r, Shape { bidders: 1, rentals: n as usize + 1, fit: n, need_refusal: true, rounds: 1, fails: 1, mode: 0, expires: "1h", small: 2, resend: false, again: false }) } /// memfit: the bidder admits a guest only when, after the memory every admitted guest has not taken yet (an unplaced accept's /// whole committed memory, a live VM's committed memory less what its cgroup has charged), the host's MemAvailable still holds /// it. On the target's first bidder (its MemAvailable and each live VM's cgroup anon read from its host, `bidder.meminfo`, and /// its room computed as the bidder computes it: MemAvailable less each live VM's commit above its anon), guests are sized so its room is /// `fit` and a half of them (fit at most 3, 1 vCPU each, so the vCPUs never bind) and fit + 1 are rented at once: exactly fit /// are Held, the last is refused at admission (a 409 naming the memory, or no bid: `declined: capacity`) and waits; one Held /// guest is released and the waiting one is Held within 5 min; all released, every record gone. fn memory_0014(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let mut h = Handle::new(); let held = |s: &S| s.st["rentals"].as_array().map_or(0, |a| a.iter().filter(|r| r["held"] == true).count()); match run.event { Event::Start => { if args["target"] == "prod" { return Ok(s.unmet("staging only: it fills a bidder's memory (prod bidders are shared)")); } let Some(b) = args["bidders"].as_array().and_then(|a| a.first()).cloned() else { return Ok(s.unmet("needs a bidder; the target has none")); }; let Some(c) = b["cluster"].as_str().map(String::from) else { return Ok(s.unmet("the target names no kcp for its first bidder: its live guests are read")); }; let d = b["did"].as_str().unwrap_or_default().to_string(); s.st["b"] = b.clone(); s.st["wait"] = json!(4); s.set("read"); h = h .call(get_record("off", &format!("at://{d}/{OFFERING}/self"))) .call(kube("bd", "get", &c, steps::sw("Bidder", "bidder"))) .call(vms("pre", &c)) .timer("deadline", 30 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in time (phase {})", s.phase()))), Event::Timer { id } if id == "offretry" => { h = h.call(get_record("off", &format!("at://{}/{OFFERING}/self", s.st["b"]["did"].as_str().unwrap_or_default()))); } Event::Timer { id } if id == "held" && s.is("held") => { return Ok(s.fail(format!("the guests that fit were not Held in 20 min: {}", s.st["rentals"]))); } Event::Timer { id } if id == "again" && s.is("again") => { return Ok(s.fail(format!("the waiting guest was not Held within 5 min of a release: {}", s.st["rentals"]))); } Event::Message { id, data, .. } if id.starts_with("req:") => { let i: usize = id[4..].parse().unwrap_or(0); let r = &mut s.st["rentals"][i]; if r["receipt"].is_null() && data["receipt"].is_string() { r["receipt"] = data["receipt"].clone(); } match data["msg"].as_str().unwrap_or_default() { "requester_submit_accept_declined" => { r["declined"].as_array_mut().map(|a| a.push(json!({ "status": data["status"], "body": data["body"] }))); } "requester_waiting_for_capacity" => r["waits"] = json!(r["waits"].as_u64().unwrap_or(0) + 1), "contract" if data["phase"] == "Held" => r["held"] = json!(true), "contract" if data["phase"] == "Failed" && r["held"] != true => { return Ok(s.fail(format!("{} failed before Held: {}", s.st["rentals"][i]["name"], data["message"]))); } "exited" if r["held"] != true && !s.is("release") => { return Ok(s.fail(format!("{} exited before Held: {data}", s.st["rentals"][i]["name"]))); } _ => {} } let fit = s.st["fit"].as_u64().unwrap_or(0) as usize; let n = held(&s); if s.is("held") && n > fit { return Ok(s.fail(format!("the bidder admitted {n} guests of {}M committed each; its memory room was {fit} (MemAvailable {}M, owed to live guests {}M)", s.st["vmMiB"], s.st["avail"], s.st["pendingMiB"]))); } let rs = s.st["rentals"].as_array().cloned().unwrap_or_default(); if s.is("held") && n == fit { if let Some((w, r)) = rs.iter().enumerate().find(|(_, r)| r["held"] != true && r["waits"].as_u64().unwrap_or(0) > 0) { for d in r["declined"].as_array().into_iter().flatten() { let body = d["body"].to_string(); if d["status"] != 409 || d["body"]["error"] != "CapacityExceeded" || !body.contains("memory") { return Ok(s.fail(format!("{} was declined {} {body}: not a 409 CapacityExceeded for memory", r["name"], d["status"]))); } } s.reading("heldS", s.since("launch")); s.reading("waiterDeclines", r["declined"].clone()); s.st["waiter"] = json!(w); let first = rs.iter().position(|r| r["held"] == true).unwrap_or(0); s.st["freed"] = json!(first); s.set("again"); s.mark("again"); h = h.call(host("free", "request.release", json!({ "name": rs[first]["name"] }))).timer("again", 5 * MIN); } } if s.is("again") && s.st["rentals"][s.st["waiter"].as_u64().unwrap_or(0) as usize]["held"] == true { s.reading("heldAfterReleaseS", s.since("again")); s.set("release"); let rs = s.st["rentals"].as_array().cloned().unwrap_or_default(); s.st["wait"] = json!(rs.len()); for (i, r) in rs.iter().enumerate() { h = h.cancel(&format!("req:{i}")).call(host(&format!("rel:{i}"), "request.release", json!({ "name": r["name"] }))); } } } Event::Result { id, result, .. } => { if id == "off" && result.output().is_err_and(|e| e.contains("RecordNotFound")) { let n = s.st["offTries"].as_u64().unwrap_or(0) + 1; s.st["offTries"] = json!(n); if n > 60 { return Ok(s.fail("the bidder's offering is not published after 10 min")); } return Ok(s.keep(h.timer("offretry", 10_000))); } let tolerated = id.starts_with("rel:") && result.output().is_err(); let o = if tolerated || id == "free" { result.output().unwrap_or_default().clone() } else { match out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), } }; match id.split(':').next().unwrap_or_default() { "off" => s.st["cores"] = json!(o["value"]["maxCpus"].as_i64().unwrap_or(0)), "bd" => s.st["overcommit"] = json!(o["spec"]["cpuOvercommit"].as_i64().filter(|n| *n > 0).unwrap_or(2)), "mi" => { s.st["avail"] = o["memAvailableMiB"].clone(); s.st["total"] = o["memTotalMiB"].clone(); s.st["anonMiB"] = o["anonMiB"].clone(); let pending: i64 = s.st["live"].as_array().into_iter().flatten() .map(|v| { let commit = v["commit"].as_i64().unwrap_or(0); let anon = o["anonMiB"][v["name"].as_str().unwrap_or_default()].as_i64(); if commit == 0 { 0 } else { anon.map_or(commit, |a| (commit - a).max(0)) } }) .sum(); s.st["pendingMiB"] = json!(pending); } "pre" => { let live = live_vms(&o); s.st["liveMiB"] = json!(live.iter().map(mem_of).sum::()); s.st["liveCpus"] = json!(live.iter().map(cpus_of).sum::()); s.st["live"] = json!(live.iter().map(|v| json!({ "name": v["metadata"]["name"], "owner": v["spec"]["owner"]["handle"], "commit": mem_of(v) })).collect::>()); let ask: Vec = live.iter().filter(|v| mem_of(v) > 0) .map(|v| json!({ "name": v["metadata"]["name"], "owner": v["spec"]["owner"]["handle"] })).collect(); h = h.call(host("mi", "bidder.meminfo", json!({ "name": s.st["b"]["name"], "vms": ask }))); } "rel" => { if landed(&mut s) { s.reading("releaseS", s.since("again")); let rs = s.st["rentals"].as_array().cloned().unwrap_or_default(); s.st["wait"] = json!(rs.len()); s.st["uris"] = json!([]); s.set("records"); for (i, r) in rs.iter().enumerate() { h = h.call(host(&format!("rec:{i}"), "request.records", json!({ "name": r["name"] }))); } } return Ok(s.keep(h)); } "rec" => { for u in o["uris"].as_array().into_iter().flatten() { s.st["uris"].as_array_mut().map(|a| a.push(u.clone())); } if landed(&mut s) { h = h.call(host("gone", "assert.recordsGone", json!({ "uris": s.st["uris"] }))); } return Ok(s.keep(h)); } "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left after the release: {} {}", o["left"], o["errors"]))); } return Ok(s.pass()); } _ => return Ok(s.keep(h)), } if s.is("read") && landed(&mut s) { let cpu_room = s.st["cores"].as_i64().unwrap_or(0) * s.st["overcommit"].as_i64().unwrap_or(2) - s.st["liveCpus"].as_i64().unwrap_or(0); let room = s.st["avail"].as_i64().unwrap_or(0) - s.st["pendingMiB"].as_i64().unwrap_or(0); let fit = 3.min(cpu_room - 1); let target = if fit > 0 { room * 2 / (2 * fit + 1) } else { 0 }; let m = if target >= 5632 { target * 10 / 11 } else { target - 512 }; if fit < 1 || m < 1024 { return Ok(s.unmet(&format!("the bidder has no room for two guests: {cpu_room} vCPUs, {room}M of MemAvailable less what its live guests have not taken"))); } let vm_mib = commit_mib(m); s.st["fit"] = json!(fit); s.st["vmMiB"] = json!(vm_mib); for k in ["avail", "total", "liveMiB", "anonMiB", "pendingMiB", "fit", "vmMiB"] { s.reading(k, s.st[k].clone()); } s.reading("mem", format!("{m}M")); s.set("held"); s.mark("launch"); let did = s.st["b"]["did"].clone(); let mut rs = vec![]; for i in 0..=fit { let n = format!("{name}-m{i}"); let a = json!(["--hold", "--cpus", "1", "--mem", format!("{m}M"), "--expires-in", "1h"]); h = h.call(steps::host_sub(&format!("req:{i}"), "request.start", json!({ "name": n, "args": a, "bidderDids": [did] }))); rs.push(json!({ "name": n, "declined": [], "waits": 0 })); } s.st["rentals"] = json!(rs); h = h.timer("held", 20 * MIN); } } _ => {} } Ok(s.keep(h)) } entries! { memfit => memory_0014, abandon => abandon_0014, overcommit => overcommit_0014, impossible => impossible_0014, run => fallback, concurrent => concurrent_0014, exhausted => exhausted_0014, giveup => giveup_0014, wait => wait_0014, backtoback => backtoback_0014 }