~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0

fixes for the causes of gate failure found since roll two, the dumbpipe v0.40.1 bump; entry 0007:ingress (staging 48)

Merged johnandersen777.bsky.social asked to merge into main from socialweb.computer/socialweb-computer-kcp:phase-2
cmd/socialweb-computer/bidder.go+1−1
@@ -723,7 +723,7 @@ func (r *bidderRun) installedCore(ctx context.Context) int {
}
prefix := r.vals.String("unit-prefix")
exes := map[string]devinstance.Executable{}
for _, x := range []struct{ name, option string }{{bidder.ExeKcp, "kcp-path"}, {bidder.ExeKine, "kine-path"}, {bidder.ExeBao, "bao-path"}, {marketbidder.DumbpipeExecutable, "dumbpipe-path"}} {
for _, x := range []struct{ name, option string }{{bidder.ExeKcp, "kcp-path"}, {bidder.ExeKine, "kine-path"}, {bidder.ExeBao, "bao-path"}, {devinstance.DumbpipeExecutable, "dumbpipe-path"}} {
e, err := r.a.optionalExe(ctx, r.vals, x.name, x.option)
if err != nil {
return r.a.fail(exitRefused, "%s: %v", verb, err)
deploy/bidder/preinstall-market.json+1−1
@@ -17,6 +17,6 @@
],
"runs": [
"cloud-init --version",
"case \"$(uname -m)\" in x86_64|amd64) _arch=x86_64; _sha=9ac5e71983eba4cf4f47e92e4694dad0f0133abba4f14dd2a2a8428cece88fef ;; aarch64|arm64) _arch=aarch64; _sha=8cd099afe80c69ac58bfbc975b28e29d41caea141f3dd396457345a739836a0e ;; *) echo \"no dumbpipe archive for $(uname -m)\" >&2; exit 1 ;; esac && curl -fsSL \"https://github.com/n0-computer/dumbpipe/releases/download/v0.39.0/dumbpipe-v0.39.0-linux-${_arch}.tar.gz\" -o /tmp/dumbpipe.tar.gz && printf '%s %s\\n' \"${_sha}\" /tmp/dumbpipe.tar.gz | sha256sum -c - && mkdir -p /tmp/dumbpipe-unpack && tar -xzf /tmp/dumbpipe.tar.gz -C /tmp/dumbpipe-unpack && install -m 0755 /tmp/dumbpipe-unpack/dumbpipe /usr/local/bin/dumbpipe && echo 0.39.0 > /usr/local/bin/dumbpipe.version && rm -rf /tmp/dumbpipe.tar.gz /tmp/dumbpipe-unpack"
"case \"$(uname -m)\" in x86_64|amd64) _arch=x86_64; _sha=66aee1e249890ee63678dfc8f18ce3671e691673c83bbd72bf9a54c4d32da85c ;; aarch64|arm64) _arch=aarch64; _sha=1529c49598ac15d9413bf7f29a09c04cf2f5dc3af1b9080a1b50855a77846c47 ;; *) echo \"no dumbpipe archive for $(uname -m)\" >&2; exit 1 ;; esac && curl -fsSL \"https://github.com/n0-computer/dumbpipe/releases/download/v0.40.1/dumbpipe-v0.40.1-linux-${_arch}.tar.gz\" -o /tmp/dumbpipe.tar.gz && printf '%s %s\\n' \"${_sha}\" /tmp/dumbpipe.tar.gz | sha256sum -c - && mkdir -p /tmp/dumbpipe-unpack && tar -xzf /tmp/dumbpipe.tar.gz -C /tmp/dumbpipe-unpack && install -m 0755 /tmp/dumbpipe-unpack/dumbpipe /usr/local/bin/dumbpipe && echo 0.40.1 > /usr/local/bin/dumbpipe.version && rm -rf /tmp/dumbpipe.tar.gz /tmp/dumbpipe-unpack"
]
}
deploy/guest-image/rootfs/usr/local/bin/dumbpipe.version+1−1
@@ -1 +1 @@
0.39.0
0.40.1
deploy/guest-image/rootfs.json+1−1
@@ -13,6 +13,6 @@
},
"runs": {
"8a1b58dd67c32c937c4c3257ece9dcbcbd53d87c92e1749906da49a6489bc49b": "cloud-init --version: the base carries cloud-init, checked in its dpkg status",
"fe76a87a62dc5484bc6e777dc04b55c15c3f2eebd2bf7345f9788fbec438b140": "dumbpipe v0.39.0: baked from the versions.json dumbpipe pin of the image's arch as the asset /usr/local/bin/dumbpipe, with dumbpipe.version from the overlay"
"9f6d4e35fa37405a63e3a37936db9d5751dba5b74e4163362590a53fe74bd7d1": "dumbpipe v0.40.1: baked from the versions.json dumbpipe pin of the image's arch as the asset /usr/local/bin/dumbpipe, with dumbpipe.version from the overlay"
}
}
docs/acceptance/0007-bidder-termination.md+36−1
@@ -188,6 +188,41 @@ The guest requester's own atirc token mint answers 403 "association not verified
(0005's note; the requester retries it, the rental does not wait on it): the entry counts those lines
(`readings.atirc_mint_403`) instead of failing fast on them.
## The market endpoint over iroh survives a new ingress instance (`0007:ingress`, staging)
A bidder's market endpoint over iroh is the Exposure `svc-bidder-bidder-iroh`, published through the IngressProxy
`ingress-iroh` (a dumbpipe listener per registration). Any new generation of that IngressProxy, and a rebuild after its
service exited, makes the core's runtime stop the service instance and build a new one with no registration; the
exposure publisher re-registers on the new instance (its fingerprint holds the instance, `component.Entry.Attempt`) and
the Exposure is Ready only once dumbpipe printed its ticket. Before the fix the publication stayed on the dead instance
and the bidder was deaf over iroh until its core restarted (docs/plans/0040-findings.md, "A BIDDER GOES DEAF OVER IROH
WHEN ITS STORED DUMBPIPE PATH CHANGES").
Test: `tests/compositions/0007` entry `ingress`; staging only (it replaces each bidder's IngressProxy), exclusive. The
trigger is the bidder root's own component override, the one kube write the test host may make on a bidder (its guard:
a patch of the MarketBidder, restored after): `spec.components["IngressProxy/ingress-iroh"].settings.ingressProxy
.keepaliveMilliseconds` (an xrpc-subscription setting the iroh provider never reads). The compose pass rewrites the
IngressProxy spec, the generation changes and the core's runtime (`internal/component/runtime.go` Reconcile: generation
changed, stop, start) replaces the instance while the core stays up: the path a compose rewrite of
`spec.iroh.dumbpipePath` takes after a switch between binaries whose pinned dumbpipe differs, without changing the pin.
A core restart (`0007:restart`) is no such trigger: a new process starts a new publisher with nothing published; an
upgrade (`0007:upgrade`) is one only when the stored spec differs (the first start of a release on the previous one's
stored path). There is no operator command for one component of an installed bidder (`dev restart <component>` reads a
dev instance's record) and no kubectl on the staging bidders. Host call `bidder.dumbpipe {name}` (the dumbpipe children
of the bidder's run bidder process: pid, executable, `--host`, read from /proc over ssh, no sudo); `request.start` takes
`bidderEndpoints` (SOCIALWEB_REQUESTER_BIDDER_ENDPOINTS of that run request). Per bidder, three rounds (keepalive 6000 ms,
6001 ms, then the override as it was), each:
- the MarketBidder patch; then, read every 3 s for at most 180 s: an `ingressproxy_iroh_started` after the patch, an
`ingressproxy_iroh_dumbpipe_listening` of `svc-bidder-bidder-iroh` after the LAST started, and the IngressProxy Ready
at its generation (`readings.rounds`: seconds to heal, lines, generation, address)
then a live dumbpipe child of the core on the address of the last listening line (`readings.dumbpipe`), and a run
limited to that bidder (`--allow-bidder-dids`, bidder endpoints `iroh` only, `--exec -- id`) exits 0 (its submitRfp
over iroh reached the bidder and was answered) and leaves no record. Any failure while an override is applied (a round
that does not heal, the deadline, a failed call) puts it back first; a run killed from outside (SIGTERM, on-fail cleanup)
is put back by the test host's end: the kube transport's ledger restores every spec the run patched (`kube.cleanup`).
## Preconditions
- Authority: the owner asked for it, or the VM is matched to a contract you are ending (stray test / orphan). Never an owner-held VM unless the owner says so
@@ -258,4 +293,4 @@ Bidder journal, in order (hat-7 example, 2026-10-31 22:51Z):
| token still answers | token was plain (pre-lease or test mint); revoke it by subhandle |
| ComputeVM `Lost` after `uninstall --force` | expected: forced uninstall does not run the finalizer path; do not use it to end one VM |
| requester resumes later and re-releases | harmless: release is idempotent; the contract is already Released |
| `--expires-in` round fails `no bids received within 30s`; bidder `bidder_payload_invalid` "the RFP payload's expiresAt ... has passed" (2026-11-12) | the CLI fixes `expiresAt` when the invocation starts, before the QR sign-in; a sign-in later than the duration expires the RFP. Sign in right when the QR url appears (it can take 2+ min) |
| `--expires-in` round fails `no bids received within 30s`; bidder `bidder_payload_invalid` (since branch bid-capacity: "a bid failed" reason `DeclinedUnservable` at submitRfp, `bidder_payload_refused` kind unservable for an RFP seen over the firehose only) "the RFP payload's expiresAt ... has passed" (2026-11-12) | the CLI fixes `expiresAt` when the invocation starts, before the QR sign-in; a sign-in later than the duration expires the RFP. Sign in right when the QR url appears (it can take 2+ min) |
docs/acceptance/0014-accept-fallback.md+9−3
@@ -47,7 +47,7 @@ from the bidders' offerings (`maxCpus`) less the guests already on their kcp, so
| `impossible` | the target's bidders | 0 | none | requests nobody can take fail fast, never wait: more vCPUs than the largest offering's maxCpus, and (when no offering sets nestedVirt) a nestedVirt request, and a request whose `--expires-in 1s` is over before the requester sends an RFP (the contract's deadline: no RFP goes out): each Failed within 120 s naming why ("maxCpus", "nestedVirt", "deadline"), nothing Held, no `requester_waiting_for_capacity`. The requester judges this over the candidates with a fresh heartbeat: a relay still lists a removed bidder's repo (its heartbeat and offering gone), and that candidate does not make the request wait |
| `giveup` | first 2 of the target | 1 | both bidders full (one pinned 1 h filler each) | one rental with `--expires-in 150s` has no room: it waits in rounds (at least one `requester_waiting_for_capacity`) and fails at its deadline with "gave up waiting for capacity after N rounds" |
| `wait` | first 2 of the target | 1 | both bidders full | one rental with `--expires-in 15m` waits; after two rounds a filler is released: the rental is Held on the freed bidder within 300 s of the release (about one backoff and a boot); readings `waitRounds`, `heldAfterFreeS` |
| `exhausted` | first 2 of the target | 3 | `min(free)/2 + 1` (one per bidder) | required: one rental is refused by both, its bids run out, its new RFP (the refusers excluded, as specified) finds no bid: the rental WAITS for capacity (`requester_waiting_for_capacity`, rounds with a backoff of 30 s, 1 min, 2 min, a fresh RFP asking every bidder each time) and is released by the test |
| `exhausted` | first 2 of the target | 3 | `min(free)/2 + 1` (one per bidder) | required: one rental is refused for capacity by both (each once: a 409 to its accept, or `declined: capacity` to its RFP when the bidder was full as the RFP came; readings `rounds[].rentals[].capacityRefusals`), its bids run out, its new RFP (the refusers excluded, as specified) finds no bid: the rental WAITS for capacity (`requester_waiting_for_capacity`, rounds with a backoff of 30 s, 1 min, 2 min, a fresh RFP asking every bidder each time) and is released by the test |
| `overcommit` | the target's first | `overcommit` + 1 (input `overcommit`, default 2), at once | 2, with room left for exactly `overcommit` of them | `overcommit` Held, the last refused 409 `CapacityExceeded` by the bidder, which finds no bid and waits for capacity (plan 0041) |
| `memfit` | the target's first | fit + 1 (fit at most 3), at once, 1 vCPU each | sized from the room the bidder computes (`bidder.meminfo`: the host's MemAvailable less, per live VM, its committed memory above its cgroup's anon) so it is fit and a half guests | fit Held, the last refused at admission (409 `CapacityExceeded` naming the memory, or no bid) and waiting; one Held guest released, the waiting one Held within 5 min (owner decision 73: a guest is admitted only when MemAvailable, less the memory every admitted guest has not taken yet, holds it) |
@@ -129,8 +129,14 @@ $B dev down $N
## Pass criteria
- Every rental Held (`exhausted`: all but one), each on one of the entry's bidders and never on a bidder that refused it
- `exhausted`: the one rental left without a VM was refused (409) by every bidder of the entry, logged
`requester_bids_exhausted_new_rfp`, and then waits for capacity (at least one `requester_waiting_for_capacity` line) instead of failing; the test releases it
- `exhausted`: the one rental left without a VM was refused for capacity by every bidder of the entry (a 409
`CapacityExceeded` to its accept, or `requester_bidder_declined` kind `capacity` to its RFP: the same check, `vmsize.go`
`capacityRefused`, at either time; which one is the order the RFP and another rental's admission reach the bidder), each
before its first wait (a waiting round's RFP asks every bidder again and does not explain the wait); after a 409 it logged
`requester_bids_exhausted_new_rfp` (with no bid at all there is no accept and nothing to ask again); and then it waits for
capacity (at least one `requester_waiting_for_capacity` line) instead of failing; the test releases it. A bidder that did not
answer (unreachable, `requester_submit_rfp_failed`, or a bid dropped after submitRfp with no answer, which the bidder no
longer does: plan 0040 Findings "THE BID DECISION MAKES NO CAPACITY DECISION") fails it
- No bidder over its vCPUs: this run's VMs on a bidder fit its free vCPUs; its kcp's live ComputeVMs' `spec.vm.cpus`
sum to at most its `maxCpus`
- Every refusal is a 409 `CapacityExceeded`; every refused accept record is kept (x3), the rule above
docs/plans/0040-findings.md+591−0
591 lines changed: not loaded until asked for.
docs/plans/0040-release-pipeline.md+102−118
@@ -3,121 +3,98 @@
Status: the only plan in flight (owner, 2026-11-17); docs/plans/0001-0039 and NEXT.yaml are history. How to run, hold
and debug CI by hand: `docs/ci-debug.md`. The state before this work: main e39097c2 (2026-11-12).
State (2026-11-21 21:20Z; updated by the lead):
- Main: CI in series on four home bidders (decision 43), the fixes of promote 11's cycle, the kcp docs, the site
from the repo (`site/`), the protocol docs.
- STAGING IN CI: 43 of 43 PASS at a7dab399, the first cycle in series (one network, 6215 s); before it at
d6a2b391, a5269317, 688c2105 (four shares).
- PROD IN CI: 30 of 30 PASS at 688c2105 (promote rc186, 04:22Z 11-19). Since: 29 of 30 at db33c9ff (0005:r2fail,
fixed on main), 27 of 30 at a7dab399 (0007:run and 0014:abandon: reads tried once; 0013:run: WebRTC between two
home-NATed hosts, STUN only).
- ROLLS: promote 10 (rc186) GREEN: stable v0.0.0 = main 688c2105. Promotes 11 (db33c9ff) and 12 (a7dab399) RED,
rolled back to rc186. Two consecutive greens are still owed. One code change per cycle; a cycle is about 3 h 15
min as measured (release 22 min, test 104 min + stand-up, promote 65 min).
- No CI cycle since promote 12. Next cycle = ONE batch (decision 56), run out of band first (51), then one pull;
the roll after it is the help text change. Batch, with state:
- `rc-reads` (the three prod failures of a7dab399, reads tried once, the nftables rule of decision 54): hand
proofs running.
- `ci-no-john` 5cd2f394 (prod gate as bob and alice, GATE_VMS=6; decision 57): ready.
- `secrets` (decisions 52, 61, 64, 66): review fixes at c36635bd SAFE on second review; in work: the guest path
rule, getSecrets over iroh and WebRTC (atlibs changes and pins; steps written into this plan by its agent).
- `cli-args` (decisions 49, 60): rework at 1f50c2f1 in second review. Prod bidders have no tail (read on all
four). The prod ssh gateway unit moved to `--ssh-gateway-spec-file` 02:58Z 11-20 (active, ports 22 and 8789,
sign-in page 200; backup `/root/swc-ssh-gateway.service.pre-0043`); the gateway entries by hand owed.
- `lexicons-fix` (decision 58): in work.
- `relay-fix` (decision 65): 97bba4a7 (the three checks) reviewed SAFE and DEPLOYED to xrpc.fedproxy.com 03:4xZ
11-20 (`_health` version 97bba4a79174; 15 subscribers back, 0 refused; before: d2eec3f38a4c). Reproduced on a
local relay first (main: takeover; fix: refused; `~/.tmp/relay-fix/out-main2`, `out-branch2`). Its two
follow-up commits (close a replaced socket; client wait) are NOT deployed: reviewed NOT SAFE, in rework.
- `route-trust` (decision 66): in work.
- `rc-cap`: redesign approved (the provision StateSync is the reservation; no check after admission), in build.
- `rc-writes` (decision 67): review fixes proven with a PDS fault proxy at 0e9c128c, delta review running; the
OAuth scope in work. `rc-pins`: rebase onto series CI and a clean run of 0007:upgrade (staging 44) in work.
- `docs-assoc`: in the batch (decision 62). `do-bidder`: paused until after roll two (decision 68).
- Branch `batch` on `swc` = main + `ci-no-john` + `lexicons-fix` + `cli-args` + `rc-writes` (ingress retry, the
OAuth scope for requester and bidder) + `relay-fix` + `relay-ws-503` + `rc-reads` 7831de98 + `fix-0001s2` +
`fix-0015`, each after its review.
- Hand staging cycle of `batch` at fc9c4dce (stg-859542, 03:36Z-05:57Z): gate: staging: 40 of 43 PASS. FAILs,
each root-caused, fixed on `batch` and passing on the held network with the fix: 0001:s2 and 0020:run (their
own `install bidder` had the 3 min default network timeout for the 1.44 GiB guest image on the home uplink;
now 20m as the stand-up), 0015:pressure (test timing: crossing 2 was scheduled from record 1, 0-15 s before
the bidder could re-arm; now from crossing 1's restore). Summary: `~/.tmp/batch-cycle/status.md`.
- `batch` is COMPLETE at e4ccd291 (contains main): also `secrets` (getSecrets over iroh and WebRTC, the path rule,
entry 0007:webrtc), `rc-reads` item 8 (a failed trust lookup is not a deny), `rc-pins` (a self-updated bidder
serves kept previous binaries; entry 0007:upgrade), `route-trust` (bids from the bidder's repo, record bytes
checked against the CID, bid and receipt signatures with a verified key binding; decisions 66, 71), `rc-cap`
(the provision StateSync is the reservation; the memory admission rule of decision 73; entry 0014:memfit).
Each merged after its review passed; `deploy/check.sh --fast` and the policy spec tests pass on the merges.
Entries now: staging 46, prod 31 (Done when on `batch` follows the list).
- Hand cycle one (batch e4ccd291, network stg-028774, 08:04-11:25Z): 43 of 46. Fails: 0007:upgrade (test host
copied binaries over the overlay; fix-0007-upgrade), 0014:abandon (product regression of the capacity redesign:
500 for an unreadable accept; fix-0014-abandon: 202 with a hold-less sync, admitted or abandoned on the attempts),
0009:run (collateral of the 0007:upgrade copies). Findings has each. Saved: `~/.tmp/final-cycle/saved/`.
- Hand cycle two (batch b2df7cc1 = e4ccd291 + both fixes, fresh network stg-665385, nothing pre-staged,
11:40-13:44Z): 46 of 46 PASS, 0 FAIL (gate dir `/home/stg/stg/gate` on its atproto VM: 46 entry results, all pass;
`~/.tmp/final-cycle2/results.txt`, `logs/staging.log`). This is the out of band evidence for the pull.
- Hand prod entries (the batch requester and test host against prod's stable bidders, as bob and alice,
14:07-15:50Z; `~/.tmp/final-cycle2/prod-hand.md`): 28 of the 29 entries that can pass against stable PASS;
0006:compat as alice FAIL (the secrets work had put whole record values back into the test host's
`repo.listCreated`; alice's compute.vm page passed the 1 MiB module input cap): fix-0006-compat (values trimmed to
the fields a caller names; test host and acc-0007 module only), then on prod as alice 0006:compat, 0006:run and
0002:run PASS (0002:run failed once before, cause not found, Findings). Not provable before the roll: 0007:run and
0007:webrtc on prod (they need the new bidder's guest agent) and the bidder side of 0013:run.
- Hand cycle three (batch 8c84935d: test host and modules with fix-0006-compat, product binary of b2df7cc1, fresh
network stg-328206, 16:31-18:37Z): gate: staging: 46 of 46 PASS, wall 6791 s (`~/.tmp/final-cycle3/logs/staging-full.log`,
gate dir copies `~/.tmp/final-cycle3/saved/`). A first try with a listed suite ran zero entries and exited 0
(Findings).
- Roll one, main 93fd8c11 (the batch, one pull, 18:40Z): `release` green (18:40-19:15Z; amd64 d51bad4f);
`test` green: `gate: staging: 46 of 46 PASS`, 46 RESULT lines PASS, wall 6973 s (19:15-21:26Z;
`knotless run list $U --commit 93fd8c115e10 --workflow test --log`); `promote` RED (21:26-22:58Z): all four
bidders heartbeat d51bad4f by 21:35Z, then `gate: prod: 28 of 31 PASS`, wall 5000 s: FAIL 0003:run (the first
ssh over webrtc, exit 255), 0007:webrtc (the guest fetched its secrets over https, not webrtc), 0013:run (ICE
failed before the datachannel opened): all three WebRTC, all with the requester inside the release VM. Rolled
back by the run itself: rc back on v0.0.0-rc186, every bidder heartbeats d4933943 again (read 22:59Z), stable
not moved.
- Root cause of the red promote: a guest's rules were loaded once by the bidder that booted it; promote's own
VM never had decision 54's rule (Findings). Fix (decision 75): the bidder reloads an adopted guest's rules;
entry 0007:fence reproduces promote's topology (staging 47). hat-10's zone opened for UDP 31000-31999
(decision 76).
- ROLL ONE GREEN, main 324f69a1 (the reload-on-adopt fix on the batch; v0.0.1-rc3; 2026-11-21): `release` green
(04:48-05:20Z; amd64 394445f3); `test` green: `gate: staging: 47 of 47 PASS`, 47 RESULT lines PASS, wall 6956 s
(05:20-07:31Z); `promote` green (07:31-08:39Z): every bidder heartbeats 394445f3, `gate: prod: 31 of 31 PASS`,
wall 3718 s, 0003:run, 0007:webrtc and 0013:run PASS, `main/324f69a1.../` copied to `releases/v0.0.1/`, stable
v0.0.1. On hat-7 the new bidder logged `guest_rules_reloaded` for the promote's own VM
(`fc-gmlkrhimouee-8b06f638`) at 07:33:58Z. Logs: `knotless run list $U --commit 324f69a10b6c --workflow
<test|promote> --log`.
- After roll one, before any pull, three things that would have broken the next rolls were found and fixed
(Findings has each): 0007:fence took its from binary from the previous rc and so failed for ever once its own
roll was green (now pinned to `release:v0.0.0-rc186`; tests only); every core start began with an empty bus link
table and callers were denied after 5 s under load (`bus: link denied`, ERROR lines; 0007:upgrade, 0006:run,
0008:run assert a clean start): the bus now waits for a workspace's ComponentLinks to be loaded once, bounded;
the ssh gateway failed a command issued before the account's requester was ready (0002:run): the session now
waits for its own requester, bounded. Each reproduced first (old 10 of 10 failing, fixed 0 of 10) and reviewed.
- Left out on purpose: the dumbpipe v0.40.1 bump (branch `dumbpipe-0401`): the full hand cycle with it found both
bidders deaf over iroh after a switch between binaries with different dumbpipe paths, in both directions
(Findings: a prod outage at a roll and at its rollback); its fix needs two releases, after this plan.
- Hand cycle of the tree to land (`stack3` 2ea5fa6e = main + the three fixes, fresh network stg-134498,
14:27-16:28Z): `gate: staging: 47 of 47 PASS`, 47 RESULT lines PASS, wall 7158 s; 0 `bus: link denied` on every
layer; gateway `requester_ready` 3, `requester_unavailable` 0; no capacity leak (0 VM units at the end)
(`~/.tmp/cycle-stack3/saved/final/`). Hand survey of `cli-help` on the same morning's base: 46 of 47 (the fail:
0002:run, the sign-in race, fixed here).
- ROLL TWO RED AT TEST, main 7b0afc69 (stack3; 2026-11-21): `release` green (16:32-16:53Z); `test` RED:
`gate: staging: 45 of 47 PASS`, wall 7381 s (16:53-19:38Z with the 30 min failure freeze); `promote` refused at
its first step (no green test): channel rc never moved, prod untouched, stable still v0.0.1. FAIL 0008:run
("bob2 not CONFIRMED": 15 start-up ERRORs "... not added to shared informer because it has stopped already" at
18:00:09): NOT the stack: the lines were written by the OLD rc186 process as it shut down (the handler addresses
resolve in the rc186 binary, not in this one), and `bidder.health` counted them for the new core because it reads
the journal by user since the process start in whole seconds. FAIL 0014:exhausted ("1 refusals, newRfp 1"): cause
not established (the run's journals and gate-run dirs were never collected: `logs.fetchRuns` rejects its own
absolute `dir`). Findings has both.
- Now (decision 78): no roll until every known cause of gate failure is fixed. In work, on branches: `health-pid`
(tests: health by the core's exact start, fetchRuns, 0014:exhausted; reviewed; full hand cycle running),
`ingress-rereg` (a replaced ingress listener's publications are re-registered; no stored build-specific path),
`bid-capacity` (no silent bid drop for capacity; a signed-in account is not refused while its access is set up).
Then one pull, proven whole by hand; then `cli-help`. Two consecutive green rolls: 0 in a row.
- Relay (deployed by hand, not in the pipeline): 97bba4a7 then e7251bbc on xrpc.fedproxy.com; takeover refused on
the live relay with throwaway identities; requests fell from about 290/s to under 1/s.
State (2026-11-22; updated by the lead; older history is in Findings and in git):
- Main: stable v0.0.1 = main 324f69a1 (promote 14, v0.0.1-rc3, green 08:39Z 11-21; amd64 394445f3). Roll two, main
7b0afc69: `release` green, `test` RED `gate: staging: 45 of 47 PASS` (19:38Z 11-21), `promote` refused at its first
step; channel rc never moved, prod untouched.
- STAGING IN CI: 45 of 47 at 7b0afc69 (red); last green 47 of 47 at 324f69a1. The list is 48 with this pull.
- PROD IN CI: 31 of 31 PASS at 324f69a1 (promote 14, as prod bob and alice).
- ROLLS: 0 green in a row. Two consecutive greens are owed; between them only the colored help of decision 74.
- Decision 78: no roll while a known cause of gate failure is unfixed. The causes found since roll two are fixed on
five branches, each reviewed, merged as `stack6` 1c028e5d (the Go tree is that of `stack5`), plus the dumbpipe
bump (decision 79): this pull's tree is `bump6` cf34bc81 = `stack6` + the bump. It lands:
- `health-pid` 8e49c3a0 (tests only): `bidder.health` reads the current core process only (roll two's 0008:run
counted the old process's shutdown errors), `logs.fetchRuns` takes its absolute dir, 0014:exhausted counts a
capacity decline. Proven: hand cycle 47 of 47 (stg-031952); in the stack5 cycle 0008:run read errors 0 for a
bidder whose previous (rc186) core logged an ERROR at its stop (0007:fence, 03:45Z 11-22).
- `bid-capacity` c26c0927: the room is decided once per RFP, when its bid is created, on every intake path
(submitRfp and the market stream); a recorded decline is what a later submitRfp answers; the requester's accept
replacement budget is per waiting round and a 409 past it waits for capacity; the ssh gateway's sign-in ends
only once the account's link exists (`bus.AwaitLink`). Proven: on stg-033371 the stream brought every RFP first
and a full bidder never bid (0014:giveup 5 of 5, 0014:wait 7 of 8, the one FAIL a sign-in timeout); all ten 0014
entries PASS in the stack5 cycle. NOT proven: the ssh sign-in wait closes a window: 0 refusals in 30 sign-ins on
the fix, but main 7b0afc69 gave 0 in 40 too (failure not reproduced; worst session wait on main 5.5 s against
the 5 s settle).
- `ingress-rereg` ce2ec862, with the new staging entry `0007:ingress`: a replaced ingress listener's publications
are re-registered, Ready only once dumbpipe listens, no stored build-specific dumbpipe path, no xrpc fallback for
a named IngressProxy that is not registered yet. Proven on stg-033371: 0007:ingress PASS twice and FAIL on main
7b0afc69 as it must, 0007:upgrade and 0007:fence PASS, 0 ERROR lines in 11 bidder core processes; PASS in the
stack5 cycle. A switch between two dumbpipe pins: the bump's proof below. NOT proven: the crash-restart path.
- `signin-retry` ae1db64f: a transient createSession failure (timeout, refused or reset connection, 5xx, 429)
keeps the MarketIdentity Pending and is retried; Failed stays for a refused credential, a DNS not-found and a
certificate error; the core passes its network client, so the intended 10 s per call applies (a 5 s client cut
it). Proven on a local market only (a proxy holding createSession). NOT exercised on staging: 0 retry lines in
the stack5 cycle (no sign-in was slow).
- `stage-budget` 1c028e5d (tests only): a host script states its budget and the test host waits that long plus
120 s (it cut every script at 120 s; the stack5 cycle's one FAIL: the 0007:upgrade stage fetch of the 594 MB
from binary took 119 s). Proven by a local harness (old code cut at 120.0 s, new code completes) and by
0007:upgrade, 0007:fence, 0007:restart, 0006:run, 0008:run by hand on stg-277608 (all PASS). NOT exercised there:
a fetch (the from binary was already on both bidders); why that fetch was slow is not established. In the
stack6 cycle 0007:upgrade fetched for real (17 s at 33 MiB/s, 40 s at 14 MiB/s) and PASSED.
- the bump (`bump6` cf34bc81, reviewed): dumbpipe 0.39.0 -> 0.40.1 (pins, guest image 7e5dc836 with 0.40.1 baked,
the cloud-init default); the guest's setup-iroh fetch is limited to 3 tries of 10 s when the image has a
dumbpipe (worst case 34 s, then the image's own runs) and is main's loop when it has none. Why: in the stack6
cycle `iroh.service` restarted once on bidder-2 (07:12:33Z) and once on the gateway VM (07:29:38Z), each
"Endpoint dropped without calling Endpoint::close", both survived; the claim to check in the stack7 cycle is 0
such restarts. Proven on stg-113817 (`~/.tmp/ingress-rereg/slot6/`): both bidders 0.39.0 -> 0.40.1 with 0 ERROR;
0007:upgrade 324f69a1 <-> bump PASS, the old core listening 8.9 and 8.1 s after its start; 0007:run and
0007:restart PASS; a guest of the bump requester on a bump bidder fetched nothing and runs dumbpipe 0.40.1.
- Hand cycles of the combined tree (full list, fresh network, CI's default input):
- stack4 7b0f1bc3 (stg-033371): 44 of 48. FAIL 0007:upgrade, 0007:ingress, 0014:giveup, 0014:wait, all four from
earlier heads of `ingress-rereg` and `bid-capacity`, each root-caused and corrected (Findings).
- stack5 df3b1c2c (stg-277608): `gate: staging: 47 of 48 PASS`, wall 7311 s (`~/.tmp/cycle-stack5/saved/final/`).
FAIL 0007:upgrade (the 120 s cut; `stage-budget`).
- stack6 1c028e5d (stg-113817): gate: staging: 48 of 48 PASS, wall 7347 s, `~/.tmp/cycle-stack6/saved/final/`
- stack7 43297eda (bump6 + cli-help), stg-513391: stopped as a survey at 10 of 48 done, 9 PASS, 1 FAIL
(09:01Z; `~/.tmp/cycle-stack7/saved/final/`); iroh.service restarts 0 and "Endpoint dropped" 0 on all four
VMs in 54 min. FAIL 0006:run:
one start-up ERROR on bidder-1's first start, "the HostProcess supervisor stopped; starting it again" (sets
`provider` and `applications` both apply APIBinding `policy`; kcp refuses one when they overlap; healed 2 s
later). On main since 2026-10-21; first hit. Fix (owner: a healed retry is not an ERROR): `bind-label`
287b641c: WARN while retrying, ERROR once a failing streak lasts the set apply's deadline (1 min). The four
network VMs fetched dumbpipe v0.40.1 in 0 to 1 s each at stand-up.
- stack8 70321d69 (bind-label + cli-help), stg-858914: stopped as a survey at 3 of 48 done, 2 PASS, 1 FAIL (0006:run on bob3: that line; 09:33Z; `~/.tmp/cycle-stack8/results.txt`). Seen at
its start: one start-up ERROR on bidder-2's first start, "Failed to watch ... the server could not find the
requested resource ... *v1alpha1.Service" (the exposures virtual workspace serves discovery about 0.6 s before
it serves the resource; the reflector retried and healed in 0.4 s). Fix: `start-warn` 00e47278: that reflector
line logs WARN while its streak is under the cache-sync deadline (2 min), ERROR after. No timing shift from the
bump was found (10 local starts each on the stack6 and stack8 release builds); the races follow host load.
Sweep of every saved journal: no other ERROR within 600 s of a bidder core start (`~/.tmp/bind-label/sweep.txt`).
- stack9 1d794a7b (this tree 00e47278 + cli-help): gate: staging: 48 of 48 PASS, wall 7632 s (stg-090117,
09:45-11:52Z; `~/.tmp/cycle-stack9/saved/final/`): iroh.service restarts 0 and "Endpoint dropped" 0 on all
four VMs; dumbpipe 0.40.1 never exited under a running core (0.39.0 did 3 times under the old binaries
inside 0007); the APIBinding race fired once on bidder-1 at WARN and 0006:run passed (startupErrors 0)
- one ERROR "Failed to watch ... context canceled" (controller marketbidder, an apibindings watch) 4 s after
the start of an OLD-binary core (324f69a1) inside 0007:upgrade on stg-090117: no health reading counted it;
first seen; the same controller code is in this tree, cause not established: being root-caused while the
roll runs (lead's call, owner asleep: his order is one green roll by noon);
This one cycle is the out of band run for both pulls (decision 79).
- Open through the next roll:
- a guest still fetches dumbpipe from GitHub at boot when its requester and its image have different versions:
CI's 4 network VMs at stand-up, an old stable CLI on updated bidders, new requesters after a bidder rollback;
setup-iroh's fallback to the image's own dumbpipe is proven locally only;
- an installed ssh gateway re-rendering its iroh-dial HostProcess to the new dumbpipe tree across the upgrade:
proven in prod's layout on hat-8 (`~/.tmp/gw-dial/`): `systemctl restart` of the `--fg` gateway re-renders
the dialer onto the 0.40.1 tree (extracted before the dialer starts), dialer down about 21 s, 0 ERROR, new
connections to 0.39.0 and 0.40.1 listeners PASS, rollback the same; an open session breaks at the restart as
on every release; one host, no real signed-in ssh session
- 0007:upgrade does not assert per-bidder iroh reachability of the from binary (follow-up);
- three outer bounds of the test host shorter than their step's stated worst case (`dev up` 200 s, requester stop
120 s, page-round 900 s): latent, never observed; raising them alone was reviewed NOT SAFE (Findings);
- provision StateSyncs are never deleted; the local market delivers no RFP over the market stream, so a change
to RFP intake is proven on staging only.
- Next: the roll of this pull (release, test, promote); then `cli-help` (decision 74) for the second roll.
- Landed without CI (docs, site): the site from `site/` with `/install.sh`, white pages, build-time highlighting,
the lexicon and XRPC reference, lexicon tables and glossary generated from `lexicons/`; the flow map; plan 0053
(rename to `computer.socialweb.*` and `goat lex publish`, after Done when; decision 59).
@@ -137,7 +114,7 @@ green only when both are):
release gate checks, the one build, publish bucket main/<sha>/binaries/{..., SHA256SUMS}
| green
test one ephemeral staging network; every entry
below (46), as staging's own accounts
below (48), as staging's own accounts
(drivers alice1..alice4)
| green
promote on the release VM, as socialweb.computer (release-workload):
@@ -220,7 +197,7 @@ DNS (`/etc/systemd/resolved.conf.d/public.conf`): the region's resolver answers
## Entries
`tests/entries.txt` (47; `tests/gate.sh` reads it): *staging* entries run on staging only (16); the rest run on both (31). 0010 (iOS) was
`tests/entries.txt` (48; `tests/gate.sh` reads it): *staging* entries run on staging only (17); the rest run on both (31). 0010 (iOS) was
removed (owner, 2026-11-17).
| Entry | Proves |
@@ -240,6 +217,7 @@ removed (owner, 2026-11-17).
| 0007:webrtc | getSecrets over the WebRTC channel (iroh off by the contract's user_data): log and contract say webrtc, files written, nothing left |
| 0007:upgrade | *staging* (every bidder changes binary): a guest placed before its bidder's self-update still gets its binaries from the new core and is Held |
| 0007:fence | *staging* (a bidder changes binary under a running guest): the new core reloads the guest's nftables rules (decision 75); from inside the guest, WebRTC to a guest of the same bidder fails before and works after; its open ssh survives |
| 0007:ingress | *staging* (replaces each bidder's IngressProxy `ingress-iroh` by its MarketBidder's component override, restored): after the last of three replacements the market Exposure listens again (dumbpipe_listening after the last started, a live dumbpipe child) and a run limited to that bidder over iroh exits 0 |
| 0008:run | a VM's timeline on its host, release, vm.terminated, records gone, orphan check on every host |
| 0009:run | the browser page rents per bidder over webrtc, a shell over the datachannel, nothing left |
| 0011:run | atirc deploy gates (binary, /chat, unauthenticated revoke 401) |
@@ -767,6 +745,12 @@ All answered 2026-11-17.
capacity and the sign-in link lag (`bid-capacity`); the v0.39.0 listener exits (the v0.40.1 bump, only after
`ingress-rereg` is stable). The tests-only fixes of the red roll (`health-pid`) wait for them: one pull when all
are proven.
79. (2026-11-22, 07:50Z) The dumbpipe v0.40.1 bump rides this roll (answer: "Bump, one shared cycle"): after the
bump's switch proof, ONE hand cycle on bump + colored help (`stack7` 43297eda) is the out of band run for both
pulls; roll one's tree (`bump6` cf34bc81) differs from the cycled tree only by the CLI help change. Decision 51
stands otherwise. The lead's two-release order for the bump (Findings, "bidders deaf over iroh": the bump only
after `ingress-rereg` is in stable) is dropped, the lead's reading, proven on stg-113817: with `ingress-rereg` in
the same release the new core resolves dumbpipe from its own record and an old core rewrites its own path.
@@ -824,7 +808,7 @@ Owner questions open:
Steps 0-15, 17 and 20 done (16, 18 and 19 moved to plan 0041: owner decision 33), then step 21; on the commit of
step 21's second roll:
1. staging: CI `test` 47 of 47 PASS, 0 FAIL on a push run: the step log's gate summary.
1. staging: CI `test` 48 of 48 PASS, 0 FAIL on a push run: the step log's gate summary.
2. prod: CI `promote` (a push run) 31 of 31 *both* entries PASS as prod bob and alice (its log; decision 57). The by-hand gate
on hat-8 is the lead's debug run of the same 31 against prod (owner decision 18), not a number of this plan.
3. Two consecutive green CI promote runs on distinct main commits: each moves stable, every `rc-operators.json` DID
docs/security/proposed-controls.md+6−0
@@ -335,3 +335,9 @@ The 256 byte cap of an image marker: `RequestedMarkerMax` (requested.go), `MaxGu
- Threat: anyone holding a valid service-auth token for the relay (any DID; aud the relay, lxm the method) asks a nonce for another DID, signs it with its own key and subscribes as that DID; the relay then routes `<that DID's label>.xrpc.fedproxy.com` (a bidder's or requester's market endpoint: submitRfp, submitAccept, submitEvent, getSecrets with the guest's workload token) to it, until the DID's own subscriber connects again and replaces it. The relay's DID allowlist (`didAllowed`), when set, limits which DIDs, not who may claim one. Read from the code by the lexicon sweep (2026-11-19, `lexicons/com/fedproxy/temp/xrpc/registration.json`), not tried.
- Status: proposed; not built.
## Guest boot: the dumbpipe fetch checks no sha256
- What: the guest's cloud-init install script checks the dumbpipe archive against the versions.json sha256 of its arch (as `deploy/bidder/preinstall-market.json` already does for the image build) and refuses a mismatch.
- Where: `internal/atproto/cloudinit/modules.go` (the dumbpipe install script: `curl`, `tar -xzf`, `install`; no `sha256sum -c`). It runs only when the image's `/usr/local/bin/dumbpipe.version` differs from `cloudinit.DumbpipeVersionDefault` (a guest booted from an older image than its bidder's cloud-init).
- Threat: a changed or substituted release asset on GitHub (or a path to it) becomes the guest's ssh listener (`iroh.service`, root) with no check; the image build and the host embed are pinned, this path is not.
- Status: proposed; not built.
docs/ARCHITECTURE.md+1−1
@@ -507,7 +507,7 @@ Each cluster has two links, one to the k3s apiserver and one to the cluster vaul
- **What runs.** Per cluster, `dumbpipe listen-tcp --host 127.0.0.1:<target> --ipv4-addr
0.0.0.0:0` twice (`clusterlink.Argv`): target = the cluster's apiserver front port for
`apiserver-link`, the vault's held port for `openbao-link`. dumbpipe v0.39.0, the same release
`apiserver-link`, the vault's held port for `openbao-link`. dumbpipe v0.40.1, the same release
the guests install (the `dumbpipe` asset in `internal/versions/versions.json`).
- **On or off.** On when `SOCIALWEB_CLUSTERPROVIDER_DUMBPIPE` names the binary; the gateway sets
it to the extracted `dumbpipe` asset (`internal/stack/gatewaykcp`). Unset leaves every cluster without
internal/app/marketbidder/marketbidder.go+1−4
@@ -48,8 +48,6 @@ const (
IrohExposureSuffix = "-iroh"
DumbpipeExecutable = "dumbpipe"
BidderProfileProdQr = "prod-qr"
BidderProfileNetwork = "market-network"
BidderProfileDevLocal = "dev-local"
@@ -501,7 +499,6 @@ func (p *pass) desired(ctx context.Context, plan compose.AccountPlan, id *v1alph
out = append(out, compose.Desired{Kind: compose.ComponentKindIngressProxy, Name: IrohIngress, Spec: v1alpha1.IngressProxySpec{
ComponentSpec: compose.NewComponentSpec(compose.ComponentCRStateDir(stateRoot, compose.ComponentKindIngressProxy, IrohIngress), nil),
Provider: v1alpha1.IngressProviderIroh,
Iroh: &v1alpha1.IngressProxyIroh{DumbpipePath: dumbpipe},
}}, compose.ExposureDesired(compose.ComponentKindBidder, BidderComponent, IrohExposureSuffix, exposurev1alpha1.ExposureModeIngress, "", "/",
IrohIngress, exposurev1alpha1.ExposureTLS{}))
}
@@ -537,7 +534,7 @@ func (p *pass) dumbpipe() (string, error) {
if err != nil {
return "", fmt.Errorf("marketbidder: reading the instance record for dumbpipe: %w", err)
}
return rec.Executables[DumbpipeExecutable].Path, nil
return rec.DumbpipePath(), nil
}
func (p *pass) guestDownloadsDir() (string, error) {
internal/app/marketrequester/marketrequester.go+1−4
internal/atproto/cloudinit/cloudinit.go+1−1
internal/atproto/cloudinit/modules.go+17−5
internal/bus/handlers.go+12−5
internal/bus/typed.go+16−4
internal/component/registry.go+1−0
internal/component/runtime.go+9−5
internal/contract/ingress/ingress.go+2−0
internal/controllers/runtime/logbridge.go+62−5
internal/devcore/devcore.go+2−1
internal/devinstance/devinstance.go+9−0
internal/exposure/exposure.go+23−3
internal/exposure/registrar.go+4−0
internal/hostprocess/supervisor.go+30−4
internal/ingress/iroh/provider.go+11−4
internal/ingress/opentunnel/provider.go+15−5
internal/market/bidder/adapters.go+6−2
internal/market/bidder/bidder.go+4−0
internal/market/bidder/bids.go+20−9
internal/market/bidder/kinds.go+13−7
internal/market/bidder/submit.go+77−37
internal/market/bidder/vmsize.go+4−1
internal/market/requester/runner.go+9−0
internal/marketidentity/atproto/account.go+1−1
internal/marketidentity/atproto/accounts.go+25−0
internal/marketidentity/atproto/oauth.go+1−1
internal/services/ingressproxy/provider.go+15−0
internal/services/ingressproxy/service.go+2−0
internal/services/sshserver/service.go+5−2
internal/services/sshserver/signin.go+16−0
internal/stack/componentcore/componentcore.go+3−1
internal/stack/componentcore/controllers.go+1−1
internal/stack/componentcore/ingress.go+18−0
internal/versions/versions.json+16−16
scripts/check-shards.sh+2−2
tests/host/src/more.rs+140−27
tests/host/src/swc.rs+18−0
tests/modules/acc-0007/src/ingress.rsnew+239−0
tests/modules/acc-0007/src/lib.rs+2−1
tests/modules/acc-0007/module.json+7−0
tests/modules/acc-0014/src/lib.rs+61−8
tests/entries.txt+1−0
tests/gate.sh+2−2
.plans.yaml+3−3
AGENTS.md+46−6
CONTRIBUTING.md+1−1