1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
plan: 0007-compute-provider-kcp
written: 2026-10-07
revised: >
2026-10-07, with the user's answers to the plan's questions (decisions below marked 'decided by the user
2026-10-07'). 2026-10-11, folded from docs/research/2026-10-11-vm-lifecycle/ (user-decisions.md,
plan-0006-0007-linux-exploration.md, linux-proofs.md, lifecycle-synthesis.md); decisions from
user-decisions.md (and the user's graceful-stop decision relayed the same day) are marked 'decided by the
user 2026-10-11'. Linux only with cross-OS seams (Windows: plan 0011, macOS: plan 0012); no
socialweb-computer code in a running VM's process tree; VMs as transient services in the user manager of
ONE shared lingering bidder account; identical guest network per guest; CI-built guest images; console to
a capped file; graceful stop then SIGKILL; tri-state Verdict with mismatch = Lost + stop now. New:
M1-GUEST-IMAGE, M2-VM-RUNTIME (replaces M2-UNPRIVILEGED-RUNNER, deleted). Dependencies on 0006 renamed
(M2-HOST-SCOPES -> M2-HOST-UNITS, M3-CLUSTER-SCOPES -> M3-CLUSTER-UNITS, M1-KCP-PORT0 -> M2-KCP-PORT0).
2026-10-12, folded from user-decisions.md (2026-10-12 lines), control-plane-handover.md and linux-proofs.md
E8: the live CtrlAltDel stop measurement (0.21 s, idle guest) recorded on M2-VM-RUNTIME; the two live
defects of today's provider (VM death unnoticed; destroyed guests' work dirs never removed) named as what
M2-VM-RUNTIME and M2-CONTROLLER must prevent, with tests and ACC7-NO-LEAK; new M3-NODE-LOSS-CLOCK (our own
outage never counts against a tenant's node) and M6-CLAIM-RESUME (split from M6-GATEWAY-REPOPULATE: a
restarted door resumes Bidding and Provisioning claims, decided by the user 2026-10-12). 2026-10-12, the
user's decision of a bidder-authored termination event (user-decisions.md, 2026-10-12 line): new
M4-TERMINATION-EVENT; ownership.not_owned and ACC7-NO-PROTOCOL-CHANGE now allow exactly one protocol
addition, com.publicdomainrelay.temp.compute.events.vm.terminated. 2026-10-12 (second fold), the user's
answers to this plan's questions (user-decisions.md, 2026-10-12 lines): per-VM subuid user namespaces
replace the shared-account blast-radius limit (M2-VM-RUNTIME, with the range reserved by
0006:M1-INSTALL); the seed is a separate pure-Go NoCloud cidata drive and the guest grows its own root
(M2-VM-RUNTIME, M1-GUEST-IMAGE; no debugfs/e2fsck/resize2fs on hosts); guest SSH host keys are
guest-reported through a new record replicating onNetwork and verified by the requester and the gateway
with StrictHostKeyChecking yes (new M4-HOST-KEYS, taking the host-key parts of M4-ONNETWORK and
M2-BACKPORT; a second protocol addition). M1-API no longer depends on 0006:M1-K8S-FORK (closed, route not
chosen). Orphan receipts (user 2026-10-12): adopted first with a usage check through a new claim phase
Adopting (M6-GATEWAY-REPOPULATE). M4-TERMINATION-EVENT's live reading recorded (still doing for the Go
side). Answered questions removed; none is left open.
2026-10-12 (third fold), the user's decisions relayed with the fold (recorded as decided by the user
2026-10-12): a bidder's kcp is ONE slot on kine-sqlite and its upgrade is a short kcp restart that VMs
never notice (0006 M2-KCP-SLOTS; M5-RUN-BIDDER, M6-BIDDER-UPGRADE, semantics corrected); guest images come
from a build-host script on any Linux host, CI shells out to it later (M1-GUEST-IMAGE); every org repo
works on main and is pushed (state_rules, ownership); the dev run mode is kcp-native now and ComputeVMs are
placed through HostProcess (M2-VM-RUNTIME, M2-CONTROLLER, M5-RUN-BIDDER); new M0-VM-USERNS (spike with
fallback: firecracker in a per-VM subuid userns with pasta, single-threaded stage vs util-linux unshare);
new M1-PROTOCOL-CHECKLIST (the ordered steps a new bidder- or requester-written NSID needs, learned live on
vm.terminated) and M1-QR-DEPLOY (did-key-associator's broken deploy.sh); M4-CONTRACT-RESTORE split: new
M4-CONTRACT-STORE-FILE lands the file-backed ContractStore, market-bidder load()/onContractChange and
persisted tombstones NOW (a guest leaked 2026-10-12T01:31Z), the ComputeVM-backed store stays in
M4-CONTRACT-RESTORE. M4-HOST-KEYS follows the checklist. No item renamed.
2026-10-12 (fourth fold), the user's paradigm correction (0006 `principle`): new `principle` block here (a
bidder handle is an application instance with its own kcp; modes are profiles; applications meet only over
the market protocol), a market-network-per-workstream state rule, M5-RUN-BIDDER restated per application instance,
NodeClaim/tenant-node items tagged `kcp_use: traditional-exception`. No item added or renamed.
2026-10-12 (fifth fold), from docs/research/2026-10-11-vm-lifecycle/multi-server-to-kcp.md and the user's
answers to its section 7 (user-decisions.md 2026-10-12; plan 0006 principle.decomposition), which override
the research where they differ. New: M1-MARKET-IDENTITY (the research's M4-MARKET-IDENTITY, folded EARLY by
the user's order and with no dependency: sign-in as an object, every interactive step as status, tests
completed through hono-pds's OAuth session minting, the ephemeral-PDS mode kept for bidder and requester),
M3-BIDDER-ACCOUNT-PDS, M3-OIDC-ISSUER-INSTANCE, M3-BIDDER-COMPOSER (no standalone mode: kcp only),
M5-BIDDER-COMPONENTS (one guest-facing DenoPod per provider: OIDC issuer, /v1/on-network, /v1/host-keys; no
guest JSR registry, stripped with every non-iroh transport by 0008 M1-STRIP-LEGACY-TRANSPORTS). Changed:
M4-BIDDER-VAULT (every relay keypair by name, the OAuth session and DPoP key, never kcp Secrets),
M5-RUN-BIDDER (a MarketBidder root object with profiles; the standalone launcher and the stdin prompt go),
M1-FC-INVENTORY, M1-COMMON-EXTRACT, M3-TS-PROVIDER and M4-HOST-KEYS (the guest JSR registry and the
websocat path are not inherited; iroh only), ACC7-NOSSH (the requester's session is over iroh). Research
block multi_server_to_kcp_2026_10_12 records the findings that change items. No item renamed or removed.
queued: >
OPEN. The plan 0005 gate is satisfied: plan 0005 is complete with all 40 items done and its acceptance
checks ACC-ZERO, ACC-MESH and ACC-NOSSH done with evidence (verified 2026-10-12). Only each item's own
depends_on (within this plan and on plans 0006 and 0008) now governs when it starts.
source: >
The user's request of 2026-10-07: from what the bidder's firecracker compute provider taught us, build
compute-provider-kcp. Its TypeScript half is a ComputeProvider the bidder uses, configured
automatically under `socialweb-computer run bidder`; it launches VMs by applying objects to kcp
(Go-native, never kubectl); VM state lives in kcp, so a killed bidder still knows its running VMs. The
user then decided: ComputeVM plus a Go controller; keep both providers, with compute-provider-firecracker
as the GOLD STANDARD whose learnings are backported into the Go side with parity tests; no docker for
Go VM runners (unprivileged, /dev/kvm through the kvm group); a lost guest is replaced through a new
RFP; bidder identity keys live in an OpenBao beside the bidder; and the core binary and deno processes
of BOTH the bidder and the gateway are upgradable while VMs keep running, each re-populating which VMs
it owns afterwards. On 2026-10-11 the user decided the lifecycle: headless Linux servers; VMs survive a
core crash, upgrade and logout; reboot = Lost; the VMM is launched directly by the service manager and no
socialweb-computer code stays in a running VM's process tree; everything else in the decisions below.
principle:
stated: >
Decided by the user 2026-10-12, the ORGANIZING PRINCIPLE of plans 0006-0008 (full text and rules: plan
0006 `principle`): kcp is a core part of APPLICATIONS, not a per-machine cluster; the virtual Kubernetes
clusters project is the one traditional use.
in_this_plan:
- "A bidder handle is an application instance = its own kcp (one slot on kine-sqlite behind a port-0 front) + the bidder manifest set + a profile. Its state is ComputeVM, OpenBaoServer and HostProcess objects in that kcp; its components (the bidder deno, the vault, each VM unit) are DenoPods and HostProcesses reconciled from them. Two handles on one host never share a kcp; the host is only where they run (/dev/kvm, the shared account, the subuid range stay about the machine)."
- "Modes are profiles over the one bidder manifest set: OAuth QR mode against a real account has no PDS pod; the market network profile gives the bidder its own PDS pod. No mode is a separate code path."
- "The bidder and its requesters/gateways talk ONLY over the atproto market protocol (RFP, bid, accept, receipts, onNetwork, vm.hostKeys, vm.terminated). A requester never reads a bidder's ComputeVMs and the gateway never reads them either; what crosses applications is a market record."
- "Every bidder listener behind the edge binds port 0 and reports into its kcp object's status (0006 principle.port0_rule)."
- "kcp_use: traditional-exception marks the items that serve the virtual Kubernetes clusters project (NodeClaims, tenant nodes): M3-NODE-LOSS-CLOCK, M6-CLAIM-RESUME, M6-GATEWAY-REPOPULATE, M6-NOSSH-JOIN."
- "Fifth fold (user 2026-10-12, 0006 principle.decomposition): the bidder is a MarketBidder root object in its own kcp (0006 M2-APP-CONTROLLER), also inside a market network, where the market network's MarketNetwork applies it into the bidder's own kcp and copies the network values onto it. Its sign-in is a MarketIdentity (M1-MARKET-IDENTITY) whose pending actions the foreground CLI renders; every hono-bidder option maps to a root-object field or profile; hono-bidder runs only as a DenoPod entry (kcp only) from M5-RUN-BIDDER on; its keys, relay keypairs, OIDC JWK and OAuth session live only in its OpenBao (M4-BIDDER-VAULT); its guest-facing endpoints become their own DenoPod (M5-BIDDER-COMPONENTS); guests are reached over iroh only."
state_rules:
- TypeScript lives in this repository (decided by the user 2026-10-12): the closure's ten repositories were vendored with their history under ts/<repository>/ and the originals are frozen; a path this plan names under ts/ is changed here, never in the original.
- Item status (todo, doing, done, blocked) lives here; the git history is the evidence. No evidence lists are added (user, 2026-10-12).
- status is one of todo, doing, done, blocked.
- An item is done when its done_when holds on main; the commits say how.
- Open items are mirrored in NEXT.yaml (P7-*).
- A decision marked "decided by Claude, user may veto" was not confirmed by the user; one marked
"decided by the user 2026-10-07", "decided by the user 2026-10-11" or "decided by the user 2026-10-12"
was.
- An item names its dependencies in depends_on; an item may not start while one of them is not done.
Cross-plan dependencies are written <plan>:<item id>.
- >
RFP flow is the spine (org-root CLAUDE.md). Every test that needs a live guest drives the requester
(requestCompute / runComputeContract) against a bidder through RFP, bid, accept and cloud-init
user_data. No test or code boots a guest any other way: no fixture-booted guest, no boot from a
boot-spec, no hand-made user_data. Live unit mechanics without a guest use stand-in executables in
transient units.
- >
Linux only (decided by the user 2026-10-11). OS-specific bodies live in GOOS-tagged files behind the
vmruntime, hostsvc and guestnet interfaces; darwin and windows compile stubs returning ErrUnsupported,
gated by 0006:M0-XOS-GATE. The Windows backend is plan 0011, the macOS backend plan 0012.
- >
Dev run mode is kcp-native now (decided by the user 2026-10-12; 0006 state rule): every VM unit is a
HostProcess child of its ComputeVM, placed on systemd-user in production and on proc (devsup) in dev
and tests; a bidder's kcp is one slot on kine-sqlite. Dev adds no provisioning path: a dev instance's
guests come up only through runComputeContract against that instance's bidder.
- >
Market network per workstream (decided by the user 2026-10-12; 0006 state rule). This plan's live tests run in the
task's own market network (a git worktree of this repository; 0008 M1-MARKET-NETWORK-PARALLEL, later
`dev up --profile market-network`), with the market network's own bidders and
requesters. The shared real bidder (bobvmbuilder on hat-8) and alice are a final soak only, after the
market network tests pass.
- >
Every org repo works on `main` and is pushed to GitHub as work lands (decided by the user 2026-10-12):
atproto-market, hono-compute-provider, typescript-helpers, did-key-associator and socialweb-computer-k8s
changes for this plan are committed on main (or merged to it from a worktree branch) and pushed when
the item or its red-first step lands.
- >
A new NSID the bidder or the requester writes follows M1-PROTOCOL-CHECKLIST, in its order, before any
live run publishes it.
ownership:
owned: >
All code in socialweb-computer-kcp, the vendored deno-kcp market network, and the atproto-market requester
libraries. Decided by the user 2026-10-07: anything in hono-compute-provider and atproto-market may be
changed; decided by the user 2026-10-12: committed on main and pushed to GitHub as work lands (replaces
"committed only, never pushed"). Also changed by this plan, same rule: typescript-helpers (oauth-scope
role sets) and did-key-associator (QR client scope, deploy script) for M1-PROTOCOL-CHECKLIST,
M1-QR-DEPLOY and M4-HOST-KEYS, and socialweb-computer-k8s (the door) for M6-*.
not_owned: >
The market protocol records and lexicons. This plan changes no existing record, NSID or wire shape: the
bid config, receipt, onNetwork and vm.delete records stay what the firecracker provider writes. The only
protocol additions are two bidder-authored records carried in the existing market.event envelope:
com.publicdomainrelay.temp.compute.events.vm.terminated (decided by the user 2026-10-12,
M4-TERMINATION-EVENT) and com.publicdomainrelay.temp.compute.events.vm.hostKeys (decided by the user
2026-10-12: guest-reported SSH host keys replicating onNetwork; the separate NSID rather than new
onNetwork fields decided by Claude, user may veto; M4-HOST-KEYS). kcp objects introduced here are local
state of one host and are never published.
research:
measured_on: "2026-10-07; hono-compute-provider c06b2ad, atproto-market 117ab30, deno-kcp 25d10f92, kcp-libs f6911d2, this repo 13349d6. Lifecycle research and proofs 2026-10-09..11 against f0460fd and hono-compute-provider 4d7da59, see vm_lifecycle_2026_10_11"
compute_provider_abc: "hono-compute-provider lib/abc/compute-provider/mod.ts:39-55 ComputeProvider {name, provision, destroy, createBidConfig, injectAcceptBundle, getDroplet?, setup?, teardown?} (OS-free; no TypeScript seam needed for other host OSes)"
firecracker_gold_standard:
- "lib/compute-provider-firecracker/mod.ts:563 guests is an in-memory Map, :643 the network index an in-memory counter, :734-749 destroy stops only guests in that Map"
- ":629-732 provision: name fc-<shortId(requester)>-<8 hex> (:641), work dir <workRoot>/<name> (:642), enrichment order: OIDC enricher (:647), onNetwork base URL (:650), the requester-named relay address else the container's gateway (:662), JSR URL (:663), guest pointed at host with the guest TLS port (:664, :521-524), /root/.curlrc --resolve (:548); disk from the contract (:608-610, :670); mem parsed (:812-827)"
- ":29-37 NSIDs and SECRETS_PREFIX /root/secrets/digitalocean.com/serviceaccount; :612-627 bid config wif.simple with to_issue exchange-firecracker-oidc, token_path ${SECRETS_PREFIX}/token (c06b2ad: the bid config names the token path the provisioning script writes); :161-166 injectAcceptBundle at the accept path; :347-515 preinstall manifest (packages, files, runs baked into the image, leftToBoot); :592-606 image ensure with reuseStale and fingerprint; :574-590 a JSR registry for guests; :751-807 POST /v1/on-network; :837-848 the issuer's droplet view (containerName for the host-key scan done from inside the guest's container, oidc-sub tags, guest IP); :860-871 issuer mounted in serve.onConnected; :697-710 rbac record created at provision and deleted at destroy"
- "lib/microvm-firecracker/mod.ts:155-163 the bidder must not hold /dev/kvm, so each guest boots in its own privileged runner container (:203-216, no restart policy), work dir chmod 0777 (:186-189), result.json polled (:218-245), stop = kill + rm (:254-277); :48-58 the host address a guest dials is the container's default gateway; compute-provider-firecracker/mod.ts:129-145, :643, :660-664 the guest's network and the host address it dials depend on the allocated index, and user_data (/etc/hosts, .curlrc --resolve, JSR URL) is rewritten with it (exploration F6)"
- "cmd/socialweb-nodeboot/main.go:26-29 boots from -spec, stops by -stop -work-dir; :283-288 refuses a second boot in a work dir already holding a guest; it re-execs itself for the namespace verb, and pasta runs inside the guest's namespace to wire the tap and start the VMM (:305-310); the namespace verb (nodevm.RunNamespace -> nodefirecracker.runVMM, namespace.go:15-58) STAYS RESIDENT as firecracker's parent with Pdeathsig and writes instance.json (exploration F2); prep is in-process: reflink clone, NoCloud seed written with host debugfs -w, grow with host e2fsck + resize2fs (F3); console unbounded into <work>/guest.console (F4); node images built ON THE BIDDER with chroot + apt-get + go build, kernel = the host's /boot kernel (F5)"
restart_defects:
- "live, hat-8 2026-10-07 (lead's reading): restarting a bidder loses its receipts; vm.delete then fails (404 through the dispatcher) and the guest leaks. In code: atproto-market lib/market-bidder/mod.ts:175 activeContracts is in memory, lib/market-bidder-compute/mod.ts:395-409 answers an unknown receipt with 400 and destroys nothing"
- "live, 13349d6: a restarted bidder ran old provider code from a stale DENO_DIR v8_code_cache_v2 (bid_config token path and node image of the previous version) until the cache was moved aside; the launcher now passes --no-code-cache (plan 0008 M1-DENO keys DENO_DIR by closure sha256)"
- "atproto-market hono-bidder/mod.ts:103-113 a fresh relay keypair per process, so the issuer URL guests were told, and the OpenBao JWT mount plan 0005 keys by issuer, move on every restart; hono-compute-provider lib/oidc-issuer-hono/mod.ts:42-53 the signing key store defaults to memory"
- "plan 0005 M3-NODE-LOSS (door half): a restarted door adopts Joining and Running claims but marks Bidding and Provisioning claims Failed, because their guest's user_data names the old door's per-process secrets server"
kcp_side_precedent:
- "kcp-libs impl/execrunner/pod.go:184-196 KCP_TOKEN, KCP_SERVER, KCP_WORKSPACE for a DenoPod with a serviceAccount; deno-kcp main.go:89 token TTL 1h"
- "socialweb-computer-k8s lib/socialweb-computer-k8s-nodeclaim-kcp/mod.ts:172-192 a TypeScript kcp REST client; api/v1alpha1/resources.go:18-22 + cmd/socialweb-apigen generate schemas, exports and TS wire types"
vm_lifecycle_2026_10_11:
files:
- "docs/research/2026-10-11-vm-lifecycle/user-decisions.md (authoritative)"
- "docs/research/2026-10-11-vm-lifecycle/plan-0006-0007-linux-exploration.md (A gaps, B seams and interfaces, C revised items, D questions)"
- "docs/research/2026-10-11-vm-lifecycle/linux-proofs.md (E0-E7, hat-8, systemd 259.9, firecracker v1.13.1)"
- "docs/research/2026-10-11-vm-lifecycle/lifecycle-synthesis.md"
proven_on_linux:
- "E1/E4b: a transient service in the user manager holding pasta -> VMM stand-in survives restart, kill -9 and stop of the bidder unit; kill -9 of pasta (MainPID) makes systemd kill the VMM (control-group); kill -9 of the VMM ends pasta with 137 and the unit is failed"
- "E4: pasta -> /bin/sh -c 'exec \"$0\" ...' trampoline -> helper creates a persistent tap with CAP_NET_ADMIN in pasta's userns, then execve's; the cgroup then holds exactly pasta + the VMM; direct exec of our binary from pasta is DENIED under SELinux enforcing (pasta_t), so the trampoline is required; pasta also makes a PID namespace in which the VMM is PID 1"
- "E4-term/E5: firecracker has no SIGTERM handler and, as PID 1 of pasta's PID namespace, ignores SIGTERM, so a plain `systemctl stop` hangs until TimeoutStopSec (45 s default) then SIGKILLs"
- "E5: --config-file boots immediately with --api-sock already bound; GET / returns {id, state, vmm_version}; PUT /actions SendCtrlAltDel is x86_64 only (guest needs i8042/atkbd) and 400 before boot; do not use --no-api; --id must be [A-Za-z0-9-]{1,64}; host_dev_name attaches to a pre-created tap, name <= 15 bytes; api-sock path < 108 bytes and a stale socket file makes firecracker exit (unlink first); log timestamps are local time with no offset (run with TZ=UTC)"
- "E2: PropExecStart(argv, false) + CollectMode=inactive keeps a crashed VM's Result; MemoryMax needs MemorySwapMax=0 or the guest swaps; OOMPolicy=stop gives Result=oom-kill; AllowedCPUs and IOWeight are not enforced in the user manager"
- "E3: Lost = ESRCH, start-ticks mismatch, UnixProcessIdUnknown, NoUnitForPID, NoSuchUnit; Unknown = only dial/auth/timeout; GetUnitByPIDFD works on the VMM child as well as on MainPID"
- "E1: StartTransientUnit on an existing name fails 'already loaded': a restarted bidder adopts, never re-creates"
live_e8_2026_10_12:
- "E8, live through the RFP flow (alice-request.sh -> socialweb-computer-bidder@bobvmbuilder.bsky.social on hat-8, firecracker provider, runner-container era, guest fc-1379be663dd9-6e867147): PUT /actions SendCtrlAltDel at 00:06:21.6827Z, the guest ran systemd shutdown to reboot.target, 'Firecracker exiting successfully. exit_code=0' at 00:06:21.8913Z: 0.21 s for an IDLE guest. The guest kernel had CONFIG_SERIO_I8042=y and CONFIG_KEYBOARD_ATKBD=y; ctrl-alt-del.target -> reboot.target, which firecracker treats as VM exit. The grace window with a workload (k3s agent + pods) is still to be measured."
- "E8 defect 1 (VM death unnoticed): after firecracker exited, the nodeboot parents stayed alive and the runner container stayed Up; the bidder never noticed the VM died and the requester's ssh session hung until closed by hand (then vm.delete 200)."
- "E8 defect 2 (work dirs leak): destroyed guests' work dirs are never removed: 32 dirs, 63 GB on disk (1.6 TB apparent, sparse rootfs) on hat-8."
control_plane_handover_2026_10_11:
- "docs/research/2026-10-11-vm-lifecycle/control-plane-handover.md 7: Joining/Running claims are durable (NodeClaim status.contract + status.grant + the cluster vault); Bidding/Provisioning die with the door only because the per-contract secrets-server keypair and grant are in memory (atproto-market lib/guest-capability-secrets/mod.ts:50-75); the accept written just before a crash is a VM no claim names (ARCHITECTURE section 9)"
- "control-plane-handover.md 6: NODE-LOSS needs no rebase across a kcp/door/provider restart (the tenant's kcm keeps writing real transitions) but DOES when our side of the path was down (cluster cold start, link restart, front restart): count from max(lastTransitionTime, pathUpSince)"
multi_server_to_kcp_2026_10_12:
file: "docs/research/2026-10-11-vm-lifecycle/multi-server-to-kcp.md (atproto-market, hono-compute-provider and socialweb-computer-kcp as on disk 2026-10-12, swck 5a030ec)"
findings_that_change_items:
- "F1 (-> M3-BIDDER-ACCOUNT-PDS, M5-RUN-BIDDER): the market network's bidder does NOT use the market network's root:bob PDS DenoPod - 70-bidder.yaml:24 passes no handle, so it runs its own in-process repo (createLocalPDSAgent) served through the dispatcher, and bob's hono-pds is only probed for readiness (accept.sh:412-414). The handle+password path also builds a LOCAL repo (hono-bidder/mod.ts:262-282)."
- "F1 (-> M3-BIDDER-ACCOUNT-PDS): --atproto-pds-url (hono-bidder/cli-args-env.ts:76-81) is read nowhere, and createRemoteAgent is imported but never called (hono-bidder/mod.ts:10)."
- "F3 (-> M4-BIDDER-VAULT): a fresh relay keypair per relay per process (hono-bidder/mod.ts:103-113, :104) - the market relay, each provider's relay and the in-process repo's relay - so every ingressRef and the OIDC issuer URL move on each restart; the bidder runs 1 + P relays (+1 for an in-process repo), not one."
- "F3 (-> M3-OIDC-ISSUER-INSTANCE): oidc-issuer-hono's signing key store is in memory AND module-global (lib/oidc-issuer-hono/mod.ts:42-57; configureOidc :63-77 is called by every createOidcIssuer :616), so two providers in one process overwrite each other's issuer URL getter."
- "F2 (-> M1-MARKET-IDENTITY): interactive steps are stdout/stdin and cannot run in a pod - the QR printed (hono-bidder/mod.ts:229-234, :720-726), association a prompt('[y/N]') on stdin (:348, :743; request-vm-ssh :430), browser OAuth spawns xdg-open (:166-171), an expired session deletes its file and Deno.exit(1) (:122-135); a DenoPod has stdin null (70-bidder.yaml:85), so the market network passes --skip-qr."
- "F6 (-> M5-RUN-BIDDER, 0008 M2-SERVICE-TEMPLATE): per-user state under $HOME is shared by every instance (private-key-hex-path and pds-state-path defaults computed with Deno.env.get in cli-args-env.ts:235-246; oauth-qr-session.ts:9-20)."
- "F9 (-> M3-BIDDER-COMPOSER): hono-bidder/mod.ts is an 825-line CLI composing every server; the in-process harnesses (atproto-market test/*_integration_test.ts) re-compose the same parts by hand."
- "Guest transports (user 2026-10-12): the guest JSR registry (compute-provider-firecracker :574-590, a TCP 0.0.0.0:0 listener per firecracker/local provider) and every non-iroh transport are stripped (0008 M1-STRIP-LEGACY-TRANSPORTS), so the Go side inherits none of them."
items:
- id: M0-VM-USERNS
milestone: M0
title: "Spike: firecracker opens /dev/kvm inside a per-VM subuid user namespace with pasta - a single-threaded vmexec stage vs util-linux unshare"
depends_on: []
decision: >
Decided by the user 2026-10-12: an M0 spike with a fallback for the per-VM subuid user namespaces
decided the same day (M2-VM-RUNTIME). unshare(CLONE_NEWUSER) refuses a multi-threaded caller (EINVAL),
and every Go process is multi-threaded, so the stage that creates the namespace must be single-threaded
or must create it with clone() for a fresh child. newuidmap/newgidmap (setuid, the distro's) are host
prerequisites, written by `install bidder`'s subuid/subgid range (0006 M1-INSTALL) and checked by
check-host. Candidates, measured on hat-8 (Fedora 44, SELinux enforcing) in a scratch branch: (A) a
single-threaded stage of ours - either a Go one-shot that starts the next stage with
SysProcAttr.Cloneflags=CLONE_NEWUSER and has the maps written by newuidmap/newgidmap on the child's pid
while the child (exec'd in the new namespace) waits on an inherited pipe before execve'ing pasta, or a
tiny static non-Go stage asset - with the stage gone from the tree after start; (B) util-linux unshare
>= 2.38 `--map-users=<inner>:<outer>:65536 --map-groups=...` (it forks a short-lived helper that runs
newuidmap and exits), a host prerequisite beside newuidmap. Each candidate, under the SELinux pasta_t
trampoline (linux-proofs E4): pasta creates its net namespace and the tap inside the subuid namespace,
firecracker opens /dev/kvm by the retained supplementary kvm group (no setgroups), the work files are
reachable by ACL only, the unit's tree after start is pasta + the VMM only, the stage adds no resident
process, and the start latency added. Measured with stand-ins first (a stand-in VMM that opens /dev/kvm
and ignores SIGTERM, in transient units of the systemd-user backend and under devsup), then ONE guest
through the RFP flow: alice-request.sh -> RFP -> bid -> accept -> cloud-init user_data against a
scratch-branch bidder whose firecracker launch goes through the chosen stage (the guest is born only
from user_data; nothing is booted by hand). Decided by Claude, user may veto: prefer (A) (no new host
prerequisite, and our stage is pinned like every asset); (B) is the fallback when (A) fails or costs
more than a small file. If NEITHER keeps /dev/kvm, the next fallback is an ACL entry on /dev/kvm for the
reserved range written by install (recorded as a deviation); if a target distro refuses unprivileged
user namespaces outright, check-host refuses that host by name and the user is asked - there is no
silent fallback to running VMMs as the shared account's uid.
files: ["scratch branch only (stage code, stand-in, measurements)", "docs/research/2026-10-11-vm-lifecycle/vm-userns-spike.md (readings)"]
test_first: "None (measurement). Readings: per candidate, uid_map/gid_map of the stand-in VMM, /dev/kvm open result, the unit's process list after start, added latency, SELinux denials (ausearch), and for the chosen stage the RFP-flow guest's boot to onNetwork with its VMM's uid_map recorded (UTC)."
done_when: "the stage chosen and recorded with readings; the fallback's status recorded; M2-VM-RUNTIME's decision updated to name the stage."
risks:
- "Fedora's pasta SELinux policy may deny pasta inside an unprivileged user namespace the trampoline does not cover; recorded with the denial, and the profile change (install-time) named."
- "The RFP-flow guest needs the live market network on hat-8; until 0008 M1-MARKET-NETWORK-PARALLEL lands the run is serialized with every other market network run."
status: done
evidence:
- id: M1-DECIDE
milestone: M1
title: Record the ComputeVM-plus-Go-controller design and the VM lifecycle in ARCHITECTURE
depends_on: []
decision: >
Decided by the user 2026-10-07: a typed ComputeVM CRD reconciled by a Go controller, not a DenoPod
per VM. The bidder runs as a DenoPod and writes ComputeVMs; kcp lists running VMs. Reasons kept on
record: deleting a DenoPod SIGKILLs its group (supervisor.go:156,:322) so no release runs; the 5m
runner timeout (pod.go:17) killed the market network's bidder mid-contract (run 4); DenoPod status has no typed
guest fields; booting needs /dev/kvm reach the TypeScript sandbox must not hold. Also recorded, decided
by the user 2026-10-11: no socialweb-computer code in a running VM's process tree (the VMM is launched
by the user manager; our code is one-shot prep; the vmexec helper execve's away); every VM is one
transient service in the user manager of ONE shared lingering account (swc-bidder), units namespaced by
handle; reboot = Lost; the tri-state Verdict (Running | Lost | Unknown) with identity mismatch on an
active unit = Lost + StopUnit immediately and an unreachable bus or socket = Unknown, never Lost;
graceful stop (SendCtrlAltDel, grace, then SIGKILL); identical guest network per guest; images built
off the bidder (2026-10-12: by a build-host script on any Linux host, CI shells out to it later);
console to a capped file in the work dir, never the host journal; explicit deno permissions
for the bidder. Decided by the user 2026-10-12, also recorded: per-VM subuid user namespaces (each VM
unit's pasta and firecracker run in their own user namespace mapped to a distinct sub-range of the
account's reserved subuid/subgid range, so a VMM escape lands in an unprivileged uid that owns no
bidder file; Unix accounts are never created on demand); the seed is a separate cidata drive and the
guest grows its own root; guest SSH host keys are guest-reported and verified, never scanned from the
host (M4-HOST-KEYS). Decided by the user 2026-10-12 (third fold), also recorded: every VM unit is a
HostProcess child of its ComputeVM placed on systemd-user (production) or proc (dev); a bidder's kcp is
one slot on kine-sqlite and its upgrade is a short kcp restart VMs never notice.
files: ["docs/ARCHITECTURE.md (section 5 subsection 'Host processes and the VM lifecycle')"]
test_first: "None (decision record)."
done_when: "ARCHITECTURE subsection committed with these citations."
risks: []
status: done
evidence:
- id: M1-FC-INVENTORY
milestone: M1
title: Inventory what compute-provider-firecracker learned, as requirements the Go side must meet
depends_on: []
decision: >
Decided by the user 2026-10-07: compute-provider-firecracker (TypeScript) is the gold standard; it
was sourced from nodeboot/nodeimage and has advanced since. This item reads it (and
microvm-firecracker, node-image-firecracker, oidc-issuer-hono's enricher, cloud-init-common's
modules it composes) and writes docs/compute-provider-requirements.yaml: one entry per learning
with id, the TypeScript source line, the behaviour, whether the Go side already meets it, and the
parity test that will prove it. At least: user_data enrichment order and idempotence; the accept
bundle at the accept path (accept.json); the onNetwork reporter, its targets and the relay.localhost
alias; SECRETS_PREFIX and the token path the provisioning script writes; .curlrc --resolve and
CURL_HOME; guest TLS port rewrite; network range, index and MAC (kept for the firecracker provider);
disk size from the contract and mem parsing; image ensure, fingerprint, reuseStale and the preinstall
manifest; the JSR registry for guests (recorded as NOT inherited: deleted with every non-iroh guest
transport by 0008 M1-STRIP-LEGACY-TRANSPORTS, decided by the user 2026-10-12); the issuer's droplet view; rbac records created and deleted;
boot timeout and result.json fields; the work dir's two writers. Added 2026-10-11 (decided by Claude,
user may veto), each a requirement that changes with no container: the host address a guest dials
(constant pasta gateway, not the runner container's gateway), console capture (capped file), graceful
stop, image provenance (built by the build-host script, not on the bidder), seed transport (a drive, not debugfs), the guest
host keys (guest-reported, not a host-side scan in the guest's namespace: today oidc-issuer-hono
getPublicKeyFromSshd, mod.ts:446-483, runs ssh-keyscan through `container exec` for provisioningValidate,
:520-544; M4-HOST-KEYS), and a boot timeout with no
result.json (no resident writer: Running is read from firecracker GET /). Added 2026-10-12: the two E8
live defects of today's provider (a VMM that exits is not noticed; destroyed guests' work dirs are never
removed) are recorded as requirements the Go side must NOT inherit, with M2-VM-RUNTIME's and
M2-CONTROLLER's tests as their parity tests.
files: ["docs/compute-provider-requirements.yaml"]
test_first: >
hono-compute-provider test/requirements_inventory_test.ts: every exported function and every
logger event name in lib/compute-provider-firecracker/mod.ts and lib/microvm-firecracker/mod.ts is
referenced by some requirement entry; red before the file exists.
done_when: "inventory committed; test green; each entry names its parity test id."
risks: ["The firecracker provider keeps moving; the test re-runs on every hono-compute-provider change so a new learning cannot land without an entry."]
status: done
evidence:
- "socialweb-computer-kcp 48348e0 docs/compute-provider-requirements.yaml (55 entries: 31 port, 9 removed-legacy, 8 superseded-by-M2-VM-RUNTIME, 3 by M1-GUEST-IMAGE, 1 each by M2-CONTROLLER/M4-HOST-KEYS/M4-TERMINATION-EVENT/M1-PROTOCOL-CHECKLIST) + 5d36404 FC-ADOPT-AFTER-RESTART; hono-compute-provider ad73463/54d1b8f test/requirements_inventory_test.ts"
- "red: NotFound before the yaml existed; green 4/4 (--allow-read only); the test also checks every cited file:line exists and every superseded-by item exists in a plan; on 2026-10-12 it caught the contract-store merge's new readHostBootId and adopt events (red) until FC-ADOPT-AFTER-RESTART was added (green)"
- "Go-side gaps found: DEFAULT_DNS_GATEWAY 172.30.0.1 vs Go 172.16.0.x plan; oidc reporter falls back to https://xrpc.fedproxy.com (must fail closed); enrichment not idempotent (store enriched user_data on the ComputeVM); work dir 0777 must not be inherited (0700)"
- id: M1-API
milestone: M1
title: ComputeVM in api/v1alpha1 with an OS-neutral status, generated schema, export, workspace type and TypeScript wire types
depends_on: [M1-DECIDE, "0005:M2-CODEGEN", "0006:M2-MANIFESTS"]
decision: >
Go types are the source of truth (plan 0005 M2-CODEGEN). ComputeVM spec: vm {cpus, mem, disk, role},
owner {handle} (shared account, decided by the user 2026-10-11: the controller and GC namespace by
handle), userDataSecret, requester {did, plc}, contract {receiptUri, receiptCid, acceptUri, acceptCid,
submitEventUrl, acceptedAt}, image {reuseStale}, stopGraceSeconds. No network index anywhere: every
guest gets the identical address plan in its own pasta namespace (decided by the user 2026-10-11), so
user_data enrichment is final at create time and there is no two-phase write. Status through /status by
two field managers (decided by Claude, user may veto), OS-neutral so the Windows and macOS backends
(plans 0011, 0012) need no schema revision: the controller owns phase (Pending, Preparing, Booting,
Running, Releasing, Lost, Failed), lostReason (HostRebooted, UnitGone, UnitFailed, OOMKilled, VMMExited,
IdentityMismatch), observedGeneration, network {guestIP, hostAddress, prefix, mac}, image {ref, sha256,
hypervisor}, runtime {backend (free string, no enum), ref, instance, pid, startTime (opaque string),
hostBootID, vmmID}, workDir, bootedAt, message, conditions (Prepared, Started, RuntimeObserved
True|False|Unknown, GuestReported) - Unknown is a CONDITION, the phase stays what it was; the provider
owns market {onNetworkUri, address, hostKeys[] {type, publicKey, fingerprint}, hostKeysUri,
reportedAt} (host keys from the guest's own report, M4-HOST-KEYS). Finalizer computevm.socialweb.computer/release;
printer columns phase, guestIP, requester, age. Export computevms; WorkspaceType bidder binds
denoruntime, computevms and hostprocesses (for the bidder's OpenBaoServer). apigen's second TypeScript
target writes ts/hono-compute-provider/lib/common/compute-provider-kcp-common/mod.ts, drift-checked when
the sibling is present. Dependencies (repointed 2026-10-12 from 0006:M1-K8S-FORK, closed as route not
chosen): the schema, export and TS wire types come from the existing apigen pipeline (0005:M2-CODEGEN,
done) on today's k8s.io v0.36.4; the bidder set lives in deploy/sets.yaml (0006:M2-MANIFESTS).
files: ["api/v1alpha1/computevm_types.go", "api/v1alpha1/resources.go", "internal/apigen", "deploy/*computevm*.yaml, deploy/workspacetype-bidder.yaml", "deploy/sets.yaml (bidder)", "ts/hono-compute-provider/lib/common/compute-provider-kcp-common/"]
test_first: "TestComputeVMRoundTripsThroughTheSharedFixtures (Go and the generated TS validator), TestTheComputeVMSchemaIsRevisionLocked, TestTheComputeVMSchemaHasNoBackendEnumAndNoNetworkIndex; red first."
done_when: "check.sh --fast green with apigen no drift; both fixture suites green."
risks: ["APIResourceSchema.spec is immutable; the first revision is reviewed field by field, including that no Linux-only field (unit, pasta pid, API socket) is in it."]
status: done
evidence:
- id: M1-COMMON-EXTRACT
milestone: M1
title: Move the firecracker provider's pure user_data and network functions into compute-provider-common
depends_on: []
decision: >
Two impls of one ABC must not import each other (org-root CLAUDE.md). Move unchanged into
lib/common/compute-provider-common: injectAcceptBundle, onNetworkUrlFromBundle, onNetworkTargets,
onNetworkReporterModule, withOnNetworkBaseUrl, withOnNetworkReporter, addressTheRequesterNamed,
pointGuestAtHost, the curlrc module, injectJsrUrl, networkFor, guestMacFor, shortId, parseMemMiB,
preinstallManifest and the NSID and path constants. Firecracker imports them; its behaviour is the
reference and does not change (it keeps networkFor and its index). compute-provider-kcp calls
pointGuestAtHost with the constant host address (M3-BIDDER-FLAG), never a container gateway. Fifth fold
2026-10-12: injectJsrUrl and the guest JSR registry belong to a stripped legacy path (0008
M1-STRIP-LEGACY-TRANSPORTS, decided by the user 2026-10-12); whichever of the two items lands second
makes sure they are not in compute-provider-common.
files: ["ts/hono-compute-provider/lib/common/compute-provider-common/mod.ts", "ts/hono-compute-provider/lib/compute-provider-firecracker/mod.ts", "hono-compute-provider/deno.json"]
test_first: "test/user_data_enrichment_golden_test.ts (goldens captured from firecracker before the move); fails to compile before, passes after; the suite stays green."
done_when: "golden test green; deno task test green."
risks: ["Helpers closing over module state (jsrPort, nextIndex) stay in the impl and are passed in."]
status: done
evidence:
- id: M1-QR-DEPLOY
milestone: M1
title: did-key-associator's deploy ships a self-contained backend, has a static-only mode and a dry-run
depends_on: []
decision: >
Decided by the user 2026-10-12: did-key-associator/scripts/deploy.sh is broken and an item owns fixing
it. Reading (2026-10-12): it scp's hono-backend/mod.ts alone to /opt/qr-fedfork/mod.ts and runs `deno
run -A --unstable-kv mod.ts` (deploy.sh:21, :48, :63), but the backend imports ts/typescript-helpers/*
and ./cli-args-env.json (did-key-associator 4b3db9c moved its config to the CLI), so running the script
would take the QR backend at qr.fedfork.com down. On 2026-10-12 only the static files were deployed by
hand (01:41:55Z), with a backup at /root/qr-static-backup/20261012T014155Z on the host. Decided by Claude,
user may veto: (1) the backend ships as ONE self-contained artefact - `deno compile` of
hono-backend/mod.ts with `--include cli-args-env.json` (the org CLAUDE.md compile-bundling rule; the
sibling typescript-helpers imports resolved at build time), falling back to a vendored tree (`deno
vendor`-style local copy with an import map) if compile cannot carry --unstable-kv; the systemd unit
runs that artefact, never a loose mod.ts; (2) `deploy.sh --static-only` copies dist/, the client
metadata and the Caddyfile and reloads caddy, touching no backend file or unit (the 2026-10-12 manual
step, made a mode; M1-PROTOCOL-CHECKLIST step 3 uses it); (3) `deploy.sh --dry-run` builds everything
locally, prints every remote path it would write, the unit diff and the backup path, and connects to
nothing; (4) every real run first copies what it replaces to /root/qr-backup/<UTC ts>/ and prints a
restore command. The backend artefact is started once locally from an empty temp dir with no sibling
repos before any upload.
files: ["did-key-associator/scripts/deploy.sh", "did-key-associator/deno.json (compile task)", "did-key-associator/test/deploy_test.ts"]
test_first: >
did-key-associator test/deploy_test.ts: 'the compiled backend starts from an empty dir with no
../typescript-helpers and answers its client-metadata route' (red on today's loose mod.ts), '--dry-run
writes nothing and lists every remote path', '--static-only names no backend file or unit'; red first.
done_when: "tests green; one real --static-only run and one full run to qr.fedfork.com with the backup path and a post-deploy check of the client metadata and the backend recorded (UTC); pushed to GitHub on main."
risks: ["The QR backend serves every account's sign-in; a full deploy runs only after a green --dry-run and with the restore command printed."]
status: done
evidence:
- "did-key-associator 10be131 (oauth-scope generator shared by build.ts and a drift test) + f9dcf22 (scripts/deploy.sh: compiled self-contained backend, --static-only, --dry-run printing every remote action, dated backup of static/Caddyfile/backend/unit excluding kv.db, health check on the xrpc route with automatic restore, no set -x / secret echo; caddy.service no longer rewritten), merged to main 2026-10-12"
- "tests red then green: deploy_test.ts 4/4 (old script ran ssh in dry-run, set -x, no compile task); compiled backend starts on port 0 from an empty dir offline and answers its xrpc route 401/400; oauth_scope_drift_test.ts 3/3 (fails at 3cde452^); merged main 7/7"
- "deployed 2026-10-12T03:26:31Z (host key ED25519 SHA256:GY2/Nh2R... verified first): `deployed mode=full backup=/root/qr-backup/20261012T032631Z`; public check: oauth-client-metadata.json carries vm.terminated, /xrpc/com.fedfork.atprotoOauthQR answers 401; the host no longer needs deno"
- "left: backend/mod.ts still reads KV_PATH via Deno.env.get (CLAUDE.md CLI rule) - follow-up when the backend joins the cli-args-env pattern"
- id: M1-PROTOCOL-CHECKLIST
milestone: M1
title: "The new-NSID checklist - lexicon, role-set scope, regenerated QR client scope deployed, every account re-signed in - with a check test"
depends_on: [M1-QR-DEPLOY]
decision: >
Decided by the user 2026-10-12, learned live on vm.terminated (M4-TERMINATION-EVENT evidence): a new
collection the bidder or the requester writes needs, IN ORDER, (1) the lexicon (atproto-market
lexicons/ + generated market-lexicons types + the NSID constant); (2) the NSID in the writer's
oauth-scope role set in typescript-helpers (5b071fe did it for the bidder's vm.terminated); (3)
did-key-associator regenerating generated/oauth-scope.js (3cde452) and redeploying its static client
metadata to qr.fedfork.com (M1-QR-DEPLOY --static-only); (4) every account that writes it re-signing in
through the QR, because a token refresh keeps the old scopes. Skipping any step makes publishing fail
with 403 ScopeMissingError (seen 2026-10-12T01:32:39Z). Decided by Claude, user may veto: the checklist
is written as docs/new-nsid-checklist.md in atproto-market (the repo that owns the lexicons) and
referenced from this plan's state rule; a check test in atproto-market imports typescript-helpers' role
sets and asserts that every collection the bidder (market-bidder, market-bidder-compute) or the
requester (requester-xrpc, compute-request-xrpc) writes via createRecord/putRecord is in that role's set,
by scanning the NSID constants those packages pass; a second check in did-key-associator asserts
generated/oauth-scope.js equals what its generator produces from the current typescript-helpers (no
drift). M4-HOST-KEYS (vm.hostKeys) is the first NSID to follow it.
files: ["atproto-market/docs/new-nsid-checklist.md", "ts/atproto-market/test/role_scope_coverage_test.ts", "did-key-associator/test/oauth_scope_drift_test.ts", "typescript-helpers (role sets, read only here)"]
test_first: >
atproto-market test/role_scope_coverage_test.ts red with a fixture collection written by the bidder and
missing from its role set, then green on the tree (vm.terminated included); did-key-associator
test/oauth_scope_drift_test.ts red with a stale generated file, then green.
done_when: "both checks green and run by each repo's test task; the checklist committed and pushed on main; the four steps recorded for vm.terminated as the worked example."
risks: ["Step 4 cannot be automated (a person scans the QR per account); the requester and bidder log ScopeMissingError with the missing NSID so the step is found, not guessed."]
status: done
evidence:
- "atproto-market acd3f3e merged to main cd09af4 2026-10-12: test/role_scope_coverage_test.ts (scans 12 bidder writes in market-bidder, market-bidder-compute, hono-bidder and 8 requester writes in requester-xrpc, compute-request-xrpc against the typescript-helpers oauth-scope role sets; a fixture proves an unscoped write is reported), `deno task test:protocol`, docs/new-nsid-checklist.md with vm.terminated as the worked example"
- "red: fails with typescript-helpers at 5b071fe^ naming vm.terminated?action=create (lib/market-bidder-compute/mod.ts:192); green at 5b071fe; merged main 5/5"
- "note: market-bidder-worker's worker.instance writes are outside the bidder/requester sets and not scanned; step 4 (re-sign-in) was exercised live 2026-10-12T01:43:50Z for bobvmbuilder"
- id: M1-GUEST-IMAGE
milestone: M1
title: Guest images (kernel + rootfs) built by a build-host script per hypervisor x arch, pinned by sha256, fetched and verified on bidders
depends_on: [M1-FC-INVENTORY, "0006:M1-ASSETS"]
decision: >
Decided by the user 2026-10-12 (third fold): images come from a build-host script that runs on any
Linux host (scripts/build-guest-image.sh); CI will shell out to the same script later; there is no CI
service now, so wherever this item says "CI" below it means that script run on a build host (hat-8
today), and reproducibility is proved by two runs of it.
Decided by the user 2026-10-11: guest images are built in CI on a Linux build host per hypervisor x
arch, pinned by sha256, shipped as assets; bidders need no root, chroot, apt or go toolchain. Decided by
Claude, user may veto: targets firecracker-x86_64 and firecracker-aarch64 here (hyperv-x86_64 is plan
0011, vz-arm64 plan 0012). Each image = a kernel with a pinned config (virtio-mmio; i8042 + atkbd on
x86_64 so SendCtrlAltDel reaches the guest), a raw ext4 rootfs with cloud-init (NoCloud datasource read
from the separate cidata drive the controller attaches, and resize_rootfs/growpart enabled so the guest
grows its own root to the disk size the controller truncates the copy to - decided by the user
2026-10-12: the rootfs is copied as-is and debugfs/e2fsck/resize2fs leave the hosts), NO SSH host key
baked in (cloud-init's ssh module generates fresh keys on each guest's first boot, ssh_deletekeys on,
so every guest's keys are its own and are what M4-HOST-KEYS reports), the guest init socialweb-nodeinit
(a guest binary, built here, not a host asset), the preinstall
manifest of M1-FC-INVENTORY baked in, and a manifest {hypervisor, arch, kernel sha256, rootfs sha256,
inputs}. cmd/socialweb-nodeimage moves to the build pipeline (scripts/build-guest-image.sh, run by CI;
a `dev`-tag verb remains for local builds); the host's /boot kernel stops being an input, so every
bidder boots the same kernel. versions.json pins each image's URL and sha256; internal/guestimage
fetches, verifies and stores images keyed by (hypervisor, arch, sha256) under the account's state dir
and refuses a mismatch. Images are not embedded in the binary (0006:M0-SIZE-BUDGET). A guest boot from
the image is proved only through the RFP flow (M6-*).
files: ["scripts/build-guest-image.sh", "cmd/socialweb-nodeimage (build pipeline; dev tag)", "cmd/socialweb-nodeinit", "internal/guestimage/", "internal/versions/versions.json (image pins)", "deploy/guest-kernel/firecracker-x86_64.config, firecracker-aarch64.config"]
test_first: >
internal/guestimage TestAnImageWhoseShaDiffersIsRefused and TestAnImageIsFetchedOnceAndReusedByHash;
build-pipeline TestTheGuestKernelConfigHasTheGracefulStopAndVirtioMmioDrivers (reads the pinned
config) and TestTheRootfsCarriesNoSshHostKeyAndEnablesGrowpart (reads the built image's file list and
cloud.cfg); red first.
done_when: "both images built twice by scripts/build-guest-image.sh on a build host with equal sha256; pins recorded; a bidder host fetches and verifies them with no root."
risks:
- "A raw ext4 rootfs with no partition table must grow through cloud-init's resizefs on /dev/vda; unproven, measured in the first M6 run (df inside the guest reported through onNetwork or the guest's own log), with a partitioned image as the fallback."
- "Moving off the host kernel changes the module set the guest relies on (iptables, bridge, overlay for k3s); the kernel config lists them and the M6 join proves them."
status: done
evidence:
- id: M1-MARKET-IDENTITY
milestone: M1
title: "EARLY: MarketIdentity - an application's own atproto identity and sign-in as an object; what the user must do is status, never stdin"
depends_on: []
decision: >
Decided by the user 2026-10-12 (multi-server-to-kcp.md section 7; user-decisions.md): QR/OAuth sign-in
becomes an identity CRD that sets the sign-in up and surfaces what the user must do (URL to visit,
approval) as status, which the foreground CLI shows; for testing, hono-pds's OAuth session minting
completes it automatically; the ephemeral-PDS mode (in-process repo + association records) stays
supported for both bidder and requester; OAuth sessions and every key live in the application's own
OpenBao, never in kcp. Folded EARLY with no dependency (the research's M4-MARKET-IDENTITY), by the
user's order: the interactive steps are what cannot run in a DenoPod today (research
multi_server_to_kcp_2026_10_12 F2). Decided by Claude, user may veto: kind MarketIdentity
(socialweb.computer/v1alpha1, apigen; distinct from the gateway's RBAC AtprotoIdentity,
api/v1alpha1/resources.go:22) with spec {mode oauthQr|oauthBrowser|pdsAccount|localRepo|oauthDelegated,
handle, pdsUrl, associateWith, approvedAssociations[], keysRef, sessionRef} and status {did, serviceDid,
phase AwaitingScan|AwaitingAuthorization|AwaitingApproval|Ready|SessionExpired, pendingAction {kind
scanQr|visitUrl|approveAssociation|signInAgain, url, callerDid, handle, expiresAt}, conditions};
keysRef and sessionRef are vault references only - no field holds a key, a token, a DPoP key or an
authorization code. localRepo is the ephemeral-PDS mode: the association request becomes pendingAction
approveAssociation and an approval is a spec patch to approvedAssociations[]. oauthBrowser: the
loopback callback listener runs only in the foreground CLI, which hands the authorization response to
the identity component through the application's vault, never through a kcp object. An expired
session is the SessionExpired phase, never Deno.exit(1). TypeScript (atproto-market, ABC-layered): an
IdentityPrompt port in lib/abc with sibling impls identity-prompt-kcp (status writes and a spec watch
over kcp REST, the shape of the door's nodeclaim-kcp), identity-prompt-memory (library tests) and
identity-prompt-tty (today's stdout/stdin/xdg-open behaviour, used only by the standalone launchers
still running and deleted with them in M5-RUN-BIDDER and 0008 M4-RUN-REQUEST, so production keeps
working until then); hono-bidder and request-vm-ssh reach every interactive step through the port.
Sessions go through a SessionStore port: today's file store, under the instance's state dir and never
$HOME, until M4-BIDDER-VAULT (bidder) and 0008 M4-RUN-REQUEST (requester) plug in the OpenBao store.
Go: internal/appcli renders pendingAction in a terminal (the QR from url, an approval prompt that
writes the spec patch, a re-sign-in on SessionExpired) for `run bidder` and `run request` (wired by
M5-RUN-BIDDER and 0008 M4-RUN-REQUEST); the door UI later reads the same status. Test driver (the
user's choice): given a pendingAction url naming a hono-pds authorization request, a helper completes
it through hono-pds's session minting (hono-pds lib/atproto-oauth-server-deno createSessionInjector,
mod.ts:147) for a test account, so the oauthQr, oauthBrowser and pdsAccount modes complete with no
human; inside a market network the QR relay is the MarketNetwork's qr DenoPod (0006 M2-APP-CONTROLLER).
files: ["api/v1alpha1/marketidentity_types.go", "internal/apigen (schema, export, TS wire types)", "ts/atproto-market/lib/abc/identity-prompt/", "ts/atproto-market/lib/identity-prompt-kcp/", "ts/atproto-market/lib/identity-prompt-memory/", "ts/atproto-market/lib/identity-prompt-tty/", "ts/atproto-market/hono-bidder/mod.ts and request-vm-ssh/mod.ts (interactive steps through the port)", "ts/atproto-market/test/helpers (hono-pds session-minting driver)", "internal/appcli/ (pendingAction renderer)"]
test_first: >
atproto-market against a fake kcp: 'the QR url appears in status.pendingAction and the scan completes
through the hono-pds session-minting driver', 'an approval patched into spec completes the association
in localRepo mode for the bidder and for the requester', 'an expired session sets SessionExpired and the
process stays up', 'no MarketIdentity write carries a key, token, DPoP key or code' (scans every status
and spec write); Go: TestTheMarketIdentitySchemaHasNoSecretField,
TestTheRendererShowsTheQrAndReturnsWhenReady (fake kcp). Red first.
done_when: "green; on a market network the bidder DenoPod runs without --skip-qr and its association is approved through the identity object (UTC reading); pushed on main in atproto-market."
risks: ["Until 0008 M4-MARKET-NETWORK-PORT gives each market network application its own kcp, the market network bidder's MarketIdentity lives in today's shared market network kcp (its DenoPod's KCP_WORKSPACE); the port writes wherever its DenoPod's kcp is, so nothing changes when it moves."]
status: done
evidence:
- id: M2-VM-RUNTIME
milestone: M2
title: The Linux VM runtime - one-shot prep, one transient service per VM whose tree is pasta -> firecracker, graceful stop
depends_on: [M0-VM-USERNS, M1-DECIDE, M1-GUEST-IMAGE, "0006:M2-HOST-UNITS", "0006:M2-HOST-BACKEND-PROC", "0006:M2-HOST-CONFORMANCE", "0006:M1-ASSETS", "0006:M2-VM-TREE-AUDIT"]
decision: >
Third fold 2026-10-12 (decided by the user: dev run mode kcp-native now, ComputeVMs placed through
HostProcess; details decided by Claude, user may veto, dev-run-mode.md 4.2 and 5): Runtime composes a
hostsvc.Manager of EITHER backend and produces the VM's UnitSpec (the HostProcess child's spec, placed
by the supervisor, 0006 M2-HOSTPROCESS-KIND; M2-CONTROLLER owns that child); Watch is Manager.Watch;
on proc the tree is devsup -> the unit's process group (userns stage gone, pasta -> firecracker), the
cgroup limits are recorded unenforced through Capabilities().EnforcesLimits=false as a ComputeVM status
condition, and the VM's memory stays capped by firecracker's machine-config; the API socket lives
under the instance's runtime dir ($XDG_RUNTIME_DIR/swc/<id8>/swvm/ in dev). The userns stage is the one
M0-VM-USERNS chose. RuntimeRef.backend names the placement backend (systemd-user | proc); the
hypervisor is status.image.hypervisor.
Replaces M2-UNPRIVILEGED-RUNNER (deleted 2026-10-11: it started each guest as a resident `internal
nodeboot` wrapper, and its test booted a guest from a fixture boot-spec, an RFP-spine violation).
Decided by the user 2026-10-07: no docker for VM runners; unprivileged through /dev/kvm via the kvm
group, never chmod. Decided by the user 2026-10-11: no socialweb code in the VM's process tree; ONE
shared lingering account; identical guest network; console to a capped file; graceful stop first
(SendCtrlAltDel over the API socket, a grace window measured live on the market network, then the unit
stops with KillSignal=SIGKILL; x86_64 only, arm64 goes straight to SIGKILL until a guest-side shutdown
path exists). Decided by Claude, user may veto (exploration B1-B3, linux-proofs E1-E5): internal/vmruntime
(interface Runtime {Capabilities, Prepare, Start, Inspect, Stop, List, Cleanup}; VMSpec, RuntimeRef,
Verdict, LostReason; ForHost() picks the backend compiled for GOOS) with firecracker_linux.go composing
hostsvc, internal/guestnet (pasta_linux.go), internal/guestdisk (clone_linux.go FICLONE with copy
fallback, truncate, seed.go pure-Go NoCloud cidata drive) and internal/fcapi (firecracker client over
the unix socket + config writer), plus unsupported_other.go. Seed, decided by the user 2026-10-12: a
SEPARATE pure-Go NoCloud cidata drive (vfat or iso9660 labelled cidata, written without any host tool);
the rootfs is copied as-is (reflink clone) and only truncated to the contract's disk; the guest grows
its own root (M1-GUEST-IMAGE); debugfs, e2fsck and resize2fs are not run on, or needed by, any host.
Prep is one-shot in the controller: clone
the rootfs, truncate to the contract's disk, write cidata.img (user-data, meta-data), fc.json
(boot-source, drives rootfs + cidata, machine-config, network-interfaces host_dev_name fctap0 + MAC),
params.json; the work dir is <state>/handles/<hid>/vms/<name> (0700, one uid); the API socket is
$XDG_RUNTIME_DIR/swvm/<short-id>.sock (< 108 bytes, unlinked before start). Per-VM user namespace,
decided by the user 2026-10-12 (replaces the shared-account blast-radius limit): `install bidder`
(0006:M1-INSTALL) reserves ONE subuid AND subgid range for the shared account (65536 ids x 256 slots);
each VM unit runs pasta and firecracker in its own user namespace mapped to its own 65536-id sub-range
(inside 0..65535 -> host base + k*65536 ..), the rootless-podman pattern through the distro's setuid
newuidmap/newgidmap, so a VMM escape lands in an unprivileged host uid that owns nothing. Decided by
Claude, user may veto: the slot k is the lowest index not named by any loaded socialweb-vm-* unit of
the ACCOUNT (every handle shares the range), chosen under an flock on <state>/vmslots.lock held until
StartTransientUnit returns (two handles' controllers never pick the same slot), and carried in the
unit's argv, so adoption reads it back from the unit and no counter is persisted. The VMM's host uid
gets access to exactly its own work files by POSIX ACL entries the controller sets in prep (an owner
may name any uid in an ACL; no chown, no privilege): rw on rootfs.ext4, r on cidata.img and fc.json, x
on the path to the work dir and the API socket dir (incl. $XDG_RUNTIME_DIR), rwx on its socket dir;
nothing else under the account's home, state dir, vault or another VM's dir names a slot uid. The
supplementary kvm group is retained across the userns entry (no setgroups call), so /dev/kvm opens by
group. The first stage of ExecStart is a one-shot that creates the user namespace, has the maps written
by newuidmap/newgidmap and execve's pasta inside it, leaving no process behind (the util-linux
`unshare --map-users` pattern: a short-lived helper child writes the maps and exits);
unshare(CLONE_NEWUSER) refuses a multi-threaded caller, so which binary runs that stage - a
single-threaded stage of ours, or the host's util-linux unshare >= 2.38 as a stated host prerequisite
beside newuidmap - is measured by M0-VM-USERNS (moved there 2026-10-12, with its fallbacks). Unit
socialweb-vm-<hid>-<name>.service (hid = first 10 hex of sha256(handle)): ExecStart = <userns stage,
slot k> --
runtime/<pasta-sha>/pasta --config-net (the identical address plan; host address = pasta's mapped
gateway, constant) -- /bin/sh -c 'exec "$0" "$1"' runtime/<vmexec-sha>/socialweb-vmexec params.json (the
trampoline SELinux requires); socialweb-vmexec (new tiny asset, cmd/socialweb-vmexec) creates the
persistent tap and bridge over netlink in pasta's namespace (the nodevm.WireNamespace logic without
exec'ing ip) and execve's runtime/<fc-sha>/firecracker --id <vm-name> --config-file fc.json --api-sock
<sock>; after start the cgroup holds pasta + firecracker only. Properties: Type=exec, Restart=no,
KillMode=control-group, KillSignal=SIGKILL (firecracker ignores SIGTERM as PID 1 of pasta's PID
namespace), TimeoutStopSec=10, MemoryMax = guest mem + VMM overhead, MemorySwapMax=0, OOMPolicy=stop,
TasksMax, CPUQuota, CPUWeight, Slice=socialweb-b-<hid>.slice (not app.slice), no AllowedCPUs, no
IOWeight, no ManagedOOMPreference, Environment=TZ=UTC, StandardOutput=append:<work>/console.log and
StandardError=append:<work>/vmm.log (tenant data never enters the host journal; the controller caps
each file by copy-truncate rotation keeping one bounded predecessor, and both go with the work dir on
release), PropExecStart(argv, false), CollectMode=inactive. Start records RuntimeRef {backend
systemd-firecracker, ref unit, instance InvocationID, pid = the firecracker pid (the cgroup's process
whose exe is the pinned firecracker), startTime, hostBootID, vmmID}, then polls GET / for {id ==
vmmID, state Running} within the boot timeout. Stop: x86_64 PUT /actions SendCtrlAltDel, wait
spec.stopGraceSeconds (default = the grace window measured in M6-BIDDER-UPGRADE, recorded in
versions.json) for the unit to go inactive, then StopUnit (SIGKILL to the cgroup); aarch64 StopUnit at
once; then record Result, ResetFailedUnit, remove the work dir and socket. The resident namespace verb,
nodevm.RunNamespace, WriteNamespaceRecord, FindVMM, WriteInstance and cmd/socialweb-nodeboot are
deleted. Live measurement (linux-proofs E8, 2026-10-12, through the RFP flow): SendCtrlAltDel -> firecracker
exit 0 took 0.21 s on an IDLE guest (the guest's ctrl-alt-del.target -> reboot.target, which firecracker
treats as VM exit); the grace window with a k3s workload is still to measure (M6-BIDDER-UPGRADE). Two live
defects of today's provider (E8) this runtime must make impossible, decided by Claude, user may veto:
(a) VM death unnoticed - after firecracker exited, the nodeboot parents and the runner container stayed
up and the bidder never noticed. Here the VMM is the only child of pasta in the unit: when firecracker
exits for ANY reason (guest reboot or poweroff, CtrlAltDel, crash, kill) pasta exits and the unit goes
inactive or failed with nothing of the VM left in its cgroup; Inspect reports that on the next call, and
Runtime exposes a Watch (D-Bus PropertiesChanged on the unit's ActiveState) so the controller learns it
within seconds instead of a pass interval. (b) Work dirs never removed - 32 dirs, 63 GB on hat-8. Here
Stop and Cleanup remove the work dir (rootfs clone, cidata, console and vmm logs) and the API socket
for every terminal outcome (released, Lost, Failed, boot timeout), and List reports work dirs under
<state>/handles/<hid>/vms/ so the controller can collect any with no ComputeVM (M2-CONTROLLER).
files: ["internal/vmruntime/", "internal/guestnet/", "internal/guestdisk/", "internal/fcapi/", "internal/vmruntime/subids_linux.go (slot choice, flock, ACL grants)", "cmd/socialweb-vmexec/ (tap stage; userns stage if chosen)", "internal/nodevm, internal/nodefirecracker (deleted or reduced)", "cmd/socialweb-nodeboot (deleted)", "docs/bidder-host.md (kvm group and subuid range via `install bidder`)"]
test_first: >
Offline: TestPrepareWritesASeedDriveAndFcJsonAndNoHostToolIsExecd, TestTheVMUnitSpecHasSigkillSwapZeroAndNoIgnoreFailure,
TestTheApiSocketPathIsUnder108BytesAndUnlinkedFirst, TestStopSendsCtrlAltDelThenWaitsTheGraceThenStopsTheUnit
(fake fc API and fake hostsvc, x86_64), TestStopOnAarch64GoesStraightToStopUnit, TestTheConsoleFileIsCappedByRotation,
TestCleanupRemovesTheWorkDirAndSocketForEveryTerminalOutcome (released, Lost, Failed, boot timeout),
TestTheSubRangeSlotIsTheLowestNotNamedByAnyLoadedVMUnitOfTheAccount (another handle's units count),
TestTwoConcurrentPreparesNeverPickTheSameSlot, TestPrepareGrantsTheSlotUidItsOwnWorkFilesByACLAndNothingElse,
TestTheSeedIsASeparateCidataDriveAndTheRootfsIsNotEdited (fc.json lists two drives; the rootfs bytes
equal the image's up to its original size).
Behind livegate on hat-8 with stand-ins (a stand-in VMM that ignores SIGTERM like firecracker, pinned as
a test asset; never a guest): TestTwoVMUnitsMapDisjointSubRanges (two stand-in VM units:
/proc/<vmm>/uid_map and gid_map name disjoint sub-ranges inside the account's reserved range),
TestTheVMMUidOwnsNoBidderFile (no file under the account's home and state dir is owned by either
stand-in VMM's host uid; from inside the stand-in, opening the bidder vault dir, the kcp state and the
other VM's rootfs is EACCES while its own rootfs opens and /dev/kvm opens by group),
TestTheVMUnitsTreeIsPastaAndTheStandInOnly (0006:M2-VM-TREE-AUDIT passes; the userns stage is gone),
TestAStopOfAVMMThatIgnoresSigtermEndsWithinTimeoutStopSec, TestTheVMUnitSurvivesSystemctlRestartOfTheBidderUnit,
TestAVMMThatExitsOnItsOwnEndsItsUnitAndLeavesNoProcess (the stand-in exits 0 and, separately, is killed:
the unit is inactive/failed, its cgroup is empty, pasta is gone, and Watch reported it within 2 s - E8
defect a). The stand-in live tests run through the 0006 M2-HOST-CONFORMANCE harness on systemd-user and
on proc (TestTheVMUnitsTreeIsPastaAndTheStandInOnly and TestAVMMThatExitsOnItsOwnEndsItsUnitAndLeavesNoProcess
on both), plus TestOnProcTheComputeVMRecordsLimitsUnenforced. Red first.
done_when: "offline and stand-in tests green; a real guest's boot, stop grace, tree audit and the no-leak readings are proved by M6-BIDDER-UPGRADE through the RFP flow."
risks:
- "E8 measured 0.21 s from SendCtrlAltDel to exit on an IDLE guest; the grace with a k3s workload is unmeasured, so a 10 s placeholder stays the default until M6-BIDDER-UPGRADE records the loaded value."
- "The CI kernel (M1-GUEST-IMAGE) must keep CONFIG_SERIO_I8042 and CONFIG_KEYBOARD_ATKBD (E8's guest had both) or SendCtrlAltDel does nothing and every stop waits the full grace."
- "pasta's mapped-gateway flag differs across pasta versions; the flag is pinned against the pinned pasta asset's own --help in this item."
- "pasta inside an already-unprivileged user namespace (mapped to a subuid) still has to create its net namespace and the tap; proved with the stand-in first, and the SELinux pasta_t trampoline (linux-proofs E4) re-measured under the extra namespace."
- "Whether firecracker's jailer-free run inside a subuid namespace keeps /dev/kvm by the retained supplementary group on every target distro is measured; if a distro drops it, the fallback is an ACL entry on /dev/kvm for the reserved range, written by install, recorded as a deviation."
status: done
evidence:
- id: M2-BACKPORT
milestone: M2
title: Bring the Go side to the firecracker provider's standard, proved by parity tests
depends_on: [M1-FC-INVENTORY, M1-COMMON-EXTRACT, M2-VM-RUNTIME]
decision: >
Decided by the user 2026-10-07: backport the gold standard into the Go side with parity tests. Each
requirement of M1-FC-INVENTORY that the Go side misses is implemented in internal/vmruntime,
internal/guestimage or internal/computevm, in the order the inventory lists. The parity boundary
(decided by Claude, user may veto, 2026-10-11; boot-spec.json was nodeboot's format and is gone): the
VMSpec handed to vmruntime.Prepare, the seed drive's bytes, fc.json, the bid config record and the
issuer's droplet view. One recorded RFP fixture (vm record, requester, accept bundle, contract) is
handed to the TypeScript firecracker provider (its microvm stubbed at that boundary) and to the Go
controller (its unit start stubbed at the same boundary); the outputs must be equal field for field,
except names and addresses the fixture marks as allocated. No guest is booted by the parity test. Pure
enrichment stays TypeScript (compute-provider-common, applied by compute-provider-kcp before the
ComputeVM is written); the Go side owns image, disk, seed, unit, stop. Guest host keys are NOT part of
this backport: the firecracker provider's host-side ssh-keyscan is deleted rather than ported, and the
guest-reported keys, their record and their verification are M4-HOST-KEYS (decided by the user
2026-10-12); the droplet view reads them from status.market.
files: ["internal/vmruntime", "internal/computevm", "testdata/compute-parity/", "ts/hono-compute-provider/test/compute_parity_test.ts", "internal/computevm/parity_test.go"]
test_first: "Both parity tests, run against the same fixture, red on every requirement the Go side misses (the failure list is the evidence); then green."
done_when: "every inventory entry marked met with its parity test green on both sides."
risks: ["The guest-reported host keys arrive after boot (M4-HOST-KEYS); the droplet view shows none until the guest's report, which the parity fixture records as the expected shape."]
status: done
evidence:
- id: M2-CONTROLLER
milestone: M2
title: The Go ComputeVM controller boots, adopts by the tri-state Verdict, and releases
depends_on: [M1-API, M2-VM-RUNTIME, "0006:M2-APPLY", "0006:M2-HOSTPROCESS-KIND"]
decision: >
Placement (decided by the user 2026-10-12: ComputeVMs are placed through HostProcess; details decided by
Claude, user may veto): each ComputeVM owns exactly ONE HostProcess child (ownerReference, placement
supervisor, restartPolicy Never) whose spec is vmruntime's VM UnitSpec; the supervisor is the only
caller of Manager.Start, does the generic identity rows (unit, InvocationID, pid + start ticks, boot id,
backend mismatch) and stops an active unit on a mismatch at once; this controller adds the VM rows
(firecracker GET / id and state, the API socket) and maps the child's status into the Verdict below.
Releasing: SendCtrlAltDel and the grace here, then the child is deleted (the supervisor stops the unit
with SIGKILL). The bidder's kcp is one slot on kine-sqlite (decided by the user 2026-10-12), so a kcp
restart is a short gap in which units keep running and the controller re-adopts every VM when kcp
answers; nothing is stopped because kcp was down.
internal/computevm, a client-go informer loop over the bidder's kcp (decided by Claude, user may veto,
2026-10-11: no dependency on the forked deno-kcp controller factory). A ComputeVM without
deletionTimestamp: Preparing (image ensure, prep), Booting (unit started), Running (GET / Running). With
deletionTimestamp: Releasing, graceful stop (M2-VM-RUNTIME), remove the work dir, drop the finalizer.
Re-population after any restart or upgrade (decided by the user 2026-10-07: the bidder re-populates from
ComputeVM plus live VMM state) by the Verdict, decided by the user 2026-10-11 (mismatch on an active unit
= Lost and StopUnit immediately; unreachable bus or socket = Unknown, retried, never Lost), evaluated in
order: bus dial/auth/timeout -> Unknown (condition RuntimeObserved=Unknown, phase unchanged, backoff);
status.runtime.hostBootID != the current boot_id -> Lost(HostRebooted), nothing to stop; unit NoSuchUnit
-> Lost(UnitGone); unit failed or inactive -> Lost(UnitFailed | OOMKilled | VMMExited from Result and
ExecMainStatus), recorded, then ResetFailedUnit; unit active with InvocationID != recorded, or the
recorded pid failing pidfd_open / start ticks / pidfd_send_signal(0), or GetUnitByPIDFD answering
UnixProcessIdUnknown, NoUnitForPID or another unit -> Lost(IdentityMismatch) and StopUnit immediately;
identity matches but the API socket is unreachable or times out -> Unknown; GET / id != vmmID or state !=
Running -> Lost(IdentityMismatch) and StopUnit immediately; all match -> Running. A Lost guest is never
booted again (decided by the user 2026-10-07: replaced through a new RFP; plan 0005 M3-NODE-LOSS does
that from the requester side), and the receipt stays releasable. Orphans: a unit with prefix
socialweb-vm-<hid>- of THIS handle and no ComputeVM is reported and stopped after a 10-minute grace;
units of other handles on the shared account and of market networks (socialweb-market-network-*) are never touched (name
filter in hostsvc.List). A user-manager restart, linger removal or `loginctl terminate-user` leaves no
units: every ComputeVM goes Lost(UnitGone), the same as a reboot. Against the two E8 live defects
(2026-10-12; decided by Claude, user may veto): (a) a VM that dies is noticed - the controller consumes
vmruntime Watch, so a unit going inactive or failed while its ComputeVM is not Releasing becomes
Lost(VMMExited | UnitFailed | OOMKilled) within seconds, not at the next pass; the provider learns it
from the ComputeVM watch (M3-TS-PROVIDER), and the requester's side replaces the node through plan 0005
M3-NODE-LOSS. (b) nothing of a VM outlives it - the finalizer is removed only after the work dir and the
API socket are gone (Cleanup verified by stat), for a Lost or Failed ComputeVM as well as a released
one; a work dir under <state>/handles/<hid>/vms/ with no ComputeVM of this handle and no unit is
removed after the same 10-minute grace as an orphan unit, and every removal is logged with its size.
files: ["internal/computevm/", "internal/stack/bidder"]
test_first: >
Offline (fake hostsvc, fake fc API, fake kcp), one test per Verdict row, plus
TestAnUnreachableBusIsUnknownNeverLost, TestAnUnreachableApiSocketWithMatchingIdentityIsUnknown,
TestIdentityMismatchStopsTheUnitImmediately, TestABootIDChangeMakesEveryVMLostWithoutStartingAnything,
TestALostGuestIsNeverBootedAgain, TestDeletingAComputeVMStopsGracefullyAndRemovesItsWorkDirBeforeTheFinalizerGoes,
TestAnotherHandlesUnitIsNeverTouched, TestANetworkUnitIsNeverTouched,
TestAVMMExitObservedByWatchMakesTheComputeVMLostWithinOneEvent (E8 defect a),
TestTheFinalizerStaysUntilTheWorkDirIsGoneForReleasedLostAndFailed and
TestAWorkDirWithNoComputeVMAndNoUnitIsRemovedAfterTheGrace (E8 defect b; a fake vms/ tree seeded with
32 orphan dirs is empty after one pass past the grace, another handle's dirs untouched),
TestEveryComputeVMOwnsExactlyOneHostProcessAndNoOtherCodePathStartsAVMUnit,
TestAnObjectPlacedByAnotherBackendIsLostNotAdopted, TestAKcpRestartStopsNoVMAndEveryVMIsReadoptedAfter;
with stand-in VMMs on proc (dev instance) and systemd-user: TestACoreRestartUnderTheProcBackendAdoptsEveryVMUnit.
Red first.
done_when: "offline tests green; live proofs are M6-*."
risks: ["A false Lost costs a tenant a VM (LXD shipped this bug); every Lost row names a definite answer from the kernel or systemd, and the tests pin that transport errors never reach a Lost row."]
status: done
evidence:
- id: M3-TS-PROVIDER
milestone: M3
title: compute-provider-kcp, the TypeScript ComputeProvider that writes ComputeVMs
depends_on: [M1-API, M1-COMMON-EXTRACT]
decision: >
ABC-layered: lib/common/compute-provider-kcp-common (generated types), lib/compute-provider-kcp (impl,
fetch to kcp, no Hono), lib/hono-factory-compute-provider-kcp (mounts the OIDC issuer and POST
/v1/on-network on the provider's serve in serve.onConnected; no guest JSR registry - stripped as a
legacy guest path by 0008 M1-STRIP-LEGACY-TRANSPORTS, decided by the user 2026-10-12; the factory is
what M5-BIDDER-COMPONENTS later runs as its own DenoPod).
provision: enrich user_data with the common functions in firecracker's order using the constant guest
host address (one write, no two-phase gate: the guest network is identical per guest, decided by the
user 2026-10-11), server-side apply the user_data Secret and the ComputeVM (fieldManager
compute-provider-kcp, spec.owner.handle set), watch until Running or Failed/Lost within the boot timeout.
destroy: DELETE and wait for 404. createBidConfig: the firecracker record's exact shape. getDroplet: from
a list+watch cache filled in setup() before the bidder serves. The kcp credential is a token file the Go
side rotates (TokenRequest every TTL/2) because the DenoPod env token expires after 1h (deno-kcp
main.go:89). It runs under the bidder DenoPod's explicit permissions (M5-RUN-BIDDER). Decided by Claude,
user may veto.
files: ["ts/hono-compute-provider/lib/compute-provider-kcp/", "ts/hono-compute-provider/lib/hono-factory-compute-provider-kcp/", "hono-compute-provider/deno.json", "internal/stack/bidder (token file)"]
test_first: "test/compute_provider_kcp_test.ts against a fake kcp: provision resolves on Running; destroy only after 404; createBidConfig equals firecracker's; setup() rebuilds getDroplet; the user_data Secret is written once. Red first."
done_when: "unit suite green; deno check of hono-bidder green."
risks: ["If watch is unavailable through the workspace path, bounded polling; which one was used is recorded."]
status: done
evidence:
- id: M3-BIDDER-FLAG
milestone: M3
title: hono-bidder --compute-provider-kcp
depends_on: [M3-TS-PROVIDER]
decision: >
New cli-args-env options with env names (no Deno.env.get): compute-provider-kcp (COMPUTE_PROVIDER_KCP),
compute-provider-kcp-server (KCP_SERVER), compute-provider-kcp-workspace (KCP_WORKSPACE),
compute-provider-kcp-token-file, compute-provider-kcp-ca-file, compute-provider-kcp-handle (the owner
handle units and work dirs are namespaced by), compute-provider-kcp-guest-host-address (the constant
address every guest dials). A wiring block shaped like firecracker's (hono-bidder/mod.ts:546-597).
Auto-configuration means run bidder's DenoPod template sets them; the bidder does not sniff its
environment. Both providers stay selectable (decided by the user 2026-10-07).
files: ["ts/atproto-market/hono-bidder/cli-args-env.ts", "ts/atproto-market/hono-bidder/mod.ts", "ts/atproto-market/hono-bidder/deno.json"]
test_first: "test/bidder_cli_kcp_provider_test.ts: --compute-provider-kcp without a server, handle or guest host address refuses at startup naming the missing options; red first."
done_when: "test green; existing bidder tests green."
risks: []
status: done
evidence:
- id: M3-BIDDER-ACCOUNT-PDS
milestone: M3
title: hono-bidder signs in to a real PDS account by handle and password, so a market network bidder uses its own PDS pod
depends_on: []
decision: >
Fifth fold 2026-10-12 (multi-server-to-kcp.md finding F1; decided by Claude, user may veto, within the
user's principle that a market network bidder has its own PDS pod): today --atproto-handle/--atproto-password
builds a LOCAL repo (hono-bidder/mod.ts:262-282), --atproto-pds-url is read nowhere
(cli-args-env.ts:76-81) and createRemoteAgent is imported but never called (:10), so the market network's bidder
ignores the root:bob PDS DenoPod (70-bidder.yaml:24; accept.sh:412-414 only probes it). That path
becomes a remote-PDS session (createRemoteAgent against --atproto-pds-url, MarketIdentity mode
pdsAccount): records go to that PDS, the bidder serves no repo, its market endpoints stay on its own
relay. The market-network profile's bidder uses its own PDS DenoPod; production stays QR on the user's PDS; the
in-process repo (localRepo, the ephemeral-PDS mode the user keeps) stays for dev-local and the
harness. The password is a vault value, never a kcp field (M4-BIDDER-VAULT; until then the existing
option path).
files: ["ts/atproto-market/hono-bidder/mod.ts", "ts/atproto-market/lib/atproto-helpers (remote agent)", "atproto-market/test/bidder_remote_pds_account_test.ts"]
test_first: >
atproto-market test/bidder_remote_pds_account_test.ts on the container harness with an in-process
hono-pds factory account: the bidder's offering is listed by that PDS's listRecords, the bidder serves
no repo, and runComputeContract still reaches receipt and onNetwork. Red today.
done_when: "green; existing bidder tests green; pushed on main in atproto-market."
risks: []
status: done
evidence:
- id: M3-OIDC-ISSUER-INSTANCE
milestone: M3
title: oidc-issuer-hono keeps no module-global state, so two issuers in one process keep their own issuer URL and key
depends_on: []
decision: >
Fifth fold 2026-10-12 (multi-server-to-kcp.md finding F3, gap G10; decided by Claude, user may veto):
configureOidc sets module globals (_getIssuerUrl, _jwkStore, _signingKey;
hono-compute-provider lib/oidc-issuer-hono/mod.ts:42-77) and every createOidcIssuer calls it (:616), so
a bidder with two providers serves the last one's issuer URL from both, and the issuer cannot move into
its own component. The state moves into the issuer instance; JwkStore stays injectable (M4-BIDDER-VAULT
plugs the OpenBao store in). A prerequisite of M5-BIDDER-COMPONENTS (decided by the user 2026-10-12).
files: ["ts/hono-compute-provider/lib/oidc-issuer-hono/mod.ts", "ts/hono-compute-provider/test/oidc_issuer_instances_test.ts"]
test_first: "two createOidcIssuer in one process: each discovery document and JWKS names its own issuer, and a token from one does not verify at the other. Red today."
done_when: "green; provider suites green; pushed on main in hono-compute-provider."
risks: []
status: done
evidence:
- "task/oidc-instance merged to main 2026-10-12: hono-compute-provider 834a125 (oidc-issuer-hono has no module globals: createOidcIssuer returns an instance owning its issuer URL getter, key store/JWKS, trusted issuers, nonce store, provisioning factory, createToken/validateToken/RBAC check; JwkStore may be async for M4-BIDDER-VAULT; providers take oidcProvisioning: true + optional oidcJwkStore/oidcNonceStore; a provisioning with an empty, unreachable or non-http(s) issuer URL is refused; no xrpc.fedproxy.com fallback, the reporter exits 1 without base_url), atproto-market 33a03c6 (rebased 52ee051, hono-bidder wiring), socialweb-computer-kcp df12363 (inventory citations, rebased 18c679a)"
- "red: two createOidcIssuer calls in one process served the same JWKS kid; green: oidc_issuer_instances_test.ts 6/6, hono-compute-provider non-integration 66/66 on merged main (defect3/9/10 incl.), atproto-market 23/23, deno check hono-bidder + request-vm-ssh clean"
- "follow-up running: digitalocean-bidder and deno-macos-runner-desktop still import the removed createOidcProvisioningEnricher (task/oidc-callers)"
- id: M3-BIDDER-COMPOSER
milestone: M3
title: The bidder's composition is a factory taking a component spec; hono-bidder becomes a thin DenoPod entry
depends_on: [M3-OIDC-ISSUER-INSTANCE, M1-MARKET-IDENTITY, "0008:M1-STRIP-LEGACY-TRANSPORTS", "0008:M2-MARKET-NETWORK"]
decision: >
Fifth fold 2026-10-12 (multi-server-to-kcp.md finding F9, gap G9; decided by Claude, user may veto,
within the user's decisions: kcp only, every option kept). hono-bidder/mod.ts's composition (825 lines:
sign-in, the market serve and relay, each provider's serve, relay and OIDC issuer, the market bidder
loops, the post-start one-shots) moves into lib/hono-factory-market-bidder-app: input = a component
spec {identity (MarketIdentity mode and refs, through the IdentityPrompt port of M1-MARKET-IDENTITY),
repo inproc|url, providers[], guest endpoints inproc|external (M5-BIDDER-COMPONENTS), the MarketNetwork
value (0008 M2-MARKET-NETWORK)}, output = the serves, relays and loops of research section 2.1. There
is NO standalone mode (decided by the user 2026-10-12: kcp only): the DenoPod template (0008
M2-SERVICE-TEMPLATE) sets the spec for one component; the in-process harnesses call the factory with an
all-inproc spec instead of composing by hand (they are library tests, not runs; decided by Claude, user
may veto), so a component split cannot pass the harness and fail the market network. It composes nothing the strip
deleted (no guest JSR registry, no non-iroh transport). bidder_prod_integration_test.ts, which spawns
the CLI against real plc.directory and xrpc.fedproxy.com, is moved onto a market network or deleted (market network per
workstream; no real account in a workstream test). The `bidder_ready` stdout line stays only until
0008 M2-DENOPOD-LISTENERS' report replaces every reader of it.
files: ["ts/atproto-market/lib/hono-factory-market-bidder-app/", "ts/atproto-market/hono-bidder/mod.ts", "ts/atproto-market/test/*_integration_test.ts"]
test_first: >
A composer test: the all-inproc spec reproduces today's listener and relay set minus the stripped
components (names compared), a spec with repo: url builds no in-process repo, two providers in one spec
serve two distinct issuers; bidder_container_integration_test switched to the composer still passes
the RFP flow. Red first.
done_when: "hono-bidder/mod.ts under 150 lines; every bidder test green; pushed on main in atproto-market."
risks: []
status: done
evidence:
- id: M3-NODE-LOSS-CLOCK
milestone: M3
kcp_use: "traditional-exception (virtual Kubernetes clusters project: tenant nodes; 0006 principle)"
title: Our own outage never counts against a tenant's node - the NODE-LOSS clock counts only from when our side of the path is back up
depends_on: []
decision: >
Decided by Claude, user may veto (control-plane-handover.md 6, Q-H6): plan 0005 M3-NODE-LOSS releases a
market node whose Ready is False/Unknown for 90 s, timed from the tenant Node's lastTransitionTime
(cmd/socialweb-clusterprovider/nodes.go:194-223). When OUR side of the path was down (a cluster cold
start, a link restart, a front restart, a user-manager restart of the gateway), that clock bills our
outage to the tenant's VM. NotReady time now counts from max(lastTransitionTime, pathUpSince), where
pathUpSince is, per cluster, when the provider last saw BOTH the cluster's apiserver answer /readyz at
status.URL (the front port after 0006:M3-APISERVER-FRONT) AND both ClusterLinks Listening, after any
moment either was not; held in memory, and a provider start counts as path-up for the clusters it
brings up or adopts. No rebase across a kcp, door or provider restart alone: the tenant's kcm keeps
running and writes real transitions, so a node that died then is released on time. The never-Ready
join grace (2 m, config.go:51) is rebased the same way.
files: ["cmd/socialweb-clusterprovider/nodes.go (loss rule)", "internal/clusterprovider (pathUpSince per cluster from the readyz probe and ClusterLink phases)"]
test_first: >
cmd/socialweb-clusterprovider (fake clock, fake tenant, fake links):
TestANodeNotReadyDuringOurLinkOutageIsKeptFor90sAfterTheLinkReturns,
TestANodeNotReadyDuringAClusterColdStartIsKeptFor90sAfterTheApiserverAnswers,
TestANodeNotReadyBeforeAProviderRestartIsStillReleasedOnTime,
TestTheJoinGraceIsRebasedToPathUpToo, TestAFlappingPathDoesNotPostponeReleaseForever. Red first (today
the first two release at 90 s from lastTransitionTime).
done_when: "offline tests green; the market network reading is taken by 0006:M3-CP-CONTINUITY's cold-start test."
risks: ["A path that flaps keeps resetting pathUpSince and would never release a really dead node; pathUpSince only moves forward after the path was continuously up for 30 s, and a test pins it."]
status: done
evidence:
- "task/door-resilience merged to main 2026-10-12: socialweb-computer-kcp a02387e/2a8b90d/7791f6d (rebased to f4dd567). The NotReady timer lives in the Go clusterprovider: each pass checks the tenant apiserver /readyz and that both cluster links are Listening; while that path is down no node is released; after an outage of >=30 s or a cold start the 90 s timer and 2 m join grace count from when the path came back; a provider start finding the path up does not reset the timer (decided by Claude, user may veto: reset only after an outage >=30 s so a flapping path still releases a dead node)"
- "5 new Go tests red then green, 2 guard tests (provider restart, flapping path) green before and after; merged main go test ./... 66/66 ok, check.sh --fast exit 0"
- "left: cold-start and link-restart NotReady readings live in a market network with a tenant cluster (nodes via runComputeContract)"
- id: M4-BIDDER-VAULT
milestone: M4
title: The bidder's keys live in an OpenBao beside it, so its identity and issuer survive restarts
depends_on: [M3-BIDDER-FLAG, "0006:M2-OPENBAO-SERVER", M3-OIDC-ISSUER-INSTANCE, M1-MARKET-IDENTITY]
decision: >
Fifth fold 2026-10-12 (multi-server-to-kcp.md F3, gap G3; decided by the user 2026-10-12: OAuth sessions
and every key live in the application's own OpenBao, never in kcp Secrets): the vault holds EVERY relay
keypair by relay name (the market relay, each provider's relay, the in-process repo's relay: 1 + P (+1)
today, all fresh per process at hono-bidder/mod.ts:104), the QR or browser OAuth session with its DPoP
key (MarketIdentity.spec.sessionRef, through M1-MARKET-IDENTITY's SessionStore port, replacing the
$HOME/.cache/pdr-market session file), a pdsAccount password (M3-BIDDER-ACCOUNT-PDS), and the requester
application's keys use the same layout (0008 M4-RUN-REQUEST). No key, session or token is ever written
to a kcp object; kcp holds the vault references.
Decided by the user 2026-10-07: the bidder key, the OIDC signing key and the relay keypair live in an
OpenBao beside the bidder, so they persist across restarts and the issuer URL stays stable.
Decided by Claude, user may veto: run bidder applies an OpenBaoServer object (loopback TLS, storage
under <state>/handles/<hid>/bao, unit socialweb-bao-<hid>.service in the shared account's user manager,
so it outlives upgrades; unsealed by the controller on every new InvocationID, 0006:M2-OPENBAO-SERVER);
a KV v2 mount bidder holds repo-key, relay-keypair and oidc-jwk; each is created once with
check-and-set 0 (a losing writer takes the winner's value, the internal/clusterlink EnsureKey pattern).
One vault per handle (keys are per identity). The bidder DenoPod gets a short-lived OpenBao token for
policy bidder-read through a token file the Go side renews; hono-bidder's cliCreateIngress takes a
keypair loader, oidc-issuer-hono a JwkStore, and the bidder's repo key a loader, each backed by that KV.
Applies to both providers when run under run bidder.
files: ["internal/stack/bidder (vault object, token file)", "ts/atproto-market/hono-bidder/mod.ts (loaders)", "ts/hono-compute-provider/lib/oidc-issuer-hono (JwkStore over OpenBao) or a sibling jwk-store-openbao package"]
test_first: >
Behind livegate: TestABidderRestartedTwiceKeepsItsDidItsIssuerUrlAndItsJwks (three starts against
one state dir; the logged did, issuer URL and JWKS are identical, and a token issued before the
restart validates after). Red against today's per-process keypair. Fifth fold:
TestAQrSessionSurvivesABidderPodRecreateWithoutARescan (the session read back from the vault),
TestEveryRelayOfTheBidderKeepsItsIngressAcrossARestart (market, each provider, repo) and
TestNoKeyOrSessionIsEverWrittenToAKcpObject (scans every object the bidder application writes).
done_when: "green; on the market network the issuer URL logged before and after a bidder restart is identical."
risks: ["The dispatcher must accept a reconnect under the same relay key after a crash (stale registration); measured on the market network."]
status: done
evidence:
- id: M4-BIDDER-BAO-SYNC
milestone: M4
title: The bidder's OpenBao holds exactly what live ComputeVMs ask for, and key rotation never orphans a cluster's jwt mount
depends_on: [M4-BIDDER-VAULT, M2-CONTROLLER, "0006:M3-BAO-SYNC"]
decision: >
User asked 2026-10-07 that OpenBao backends and roles stay in sync with what is active (plan 0005
M3-BAO-SYNC: 5 jwt-* mounts in one cluster vault, one per issuer a bidder ever had). Decided by
Claude, user may veto. (1) The bidder vault is reconciled by internal/baosync (0006:M3-BAO-SYNC):
desired = the fixed identity set (mount bidder: repo-key, relay-keypair, oidc-jwk; policy
bidder-read; the DenoPod's token role) plus, for each live ComputeVM, whatever per-guest policy,
role or KV path the provider issues under the owned prefix guests/<computevm>; a per-guest entry
whose ComputeVM is gone is deleted; the pass runs every controller pass, at startup, after an
upgrade and when the vault unit's InvocationID changed. (2) One stable issuer per bidder is the
cluster-side invariant: the issuer URL derives from the relay keypair, so a restart, upgrade or OIDC
signing-key rotation never changes it. Signing-key rotation is explicit (`socialweb-computer run bidder
rotate-signing-key`) and keeps the issuer URL: the new key is added to the JWKS, tokens are signed with
it, and the old key stays published until the longest token TTL has passed, so no cluster needs a new
mount. Relay-keypair rotation changes the issuer and therefore the jwt-<sha12(issuer)> mount on every
cluster; it is a separate explicit command, refused while any ComputeVM is Running unless --drain,
which waits for each guest to be released. Clusters drop the old mount through 0005 M3-BAO-SYNC's rule
(a jwt-* mount with no role left is disabled) once its last node is released, never earlier.
files: ["internal/stack/bidder (bidder vault reconcile, rotation subcommands)", "internal/baosync (bidder desired set)", "ts/hono-compute-provider/lib/oidc-issuer-hono (JWKS with overlapping keys)"]
test_first: >
internal/stack/bidder TestTheBidderVaultSeededWithStaleGuestEntriesIsReducedToTheLiveComputeVMs,
TestASigningKeyRotationKeepsTheIssuerUrlAndTokensSignedBeforeItStillValidate,
TestARelayKeyRotationIsRefusedWhileAGuestRuns; behind livegate on the market network:
TestBidderRestartsAndUpgradesLeaveOneJwtMountPerBidderInTheClusterVault (three restarts and one
upgrade of the bidder while its node stays joined: the cluster vault holds one jwt-* mount for it).
Red first (today every restart adds a mount).
done_when: "tests green; the cluster-vault reading on hat-8 recorded: one jwt-* mount per bidder with a live node."
risks: ["oidc-issuer-hono publishes a single key today; overlapping keys change its JWKS shape, which OpenBao's JWT auth already accepts (it verifies by kid)."]
status: done
evidence:
- id: M4-CONTRACT-STORE-FILE
milestone: M4
title: "NOW: a restarted TypeScript bidder keeps its receipts and tombstones - market-bidder ContractStore load()/save and a file-backed store in compute-provider-firecracker"
depends_on: []
decision: >
Split from M4-CONTRACT-RESTORE by the user 2026-10-12: the file-backed ContractStore and market-bidder's
load()/onContractChange land now, independent of the Go path, and the terminated-receipt tombstones are
persisted too. Live reason: bidder restarts during testing kept losing receipts; a guest leaked on
2026-10-12T01:31Z (vm.delete after a restart hit an unknown receipt; first recorded 2026-10-07). Today
(atproto-market main bdd423b) activeContracts is in memory (market-bidder/mod.ts), onContractChange is
an event hook with no store behind it (abc/market-bidder/mod.ts:93), and TerminatedReceipts is a bounded
in-memory set (abc/market-bidder/mod.ts:37, 1024 entries). Decided by Claude, user may veto: the ABC
gains interface ContractStore {load(): Promise<StoredContracts>; save(event: ContractEvent):
Promise<void>; saveTombstone(t: Tombstone): Promise<void>} where StoredContracts carries the active
contracts (receipt uri+cid, accept uri+cid, acceptAuthor, acceptedAt, providerId, submitEventUrl) and
the tombstones (receipt, reason, at); createMarketBidder({contractStore?}) awaits load() BEFORE it
subscribes to any stream, seeds activeContracts (providerIdPromise = Promise.resolve(providerId)) and
TerminatedReceipts, and awaits save() inside the handler that changes a contract (the M2-CLAIM-RECEIPT
"every handler awaits its write" pattern), so a contract the bidder acknowledged is on disk first. The
file impl (a sibling impl package, e.g. lib/market-bidder-contract-store-fs, ABC-layered, used by the
firecracker provider's wiring) writes one JSON file per receipt under the provider's work root by
temp-file + rename + fsync of the dir, tombstones in the same tree with a TTL longer than the longest
requester retry, and drops a contract file only after its destroy completed and its tombstone exists.
On load, a stored contract whose guest is gone (the provider's own probe) is tombstoned with
vmm_exited and released once. No market record changes; both providers stay (decided by the user
2026-10-07); the ComputeVM-backed store for compute-provider-kcp stays M4-CONTRACT-RESTORE.
files: ["ts/atproto-market/lib/abc/market-bidder/mod.ts (ContractStore, Tombstone)", "ts/atproto-market/lib/market-bidder/mod.ts (load before subscribe, awaited save)", "ts/atproto-market/lib/market-bidder-compute/mod.ts (tombstones through the store)", "ts/atproto-market/lib/market-bidder-contract-store-fs/ (new impl package)", "ts/atproto-market/hono-bidder/mod.ts and cli-args-env (store path option)", "ts/hono-compute-provider/lib/compute-provider-firecracker (guest probe on load)"]
test_first: >
atproto-market test/market_bidder_contract_restore_test.ts: a bidder seeded from a ContractStore
answers vm.delete for that receipt with 200 and destroys once (today 400 unknown receipt); 'a tombstone
survives a restart: a late vm.delete after the restart answers 200 already-terminated';
'load() completes before the first stream subscription'; 'a crash between accept and save leaves no
contract the bidder acknowledged unsaved' (save awaited before the ack); the fs store's own tests (atomic
write, a torn temp file ignored on load, tombstone TTL). Live on hat-8 through the RFP flow
(alice-request.sh -> RFP -> bid -> accept -> cloud-init, firecracker provider): rent a guest, restart the
bidder unit, vm.delete answers 200, the guest and its work dir are gone; and a guest powered off, the
bidder restarted, then a late vm.delete answers 200 already-terminated. Red first.
done_when: "suites green; both live readings recorded (UTC) with no leaked guest or work dir; pushed on main in atproto-market and hono-compute-provider."
risks: ["A store written on a slow disk delays the accept ack by the fsync; measured and recorded."]
status: done
evidence:
- "merged to main 2026-10-12: atproto-market c78cf31 (ContractStore ABC, load before subscribe, save before ack, tombstones persisted, lib/market-bidder-contract-store-fs: one 0600 file per receipt, temp+fsync+rename+dir fsync, corrupt file refuses startup naming it; hono-bidder --contract-store-dir default <work-root>/contracts), hono-compute-provider ac3a1f5 (ComputeProvider.adopt, guest.json with host boot id, adopted guests re-watched, gone-at-adopt -> vmm_exited|host_reboot)"
- "tests red then green: market_bidder_contract_restore_test.ts 8/8 (red: vm.delete 400 unknown receipt, subscribe before load, ack before save), contract_store_fs_test.ts 7/7, firecracker_adopt_test.ts 4/4; regression 43 + 45 green; re-run on merged main 32/32 + 12/12"
- "left: the two live readings (in a market network once M1-MARKET-NETWORK-PARALLEL lands); known gap - a restart mid-provision ends the contract vmm_exited and a still-booting guest may leak (closed by the Go controller); fsync delay unmeasured"
- id: M4-CONTRACT-RESTORE
milestone: M4
title: A restarted bidder rebuilds its contracts from the ComputeVMs and still honours vm.delete
depends_on: [M4-CONTRACT-STORE-FILE, M3-BIDDER-FLAG, M2-CONTROLLER]
decision: >
Closes the live finding (restart loses receipts, vm.delete fails, guest leaks) for compute-provider-kcp.
Since 2026-10-12 (third fold, split by the user) the ContractStore ABC, market-bidder's load()/save and
the firecracker provider's file store are M4-CONTRACT-STORE-FILE; this item implements the same
ContractStore on ComputeVM spec.contract in compute-provider-kcp, seeding providerIdPromise =
Promise.resolve(name) and acceptToContract, with tombstones from Lost/released ComputeVMs. No market
record changes.
files: ["ts/hono-compute-provider/lib/compute-provider-kcp (ContractStore over ComputeVMs)"]
test_first: >
atproto-market test/market_bidder_contract_restore_test.ts run against the ComputeVM-backed store (fake
kcp): a bidder seeded from it answers vm.delete with 200 and destroys once; red first.
done_when: "green for the ComputeVM store; existing market-bidder suite green."
risks: []
status: done
evidence:
- id: M4-ONNETWORK
milestone: M4
title: onNetwork and the guest's host-key report are recorded on the ComputeVM as well as in the market records
depends_on: [M3-TS-PROVIDER, M4-HOST-KEYS]
decision: >
The factory's POST /v1/on-network keeps firecracker's behaviour and also applies status.market
{onNetworkUri, address, reportedAt} under fieldManager compute-provider-kcp; its POST /v1/host-keys
(M4-HOST-KEYS, same route shape) applies status.market {hostKeys[], hostKeysUri}. The host keys come
only from the guest's own report; nothing here scans the guest. A failed status write never fails the
guest's request.
files: ["ts/hono-compute-provider/lib/hono-factory-compute-provider-kcp"]
test_first: "test/compute_provider_kcp_onnetwork_test.ts (address lands in status.market from on-network and host keys from host-keys; a failed status write still answers the guest 200); red first."
done_when: "green."
risks: []
status: done
evidence:
- id: M4-TERMINATION-EVENT
milestone: M4
title: The bidder tells the requester when and why a guest ended - vm.terminated in the market.event envelope
depends_on: []
decision: >
Decided by the user 2026-10-12 (docs/research/2026-10-11-vm-lifecycle/user-decisions.md, 2026-10-12
line): new market lexicon com.publicdomainrelay.temp.compute.events.vm.terminated (record, key tid)
{reason (known values guest_poweroff, vmm_exited, oom, host_reboot, identity_mismatch,
deleted_on_request), exitCode?, observedAt, request? (strongRef to the requester's vm.delete it
acknowledges), createdAt}, written by the BIDDER in its own repo and wrapped in the existing
com.publicdomainrelay.temp.market.event envelope {receipt, payload, signatures (the bidder's inline
badge.blue signature)}, exactly like vm.onNetwork; submitted to the accept's submitEvent when it names
one, firehose otherwise. Requesters end the contract on it: the CLI ends the session and skips the
redundant vm.delete; the door claims the node Lost at once and re-RFPs (plan 0005 M3-NODE-LOSS through
M3-NODE-LOSS-CLOCK and the M6-* recovery items) instead of waiting for NotReady. Closes the requester
half of E8 defect 1 (live hat-8 2026-10-12T00:57Z: the bidder released a powered-off guest in ~4 s
after hono-compute-provider 96147f0 / atproto-market 148756b, but request-vm-ssh hung until closed by
hand and its vm.delete then hit 'submitEvent: unknown receipt' on the bidder's firehose path). Decided
by Claude, user may veto: the Go controller (M2-CONTROLLER) emits the same event, through the TS
provider's onGuestGone (M3-TS-PROVIDER), from every Lost verdict - at re-population (HostRebooted ->
host_reboot, UnitGone/UnitFailed/VMMExited -> vmm_exited, OOMKilled -> oom, IdentityMismatch ->
identity_mismatch) and from a unit failure seen through Watch - so a requester learns of a Lost
ComputeVM without polling; a requester's later vm.delete for it answers 200 (tombstone), never 'unknown
receipt'.
files: ["ts/atproto-market/lexicons/com/publicdomainrelay/temp/compute/events/vm/terminated.json", "ts/atproto-market/lib/common/market-lexicons, lib/common/market-common (NSID, VM_TERMINATED_REASONS)", "ts/atproto-market/lib/abc/market-bidder (TerminatedReceipts)", "ts/atproto-market/lib/market-bidder-compute (publish on onGuestGone and as the vm.delete ack)", "ts/atproto-market/lib/abc/compute-request (terminationOf, ContractState.termination, onTerminated)", "ts/atproto-market/lib/abc/requester (setOnTerminated, ssh abort signal)", "ts/atproto-market/lib/compute-request-xrpc (contract watch)", "ts/atproto-market/lib/requester-xrpc (submitEvent handler, runComputeContract)", "ts/hono-compute-provider/lib/compute-provider-kcp (onGuestGone from ComputeVM Lost)", "internal/computevm (Lost reason to the provider)"]
test_first: >
atproto-market test/vm_terminated_lexicon_test.ts, test/vm_terminated_bidder_test.ts,
test/vm_terminated_requester_test.ts (red first, then green). Go side, still to write:
internal/computevm TestEveryLostVerdictCarriesTheTerminatedReason and
hono-compute-provider test/compute_provider_kcp_test.ts 'a ComputeVM going Lost fires onGuestGone with
its reason'; live on the market network (guest born via runComputeContract): a guest powered off from inside
ends the requester's ssh within seconds with vm_session_ended_by_bidder and no vm.delete is submitted.
done_when: "TS side green (done); Go controller and compute-provider-kcp emit it from Lost verdicts at re-population and from unit failure, tests green; the live reading recorded with UTC timestamps."
risks: ["Tombstones are in memory: a bidder restarted between a guest's end and the requester's vm.delete answers 'unknown receipt' until M4-CONTRACT-STORE-FILE persists contracts and the terminated set (split out 2026-10-12 to land now)."]
status: done
evidence:
- "2026-10-12 atproto-market (branch pre-iroh) eb2aa82: lexicon terminated.json, generated market-lexicons types, COMPUTE_EVENTS_VM_TERMINATED_NSID and VM_TERMINATED_REASONS; test/vm_terminated_lexicon_test.ts red (missing exports) then 4/4 green."
- "2026-10-12 atproto-market 7c4a8f4: market-bidder-compute publishes a signed market.event wrapping vm.terminated from onGuestGone (terminatedReasonOf: 'runner container exited' code 0 -> guest_poweroff, else vmm_exited; oom, host_reboot, identity_mismatch passed through) and as the vm.delete ack (deleted_on_request + request strongRef), submitted to the accept's submitEvent, firehose otherwise; bounded TerminatedReceipts (1024) makes a late or duplicate vm.delete answer 200 (no 'unknown receipt', no second destroy or event); concurrent HTTP + firehose deliveries destroy once. test/vm_terminated_bidder_test.ts red (missing terminatedReasonOf/TerminatedReceipts/submitEventUrl) then 10/10 green; market_bidder_guest_gone_test.ts still green."
- "2026-10-12 atproto-market bdd423b: requester contract watch takes vm.terminated from submitEvent (RequesterPDS.setOnTerminated) and the firehose, binds it to the receipt and its bidder (compute-request-abc terminationOf) and verifies the envelope signature as receipt_verified does; sets ContractState.termination, fires handlers.onTerminated / awaitComputeNetwork onTerminated and the handle's terminated promise (phases and outcomes unchanged, so the door's mirrored types hold; outcome returned); releaseCompute skips the vm.delete (vm_delete_skipped reason terminated_by_bidder); runComputeContract aborts the ssh poll, session or hold (vm_session_ended_by_bidder) and returns result.terminated. test/vm_terminated_requester_test.ts red (missing terminationOf, onTerminated, termination) then 7/7 green, incl. forged (tampered, unsigned) and unbound (another repo, another receipt) events rejected; compute_request_test.ts and the door's node_market_test.ts (32) green; deno check hono-bidder/mod.ts and request-vm-ssh/mod.ts green."
- "Remaining (Go side): M2-CONTROLLER Lost verdicts and Watch unit failures -> compute-provider-kcp onGuestGone with the mapped reason; door acting on it is M3-NODE-LOSS-CLOCK / M6-*."
- "live reading, hat-8 2026-10-12, through the RFP flow (alice-request.sh -> socialweb-computer-bidder@bobvmbuilder.bsky.social, firecracker provider): CtrlAltDel 01:44:39.873Z; microvm_exited 01:44:42.367Z; work dir removed / bidder_guest_gone 01:44:44.897Z; vm.terminated created on the PDS 01:44:44.976Z and submitted to the requester 01:44:45.125Z; requester vm_terminated_by_bidder, vm_session_ended_by_bidder and vm_delete_skipped 01:44:45.429Z. 5.6 s from guest poweroff to session end, no vm.delete sent. Earlier, 01:33:07Z: a late vm.delete answered 'vm.delete for already-terminated receipt' on both the submitEvent and firehose paths (tombstone idempotency seen live)."
- "prerequisites that landed for the live run: typescript-helpers 5b071fe added the vm.terminated scope to the bidder role set (before it, publish failed 01:32:39Z with 403 ScopeMissingError); did-key-associator 3cde452 regenerated the QR client scope, only its static files deployed to qr.fedfork.com at 01:41:55Z; the bidder re-signed in via QR at 01:43:50Z. Any new bidder-authored NSID (M4-HOST-KEYS) needs the same two scope changes and a re-sign-in."
- "status stays doing: the Go controller and compute-provider-kcp must still emit vm.terminated from Lost verdicts and unit failures (tests named in test_first)."
- id: M4-HOST-KEYS
milestone: M4
title: Guest SSH host keys are guest-reported in a bidder-signed vm.hostKeys record and every ssh to a guest verifies them - no host-side scan, no TOFU
depends_on: [M1-PROTOCOL-CHECKLIST, M1-QR-DEPLOY]
decision: >
Third fold 2026-10-12 (decided by the user): vm.hostKeys follows M1-PROTOCOL-CHECKLIST in order -
lexicon, the bidder role set in typescript-helpers, did-key-associator's regenerated client scope
deployed with M1-QR-DEPLOY's --static-only mode, the bidder re-signed in through the QR - before any
live run publishes it. The door's ContractState.hostKeys rides in M6-CLAIM-RESUME's one NodeClaim
schema revision, so the door half lands with or after that revision.
Decided by the user 2026-10-12: guest SSH host keys are guest-reported, replicating how onNetwork works:
new atproto record(s) plus plumbing so the requester receives the host key(s) with the ssh command;
request-vm-ssh and the gateway/door validate against the sent key as the known host key (no TOFU, no
host-side scan). Decided by Claude, user may veto: ONE new record
com.publicdomainrelay.temp.compute.events.vm.hostKeys (record, key tid) {keys [{type (ssh-ed25519,
ecdsa-sha2-nistp256, ssh-rsa), publicKey (the base64 blob of a known_hosts line), fingerprint
(SHA256:...)}], observedAt, createdAt}, written by the BIDDER in its own repo and wrapped in the
existing com.publicdomainrelay.temp.market.event envelope {receipt, payload, signatures (the bidder's
inline signature)}, submitted to the accept's submitEvent when it names one, firehose otherwise -
exactly the vm.onNetwork and vm.terminated (M4-TERMINATION-EVENT) path. A separate NSID rather than new
fields on onNetwork, because (a) onNetwork's shape stays field-for-field what the firecracker provider
writes (ACC7-NO-PROTOCOL-CHANGE), (b) the keys exist only after cloud-init's ssh module, which is not
the moment the network report fires, and (c) vm.terminated set the precedent of a new event NSID in the
same envelope. Guest side (delivered only by user_data, the RFP spine): a hostKeysReporterModule in
atproto-market lib/common/cloud-init-common's registry runs after cloud-init's ssh module, reads
/etc/ssh/ssh_host_*_key.pub (the guest image bakes no host key, M1-GUEST-IMAGE), signs its single-use
provisioning token with the ed25519 host key (`ssh-keygen -Y sign -n prove-sshd`, the proof today's
provisioning exchange already uses) and POSTs {keys, signature, token} to the bidder's /v1/host-keys
(same base URL and auth as /v1/on-network). Bidder side: the report is accepted only with a valid
proof of possession over the token the bidder issued for that guest (so only the guest born from that
user_data can report), at most once per receipt (a second, different report is refused and logged),
then published as vm.hostKeys. The host-side scan is deleted: hono-compute-provider
lib/oidc-issuer-hono getPublicKeyFromSshd (mod.ts:446-483, ssh-keyscan, through `container exec` into
the runner container) goes, and provisioningValidate (:520-544) verifies the guest's signature against
the reported ed25519 key bound to the same token. Requester side: the contract watch takes vm.hostKeys
like vm.terminated (envelope signature verified, bound to the receipt and its bidder; forged or unbound
events rejected) into ContractState.hostKeys; sshTunnelArgs (atproto-market lib/requester-xrpc/mod.ts:609-620,
today StrictHostKeyChecking=no and UserKnownHostsFile=/dev/null) writes a per-contract known_hosts with
exactly the reported keys under a HostKeyAlias derived from the receipt and passes
StrictHostKeyChecking=yes; request-vm-ssh waits (bounded) for the keys and refuses to ssh without them.
Gateway side: the door keeps ContractState.hostKeys on the NodeClaim (in the one NodeClaim schema
revision of M6-CLAIM-RESUME) and every ssh command it prints or runs (today
socialweb-computer-k8s hono-socialweb-computer-k8s/mod.ts:177, StrictHostKeyChecking=no) and the ssh
door (0008:M4-RUN-GATEWAY-SSH) use the same known_hosts + StrictHostKeyChecking=yes. The record is also
the guest's signed identity in M6-GATEWAY-REPOPULATE's adoption match. Both providers emit it (the
firecracker provider now; compute-provider-kcp through M4-ONNETWORK). Like vm.terminated, the new NSID
needs its scope in the bidder role set (typescript-helpers) and the QR client scope
(did-key-associator), and the bidder re-signs in. Fifth fold 2026-10-12 (decided by the user: iroh is
the only guest transport): after 0008 M1-STRIP-LEGACY-TRANSPORTS the only ssh path to a guest is the
iroh (dumbpipe) ProxyCommand; the per-contract known_hosts and StrictHostKeyChecking=yes are written
once, on that path (the websocat ProxyCommand of sshTunnelArgs is deleted, not fixed), so the
requester half lands after the strip (execution_order shared files).
files: ["ts/atproto-market/lexicons/com/publicdomainrelay/temp/compute/events/vm/hostKeys.json", "ts/atproto-market/lib/common/market-lexicons, lib/common/market-common (NSID)", "ts/atproto-market/lib/common/cloud-init-common (hostKeysReporterModule)", "ts/atproto-market/lib/market-bidder-compute (POST /v1/host-keys, proof check, publish)", "ts/atproto-market/lib/abc/compute-request, lib/compute-request-xrpc (hostKeysOf, ContractState.hostKeys)", "ts/atproto-market/lib/requester-xrpc (sshTunnelArgs known_hosts + StrictHostKeyChecking=yes)", "ts/atproto-market/request-vm-ssh", "ts/hono-compute-provider/lib/oidc-issuer-hono (scan deleted, reported key verified)", "ts/hono-compute-provider/lib/compute-provider-firecracker", "socialweb-computer-k8s door (NodeClaim hostKeys, ssh command)", "typescript-helpers (bidder role scope)", "did-key-associator (QR client scope)"]
test_first: >
atproto-market test/vm_hostkeys_lexicon_test.ts; test/vm_hostkeys_bidder_test.ts (a report without a
valid signature over the issued token is refused; a second, different report for the receipt is
refused; the published record is wrapped and signed and goes to submitEvent, firehose otherwise);
test/vm_hostkeys_requester_test.ts (forged and unbound events rejected; ContractState.hostKeys set;
sshTunnelArgs carries StrictHostKeyChecking=yes and a known_hosts holding exactly the reported keys; no
ssh is attempted without keys); cloud-init-common test (the module renders after the ssh module and
posts keys + signature); hono-compute-provider test 'provisioningValidate verifies against the reported
key and nothing in lib execs ssh-keyscan'. Live on the market network, guest born via runComputeContract
(alice-request.sh -> RFP -> bid -> accept -> cloud-init): request-vm-ssh connects with
StrictHostKeyChecking=yes against the reported key, and the same run with a known_hosts holding a
different key fails host-key verification and never authenticates. Red first.
done_when: "TS suites green; the live reading recorded with UTC timestamps (keys reported, record created and submitted, ssh verified); no ssh-keyscan and no StrictHostKeyChecking=no left on any path that reaches a guest."
risks:
- "A guest that never reports keys cannot be ssh'd; that is the intended fail-closed behaviour, and the requester logs which receipt is missing its report."
- "Keys are fixed for a guest's life (no rekey on a live guest); a rebuilt guest is a new receipt."
status: done
evidence:
- id: M5-RUN-BIDDER
milestone: M5
title: socialweb-computer run bidder as a user unit of the shared lingering bidder account, no docker, least-privilege deno
depends_on: [M2-CONTROLLER, M3-BIDDER-FLAG, M4-BIDDER-VAULT, M4-BIDDER-BAO-SYNC, M4-CONTRACT-RESTORE, "0008:M2-SERVICE-TEMPLATE", "0006:M2-KCP-PORT0", "0006:M1-INSTALL", "0006:M2-HOST-CONTRACT", "0006:M2-BOOTSTRAP-LAYER", "0006:M2-HOSTPROCESS-KIND", "0006:M2-APP-CONTROLLER", M1-MARKET-IDENTITY, M3-BIDDER-COMPOSER, M3-BIDDER-ACCOUNT-PDS]
decision: >
Fifth fold 2026-10-12 (multi-server-to-kcp.md 4.3, 5.2, gap G2; the user's decisions of 2026-10-12:
every option kept kcp-native, sign-in as an identity object, kcp only, secrets only in OpenBao): `run
bidder` follows 0006 M1-CLI's application-verb contract - it applies a MarketBidder root object (0006
M2-APP-CONTROLLER) with --profile prod-qr|market network|dev-local into the handle's kcp and watches it. prod-qr
renders no PDS and a MarketIdentity in mode oauthQr; market network renders the bidder's own PDS DenoPod and mode
pdsAccount (M3-BIDDER-ACCOUNT-PDS; finding F1: today's market network bidder ignores bob's PDS); dev-local renders
mode localRepo (the ephemeral-PDS mode, kept by the user). Every hono-bidder option is a MarketBidder
field or a profile (0006 M2-APP-CONTROLLER's option map). The foreground renders MarketIdentity pending
actions (QR, association approval, session expired; M1-MARKET-IDENTITY's renderer) and returns when
Ready, or stays with --fg. hono-bidder runs ONLY as the bidder DenoPod's entry from this item on: it
exits 2 naming `socialweb-computer run bidder` when started without its DenoPod's kcp environment, and
the standalone launcher (deploy/bidder/socialweb-computer-bidder-run), the identity-prompt-tty impl and
the org-root bob-bidder.sh path are deleted in the same change. State paths move out of $HOME (finding
F6) into the instance's state dir and vault. Inside a market network the bidder is a child application
whose MarketBidder the market network's MarketNetwork applies into the bidder's own kcp with the network values
copied on (0006 M2-APP-CONTROLLER); the market network's bidder runs the firecracker provider until this item makes
compute-provider-kcp the default member, then the market-network profile switches.
Principle (user 2026-10-12, 0006 `principle`): each bidder handle is one application instance - its own
kcp, the bidder manifest set and a profile (production QR mode against the handle's real account: no
PDS pod; the market-network profile: its own PDS pod). Handles on one host share the machine (the account,
/dev/kvm, the subuid range), never a kcp, and reach requesters and gateways only over the atproto
market protocol. Third fold 2026-10-12 (decided by the user): the handle's kcp is ONE slot on kine-sqlite behind a
port-0 front, in the bootstrap layer (installed units socialweb-kcp-<hid>-kine@a, socialweb-kcp-<hid>@a,
socialweb-kcp-<hid>-front written by `install bidder --handle`; devsup units in dev), and a kcp upgrade
is a short restart of that slot that VMs never notice (0006 M2-KCP-SLOTS, M3-KCP-ROLL); the bidder
vault and every VM are HostProcess objects placed by the supervisor. Dev run mode (decided by the user
2026-10-12; details decided by Claude, user may veto, dev-run-mode.md 5): `run bidder
--host-backend=proc --instance x` runs with no shared account, no install and no linger (the account
check, the QR-guard's account binding and check-host run only under host-backend=systemd-user), and
`--host-backend` is required (no default). The deno denies cover devsup's socket because it lives under
$XDG_RUNTIME_DIR.
Decided by the user 2026-10-11: ONE shared lingering account (swc-bidder) for all bidder handles, units
namespaced by handle; the bidder lives in that account's USER manager; explicit deno permissions for the
bidder (no -A, no --allow-run, no /dev/kvm, no D-Bus socket path), enforced in the DenoPod template and
tested. `socialweb-computer install bidder --handle <h>` (0006:M1-INSTALL, the one explicit root step,
decided by the user 2026-10-12) creates the account (never created on demand anywhere else), reserves
its subuid/subgid range for per-VM user namespaces (M2-VM-RUNTIME), adds it to kvm BEFORE linger and the
first user@ start, writes the delegation drop-in, enables linger, installs the user unit file
socialweb-computer-bidder@.service and enables the handle's instance. `run bidder --handle <h>` (ExecStart=<state>/bin/current/socialweb-computer
run bidder --handle %i; Restart=always; default KillMode=control-group - the 2026-10-07 KillMode=mixed is
removed: VM and vault units are separate cgroups and survive any KillMode, and mixed would SIGKILL deno
with no graceful stop): the handle's single kcp slot on kine-sqlite behind a port-0 front
(0006:M2-KCP-SLOTS, M2-KCP-PORT0; corrected 2026-10-12 from "kcp slots on kine -> postgres", which the
user vetoed), the owned deno-kcp provider, the bidder set
(computevms, hostprocesses, workspace root:bidder, a ServiceAccount limited to computevms,
computevms/status and Secrets there), the bidder vault, token rotation, the ComputeVM controller, and the
bidder DenoPod with --compute-provider-kcp and every operator option. The DenoPod's deno argv grants no
--allow-run, --allow-ffi or -A; --allow-read/--allow-write only its state dir and the TypeScript tree;
--deny-read/--deny-write of /dev/kvm and $XDG_RUNTIME_DIR (bus and systemd private socket); because deno
runs as the same uid as the user manager and in group kvm, deno's permission sandbox is the only
boundary there is. Keeps plan 0005 M1-BIDDER-UNIT's guarantees (key written before first start; a QR
sign-in into another account refused), with the QR guard moved from the bash launcher into Go
(internal/stack/bidder); deploy/bidder/socialweb-computer-bidder-run and the system unit are deleted.
Decided by Claude, user may veto, except where marked.
files: ["internal/stack/bidder", "deploy/sets.yaml", "deploy/denopods/bidder.values.yaml", "deploy/bidder/socialweb-computer-bidder@.service (user unit)", "deploy/bidder/socialweb-computer-bidder-run (deleted)", "scripts/deploy-firecracker-bidder.sh (copies the binary, runs install bidder / upgrade)"]
test_first: >
test/bidder_unit_test.go TestTheBidderUserUnitVerifies (`systemd-analyze --user verify` clean),
TestTheBidderUnitHasNoKillModeMixed, TestRunBidderRefusesAnAccountOtherThanItsHandle,
TestTheBidderDenoPodArgvGrantsNoRunFfiBusOrKvm; behind livegate on hat-8:
TestDenoInTheBidderPodCannotOpenDevKvmOrDialTheUserBus (a probe entry in the pod: Deno.open('/dev/kvm')
and a unix connect to $XDG_RUNTIME_DIR/systemd/private and the bus are PermissionDenied); on proc in a
dev instance: TestDenoInTheBidderPodCannotDialTheDevsupSocket and TestRunBidderOnProcNeedsNoAccount;
TestRunBidderWithoutAHostBackendExits2. Fifth fold: TestRunBidderRendersTheQrFromStatusAndReturnsWhenReady,
TestRunBidderSwitchingProfileFromNetworkToProdQrRemovesThePdsDenoPod, and atproto-market 'hono-bidder
outside a DenoPod exits 2 naming socialweb-computer run bidder'. Red first.
done_when: "tests green; on hat-8 the unit runs under the shared account and `kubectl get computevms` on its kcp answers."
risks: ["Handles on one host still share a uid for their cores and deno processes (shared account, decided by the user 2026-10-11); a VMM escape no longer reaches them, because each VMM runs in its own subuid sub-range owning no bidder file (decided by the user 2026-10-12, M2-VM-RUNTIME). deno's permission sandbox remains the only boundary between a bidder's deno and its own user manager."]
status: done
evidence:
- id: M5-BIDDER-COMPONENTS
milestone: M5
title: The bidder's guest-facing endpoints (OIDC issuer, /v1/on-network, /v1/host-keys) as their own DenoPod per provider
depends_on: [M5-RUN-BIDDER, M4-BIDDER-VAULT, M3-OIDC-ISSUER-INSTANCE, M4-ONNETWORK, M4-HOST-KEYS]
decision: >
Decided by the user 2026-10-12 (multi-server-to-kcp.md section 7): the bidder's guest-facing endpoints
become their own DenoPods after M4-BIDDER-VAULT and the per-instance OIDC fix (M3-OIDC-ISSUER-INSTANCE).
Guests dial the provider relay for OIDC token exchange, onNetwork and host keys for their whole life,
so a bidder-core restart or upgrade must not move or interrupt them. Decided by Claude, user may veto:
one DenoPod `guest-<provider>` per provider owns that provider's relay (keypair from the vault), the
OIDC issuer (JWK from the vault), POST /v1/on-network and POST /v1/host-keys (writing ComputeVM
status.market, M4-ONNETWORK) - one DenoPod, not one per endpoint, because a relay and the Hono app
that answers it are one component (createIngress delivers requests in-process). The bidder core keeps
the market relay and the market loops. There is no `guest-jsr` DenoPod: the guest JSR registry is
stripped with every non-iroh guest path (0008 M1-STRIP-LEGACY-TRANSPORTS, decided by the user
2026-10-12). The MarketBidder profiles render the guest DenoPods (0006 M2-APP-CONTROLLER).
files: ["hono-compute-provider/hono-bidder-guest/ (thin CLI over lib/hono-factory-compute-provider-kcp)", "deploy/apps/bidder/*.values.yaml"]
test_first: "Behind livegate on a market network (guest born via runComputeContract): TestABidderCoreRestartLeavesTheGuestIssuerAndOnNetworkServing (the core DenoPod restarted while the guest runs: the issuer URL and JWKS unchanged, a token issued before validates after, the next onNetwork POST accepted). Red first."
done_when: "green with readings (UTC); pushed on main in hono-compute-provider."
risks: []
status: done
evidence:
- id: M6-BIDDER-UPGRADE
milestone: M6
title: Upgrade the bidder's binary and deno processes while its VMs keep running; measure the stop grace
depends_on: [M5-RUN-BIDDER, "0008:M4-MARKET-NETWORK-PORT"]
decision: >
Decided by the user 2026-10-07. On the market network with run bidder: rent a guest through the RFP flow
(requester runComputeContract -> RFP -> bid -> accept -> cloud-init); 0006:M2-VM-TREE-AUDIT passes on its
unit; then `socialweb-computer upgrade --binary <a build with a different closure>`; read: the guest's
unit never restarted (same InvocationID, firecracker pid and start ticks), every DenoPod restarted on
the new code (version marker), ComputeVM still Running, the same issuer URL, the guest's next onNetwork
POST accepted, the tree audit passes again, then releaseCompute's vm.delete 200 with the graceful stop
timed. Decided by the user 2026-10-11: the stop grace window is measured live here through the RFP flow
(time from SendCtrlAltDel to the unit going inactive, over at least five releases) and recorded; the
measured value (with margin) becomes the default stopGraceSeconds in versions.json. E8 (2026-10-12)
already read 0.21 s on an IDLE guest; the readings here are taken with the k3s agent joined and pods
running, which is the window that sets the default. After each release: no unit, no cgroup and no
work dir of the released VM remain (E8 defects a and b; ACC7-NO-LEAK). Also (decided by the user
2026-10-12: a bidder kcp upgrade is a short restart of its one slot): `units roll kcp` on the bidder to a
different kcp asset while the guest runs - the VM unit's InvocationID, firecracker pid and start ticks
unchanged, the ComputeVM re-adopted Running, the kcp-down seconds recorded, the guest's next onNetwork
accepted.
files: ["internal/marketnetwork/live_test.go (TestABidderUpgradeLeavesItsVMRunningAndStillReleasesIt)", "internal/versions/versions.json (stop grace)"]
test_first: "The live test, red until upgrade and re-population exist (today the guest leaks on restart)."
done_when: "PASS recorded with UTC timestamps; grace readings under a k3s workload recorded; vms/ work-dir count equals live ComputeVMs after every release; zero ssh sessions to the guest."
risks: ["Needs the bidder account in kvm; the market network's fixed ports are gone once 0008:M1-MARKET-NETWORK-PARALLEL lands (a dependency of 0008:M4-MARKET-NETWORK-PORT)."]
status: done
evidence:
- id: M6-CLAIM-RESUME
milestone: M6
kcp_use: "traditional-exception (virtual Kubernetes clusters project: NodeClaims; 0006 principle)"
title: A restarted door resumes Bidding and Provisioning claims - per-claim secrets-server key in OpenBao, served state on the NodeClaim
depends_on: []
decision: >
Split from M6-GATEWAY-REPOPULATE 2026-10-12. Decided by the user 2026-10-12: in-flight (Bidding,
Provisioning) claims resume across a door restart, with a per-claim secrets-server key in OpenBao and
served-state on the NodeClaim. Today the per-contract secrets server's Secp256k1 keypair, ingress
subdomain and grant live in memory (atproto-market lib/guest-capability-secrets/mod.ts:50-75), so a
restarted door marks those claims Failed (socialweb-computer-k8s node-market ORPHANED_PHASES, :487-495)
and the guest's user_data names a secrets URL nobody serves. Decided by Claude, user may veto
(control-plane-handover.md 7): (1) the keypair is created once per claim in the cluster's OpenBao at KV
socialweb-door/claims/<claim>/secrets-ingress with check-and-set 0 (a losing writer takes the winner's
value, the clusterlink EnsureKey pattern), so the ingress subdomain and the URL in the guest's user_data
are a function of that key; (2) NodeClaim status.secrets {ingressDid, keyRef, url, servedAt} records
what was served; (3) on restart the door re-creates the ingress from the key, rebuilds the authorizer
grant from status.grant and the bundle (ticket, CA, claim.json are deterministic), and resumes
awaitComputeNetwork, so the claim reaches Joining; (4) write-ahead status.contract.acceptPending
{bidUri} before the accept record is written, so a restart between accept intent and accept record
looks for an accept naming that RFP in the requester's repo and completes or releases exactly once.
One NodeClaim schema revision carries status.secrets, acceptPending and M6-GATEWAY-REPOPULATE's
Adopting phase and M4-HOST-KEYS' ContractState.hostKeys (APIResourceSchema.spec is immutable;
versions.json our-crds). The KV path is in
0006:M3-BAO-SYNC's desired set (live claim -> kept; claim gone -> deleted). Bidding/Provisioning leave
ORPHANED_PHASES.
files: ["ts/atproto-market/lib/guest-capability-secrets (key loader instead of a fresh keypair)", "socialweb-computer-k8s/lib/socialweb-computer-k8s-node-market (resume path, acceptPending)", "api/v1alpha1 NodeClaim (status.secrets, status.contract.acceptPending; one schema revision)", "socialweb-computer-k8s door OpenBao key store"]
test_first: >
socialweb-computer-k8s test/node_market_test.ts: 'a door restarted during Provisioning re-serves the
same secrets URL and the claim reaches Joining', 'a door restarted during Bidding resumes the RFP it
already posted and posts no second one', 'a door restarted between accept intent and accept record
completes or releases exactly once'; atproto-market test: 'a secrets server built from a stored key
answers on the same ingress subdomain'. On the market network (node born via runComputeContract):
TestADoorRestartMidProvisioningStillJoinsTheNode (the door process killed after accept and before
onNetwork; the node joins, one claim, one receipt, no vm.delete). Red first (today the claim is marked
Failed).
done_when: "unit tests green; the market network test green with UTC readings; no claim in Bidding or Provisioning is failed by a door restart."
risks: ["The dispatcher must accept a re-registration of the same ingress key after a crash (stale registration); measured on the market network, the M4-BIDDER-VAULT risk."]
status: done
evidence:
- "task/door-resilience merged to main 2026-10-12: NodeClaim schema revision 4 (phases Lost and Adopting, status.secrets, contract.acceptPending, contract.termination; export re-pointed, TS types regenerated); per-claim secrets-ingress key written once (CAS 0) at secret/socialweb-door/claims/<claim>/secrets-ingress through a new OpenBao store (in-memory fallback) with door options openbao-addr/-token-file/-ca-file/-kv-mount; served URL recorded before the RFP, acceptPending before the accept; a restarted door holding the key resumes Bidding/Provisioning claims (no second RFP/accept, same secrets URL, grant re-installed, network wait resumed, reaches Joining); also vm.terminated: a terminated guest ends its claim Lost at once, released with no vm.delete, one replacement rented. atproto-market aed08e45/95b5ea6 (rebased to 1c4073b)"
- "tests red then green: atproto-market stored-key secrets 1/1, resume 7/7, 77 related green; door node_market 8 new red, 40/40 on merged main; OpenBao store 4/4; full door suite 142 passed (1 live skipped); three deliberate regressions each caught"
- "left: prod door OpenBao token + policy for secret/socialweb-door/claims/* (until then the in-memory store, so a restart still fails in-flight claims); TestADoorRestartMidProvisioningStillJoinsTheNode and a live guest poweroff (Lost, one replacement RFP, no vm.delete) in a market network; M4-HOST-KEYS hostKeys needs a further NodeClaim schema revision"
- id: M6-GATEWAY-REPOPULATE
milestone: M6
kcp_use: "traditional-exception (virtual Kubernetes clusters project: NodeClaims; 0006 principle)"
title: After an upgrade the gateway re-populates every VM it owns, including ones rented from bidders elsewhere
depends_on: ["0008:M3-DOOR-DENOPOD", "0006:M3-CLUSTER-UNITS", M4-BIDDER-VAULT, M6-CLAIM-RESUME, M4-HOST-KEYS, M3-NODE-LOSS-CLOCK]
decision: >
Decided by the user 2026-10-07: the gateway's core binary and deno processes are upgradable while VMs
keep running, and afterwards it re-populates which VMs it owns from NodeClaims, receipts and market
records. Decided by the user 2026-10-12: control-plane continuity across upgrades of the finished system
is blue/green behind a per-cluster front (0006:M3-CP-CONTINUITY), and in-flight claims resume
(M6-CLAIM-RESUME). Decided by Claude, user may veto: the door builds one ownership table at start and on
a timer from (1) every NodeClaim's status.contract, (2) the accept records its requester repos wrote
and the vm.delete events they submitted, (3) the receipts in the winners' repos that reference those
accepts, (4) onNetwork records, and (5) the tenant clusters' Node objects. The record scan (2)-(4) runs
only for claims carrying status.contract.acceptPending and for receipts with no claim at all; every
other claim is resolved from the claim alone. Rules: a receipt with a claim and a joined Node is
adopted (no write); a receipt with a claim and no Node follows plan 0005 M3-NODE-LOSS with the rebased
clock (M3-NODE-LOSS-CLOCK); a vm.delete already submitted is never submitted twice. A receipt with NO
claim (the claim lost while the VM lives) is ADOPTED FIRST with a usage check (decided by the user
2026-10-12, replacing the 2026-10-11 release-unless-joined default): the door writes a claim in the new
phase Adopting and matches the guest to a tenant Node only when all three agree - the Node's name equals
the guest name (vmName) in the door's own compute.vm record for that receipt, the cluster's OpenBao
holds role node-<name>, and the receipt's bidder-signed vm.hostKeys record (M4-HOST-KEYS, verified
and bound to the receipt) is present as the guest's signed identity; any disagreement is no match.
Then: joined and Ready -> adopted into that cluster (the claim moves to the phase a joined claim has,
no vm.delete, no RFP); not joined, but its user_data targets a cluster that still exists (the cluster
named by the ticket and claim.json the door's vm record carried) -> wait the join grace (the 2 m
never-Ready grace, rebased to pathUpSince by M3-NODE-LOSS-CLOCK) and adopt on join; the cluster is
gone, or the grace passes with no join -> release once (releaseCompute, one vm.delete). Control planes
stay up across the upgrade (0006:M3-CLUSTER-UNITS), so joined nodes stay Ready.
files: ["socialweb-computer-k8s/lib/socialweb-computer-k8s-node-market (ownership table, Adopting phase, adoption match)", "api/v1alpha1 NodeClaim (phase Adopting, in M6-CLAIM-RESUME's schema revision)", "ts/atproto-market/lib (requester record listing helpers)"]
test_first: >
socialweb-computer-k8s test/node_market_test.ts: 'an unclaimed receipt whose guest is joined and Ready
is adopted into that cluster with no vm.delete and no RFP', 'an unclaimed receipt not yet joined whose
user_data targets a live cluster waits the join grace and is adopted on join', 'an unclaimed receipt
whose cluster is gone is released exactly once', 'an unclaimed receipt with no join within the grace is
released exactly once', 'a Node whose name matches but whose vault role or vm.hostKeys record does not
is never adopted', 'the join grace counts from pathUpSince after our own outage', 'the record scan
touches only acceptPending claims and unclaimed receipts', 'a restarted door never submits a second
vm.delete'; on the market network (nodes born via runComputeContract):
TestAGatewayUpgradeKeepsEveryMarketNodeReadyAndOwned and TestAnUnclaimedJoinedNodeIsAdoptedNotReleased
(the NodeClaim object of a joined node removed through the kcp API with its finalizer stripped, then a
door restart: the claim returns via Adopting, the node stays Ready, no vm.delete). Red first.
done_when: "unit tests green; live: a two-node market cluster stays Ready across a gateway upgrade, both claims adopted, no RFP and no vm.delete sent by the restart; the unclaimed-joined-node market network reading recorded (UTC)."
risks: ["Listing market records across winners' repos is network-bound; the table is built from claims first and the record scan only fills the acceptPending and unclaimed gaps, bounded per pass."]
status: done
evidence:
- id: M6-LIVE-RECOVERY
milestone: M6
title: A killed, restarted or logged-out bidder still releases the VM it rented; a user-manager restart is a reboot
depends_on: [M5-RUN-BIDDER, "0008:M4-MARKET-NETWORK-PORT"]
decision: >
On the market network, each case after renting a guest through the RFP flow: (a) `systemctl --user kill -s
KILL` of the bidder unit's main process and its automatic restart; (b) `systemctl --user restart` of
the bidder unit; (c) every login session of the bidder account ended with linger on. Each reads:
ComputeVM Running with the same unit InvocationID and firecracker pid + start ticks, the tree audit
passing, same issuer URL, onNetwork accepted, vm.delete 200, VM gone. (d) Reboot semantics without a
reboot (decided by the user 2026-10-11: reboot = Lost): `systemctl restart user@<uid>` (or
`loginctl terminate-user`) as root stops every unit; the restarted bidder marks each ComputeVM
Lost(UnitGone) and starts nothing, and the requester side replaces it through a new RFP (plan 0005
M3-NODE-LOSS) and releases the old receipt, which deletes the Lost ComputeVM. Case (a) is written first
against the firecracker provider before M4-CONTRACT-STORE-FILE, where it fails (vm.delete refused, guest
leaks; seen live 2026-10-12T01:31Z).
files: ["internal/marketnetwork/live_test.go (TestAKilledBidderStillReleasesTheVMItRentedBeforeTheKill, TestARestartedBidderAdoptsItsVM, TestALoggedOutLingeringBidderKeepsItsVM, TestAUserManagerRestartMakesEveryVMLostAndTheRequesterReplacesIt)"]
test_first: "Case (a): FAIL with the firecracker provider today (recorded), PASS with it after M4-CONTRACT-STORE-FILE and with compute-provider-kcp after M4-CONTRACT-RESTORE; cases (b)-(d) red until M5-RUN-BIDDER."
done_when: "readings recorded for all four cases; zero ssh sessions."
risks: ["Case (d) stops every unit of the account on hat-8, including any other market network's; it runs on a dedicated test account."]
status: done
evidence:
- id: M6-NOSSH-JOIN
milestone: M6
kcp_use: "traditional-exception (virtual Kubernetes clusters project: tenant nodes; 0006 principle)"
title: A market node rented through compute-provider-kcp joins a tenant cluster with no ssh, across a bidder upgrade
depends_on: [M6-BIDDER-UPGRADE]
decision: >
Plan 0005's join (M3-UNITS, M3-JWT) needs a stable issuer from the winning bidder, which
M4-BIDDER-VAULT provides. A door-driven NodeClaim rented from a run bidder host joins, runs a pod,
stays Ready through a bidder upgrade, and is released through the claim's deletion. The guest boots
the build-host-script image (M1-GUEST-IMAGE), so this is also the proof that its kernel carries what k3s needs.
files: ["the live test that owns the door-driven join at the time"]
test_first: "The join test upgrading the bidder mid-run; fails while the issuer moves, passes after."
done_when: "node Ready, pod Running, bidder upgraded, node still Ready, release vm.delete 200, zero ssh sessions."
risks: []
status: done
evidence:
semantics:
bidder_deno_restarts_or_upgrades: "DenoPods restart on the new closure; setup() rebuilds getDroplet from ComputeVMs; ContractStore seeds activeContracts; the vault keeps the did, issuer URL and keys; VMs never notice."
bidder_core_binary_upgraded_or_killed: "the controller and the deno processes stop with the bidder unit (control-group); VM units, the vault unit and the handle's kcp units (kine on sqlite, the one slot, front; 0006:M2-KCP-SLOTS, corrected 2026-10-12) are separate units and keep running; on start the controller re-populates by the Verdict (unit, InvocationID, firecracker pid + start ticks, GET / id and state)."
bidder_kcp_upgraded: "the handle's one kcp slot is restarted on the new asset (sqlite store copied first); kcp is down for its restart (~6 s measured on kine); VM units and the vault keep running; once kcp answers the controller re-adopts every VM by the Verdict; no VM is stopped because kcp was down (decided by the user 2026-10-12)."
dev_instance: "under host-backend=proc every unit above is a child of the instance's devsup instead of the user manager: units survive a core restart and are adopted; every unit dies with devsup (dev down, crash), which is read as a user-manager stop (every ComputeVM Lost(UnitGone)); guests still come up only through runComputeContract (decided by the user 2026-10-12)."
bidder_logout: "with linger, logging out of every session changes nothing."
user_manager_stops: "user@<uid> restart, `loginctl terminate-user`, or linger removed: every unit stops; treated as a reboot - every ComputeVM Lost(UnitGone), none booted again."
gateway_upgraded: "control planes in their own member units keep serving (0006:M3-CLUSTER-UNITS); k3s/kine upgrades are blue/green slots behind the per-cluster front (0006:M3-CP-SLOTS); the door resumes Bidding and Provisioning claims (M6-CLAIM-RESUME) and re-populates its ownership table from claims, receipts, market records and Nodes; joined nodes stay Ready; no RFP and no vm.delete is caused by the restart."
our_path_down: "a cluster cold start, link restart or front restart is our outage: NotReady time counts only from when the apiserver and both ClusterLinks are back (M3-NODE-LOSS-CLOCK), so it never releases a tenant's node whose path returns."
vmm_exits_on_its_own: "guest reboot, poweroff, crash or kill: firecracker exits, pasta exits, the unit is inactive or failed with nothing left in its cgroup; the controller sees it through Watch within seconds, marks the ComputeVM Lost, and the requester replaces the node (plan 0005 M3-NODE-LOSS); the work dir is removed before the finalizer goes."
kcp_state_lost_on_a_bidder: "every socialweb-vm unit of this handle becomes an orphan and is stopped after its grace; its receipts can no longer be honoured - a stated limit; the bidder state dir is durable state, backed up with its vault."
host_reboots: "transient units are gone, so every VM is gone; on start every ComputeVM whose hostBootID differs is Lost(HostRebooted) without probing; the requester replaces them through new RFPs (plan 0005 M3-NODE-LOSS) and releases the old receipts, which delete the Lost ComputeVMs."
identity_mismatch: "an active unit whose InvocationID, VMM pid, start ticks or firecracker id differs from the record is Lost(IdentityMismatch) and its unit is stopped at once; an unreachable bus or API socket is Unknown and retried, never Lost."
vm_delete: "bidder onVmDelete -> provider.destroy -> DELETE ComputeVM -> finalizer: SendCtrlAltDel (x86_64), grace, StopUnit with SIGKILL, work dir (console included) removed -> 404 -> 200 to the requester."
acceptance:
where: hat-8 market network, then a door-driven join
checks:
- id: ACC7-SOURCE-OF-TRUTH
text: "`kubectl get computevms` on the bidder's kcp lists exactly the guests whose VMMs run, before and after a kill and after an upgrade of the bidder."
status: done
evidence: []
- id: ACC7-UPGRADE
text: A bidder upgrade and a gateway upgrade each leave every VM running (same unit InvocationID, VMM pid and start ticks) and every market node Ready, and each side re-populates what it owns.
status: done
evidence: []
- id: ACC7-NO-RESIDENT-CODE
text: "In every live run of this plan, 0006:M2-VM-TREE-AUDIT passed on every VM unit after boot, after each restart and after each upgrade: only the pinned pasta and firecracker sha256s run in a VM unit (decided by the user 2026-10-11)."
status: done
evidence: []
- id: ACC7-PARITY
text: Every requirement in docs/compute-provider-requirements.yaml is met by the Go side (the TypeScript firecracker provider and its parity tests were retired by 0009 M4-LEGACY-BIDDER-CUT after M2-BACKPORT proved them green on both).
status: done
evidence: []
- id: ACC7-BAO-SYNC
text: "Across restarts, upgrades and a signing-key rotation of a bidder, each cluster OpenBao holds exactly one jwt-* mount per bidder with a live node, and the bidder's own OpenBao holds no per-guest entry without a live ComputeVM (decided by Claude, user may veto)."
status: done
evidence: []
- id: ACC7-NO-PROTOCOL-CHANGE
text: "The bid config, receipt, onNetwork and vm.delete records written in the runs are field-for-field the shapes of their lexicons and of the M2-BACKPORT parity fixtures (the firecracker provider is retired); the only protocol additions are com.publicdomainrelay.temp.compute.events.vm.terminated (M4-TERMINATION-EVENT) and com.publicdomainrelay.temp.compute.events.vm.hostKeys (M4-HOST-KEYS), both bidder-authored in the market.event envelope (decided by the user 2026-10-12), written by the one compute provider that remains."
status: done
evidence: []
- id: ACC7-NOSSH
text: "Zero ssh sessions to any guest by bidder, gateway, join or test-harness code in every log of every run of this plan; the only ssh to a guest is the requester's own session (over the iroh ProxyCommand, the only guest transport since 0008 M1-STRIP-LEGACY-TRANSPORTS) in M4-HOST-KEYS' live proof, named in its evidence."
status: done
evidence: []
- id: ACC7-HOST-KEYS
text: "Every ssh to a guest (request-vm-ssh, the door's ssh command, the ssh door) runs with StrictHostKeyChecking=yes against the guest-reported keys of the receipt's vm.hostKeys record, a wrong key fails verification, and no ssh-keyscan runs on any host (decided by the user 2026-10-12)."
status: done
evidence: []
- id: ACC7-VM-USERNS
text: "In every live run, each VM unit's pasta and firecracker run in their own subuid sub-range of the account's reserved range, no two live VMs share a sub-range, and no file of the bidder (state dir, vault, kcp store, other VMs) is owned by or readable by a VMM's host uid (decided by the user 2026-10-12)."
status: done
evidence: []
- id: ACC7-NO-LEAK
text: "In every live run, a VMM that exits on its own makes its ComputeVM Lost within seconds and leaves no process, and after every release, Lost or Failed outcome the bidder's vms/ directory holds exactly the live ComputeVMs' work dirs (the two E8 defects of 2026-10-12 never reproduce)."
status: done
evidence: []
- id: ACC7-CLAIM-RESUME
text: "A door restarted while a claim is Bidding or Provisioning (node born via runComputeContract on the market network) resumes it to Joining with the same secrets URL, one RFP, one receipt and no vm.delete (decided by the user 2026-10-12)."
status: done
evidence: []
- id: ACC7-SCOPES
text: "Every collection the bidder or the requester writes is in its oauth-scope role set and in the client scope deployed at qr.fedfork.com (M1-PROTOCOL-CHECKLIST's check tests green), and no live run of this plan logged a 403 ScopeMissingError (decided by the user 2026-10-12)."
status: done
evidence: []
- id: ACC7-RESTART-KEEPS-RECEIPTS
text: "A bidder restarted with a guest rented (firecracker provider via M4-CONTRACT-STORE-FILE, compute-provider-kcp via M4-CONTRACT-RESTORE) answers vm.delete 200 and destroys the guest once; a late vm.delete for a guest that ended before the restart answers 200 already-terminated; no guest or work dir leaks (decided by the user 2026-10-12)."
status: done
evidence: []
questions_for_user: []