1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
name: CI
on:
push:
branches:
- main
- "task/**"
tags:
- "v*"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: false
jobs:
gate:
runs-on: ubuntu-latest
timeout-minutes: 75
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 20
- uses: ./.github/actions/toolchain
- name: gate
run: |
set -euo pipefail
timeout -k 10 600 go mod download
timeout -k 10 900 go build ./...
timeout -k 10 900 go vet ./...
timeout -k 10 2400 deploy/check.sh --fast
test -z "$(git status --porcelain)"
pins:
runs-on: [self-hosted, linux, x64, socialweb-computer-kcp]
timeout-minutes: 45
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
fetch-tags: true
- uses: ./.github/actions/toolchain
- uses: ./.github/actions/containers
- name: pins
run: |
set -euo pipefail
timeout -k 10 2580 make check-pins BUILD_ROOT="$CI_STATE/pins-$GITHUB_RUN_ID-$GITHUB_JOB"
binaries:
needs: pins
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
runs-on: [self-hosted, linux, x64, socialweb-computer-kcp]
timeout-minutes: 90
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
fetch-tags: true
- uses: ./.github/actions/toolchain
- uses: ./.github/actions/containers
- name: binaries
run: |
set -euo pipefail
timeout -k 10 5280 scripts/build-socialweb-computer.sh --platform linux/amd64 --platform linux/arm64 --commit "$GITHUB_SHA" --build-root "$CI_STATE/release-build-$GITHUB_RUN_ID-$GITHUB_JOB"
rel="$CI_STATE/release-build-$GITHUB_RUN_ID-$GITHUB_JOB/release/$GITHUB_SHA"
ls -la "$rel"
want="$(jq -r '.assets.deno.version' internal/versions/versions.json)"
jq -e --arg v "$want" '(.builds | length) > 0 and all(.builds[]; .denoEmbedded.version == $v and (.denoEmbedded.zips | keys | sort) == ["linux/amd64","linux/arm64"])' "$rel/release.json"
rm -rf "$OUT/binaries" && mkdir -p "$OUT/binaries" && cp -a "$rel"/. "$OUT/binaries/"
(cd "$OUT/binaries" && sha256sum -c --quiet SHA256SUMS)
- name: node downloads by sha256
run: |
set -euo pipefail
blob="$CI_STATE/release-build-$GITHUB_RUN_ID-$GITHUB_JOB/src/1/internal/assets/blobs/linux_amd64/socialweb-computer-k8s-downloads.tar.zst"
tree="$(mktemp -d "$RUNNER_TEMP/k8s-downloads.XXXXXX")"
zstd -dc "$blob" | tar -x -C "$tree"
rm -rf "$OUT/downloads" && mkdir -p "$OUT/downloads"
for side in "$tree"/*.sha256; do
name="$(basename "$side" .sha256)"
want="$(cut -d' ' -f1 "$side")"
got="$(sha256sum "$tree/$name" | cut -d' ' -f1)"
echo "downloads/$want/$name (sha256 $got)"
test "$got" = "$want"
mkdir -p "$OUT/downloads/$want" && cp "$tree/$name" "$OUT/downloads/$want/$name"
done
test "$(find "$OUT/downloads" -type f | wc -l)" -gt 0
- uses: actions/upload-artifact@v7
with:
name: binaries
path: ${{ runner.temp }}/ci/out/${{ github.sha }}/binaries/
if-no-files-found: error
retention-days: 7
- uses: actions/upload-artifact@v7
with:
name: node-downloads
path: ${{ runner.temp }}/ci/out/${{ github.sha }}/downloads/
if-no-files-found: error
retention-days: 7
guest-image:
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 120
env:
TARGET: firecracker-x86_64
outputs:
published: ${{ steps.published.outputs.published }}
key: ${{ steps.published.outputs.key }}
target: ${{ steps.published.outputs.target }}
steps:
- uses: actions/checkout@v7
- name: is the pinned guest image published
id: published
env:
SPACES_BUCKET: ${{ vars.SPACES_BUCKET }}
SPACES_REGION: ${{ vars.SPACES_REGION }}
run: |
set -euo pipefail
pin="$(jq -er --arg t "$TARGET" '.guestImages.targets[$t].outputs["image.json"].sha256' internal/versions/versions.json)"
key="guest-images/$TARGET/$pin"
img="$RUNNER_TEMP/ci/out/$GITHUB_SHA/guest-image"
echo "key=$key" >>"$GITHUB_OUTPUT"
echo "target=$TARGET" >>"$GITHUB_OUTPUT"
build() {
echo "guest-image: building $TARGET: $*"
rm -rf "$img"
echo "published=false" >>"$GITHUB_OUTPUT"
exit 0
}
{ [ -n "$SPACES_BUCKET" ] && [ -n "$SPACES_REGION" ]; } || build "no SPACES_BUCKET or SPACES_REGION to look $key up in"
url="https://${SPACES_BUCKET}.${SPACES_REGION}.digitaloceanspaces.com/$key"
code="$(curl -s -o /dev/null -I -w '%{http_code}' --max-time 60 "$url/image.json" || true)"
[ "$code" = 200 ] || build "$url/image.json answered ${code:-nothing}, so the image of these pins is not published"
rm -rf "$img" && mkdir -p "$img/$TARGET"
for f in image.json vmlinux rootfs.ext4; do
curl -sSfL --retry 3 --max-time 1800 -o "$img/$TARGET/$f" "$url/$f" || build "fetching $url/$f failed"
want="$(jq -er --arg t "$TARGET" --arg f "$f" '.guestImages.targets[$t].outputs[$f].sha256' internal/versions/versions.json)"
got="$(sha256sum "$img/$TARGET/$f" | cut -d' ' -f1)"
echo "$TARGET/$f sha256 $got (pinned $want) from $url/$f"
[ "$got" = "$want" ] || build "$url/$f hashes to $got, not its pin $want"
[ "$f" = image.json ] || continue
while read -r name recorded current; do
echo "$TARGET input $name sha256 $current (recorded in the published image.json $recorded)"
[ "$recorded" = "$current" ] || build "input $name changed since $key was built and its pins did not, so it is built to check them"
done < <(jq -r --slurpfile v internal/versions/versions.json --arg t "$TARGET" '
.inputs.kernelSource.sha256 as $ks | .inputs.base.sha256 as $bs |
"kernelSource \($ks) \($v[0].guestImages.kernel.source.sha256)",
"base \($bs) \($v[0].guestImages.targets[$t].base.sha256)",
(.inputs[] | select(.path != null) | "\(.path) \(.sha256) PATH")' "$img/$TARGET/image.json" |
while read -r name recorded current; do
if [ "$current" = PATH ]; then current="$(sha256sum "$name" 2>/dev/null | cut -d' ' -f1 || true)"; fi
echo "$name $recorded ${current:-absent}"
done)
done
echo "guest-image: not building $TARGET: every output is published at $url and matches its pin"
echo "published=true" >>"$GITHUB_OUTPUT"
- if: steps.published.outputs.published != 'true'
uses: ./.github/actions/toolchain
- if: steps.published.outputs.published != 'true'
uses: ./.github/actions/containers
- name: guest-image
if: steps.published.outputs.published != 'true'
run: |
set -euo pipefail
work="$CI_STATE/guest-image-work" && img="$OUT/guest-image"
rm -rf "$img" && mkdir -p "$work" "$img"
timeout -k 10 7000 scripts/build-guest-image.sh --target "$TARGET" --work "$work" --out "$img"
for f in vmlinux rootfs.ext4 image.json; do
want="$(jq -er --arg t "$TARGET" --arg f "$f" '.guestImages.targets[$t].outputs[$f].sha256' internal/versions/versions.json)"
got="$(sha256sum "$img/$TARGET/$f" | cut -d' ' -f1)"
echo "$TARGET/$f sha256 $got (pinned $want)"
test "$got" = "$want"
done
- if: steps.published.outputs.published != 'true'
uses: actions/upload-artifact@v7
with:
name: guest-image
path: ${{ runner.temp }}/ci/out/${{ github.sha }}/guest-image/
if-no-files-found: error
retention-days: 7
publish:
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 90
needs: [gate, binaries, guest-image]
steps:
- uses: actions/checkout@v7
- uses: ./.github/actions/toolchain
- uses: actions/download-artifact@v8
with:
name: binaries
path: ${{ runner.temp }}/ci/out/${{ github.sha }}/binaries
- uses: actions/download-artifact@v8
with:
name: node-downloads
path: ${{ runner.temp }}/ci/out/${{ github.sha }}/downloads
- if: needs.guest-image.outputs.published != 'true'
uses: actions/download-artifact@v8
with:
name: guest-image
path: ${{ runner.temp }}/ci/guest-image
- name: publish the guest image
env:
SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }}
SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}
SPACES_BUCKET: ${{ vars.SPACES_BUCKET }}
SPACES_REGION: ${{ vars.SPACES_REGION }}
PUBLISHED: ${{ needs.guest-image.outputs.published }}
KEY: ${{ needs.guest-image.outputs.key }}
TARGET: ${{ needs.guest-image.outputs.target }}
run: |
set -euo pipefail
test -n "$KEY" && test -n "$TARGET"
if [ "$PUBLISHED" = true ]; then
echo "guest-image: $KEY is already published and the guest-image job verified it against the pins, so nothing is uploaded"
exit 0
fi
test -n "$SPACES_ACCESS_KEY_ID" && test -n "$SPACES_SECRET_ACCESS_KEY"
gi="$RUNNER_TEMP/ci/guest-image/$TARGET"
for f in vmlinux rootfs.ext4 image.json; do
want="$(jq -er --arg t "$TARGET" --arg f "$f" '.guestImages.targets[$t].outputs[$f].sha256' internal/versions/versions.json)"
got="$(sha256sum "$gi/$f" | cut -d' ' -f1)"
echo "$TARGET/$f sha256 $got (pinned $want)"
test "$got" = "$want"
done
echo "guest-image: uploading the built $TARGET to $KEY"
timeout -k 10 4800 deno run -A --no-lock scripts/publish-spaces.ts --dir "$gi" --prefix "$KEY" --content-addressed
- name: publish
env:
SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }}
SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}
SPACES_BUCKET: ${{ vars.SPACES_BUCKET }}
SPACES_REGION: ${{ vars.SPACES_REGION }}
run: |
set -euo pipefail
test -n "$SPACES_ACCESS_KEY_ID" && test -n "$SPACES_SECRET_ACCESS_KEY"
if [ "$GITHUB_REF_TYPE" = tag ]; then
prefix="releases/$GITHUB_REF_NAME"; also=""
else
prefix="main/$GITHUB_SHA"; also="main/latest"
fi
dl="$OUT/downloads"
timeout -k 10 1800 deno run -A --no-lock scripts/publish-spaces.ts --dir "$dl" --prefix downloads --content-addressed
rm -rf "$dl"
timeout -k 10 4800 deno run -A --no-lock scripts/publish-spaces.ts --dir "$OUT" --prefix "$prefix" ${also:+--also "$also"}
- name: the published guest image is where the binary looks for it
env:
SPACES_BUCKET: ${{ vars.SPACES_BUCKET }}
SPACES_REGION: ${{ vars.SPACES_REGION }}
KEY: ${{ needs.guest-image.outputs.key }}
TARGET: ${{ needs.guest-image.outputs.target }}
run: |
set -euo pipefail
base="https://${SPACES_BUCKET}.${SPACES_REGION}.digitaloceanspaces.com"
for f in image.json vmlinux rootfs.ext4; do
url="$base/$KEY/$f"
want="$(jq -er --arg t "$TARGET" --arg f "$f" '.guestImages.targets[$t].outputs[$f].bytes' internal/versions/versions.json)"
head="$(curl -sS -I --max-time 60 "$url")"
code="$(printf '%s\n' "$head" | awk 'NR==1 {print $2}')"
size="$(printf '%s\n' "$head" | tr -d '\r' | awk 'tolower($1)=="content-length:" {print $2}')"
echo "$code $url $size bytes (pinned $want; readable without credentials: a bidder fetches its own image from here)"
test "$code" = 200
test "$size" = "$want"
done
if [ "$GITHUB_REF_TYPE" != tag ]; then
latest="$base/main/latest/binaries/socialweb-computer-linux-amd64"
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 60 -r 0-0 "$latest")"
echo "$code $latest (the newest main binary, readable without credentials)"
test "$code" = 206
fi
- name: the published binary fetches its own guest image
env:
SPACES_BUCKET: ${{ vars.SPACES_BUCKET }}
SPACES_REGION: ${{ vars.SPACES_REGION }}
run: |
set -euo pipefail
base="https://${SPACES_BUCKET}.${SPACES_REGION}.digitaloceanspaces.com"
if [ "$GITHUB_REF_TYPE" = tag ]; then
bin="$base/releases/$GITHUB_REF_NAME/binaries/socialweb-computer-linux-amd64"
else
bin="$base/main/latest/binaries/socialweb-computer-linux-amd64"
fi
probe="$RUNNER_TEMP/probe"
mkdir -p "$probe/run"
curl -sSfL --max-time 900 -o "$probe/socialweb-computer" "$bin"
chmod 0755 "$probe/socialweb-computer"
XDG_RUNTIME_DIR="$probe/run" "$probe/socialweb-computer" run bidder \
--handle 'NOT A HANDLE' --host-backend proc --profile dev-local --state-dir "$probe/state" \
>"$probe/out.txt" 2>&1 || true
cat "$probe/out.txt"
if grep -qE 'guestimagerelease|guest-image-source is required' "$probe/out.txt"; then
echo "the published binary cannot fetch the guest image of its own build"
exit 1
fi
echo "the published binary resolved its own guest image"
