chadtmiller.com/hatk
AT Protocol app framework
1
0
Checks and npm releases on knotless
.knotless/workflows/ci.ymlnew+26−0
| @@ -0,0 +1,26 @@ | ||
# The gate: typecheck, lint, format and the full test suite with its coverage | ||
# threshold, on every push and every pull into main. `npm test` fails the run | ||
# if coverage drops below the floor in packages/hatk/vitest.config.ts, so a | ||
# change that guts a tested path cannot land quietly. | ||
# | ||
# node:26, because package.json requires node >=25, on Debian trixie (glibc | ||
# 2.41), not bookworm (2.36): better-sqlite3's prebuilt binary needs glibc >= | ||
# 2.38, so on bookworm every sqlite-backed test dies with a GLIBC error. | ||
# | ||
# `npm ci` rather than `npm install`: it installs exactly the lockfile, the only | ||
# record of the Linux binaries for better-sqlite3 and @duckdb/node-bindings. | ||
# Never regenerate that lockfile on macOS over an existing node_modules: it | ||
# drops every non-darwin platform package and breaks this run. | ||
on: | ||
push: { refs: [refs/heads/*] } | ||
pull: { base: [refs/heads/main] } | ||
image: node:26-trixie | ||
cache: [/root/.npm] | ||
steps: | ||
- name: install | ||
run: npm ci | ||
- name: check | ||
run: npm run check | ||
- name: test | ||
timeout: 30 | ||
run: npm test | ||
.knotless/workflows/release.ymlnew+57−0
| @@ -0,0 +1,57 @@ | ||
# Publishes @hatk/hatk to npm, under the `alpha` tag, when main's version has | ||
# not been published yet. Bump `version` in packages/hatk/package.json in a | ||
# pull; once it lands, this publishes it. A version on npm already is left | ||
# alone, so the run passes at once. | ||
# | ||
# It runs the checks itself before publishing rather than waiting on ci's, so | ||
# nothing is published from a commit that does not pass them. | ||
# | ||
# NPM_TOKEN is an npm granular access token that may publish @hatk/hatk, | ||
# granted to pushes to main alone in the runner's secrets file. Asked for on | ||
# any other branch, | ||
# knotless rerun knotless://chadtmiller.com/hatk --ref refs/heads/<branch> --workflow release | ||
# it checks and packs everything and publishes nothing, to try it. | ||
on: | ||
push: { refs: [refs/heads/main], paths: [packages/hatk/package.json] } | ||
asked: { refs: [refs/heads/*] } | ||
superseded: finish | ||
image: node:26-trixie | ||
cache: [/root/.npm] | ||
secrets: [NPM_TOKEN] | ||
steps: | ||
- name: version | ||
run: | | ||
version=$(node -p 'require("./packages/hatk/package.json").version') | ||
echo "$version" > ~/.version | ||
if [ -n "$(npm view "@hatk/hatk@$version" version 2>/dev/null)" ]; then | ||
echo "@hatk/hatk@$version is published already" | ||
touch ~/.done | ||
elif [ "$KNOTLESS_RUN_REF" != refs/heads/main ]; then | ||
echo "on $KNOTLESS_RUN_REF: checking $version, publishing nothing" | ||
else | ||
echo "publishing @hatk/hatk@$version" | ||
fi | ||
- name: install | ||
run: | | ||
[ ! -e ~/.done ] || exit 0 | ||
npm ci | ||
- name: check | ||
run: | | ||
[ ! -e ~/.done ] || exit 0 | ||
npm run check | ||
- name: test | ||
timeout: 30 | ||
run: | | ||
[ ! -e ~/.done ] || exit 0 | ||
npm test | ||
- name: publish | ||
run: | | ||
[ ! -e ~/.done ] || exit 0 | ||
if [ "$KNOTLESS_RUN_REF" != refs/heads/main ]; then | ||
npm publish -w @hatk/hatk --tag alpha --dry-run | ||
exit 0 | ||
fi | ||
[ -n "$NPM_TOKEN" ] || { echo "NPM_TOKEN is not granted: not publishing"; exit 1; } | ||
echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/.npmrc | ||
npm publish -w @hatk/hatk --tag alpha | ||
rm ~/.npmrc | ||
.tangled/workflows/ci.ymldeleted+0−53
| @@ -1,53 +0,0 @@ | ||
# The gate: typecheck, lint, format and the full test suite with its coverage | ||
# threshold, on every push. `npm test` fails the build if coverage drops below | ||
# the floor in packages/hatk/vitest.config.ts, so a change that guts a tested | ||
# path cannot land quietly. | ||
# | ||
# A push constraint needs `branch` or `tag`: with neither, spindle matches | ||
# nothing and the workflow silently never fires. `manual` is how to exercise | ||
# this without pushing. | ||
when: | ||
- event: ['push', 'manual'] | ||
branch: ['**'] | ||
| ||
# NixOS gives a Node project nothing directly, so run the checks inside an | ||
# official node image, the same shape pds.js uses. | ||
engine: microvm | ||
image: nixos | ||
| ||
virtualisation: | ||
docker: true | ||
| ||
steps: | ||
- name: 'Check and test' | ||
command: | | ||
set -eo pipefail | ||
| ||
# The checkout is the current directory; bind-mount it into a node | ||
# container. node_modules lands back in the workspace, but the microvm is | ||
# torn down after the run, so nothing leaks between builds. | ||
# | ||
# node:26, because package.json requires node >=25. | ||
# | ||
# Debian trixie (glibc 2.41), not bookworm (2.36): better-sqlite3's | ||
# prebuilt native binary needs glibc >= 2.38, so on bookworm every | ||
# sqlite-backed test dies with a GLIBC version error. | ||
# | ||
# `npm ci` rather than `npm install`: it installs exactly the lockfile, | ||
# which is the only record of the Linux binaries for better-sqlite3 and | ||
# @duckdb/node-bindings. Never regenerate that lockfile on macOS — doing | ||
# so over an existing node_modules drops every non-darwin platform | ||
# package and breaks this build. | ||
docker run --rm \ | ||
-v "$PWD":/workspace \ | ||
-w /workspace \ | ||
node:26-trixie \ | ||
bash -c ' | ||
set -eo pipefail | ||
echo "==> npm ci" | ||
npm ci | ||
echo "==> check (tsc + oxlint + oxfmt)" | ||
npm run check | ||
echo "==> test (vitest + coverage gate)" | ||
npm test | ||
' | ||
packages/hatk/test/clef.test.tsnew+166−0
| @@ -0,0 +1,166 @@ | ||
import { afterEach, beforeEach, expect, test, vi } from 'vitest' | ||
import { askClef, configureClef } from '../src/classifiers.ts' | ||
| ||
// askClef on its own: what it sends, how each answer type becomes a 0–1 score, | ||
// and which failures it retries. Classifier scans over it are in | ||
// classifiers.test.ts. | ||
| ||
const QUESTIONS = { q: { type: 'noul' as const, instructions: 'Is it?' } } | ||
| ||
function ok(body: unknown) { | ||
return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } }) | ||
} | ||
| ||
beforeEach(() => { | ||
configureClef({ apiToken: 'test-token', model: 'clef', endpoint: 'https://model.test/v1' }) | ||
vi.useFakeTimers({ toFake: ['setTimeout'] }) | ||
}) | ||
| ||
afterEach(() => { | ||
vi.useRealTimers() | ||
vi.unstubAllGlobals() | ||
}) | ||
| ||
/** Run askClef to its end, letting every backoff it sleeps through pass at once. */ | ||
async function settle<T>(p: Promise<T>): Promise<T> { | ||
const done = p.then( | ||
(v) => ({ v }), | ||
(e) => ({ e }), | ||
) | ||
await vi.runAllTimersAsync() | ||
const r: any = await done | ||
if ('e' in r) throw r.e | ||
return r.v | ||
} | ||
| ||
test('without Clef configured, asking is refused', async () => { | ||
configureClef(null) | ||
await expect(askClef({}, QUESTIONS)).rejects.toThrow(/not configured/) | ||
}) | ||
| ||
test('images go in the request only when there are some', async () => { | ||
const fetchMock = vi.fn(async (_url: string, _init: RequestInit) => ok({ answers: {} })) | ||
vi.stubGlobal('fetch', fetchMock) | ||
await askClef({ a: 1 }, QUESTIONS, { images: [] }) | ||
await askClef({ a: 1 }, QUESTIONS, { images: ['data:image/png;base64,AA'] }) | ||
const bodies = fetchMock.mock.calls.map((c) => JSON.parse(c[1].body as string)) | ||
expect(bodies[0]).not.toHaveProperty('images') | ||
expect(bodies[1].images).toEqual(['data:image/png;base64,AA']) | ||
expect(fetchMock.mock.calls[0][1]).toMatchObject({ headers: { authorization: 'Bearer test-token' } }) | ||
}) | ||
| ||
test('a bare response, without the Workers AI envelope, reads the same', async () => { | ||
vi.stubGlobal('fetch', async () => ok({ answers: { q: { type: 'noul', noul: 0.3 } } })) | ||
const out = await askClef({}, QUESTIONS) | ||
expect(out).toEqual({ answers: { q: { score: 0.3, type: 'noul' } }, model: 'clef', inputTokens: 0 }) | ||
}) | ||
| ||
test('every answer type becomes one 0–1 score', async () => { | ||
vi.stubGlobal('fetch', async () => | ||
ok({ | ||
result: { | ||
model: 'clef-2', | ||
usage: { input_tokens: 42 }, | ||
answers: { | ||
noul: { type: 'noul', noul: 0.7 }, | ||
noulMissing: { type: 'noul' }, | ||
choice: { type: 'choice', choice: 'b', probabilities: { a: 0.2, b: 0.8 }, confidence: 0.9 }, | ||
choiceUnknown: { type: 'choice', choice: 'z', probabilities: { a: 1 } }, | ||
choiceBare: { type: 'choice', choice: 'a' }, | ||
score: { type: 'score', score: 3, probabilities: { 0: 0, 1: 0, 2: 0, 3: 0, 4: 1 }, confidence: 0.5 }, | ||
scoreOneLevel: { type: 'score', score: 0.4, probabilities: { 0: 1 } }, | ||
scoreBare: { type: 'score' }, | ||
other: { type: 'list' }, | ||
nothing: null, | ||
}, | ||
}, | ||
success: true, | ||
}), | ||
) | ||
const { answers, model, inputTokens } = await askClef({}, QUESTIONS) | ||
expect(model).toBe('clef-2') | ||
expect(inputTokens).toBe(42) | ||
expect(answers).toEqual({ | ||
noul: { score: 0.7, type: 'noul' }, | ||
noulMissing: { score: 0, type: 'noul' }, | ||
choice: { score: 0.8, type: 'choice', choice: 'b', confidence: 0.9 }, | ||
choiceUnknown: { score: 0, type: 'choice', choice: 'z', confidence: undefined }, | ||
choiceBare: { score: 0, type: 'choice', choice: 'a', confidence: undefined }, | ||
score: { score: 0.75, type: 'score', choice: '3', confidence: 0.5 }, | ||
scoreOneLevel: { score: 0.4, type: 'score', choice: '0.4', confidence: undefined }, | ||
scoreBare: { score: 0, type: 'score', choice: 'undefined', confidence: undefined }, | ||
other: { score: 0, type: 'list' }, | ||
nothing: { score: 0, type: 'unknown' }, | ||
}) | ||
}) | ||
| ||
test('a response Workers AI marks failed is not retried', async () => { | ||
const fetchMock = vi.fn(async () => ok({ success: false, errors: [{ message: 'bad model' }] })) | ||
vi.stubGlobal('fetch', fetchMock) | ||
await expect(askClef({}, QUESTIONS)).rejects.toThrow(/bad model/) | ||
expect(fetchMock).toHaveBeenCalledTimes(1) | ||
}) | ||
| ||
test('a rejected request is not retried', async () => { | ||
const fetchMock = vi.fn(async () => new Response('no such question type', { status: 400 })) | ||
vi.stubGlobal('fetch', fetchMock) | ||
await expect(askClef({}, QUESTIONS)).rejects.toThrow(/Clef 400: no such question type/) | ||
expect(fetchMock).toHaveBeenCalledTimes(1) | ||
}) | ||
| ||
test('a rate limit waits as long as retry-after says, then tries again', async () => { | ||
const fetchMock = vi | ||
.fn() | ||
.mockResolvedValueOnce(new Response('slow down', { status: 429, headers: { 'retry-after': '7' } })) | ||
.mockResolvedValueOnce(new Response('busy', { status: 503 })) | ||
.mockResolvedValueOnce(ok({ answers: { q: { type: 'noul', noul: 1 } } })) | ||
vi.stubGlobal('fetch', fetchMock) | ||
const sleeps: number[] = [] | ||
const real = globalThis.setTimeout | ||
vi.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms: number) => { | ||
sleeps.push(ms) | ||
return real(fn, 0) | ||
}) as any) | ||
const out = await settle(askClef({}, QUESTIONS)) | ||
expect(out.answers.q.score).toBe(1) | ||
// retry-after in seconds, then backoff for the second attempt. | ||
expect(sleeps).toEqual([7000, 1000]) | ||
}) | ||
| ||
test('a server error on the last attempt is the answer', async () => { | ||
const fetchMock = vi.fn(async () => new Response('down', { status: 502 })) | ||
vi.stubGlobal('fetch', fetchMock) | ||
await expect(settle(askClef({}, QUESTIONS, { attempts: 2 }))).rejects.toThrow(/Clef 502: down/) | ||
expect(fetchMock).toHaveBeenCalledTimes(2) | ||
}) | ||
| ||
test('a network failure is retried, and the last one thrown', async () => { | ||
const fetchMock = vi.fn(async () => { | ||
throw new TypeError('fetch failed') | ||
}) | ||
vi.stubGlobal('fetch', fetchMock) | ||
await expect(settle(askClef({}, QUESTIONS, { attempts: 3 }))).rejects.toThrow('fetch failed') | ||
expect(fetchMock).toHaveBeenCalledTimes(3) | ||
}) | ||
| ||
test('a network failure that clears is answered', async () => { | ||
const fetchMock = vi | ||
.fn() | ||
.mockRejectedValueOnce(new TypeError('fetch failed')) | ||
.mockResolvedValueOnce(ok({ answers: {} })) | ||
vi.stubGlobal('fetch', fetchMock) | ||
await expect(settle(askClef({}, QUESTIONS))).resolves.toMatchObject({ answers: {} }) | ||
}) | ||
| ||
test('an abort is not retried', async () => { | ||
const fetchMock = vi.fn(async () => { | ||
throw new DOMException('aborted', 'AbortError') | ||
}) | ||
vi.stubGlobal('fetch', fetchMock) | ||
await expect(askClef({}, QUESTIONS)).rejects.toThrow('aborted') | ||
expect(fetchMock).toHaveBeenCalledTimes(1) | ||
}) | ||
| ||
test('no attempts at all fails plainly', async () => { | ||
await expect(askClef({}, QUESTIONS, { attempts: 0 })).rejects.toThrow('Clef request failed') | ||
}) | ||
packages/hatk/test/config-load.test.ts+77−0
| @@ -30,6 +30,13 @@ const ENV_KEYS = [ | ||
'CDN_SALT', | ||
'ADMINS', | ||
'OAUTH_ISSUER', | ||
'CLOUDFLARE_ACCOUNT_ID', | ||
'CLOUDFLARE_AI_TOKEN', | ||
'CLEF_MODEL', | ||
'CLEF_SCORE_ON_INDEX', | ||
'SPACES_WATCH', | ||
'SPACES_RECONCILE_INTERVAL', | ||
'SPACES_SERVICE_DID', | ||
] | ||
const savedEnv: Record<string, string | undefined> = {} | ||
| ||
| @@ -296,3 +303,73 @@ test('an oauth block with no clients still enables oauth with an empty client li | ||
conditionalScopes: [], | ||
}) | ||
}) | ||
| ||
test('clef needs both an account and a token, from the file or the environment', async () => { | ||
expect((await loadConfig(writeConfig(`export default { clef: { accountId: 'acct' } }`))).clef).toBeNull() | ||
| ||
const file = writeConfig( | ||
`export default { clef: { accountId: 'acct', apiToken: 'tok', model: 'clef-2', scoreOnIndex: true } }`, | ||
) | ||
expect((await loadConfig(file)).clef).toEqual({ | ||
accountId: 'acct', | ||
apiToken: 'tok', | ||
model: 'clef-2', | ||
scoreOnIndex: true, | ||
}) | ||
| ||
process.env.CLOUDFLARE_ACCOUNT_ID = 'env-acct' | ||
process.env.CLOUDFLARE_AI_TOKEN = 'env-tok' | ||
process.env.CLEF_MODEL = 'clef-env' | ||
process.env.CLEF_SCORE_ON_INDEX = 'false' | ||
expect((await loadConfig(file)).clef).toEqual({ | ||
accountId: 'env-acct', | ||
apiToken: 'env-tok', | ||
model: 'clef-env', | ||
scoreOnIndex: false, | ||
}) | ||
| ||
// The environment alone is enough, with the defaults for the rest. | ||
expect((await loadConfig(writeConfig(`export default {}`))).clef).toEqual({ | ||
accountId: 'env-acct', | ||
apiToken: 'env-tok', | ||
model: 'clef-env', | ||
scoreOnIndex: false, | ||
}) | ||
delete process.env.CLEF_MODEL | ||
delete process.env.CLEF_SCORE_ON_INDEX | ||
expect((await loadConfig(writeConfig(`export default {}`))).clef).toMatchObject({ | ||
model: 'clef', | ||
scoreOnIndex: false, | ||
}) | ||
}) | ||
| ||
test('spaces are off unless configured, and the environment overrides what to watch', async () => { | ||
expect((await loadConfig(writeConfig(`export default {}`))).spaces).toBeNull() | ||
expect((await loadConfig(writeConfig(`export default { spaces: {} }`))).spaces).toEqual({ | ||
types: [], | ||
watch: [], | ||
reconcileInterval: 300, | ||
serviceDid: undefined, | ||
serviceFragment: 'atproto_space_syncer', | ||
}) | ||
| ||
const file = writeConfig( | ||
`export default { spaces: { types: ['a.b.c'], watch: ['at://x'], reconcileInterval: 60, serviceDid: 'did:web:file', serviceFragment: 'syncer' } }`, | ||
) | ||
expect((await loadConfig(file)).spaces).toEqual({ | ||
types: ['a.b.c'], | ||
watch: ['at://x'], | ||
reconcileInterval: 60, | ||
serviceDid: 'did:web:file', | ||
serviceFragment: 'syncer', | ||
}) | ||
| ||
process.env.SPACES_WATCH = ' at://y , ,at://z' | ||
process.env.SPACES_RECONCILE_INTERVAL = '30' | ||
process.env.SPACES_SERVICE_DID = 'did:web:env' | ||
expect((await loadConfig(file)).spaces).toMatchObject({ | ||
watch: ['at://y', 'at://z'], | ||
reconcileInterval: 30, | ||
serviceDid: 'did:web:env', | ||
}) | ||
}) | ||
packages/hatk/test/spaces-notify-routes.test.ts+4−2
| @@ -103,7 +103,7 @@ test('an unverified notice changes nothing', async () => { | ||
expect(handleWriteNotice).not.toHaveBeenCalled() | ||
}) | ||
| ||
test('a refused notice says so in this instance\'s own log', async () => { | ||
test("a refused notice says so in this instance's own log", async () => { | ||
// Otherwise the refusal is recorded only by the authority, whose log belongs | ||
// to somebody else, and an appview that never syncs on a notice looks merely | ||
// slow rather than broken. Found exactly that way, on a live deployment. | ||
| @@ -130,7 +130,9 @@ test('a malformed notice is refused before it is verified', async () => { | ||
| ||
test('an instance that receives no notices has no endpoint to speak of', async () => { | ||
spaceServiceId.mockReturnValue(null) | ||
const res = await handler()(post('/xrpc/com.atproto.space.notifyWrite', { space: SPACE, repo: WRITER, repoRev: '3a' })) | ||
const res = await handler()( | ||
post('/xrpc/com.atproto.space.notifyWrite', { space: SPACE, repo: WRITER, repoRev: '3a' }), | ||
) | ||
expect(res.status).toBe(404) | ||
expect(verifyNotice).not.toHaveBeenCalled() | ||
}) | ||