// Runs the reference PDS (@atproto/pds via @atproto/dev-env) and a PLC // directory on free ports, creates test accounts, prints one JSON line // describing them, and shuts down when stdin closes. As knotless's // harness/pds.mjs. // // node harness/pds.mjs [--accounts alice,bob,carol] `just pds` // // The line: { pds, plc, spaces, hud: { host, cert, key }, accounts: [{ // handle, did, password }], lexicons: { handle, did, password, db } }, // every account's password `password`; `spaces`, whether the PDS serves // spaces (describeServer's spacesAlpha); `lexicons`, the account that // publishes the permission sets the PDS resolves (below), and `db`, the // PDS's account database, where it keeps the lexicons it resolved. // // A sign-in asking for a permission set (`include:`) has the PDS // resolve the set's lexicon: `_lexicon.` in DNS names an // account, whose `com.atproto.lexicon.schema/` record is the set. // No DNS here names one, so the PDS is told an authority for every NSID // (its PDS_LEXICON_AUTHORITY_DID): `lexicons.test`, an account of its own, // which publishes the permission sets of the operator's apps the // harnesses serve (crates/hud-core/src/testdata/grain: Grain's), read // from its own repository. A flow republishes one to change it; the PDS // keeps a set it resolved five minutes, which a flow can age in `db`'s // `lexicon` table. // // HUD_DEV_ENV names a built dev-env package to run instead of the npm one: // harness/spaces/node_modules/@atproto/dev-env, the spaces alpha (`cd // harness/spaces && npm ci` once), where HUD_SPACES=on keeps a person's // canvases in their own spaces. The alpha's PDS migrates an account's // database every time it opens it, which costs a read more than the read: // here each is migrated the first time it is opened only. Its // authorization page resolves each space type a sign-in's scope names, by // DNS and PLC, which reach no lexicon of HUD's here: so HUD's space types // (schemas/) are put in its store of the lexicons it resolved. // // HUD is an OAuth client of the PDS, which fetches a client's metadata // only over https at a hostname, and refuses a client_id whose TLD is a // local one (.test, .local, .example and the rest). So HUD is served at // harness/tls.mjs's HOST, with its certificate, signed by this checkout's // own authority, which this process trusts, and this process resolves that // name to 127.0.0.1; harness/tls-proxy.mjs fronts HUD with it. // // The PDS is plain http at localhost, and sets its sign-in page's cookies // (its CSRF token's, its device's) Secure. Chromium keeps a Secure cookie // from http://localhost; WebKit does not, so in Safari or the iOS app's // sign-in sheet the page cannot send its CSRF token back, and every // password is "invalid". So here the PDS's cookies go without Secure, as // grain's local PDS's CSRF cookie does (grain-web-auth's patches/). import { AsyncLocalStorage } from 'node:async_hooks'; import { readdirSync, readFileSync, realpathSync } from 'node:fs'; import { ServerResponse } from 'node:http'; import { createRequire } from 'node:module'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import dns from 'node:dns'; import tls from 'node:tls'; import { HOST as HUD_HOST, certificates } from './tls.mjs'; /// Each Set-Cookie a plain-http answer of this process gives, without Secure. function insecureCookies() { const plain = (cookie) => String(cookie).replace(/;\s*secure(?=;|$)/gi, ''); const each = (value) => (Array.isArray(value) ? value.map(plain) : plain(value)); for (const method of ['setHeader', 'appendHeader']) { const original = ServerResponse.prototype[method]; ServerResponse.prototype[method] = function (name, value) { const http = !this.req?.socket?.encrypted; return original.call(this, name, http && String(name).toLowerCase() === 'set-cookie' ? each(value) : value); }; } } insecureCookies(); const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); // The PDS's better-sqlite3 is built for the Node .node-version names; on // another major it fails to load, saying only that two ABI numbers differ. const pinned = readFileSync(join(root, '.node-version'), 'utf8').trim(); if (process.versions.node.split('.')[0] !== pinned.split('.')[0]) { console.error( `harness/pds.mjs runs on Node ${pinned}, as .node-version says, not ${process.version}: put a Node ${pinned.split('.')[0]} first on PATH`, ); process.exit(1); } const devEnv = process.env.HUD_DEV_ENV && resolve(root, process.env.HUD_DEV_ENV); const { TestNetworkNoAppView } = await import(devEnv ? pathToFileURL(join(devEnv, 'dist/index.js')).href : '@atproto/dev-env'); if (devEnv) await migrateOnce(); await skipFsync(); /// The directory of the PDS's compiled modules that the dev-env runs. function pdsDist() { const from = devEnv ? pathToFileURL(join(devEnv, 'package.json')) : import.meta.resolve('@atproto/dev-env'); return dirname(realpathSync(createRequire(from).resolve('@atproto/pds'))); } /// Has the PDS's SQLite databases commit without waiting for the disk: a /// test network has nothing to keep through a crash, and each write /// otherwise waits on several fsyncs. async function skipFsync() { const { Database } = await import(pathToFileURL(join(pdsDist(), 'db/db.js')).href); const sqlite = Database.sqlite; Database.sqlite = (location, opts) => sqlite.call(Database, location, { ...opts, pragmas: { ...opts?.pragmas, synchronous: 'OFF' } }); } /// Has the alpha's PDS migrate each account's database on its first /// opening alone, rather than on every one: the same modules it loads, /// patched before it starts. async function migrateOnce() { const load = (file) => import(pathToFileURL(join(pdsDist(), file)).href); const { ActorStore } = await load('actor-store/actor-store.js'); const { Migrator } = await load('db/migrator.js'); const opened = new Set(); const again = new AsyncLocalStorage(); const open = ActorStore.prototype.openDb; ActorStore.prototype.openDb = async function (did) { const { dbLocation } = await this.getLocation(did); const db = await again.run(opened.has(dbLocation), () => open.call(this, did)); opened.add(dbLocation); return db; }; const migrate = Migrator.prototype.migrateToLatestOrThrow; Migrator.prototype.migrateToLatestOrThrow = function () { return again.getStore() ? Promise.resolve([]) : migrate.call(this); }; } /// The account that publishes the permission sets the PDS resolves, made /// on it, and the PDS told it is the authority of every NSID; each /// permission set under crates/hud-core/src/testdata published as its /// `com.atproto.lexicon.schema` record. Its `{ handle, did, password }`. async function publishSets(net) { const password = 'password'; const res = await fetch(`${net.pds.url}/xrpc/com.atproto.server.createAccount`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ email: 'lexicons@test.com', handle: 'lexicons.test', password }), }); if (!res.ok) throw new Error(`createAccount lexicons: ${await res.text()}`); const { did, handle, accessJwt } = await res.json(); net.pds.ctx.cfg.lexicon = { ...net.pds.ctx.cfg.lexicon, didAuthority: did }; const files = (dir) => readdirSync(dir, { withFileTypes: true }).flatMap((e) => e.isDirectory() ? files(join(dir, e.name)) : e.name.endsWith('.json') ? [join(dir, e.name)] : [], ); const sets = files(join(root, 'crates/hud-core/src/testdata')) .map((file) => JSON.parse(readFileSync(file, 'utf8'))) .filter((doc) => doc.defs?.main?.type === 'permission-set'); if (sets.length === 0) throw new Error('no permission sets under crates/hud-core/src/testdata'); for (const doc of sets) { const put = await fetch(`${net.pds.url}/xrpc/com.atproto.repo.putRecord`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${accessJwt}` }, body: JSON.stringify({ repo: did, collection: 'com.atproto.lexicon.schema', rkey: doc.id, record: { $type: 'com.atproto.lexicon.schema', ...doc }, }), }); if (!put.ok) throw new Error(`publishing ${doc.id}: ${await put.text()}`); } return { handle, did, password }; } /// Puts HUD's space lexicons (schemas/) in the alpha PDS's store of the /// lexicons it resolved, as resolved now and fresh for good. async function knowSpaces(db) { const dir = join(root, 'schemas'); const docs = readdirSync(dir) .filter((file) => file.endsWith('.json')) .map((file) => JSON.parse(readFileSync(join(dir, file), 'utf8'))) .filter((doc) => doc.defs?.main?.type === 'space'); if (docs.length === 0) throw new Error(`no space lexicons under ${dir}`); const now = new Date().toISOString(); for (const doc of docs) { const row = { createdAt: now, updatedAt: '9999-01-01T00:00:00.000Z', lastSucceededAt: now, uri: `at://lexicons.hud.place/com.atproto.lexicon.schema/${doc.id}`, lexicon: JSON.stringify(doc), }; await db.db .insertInto('lexicon') .values({ nsid: doc.id, ...row }) .onConflict((oc) => oc.column('nsid').doUpdateSet(row)) .execute(); } } const { ca, cert, key } = certificates(); tls.setDefaultCACertificates([...tls.getCACertificates('default'), readFileSync(ca, 'utf8')]); const lookup = dns.lookup; dns.lookup = (hostname, options, callback) => { if (typeof options === 'function') [callback, options] = [options, {}]; if (hostname !== HUD_HOST) return lookup(hostname, options, callback); return options?.all ? callback(null, [{ address: '127.0.0.1', family: 4 }]) : callback(null, '127.0.0.1', 4); }; const arg = (name) => { const i = process.argv.indexOf(name); return i === -1 ? undefined : process.argv[i + 1]; }; const names = (arg('--accounts') ?? 'alice,bob,carol').split(','); const net = await TestNetworkNoAppView.create({ pds: { hostname: 'localhost' } }); const accounts = []; for (const name of names) { const password = 'password'; const res = await fetch(`${net.pds.url}/xrpc/com.atproto.server.createAccount`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ email: `${name}@test.com`, handle: `${name}.test`, password }), }); if (!res.ok) throw new Error(`createAccount ${name}: ${await res.text()}`); const { did, handle } = await res.json(); accounts.push({ handle, did, password }); } const described = await fetch(`${net.pds.url}/xrpc/com.atproto.server.describeServer`).then((res) => res.json()); const spaces = described.spacesAlpha === true; if (spaces) await knowSpaces(net.pds.ctx.accountManager.db); const lexicons = { ...(await publishSets(net)), db: net.pds.ctx.cfg.db.accountDbLoc }; console.log( JSON.stringify({ pds: net.pds.url, plc: net.plc.url, spaces, hud: { host: HUD_HOST, cert, key, ca }, accounts, lexicons, }), ); const shutdown = async () => { await net.close(); process.exit(0); }; process.stdin.on('end', shutdown); process.stdin.on('close', shutdown); process.stdin.resume(); process.on('SIGTERM', shutdown); process.on('SIGINT', shutdown);