chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
The harness PDS keeps its databases in memory where it can
docs/development.md+5−1
| @@ -26,7 +26,11 @@ Tests run against the reference PDS from npm, started by | ||
permissioned-data alpha build of that PDS, for private repositories. The | ||
alpha migrates an account's database every time it opens it, which made | ||
the same tests cost its PDS about a fifth more CPU than the stock one; the | ||
harness has it migrate each database only the first time. Every line and | ||
harness has it migrate each database only the first time. The PDS commits | ||
to SQLite on every write, and a commit waits on the disk (20 ms an fsync on | ||
the runner's), so where `/dev/shm` has 2 GiB free the harness keeps the | ||
PDS's databases there, as memory, and removes them when it stops | ||
(`KNOTLESS_PDS_DISK=1` keeps them on disk). Every line and | ||
function is covered across the two runs, and CI fails below that. | ||
| ||
```sh | ||
harness/pds.mjs+28−1
| @@ -18,7 +18,15 @@ | ||
import { fileURLToPath, pathToFileURL } from 'node:url' | ||
import { dirname, join, resolve } from 'node:path' | ||
import { execFileSync } from 'node:child_process' | ||
import { mkdtempSync, readdirSync, readFileSync, realpathSync } from 'node:fs' | ||
import { | ||
existsSync, | ||
mkdtempSync, | ||
readdirSync, | ||
readFileSync, | ||
realpathSync, | ||
rmSync, | ||
statfsSync, | ||
} from 'node:fs' | ||
import { createRequire } from 'node:module' | ||
import { AsyncLocalStorage } from 'node:async_hooks' | ||
import { tmpdir } from 'node:os' | ||
| @@ -37,6 +45,25 @@ if (process.versions.node.split('.')[0] !== pinned.split('.')[0]) { | ||
) | ||
process.exit(1) | ||
} | ||
| ||
// The PDS keeps its databases under the temp directory, and a SQLite commit | ||
// waits on the disk each time: 20ms an fsync on the runner's disk, which | ||
// every write a test makes pays several times over, where memory takes | ||
// none. So on a host with room in /dev/shm, the PDS's data goes there, and | ||
// is removed when the harness stops. KNOTLESS_PDS_DISK=1 keeps it on disk. | ||
const shm = '/dev/shm' | ||
if (!process.env.KNOTLESS_PDS_DISK && existsSync(shm)) { | ||
try { | ||
const { bavail, bsize } = statfsSync(shm) | ||
if ((bavail * bsize) / 2 ** 30 >= 2) { | ||
const data = mkdtempSync(join(shm, 'knotless-pds-')) | ||
process.env.TMPDIR = data | ||
process.on('exit', () => rmSync(data, { recursive: true, force: true })) | ||
} | ||
} catch { | ||
// Not writable: the data stays where the temp directory is. | ||
} | ||
} | ||
const devEnv = process.env.KNOTLESS_DEV_ENV | ||
const { TestNetworkNoAppView } = await import( | ||
devEnv ? pathToFileURL(resolve(root, devEnv, 'dist/index.js')).href : '@atproto/dev-env' | ||