chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
Spike: the test check measures where its time goes
.knotless/workflows/test.yml+36−2
| @@ -27,6 +27,15 @@ env: | ||
ATPROTO_COMMIT: 11c5d98cd50261c5e1201fedc5c6c05a7b8d1ccf | ||
# Bumped when the steps that build the alpha change, to build it again. | ||
ALPHA_BUILD: "2" | ||
# Spike knobs (ci/test-speed): a cold build of the workspace's crates, | ||
# no stored runs, both nets, and the run's parallelism. | ||
EXP_COLD: "1" | ||
EXP_NOSTORE: "1" | ||
EXP_NETS: "stock,spaces" | ||
EXP_JOBS: "" | ||
EXP_HARNESSES: "" | ||
# Skips the test step and what reads its report, to measure the replay. | ||
EXP_SKIP_TEST: "" | ||
steps: | ||
# Node into /opt/node, the harness's packages when their lockfile | ||
# changes, and cargo-llvm-cov into /opt/tools, each kept in the cache. | ||
| @@ -73,18 +82,43 @@ steps: | ||
# A pull's run, its pre-flight, goes against the stock PDS alone, to | ||
# be quick, and against the alpha too only when the gate falls short | ||
# without it; the merge queue's batch runs both. | ||
COV_NETS=${KNOTLESS_RUN_PULL:+stock} COV_STORE=/opt/testcache COV_JOBS=$(( $(nproc) / 2 )) COV_HARNESSES=2 COV_TOOLCHAIN=$(rustup show active-toolchain | cut -d' ' -f1) \ | ||
node harness/cov.mjs /opt/atproto/packages/dev-env | ||
[ -n "$EXP_SKIP_TEST" ] && exit 0 | ||
echo "nproc $(nproc); cgroup cpu.max $(cat /sys/fs/cgroup/cpu.max 2>/dev/null); host cpus $(grep -c ^processor /proc/cpuinfo); mem $(free -g | awk '/Mem/{print $2}')G" | ||
[ -n "$EXP_COLD" ] && touch crates/*/src/lib.rs crates/*/src/main.rs crates/*/src/bin/*.rs 2>/dev/null || true | ||
store=/opt/testcache; [ -n "$EXP_NOSTORE" ] && store= | ||
harness/exp/sample.sh target/cpu.log & sampler=$! | ||
harness/exp/proc.sh target/proc.log & procs=$! | ||
rm -f target/cpu.log; COV_TIMINGS=1 COV_NETS=${EXP_NETS:-${KNOTLESS_RUN_PULL:+stock}} COV_STORE=$store COV_JOBS=${EXP_JOBS:-$(( $(nproc) / 2 ))} COV_HARNESSES=${EXP_HARNESSES:-2} COV_TOOLCHAIN=$(rustup show active-toolchain | cut -d' ' -f1) \ | ||
node harness/cov.mjs /opt/atproto/packages/dev-env || { kill $sampler; cat target/cpu.log; exit 1; } | ||
kill $sampler $procs; sleep 3; echo "cpu seconds by command:"; cat target/proc.log; awk '{split($3,a,"=");s+=a[2]*5; split($4,b,"=");t+=b[2]*5} END{print "cgroup cpu-s", s, "throttled core-s", t}' target/cpu.log | ||
# Spike: the same again, warm, its PDSes not syncing SQLite. | ||
- name: test sync off | ||
run: | | ||
export PATH=/opt/node/bin:/opt/tools:$PATH KNOTLESS_SQLITE_SYNC_OFF=1 | ||
harness/exp/sample.sh target/cpu2.log & sampler=$! | ||
COV_TIMINGS= COV_NETS=stock,spaces COV_JOBS=$(( $(nproc) / 2 )) COV_HARNESSES=2 COV_TOOLCHAIN=$(rustup show active-toolchain | cut -d' ' -f1) \ | ||
node harness/cov.mjs /opt/atproto/packages/dev-env | grep -v -E '^(ok |\s+Compiling)' | ||
kill $sampler; awk '{split($3,a,"=");s+=a[2]*5; split($4,b,"=");t+=b[2]*5} END{print "cgroup cpu-s", s, "throttled core-s", t}' target/cpu2.log | ||
# Spike: and again as it is, warm, to bracket it. | ||
- name: test sync on | ||
run: | | ||
export PATH=/opt/node/bin:/opt/tools:$PATH | ||
harness/exp/sample.sh target/cpu3.log & sampler=$! | ||
COV_TIMINGS= COV_NETS=stock,spaces COV_JOBS=$(( $(nproc) / 2 )) COV_HARNESSES=2 COV_TOOLCHAIN=$(rustup show active-toolchain | cut -d' ' -f1) \ | ||
node harness/cov.mjs /opt/atproto/packages/dev-env | grep -v -E '^(ok |\s+Compiling)' | ||
kill $sampler; awk '{split($3,a,"=");s+=a[2]*5; split($4,b,"=");t+=b[2]*5} END{print "cgroup cpu-s", s, "throttled core-s", t}' target/cpu3.log | ||
- name: answers | ||
run: | | ||
# Every XRPC answer knotless-serve's tests received, fresh or kept | ||
# with a stored run, as its lexicon says: the pages' types are made | ||
# from the lexicons, so this is what holds the server to them. | ||
export PATH=/opt/node/bin:$PATH | ||
[ -n "$EXP_SKIP_TEST" ] && exit 0 | ||
node harness/answers.mjs target/llvm-cov-target/cov-report/answers | ||
- name: plans | ||
run: | | ||
# No query the tests ran scans a table that grows with the network, | ||
# as SQLite would plan it at prod's size, unless its SQL says so. | ||
export PATH=/opt/node/bin:$PATH | ||
[ -n "$EXP_SKIP_TEST" ] && exit 0 | ||
node harness/plans.mjs target/llvm-cov-target/cov-report/plans | ||
harness/exp/replay/1.patchnew+404−0
404 lines changed: not loaded until asked for.
harness/exp/replay/2.patchnew+692−0
692 lines changed: not loaded until asked for.
harness/exp/replay/3.patchnew+1727−0
1727 lines changed: not loaded until asked for.
harness/exp/replay/4.patchnew+37−0
| @@ -0,0 +1,37 @@ | ||
diff --git a/crates/knotless/src/remote.rs b/crates/knotless/src/remote.rs | ||
index 116aac76..70cabd23 100644 | ||
--- a/crates/knotless/src/remote.rs | ||
+++ b/crates/knotless/src/remote.rs | ||
@@ -260,8 +260,17 @@ impl Remote { | ||
| ||
/// `raw`, read from here: in a space, with the references it holds in | ||
/// the public form given the space. | ||
- fn parse<T: DeserializeOwned>(&self, mut raw: RawRecord) -> Result<Stored<T>> { | ||
- if let Some(space) = self.space_ref() { | ||
+ fn parse<T: DeserializeOwned>(&self, raw: RawRecord) -> Result<Stored<T>> { | ||
+ self.parse_as(raw, true) | ||
+ } | ||
+ | ||
+ /// [`Remote::parse`], or with `resolve` false, `raw` as it was written. | ||
+ fn parse_as<T: DeserializeOwned>( | ||
+ &self, | ||
+ mut raw: RawRecord, | ||
+ resolve: bool, | ||
+ ) -> Result<Stored<T>> { | ||
+ if let Some(space) = self.space_ref().filter(|_| resolve) { | ||
crate::short_form::resolve_refs(&mut raw.value, &space); | ||
} | ||
parse(raw) | ||
@@ -725,11 +734,7 @@ impl Remote { | ||
let out: ListOut = self.read("listRecords", ¶ms).await?; | ||
let full = out.records.len() == PAGE; | ||
for raw in out.records { | ||
- all.push(if resolve { | ||
- self.parse(raw)? | ||
- } else { | ||
- parse(raw)? | ||
- }); | ||
+ all.push(self.parse_as(raw, resolve)?); | ||
} | ||
match out.cursor { | ||
Some(next) if full => cursor = Some(next), | ||
harness/exp/replay/5.patchnew+40−0
| @@ -0,0 +1,40 @@ | ||
diff --git a/crates/knotless-serve/src/index/short_form.rs b/crates/knotless-serve/src/index/short_form.rs | ||
index 2a2c6377..408e9e8f 100644 | ||
--- a/crates/knotless-serve/src/index/short_form.rs | ||
+++ b/crates/knotless-serve/src/index/short_form.rs | ||
@@ -87,8 +87,8 @@ impl Index { | ||
return Ok(()); | ||
} | ||
let mut stmt = db.prepare( | ||
- // scans: space_record, once, as the index moves to full URIs | ||
- "SELECT space, did, collection, rkey FROM space_record", | ||
+ "SELECT space, did, collection, rkey FROM space_record | ||
+ -- scans: space_record, once, as the index moves to full URIs", | ||
)?; | ||
let rows = stmt.query_map([], |row| { | ||
Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) | ||
@@ -282,14 +282,20 @@ mod tests { | ||
db.query_row(sql, [], |row| row.get::<_, String>(0)) | ||
.unwrap() | ||
}; | ||
- assert_eq!(one("SELECT subject FROM comment"), opened); | ||
assert_eq!( | ||
- one("SELECT uri FROM reaction"), | ||
+ one("SELECT subject FROM comment -- scans: comment, a test's one row"), | ||
+ opened | ||
+ ); | ||
+ assert_eq!( | ||
+ one("SELECT uri FROM reaction -- scans: reaction, a test's one row"), | ||
full("did:b", nsid::REACTION, "h") | ||
); | ||
- assert_eq!(one("SELECT subject FROM reaction"), opened); | ||
assert_eq!( | ||
- one("SELECT uri FROM push"), | ||
+ one("SELECT subject FROM reaction -- scans: reaction, a test's one row"), | ||
+ opened | ||
+ ); | ||
+ assert_eq!( | ||
+ one("SELECT uri FROM push -- scans: push, a test's one row"), | ||
full("did:o", nsid::CLAIM, "k.m.0") | ||
); | ||
drop(db); | ||
harness/exp/replay/base.tar.gznew+0−0
A binary file, not shown.
harness/exp/cpuprof.mjsnew+29−0
| @@ -0,0 +1,29 @@ | ||
// Spike: self time by function, summed over the .cpuprofile files in a | ||
// directory, as node --cpu-prof wrote them: the top functions, and the | ||
// top packages. | ||
import { readdirSync, readFileSync } from 'node:fs' | ||
import { join } from 'node:path' | ||
const dir = process.argv[2] | ||
const byFn = new Map() | ||
const byPkg = new Map() | ||
let total = 0 | ||
for (const f of readdirSync(dir).filter((f) => f.endsWith('.cpuprofile'))) { | ||
const p = JSON.parse(readFileSync(join(dir, f), 'utf8')) | ||
const nodes = new Map(p.nodes.map((n) => [n.id, n])) | ||
const dt = new Map() | ||
p.samples.forEach((id, i) => dt.set(id, (dt.get(id) ?? 0) + (p.timeDeltas[i] ?? 0))) | ||
for (const [id, us] of dt) { | ||
const { functionName, url, lineNumber } = nodes.get(id).callFrame | ||
const pkg = url.match(/node_modules\/((?:@[^/]+\/)?[^/]+)/)?.[1] ?? (url.includes('/packages/') ? url.match(/packages\/([^/]+)/)[1] : url ? url.replace(/^.*\//, '') : `(${functionName || 'native'})`) | ||
const k = `${functionName || '(anon)'} ${url.replace(/^.*node_modules\//, '')}:${lineNumber}` | ||
byFn.set(k, (byFn.get(k) ?? 0) + us) | ||
byPkg.set(pkg, (byPkg.get(pkg) ?? 0) + us) | ||
total += us | ||
} | ||
} | ||
const show = (m, n) => [...m].sort((a, b) => b[1] - a[1]).slice(0, n).forEach(([k, us]) => console.log(`${(us / 1e6).toFixed(1).padStart(8)}s ${((100 * us) / total).toFixed(1).padStart(5)}% ${k}`)) | ||
console.log(`total ${(total / 1e6).toFixed(1)}s sampled`) | ||
console.log('by package:') | ||
show(byPkg, 25) | ||
console.log('by function:') | ||
show(byFn, 40) | ||
harness/exp/proc.shnew+17−0
| @@ -0,0 +1,17 @@ | ||
#!/bin/sh | ||
# Spike: CPU seconds by command, of every process in this container, alive | ||
# sampled every 2s (one that lives less is missed: the cgroup's total | ||
# less these is theirs); the last sample of each pid counts. Writes | ||
# the totals to $1 when killed. | ||
out=$1 | ||
tmp=/tmp/proc-samples | ||
: > $tmp | ||
trap 'awk "{t[\$1]=\$2; c[\$1]=\$3} END{for(p in t) s[c[p]]+=t[p]; for(k in s) printf \"%8.1f %s\n\", s[k], k}" $tmp | sort -rn | head -25 > "$out"; exit 0' TERM INT | ||
hz=$(getconf CLK_TCK) | ||
while :; do | ||
# utime and stime, fields 14 and 15, after the command in parentheses. | ||
awk -v hz=$hz '{ pid=FILENAME; sub("/proc/","",pid); sub("/stat","",pid); | ||
i=index($0,") "); c=substr($0,index($0,"(")+1,i-index($0,"(")-1); gsub(/ /,"_",c); | ||
split(substr($0,i+2),f," "); printf "%s %.2f %s\n", pid, (f[12]+f[13])/hz, c }' /proc/[0-9]*/stat 2>/dev/null >> $tmp | ||
sleep 2 | ||
done | ||
harness/exp/replay.shnew+29−0
| @@ -0,0 +1,29 @@ | ||
#!/bin/sh | ||
# Spike: replays main's last five commits to crates/, lexicons/ and | ||
# web/src/ one at a time through cov.mjs with reuse across builds, over a | ||
# store of its own, to see what that saves. `replay.sh base` takes the | ||
# checkout back before them and runs everything, both nets, filling the | ||
# store; `replay.sh <n>` applies the nth and runs as a pull's pre-flight | ||
# does, the stock PDS first. | ||
set -e | ||
export PATH=/opt/node/bin:/opt/tools:$PATH | ||
S=/tmp/replay-store | ||
P=harness/exp/replay | ||
if [ "$1" = base ]; then | ||
# The tree before the five, whatever main is now: its Rust, lexicons and | ||
# pages, fonts and pictures left as they are. | ||
find crates lexicons web/src -type f ! -path 'crates/knotless-serve/fonts/*' ! -path 'web/src/lib/assets/*' -delete | ||
tar xzf $P/base.tar.gz | ||
nets=stock,spaces | ||
else | ||
git apply $P/$1.patch | ||
nets=stock | ||
fi | ||
start=$(date +%s) | ||
set +e | ||
CARGO_BUILD_JOBS=4 COV_DIFF=1 COV_NETS=$nets COV_STORE=$S COV_JOBS=$(( $(nproc) / 2 )) COV_HARNESSES=2 COV_TOOLCHAIN=$(rustup show active-toolchain | cut -d' ' -f1) \ | ||
node harness/cov.mjs /opt/atproto/packages/dev-env > /tmp/replay-$1.log 2>&1 | ||
status=$? | ||
grep -v -E '^(ok |\s+Compiling|\S+\.rs\s+[0-9]| crates/)' /tmp/replay-$1.log | grep -v -E '^(test |running |---- |failures:|$)' | head -120 | ||
echo "uncovered lines listed: $(grep -c '^ crates/' /tmp/replay-$1.log)" | ||
echo "REPLAY $1: exit $status in $(( $(date +%s) - start ))s" | ||
harness/exp/sample.shnew+21−0
| @@ -0,0 +1,21 @@ | ||
#!/bin/sh | ||
# Spike: samples host CPU, this container's cgroup CPU and throttling every | ||
# 5s, one line each, to $1 until killed. | ||
out=$1 | ||
prev() { awk '/^cpu /{print $2+$3+$4+$5+$6+$7+$8, $5+$6}' /proc/stat; } | ||
cg=/sys/fs/cgroup/cpu.stat | ||
set -- $(prev); t0=$1; i0=$2 | ||
u0=$(awk '/^usage_usec/{print $2}' $cg 2>/dev/null || echo 0) | ||
th0=$(awk '/^throttled_usec/{print $2}' $cg 2>/dev/null || echo 0) | ||
s=0 | ||
while :; do | ||
sleep 5; s=$((s+5)) | ||
set -- $(prev); t1=$1; i1=$2 | ||
u1=$(awk '/^usage_usec/{print $2}' $cg 2>/dev/null || echo 0) | ||
th1=$(awk '/^throttled_usec/{print $2}' $cg 2>/dev/null || echo 0) | ||
host=$(awk -v a=$((t1-t0)) -v b=$((i1-i0)) 'BEGIN{printf "%.0f", a? 100*(a-b)/a : 0}') | ||
cores=$(awk -v u=$((u1-u0)) 'BEGIN{printf "%.1f", u/5e6}') | ||
thr=$(awk -v u=$((th1-th0)) 'BEGIN{printf "%.1f", u/5e6}') | ||
echo "$s host_busy=${host}% ours_cores=$cores throttled=$thr load=$(cut -d' ' -f1 /proc/loadavg)" >> "$out" | ||
t0=$t1; i0=$i1; u0=$u1; th0=$th1 | ||
done | ||
harness/cov.mjs+64−4
| @@ -74,6 +74,7 @@ import { | ||
import { availableParallelism } from 'node:os' | ||
import { basename, dirname, join, relative, resolve } from 'node:path' | ||
import { fileURLToPath } from 'node:url' | ||
import { coveredOf, inputsOf, reuser, writeMap } from './reuse.mjs' | ||
| ||
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') | ||
const [spaces, ignore = '(tests/|harness/|examples/|rustlib/)'] = process.argv.slice(2) | ||
| @@ -82,6 +83,9 @@ const cargo = ['run', toolchain, 'cargo'] | ||
const jobs = Number(process.env.COV_JOBS) || Math.max(2, Math.floor(availableParallelism() / 2)) | ||
const only = process.env.COV_ONLY?.split(',') | ||
const store = process.env.COV_STORE && resolve(process.env.COV_STORE) | ||
// COV_DIFF=1: a run whose binary changed only where it never went is | ||
// reused too (harness/reuse.mjs). | ||
const diffing = !!(store && process.env.COV_DIFF) | ||
const netsRun = process.env.COV_NETS ? process.env.COV_NETS.split(',') : ['stock', 'spaces'] | ||
const partial = netsRun.length < 2 | ||
const keep = Number(process.env.COV_STORE_KEEP) || 300 | ||
| @@ -136,7 +140,7 @@ const harness = (name, devEnv) => { | ||
const log = openSync(logged, 'w') | ||
child = spawn('node', [join(root, 'harness/pds.mjs'), '--accounts', 'seed'], { | ||
cwd: root, | ||
env: { ...process.env, KNOTLESS_DEV_ENV: devEnv }, | ||
env: { ...process.env, KNOTLESS_DEV_ENV: devEnv, ...(process.env.COV_CPUPROF ? { NODE_OPTIONS: `--cpu-prof --cpu-prof-dir=${process.env.COV_CPUPROF}` } : {}) }, | ||
stdio: ['pipe', 'pipe', log], | ||
}) | ||
let out = '' | ||
| @@ -167,7 +171,7 @@ const stop = () => Object.values(nets).flat().forEach((n) => n.stop()) | ||
process.on('exit', stop) | ||
| ||
// Every test binary, and the package it is in: cargo runs each there. | ||
const built = spawnSync('rustup', [...cargo, 'test', '--workspace', '--no-run', '--message-format=json'], { | ||
const built = spawnSync('rustup', [...cargo, 'test', '--workspace', '--no-run', '--message-format=json', ...(process.env.COV_TIMINGS ? ['--timings'] : [])], { | ||
cwd: root, | ||
env, | ||
encoding: 'utf8', | ||
| @@ -196,6 +200,20 @@ const made = built.stdout | ||
const tests = made.filter((m) => m.test && (!only || only.includes(m.name.split(' ')[0]))) | ||
const bins = made.filter((m) => !m.test) | ||
console.log(`built ${tests.length} test binaries in ${took()}`) | ||
// COV_TIMINGS=1: where the build's time went, unit by unit, from cargo's | ||
// own --timings report. | ||
if (process.env.COV_TIMINGS) { | ||
try { | ||
const html = readFileSync(join(target, 'cargo-timings/cargo-timing.html'), 'utf8') | ||
const units = JSON.parse(html.match(/const UNIT_DATA = (\[.*?\]);\n/s)[1]) | ||
const end = Math.max(...units.map((u) => u.start + u.duration)) | ||
console.log(`timings: ${units.length} units, ${end.toFixed(1)}s wall`) | ||
for (const u of units.sort((a, b) => b.duration - a.duration).slice(0, 25)) | ||
console.log(` ${u.duration.toFixed(1).padStart(6)}s at ${u.start.toFixed(1).padStart(6)}s ${u.name} ${u.target.trim() || 'lib'} ${u.mode}${u.rmeta_time ? ` (rmeta ${u.rmeta_time.toFixed(1)}s)` : ''}`) | ||
} catch (e) { | ||
console.log(`timings: unread, ${e.message}`) | ||
} | ||
} | ||
| ||
const sha = (file) => | ||
new Promise((ok, fail) => { | ||
| @@ -232,7 +250,7 @@ const profiled = (file) => { | ||
const hashOf = (...parts) => createHash('sha256').update(parts.join('\0')).digest('hex') | ||
const version = (cmd) => execFileSync(cmd, ['--version'], { encoding: 'utf8' }).trim() | ||
const harnessed = hashOf( | ||
...['harness/cov.mjs', 'harness/pds.mjs', 'harness/tls-proxy.mjs', 'harness/authorize.mjs', 'harness/package-lock.json'].map((f) => | ||
...['harness/cov.mjs', 'harness/reuse.mjs', 'harness/pds.mjs', 'harness/tls-proxy.mjs', 'harness/authorize.mjs', 'harness/package-lock.json'].map((f) => | ||
readFileSync(join(root, f), 'utf8'), | ||
), | ||
version('node'), | ||
| @@ -246,6 +264,10 @@ const netOf = { | ||
// own package always, as one may be stopped before it writes a profile, | ||
// and those it was seen to run. | ||
const keyOf = (t, net) => hashOf(t.sha, harnessed, net, netOf[net]) | ||
// What a run rests on besides its binaries, for reuse across builds. | ||
const ctxOf = (net) => hashOf(harnessed, net, netOf[net], rustc('-vV'), process.env.RUSTFLAGS ?? '') | ||
// This build, by its executables. | ||
const buildId = hashOf(...made.map((m) => m.sha).sort()) | ||
const depOf = (m) => `${m.test ? 'test' : 'bin'} ${m.name}` | ||
const own = (t) => Object.fromEntries(bins.filter((b) => b.pkg === t.pkg).map((b) => [depOf(b), b.sha])) | ||
const now = Object.fromEntries(made.map((m) => [depOf(m), m.sha])) | ||
| @@ -314,6 +336,29 @@ const timeKey = (t) => `${t.net} ${t.name}` | ||
const limitOf = (t) => Number(process.env.COV_TIMEOUT) * 1000 || Math.max(600_000, 3 * (times[timeKey(t)] ?? 0)) | ||
const all = Object.keys(netOf).flatMap((net) => tests.map((t) => ({ ...t, net, key: keyOf(t, net) }))) | ||
const reused = all.filter(stored) | ||
// Those reused across builds, where the change missed what they ran. | ||
const diffReused = [] | ||
if (diffing) { | ||
const t0 = Date.now() | ||
const inputs = inputsOf(root) | ||
const find = reuser({ root, store, inputs }) | ||
const whys = [] | ||
for (const r of all.filter((r) => !reused.includes(r))) { | ||
const found = find(r.net, r.name, ctxOf(r.net)) | ||
if (!found.run) { | ||
whys.push(` ${r.net} ${r.name}: ${found.why}`) | ||
continue | ||
} | ||
profile(join(found.run.dir, 'run.profdata')) | ||
answer(join(found.run.dir, 'answers')) | ||
plan(join(found.run.dir, 'plans')) | ||
utimesSync(found.run.dir, new Date(), new Date()) | ||
diffReused.push(r) | ||
} | ||
console.log(`${diffReused.length} more reused across builds, in ${Date.now() - t0}ms: ${diffReused.map((r) => `${r.net} ${r.name}`).join(', ')}`) | ||
if (whys.length) console.log(` not reused across builds:\n${whys.join('\n')}`) | ||
reused.push(...diffReused) | ||
} | ||
// The longest first, as the last run timed them, so that the last to | ||
// finish is not one started late: the integration tests before the rest | ||
// where there is no time yet. | ||
| @@ -393,7 +438,8 @@ const keepRun = (r, line) => { | ||
if (raws.length) execFileSync(join(tools, 'llvm-profdata'), ['merge', '-sparse', ...raws, '-o', join(tmp, 'run.profdata')]) | ||
if (existsSync(r.answers)) cpSync(r.answers, join(tmp, 'answers'), { recursive: true }) | ||
if (existsSync(r.plans)) cpSync(r.plans, join(tmp, 'plans'), { recursive: true }) | ||
writeFileSync(join(tmp, 'result.json'), JSON.stringify({ deps, line, profiled: raws.length > 0, at: new Date().toISOString() })) | ||
const across = diffing && raws.length ? { name: r.name, net: r.net, ctx: ctxOf(r.net), map: buildId, covered: coveredOf(tools, join(tmp, 'run.profdata')) } : {} | ||
writeFileSync(join(tmp, 'result.json'), JSON.stringify({ deps, line, profiled: raws.length > 0, at: new Date().toISOString(), ...across })) | ||
rmSync(join(store, r.key), { recursive: true, force: true }) | ||
renameSync(tmp, join(store, r.key)) | ||
} | ||
| @@ -464,6 +510,11 @@ await runAll(runs) | ||
// The harnesses are kept while a run may yet be made again (see below). | ||
if (!reused.length) stop() | ||
console.log(`ran ${runs.length} in ${took()}`) | ||
if (diffing) { | ||
const t0 = Date.now() | ||
writeMap({ root, tools, store, id: buildId, exes: [...new Set(made.map((m) => m.exe))], inputs: inputsOf(root) }) | ||
console.log(`this build's map in ${Date.now() - t0}ms`) | ||
} | ||
| ||
// The store keeps the runs used last, up to its bound. | ||
if (store && existsSync(store)) { | ||
| @@ -545,6 +596,15 @@ const missed = (short) => { | ||
} | ||
| ||
let status = gate() | ||
// Short with runs reused across builds: those are made again first, as a | ||
// change may move what a function it left alone reaches. | ||
if (status !== 0 && diffReused.length) { | ||
console.log(`short with ${diffReused.length} reused across builds: running them again`) | ||
for (const r of diffReused) bring(r.net) | ||
for (const n of Object.values(nets).flat()) n.file = await n.up | ||
await runAll(diffReused) | ||
status = gate() | ||
} | ||
// A line some runs reach and others do not, as a race decides, can be | ||
// one no stored run reached: the stored runs are made again, fresh, and | ||
// the report read again, rather than fail a change that did not touch it. | ||
harness/pds.mjs+15−0
| @@ -42,6 +42,21 @@ const { TestNetworkNoAppView } = await import( | ||
: '@atproto/dev-env' | ||
) | ||
if (devEnv) await migrateOnce(resolve(root, devEnv)) | ||
if (process.env.KNOTLESS_SQLITE_SYNC_OFF) await syncOff(devEnv ? resolve(root, devEnv) : dirname(dirname(createRequire(import.meta.url).resolve('@atproto/dev-env')))) | ||
| ||
/// Has every SQLite database the PDS at the dev-env in `dir` opens skip | ||
/// fsync: the PDS opens and closes an account's database on each read | ||
/// and write, and syncs each commit and each close's checkpoint, which a | ||
/// PDS thrown away after the tests never needs. | ||
async function syncOff(dir) { | ||
const index = createRequire(join(dir, 'package.json')).resolve('@atproto/pds') | ||
const dist = dirname(realpathSync(index)) | ||
const { Database } = await import(pathToFileURL(join(dist, 'db/db.js')).href) | ||
const sqlite = Database.sqlite | ||
Database.sqlite = function (location, opts) { | ||
return sqlite.call(this, location, { ...opts, pragmas: { synchronous: 'OFF', ...opts?.pragmas } }) | ||
} | ||
} | ||
| ||
/// Has the PDS that the dev-env at `dir` runs migrate each account's | ||
/// database on its first opening alone, rather than on every one: the | ||
harness/reuse.mjsnew+214−0
| @@ -0,0 +1,214 @@ | ||
// Reuse of a stored run whose test binary changed, but not where the run | ||
// went: what harness/cov.mjs does with COV_DIFF=1. | ||
// | ||
// A stored run is reused byte for byte when its binary is the same. Any | ||
// change to the core crate changes every binary, so every run is made | ||
// again, though most never reach the lines it changed. Here a stored run | ||
// keeps the functions it ran, by name, and its build keeps a map: where | ||
// each function is, by file and lines, and each file's text. A later | ||
// build's run of the same binary on the same net is reused when no line | ||
// changed since, by a diff of those texts, lies in a function it ran, or | ||
// outside every function of a file it ran in (a const, a use, a macro, a | ||
// new function), and nothing else the build reads changed (Cargo.lock, a | ||
// file brought in with include_str!, a fixture): those are hashed whole, | ||
// and any change to them reuses nothing. Changes only to comments and | ||
// blank lines count as none. | ||
// | ||
// Its profile is then counted as a stored one is: llvm-cov reads the | ||
// counts of each function whose hash is as the new build's, and those | ||
// changed are counted by the runs that are made again. | ||
| ||
import { execFileSync, spawnSync } from 'node:child_process' | ||
import { createHash } from 'node:crypto' | ||
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' | ||
import { join, relative } from 'node:path' | ||
| ||
const sha = (b) => createHash('sha256').update(b).digest('hex') | ||
| ||
/// What a build reads besides its Rust sources, hashed whole: every file | ||
/// of the checkout but those under crates/ ending .rs, and those nothing | ||
/// builds or tests read. | ||
export const inputsOf = (root) => { | ||
const skip = new Set(['target', 'node_modules', '.git', '.svelte-kit', 'build', 'dist', '.lexicons', 'docs', 'docs-site', '.changeset']) | ||
const files = [] | ||
const walk = (dir) => { | ||
for (const e of readdirSync(dir, { withFileTypes: true })) { | ||
const p = join(dir, e.name) | ||
const rel = relative(root, p) | ||
if (skip.has(e.name) || rel === 'web/src/lib/lexicons') continue | ||
if (e.isDirectory()) walk(p) | ||
else if (e.isFile() && !(rel.startsWith('crates/') && rel.endsWith('.rs')) && !/^[A-Z]+\.md$/.test(rel)) files.push(rel) | ||
} | ||
} | ||
walk(root) | ||
const h = createHash('sha256') | ||
for (const f of files.sort()) h.update(`${f}\0${sha(readFileSync(join(root, f)))}\0`) | ||
return h.digest('hex') | ||
} | ||
| ||
/// The functions a run's merged profile ran, by name. | ||
export const coveredOf = (tools, profdata) => { | ||
const out = execFileSync(join(tools, 'llvm-profdata'), ['show', '--all-functions', '--covered', profdata], { | ||
encoding: 'utf8', | ||
maxBuffer: 1 << 28, | ||
}) | ||
return [...new Set(out.split('\n').filter((l) => /^\S/.test(l) && !/^(Instrumentation level|Total functions|Functions shown|Maximum)/.test(l)).map((l) => l.trim()))] | ||
} | ||
| ||
/// The map of a build: where each function of ours is, every function's | ||
/// lines in each file, and each file's text, kept in the store by hash. | ||
export const writeMap = ({ root, tools, store, id, exes, inputs }) => { | ||
const dir = join(store, '.maps') | ||
const file = join(dir, `${id}.json`) | ||
if (existsSync(file)) return | ||
mkdirSync(join(store, '.files'), { recursive: true }) | ||
const [first, ...rest] = exes | ||
const r = spawnSync(join(tools, 'llvm-cov'), ['export', '--format=text', '--empty-profile', '--skip-expansions', first, ...rest.flatMap((e) => ['-object', e])], { | ||
encoding: 'utf8', | ||
maxBuffer: 1 << 30, | ||
}) | ||
if (r.status !== 0) return console.log(`no map of this build: llvm-cov export failed\n${r.stderr.slice(-2000)}`) | ||
const fns = {} | ||
const inside = {} | ||
const files = {} | ||
for (const f of JSON.parse(r.stdout).data[0].functions) { | ||
const path = relative(root, f.filenames[0]) | ||
if (path.startsWith('..') || path.startsWith('/')) continue | ||
// Regions of the function's own file: line start, column, line end. | ||
const own = f.regions.filter((g) => g[5] === 0) | ||
if (!own.length) continue | ||
const range = [Math.min(...own.map((g) => g[0])), Math.max(...own.map((g) => g[2]))] | ||
fns[f.name] = [path, ...range] | ||
;(inside[path] ??= new Set()).add(range.join(':')) | ||
if (!files[path]) { | ||
const text = readFileSync(join(root, path)) | ||
files[path] = sha(text) | ||
const blob = join(store, '.files', files[path]) | ||
if (!existsSync(blob)) writeFileSync(blob, text) | ||
} | ||
} | ||
// Every Rust file, functions or not: a const or a macro changed in one | ||
// with none changes what runs elsewhere. | ||
const walk = (d) => { | ||
for (const e of readdirSync(d, { withFileTypes: true })) { | ||
const p = join(d, e.name) | ||
if (e.isDirectory()) e.name !== 'target' && walk(p) | ||
else if (e.name.endsWith('.rs')) { | ||
const path = relative(root, p) | ||
if (files[path]) continue | ||
const text = readFileSync(p) | ||
files[path] = sha(text) | ||
const blob = join(store, '.files', files[path]) | ||
if (!existsSync(blob)) writeFileSync(blob, text) | ||
} | ||
} | ||
} | ||
walk(join(root, 'crates')) | ||
mkdirSync(dir, { recursive: true }) | ||
const tmp = `${file}.${process.pid}` | ||
const ranges = Object.fromEntries(Object.entries(inside).map(([p, s]) => [p, [...s].map((k) => k.split(':').map(Number))])) | ||
writeFileSync(tmp, JSON.stringify({ inputs, files, fns, inside: ranges })) | ||
renameSync(tmp, file) | ||
} | ||
| ||
/// What a change did to the old text, in its numbering, leaving out hunks | ||
/// that only change comments and blank lines: the lines it changed or | ||
/// took out, and the gaps it put lines into, each as the line before. | ||
const touched = (root, store, oldSha, path) => { | ||
const r = spawnSync('git', ['diff', '--no-index', '-U0', '--no-color', join(store, '.files', oldSha), join(root, path)], { encoding: 'utf8', maxBuffer: 1 << 26 }) | ||
const changed = [] | ||
const gaps = [] | ||
for (const h of r.stdout.split(/^(?=@@ )/m).filter((h) => h.startsWith('@@ '))) { | ||
const [, a, b = '1'] = h.match(/^@@ -(\d+)(?:,(\d+))? /) | ||
const body = h.split('\n').slice(1).filter((l) => /^[-+]/.test(l)) | ||
if (body.every((l) => /^[-+]\s*(\/\/.*)?$/.test(l))) continue | ||
const start = Number(a) | ||
const count = Number(b) | ||
if (count === 0) gaps.push(start) | ||
for (let n = start; n < start + count; n++) changed.push(n) | ||
} | ||
return { changed, gaps } | ||
} | ||
| ||
/// A reuser for this build: given a run's net, binary name and context, | ||
/// the newest stored run of it whose functions the change since missed, | ||
/// with why not if none. | ||
export const reuser = ({ root, store, inputs }) => { | ||
const entries = [] | ||
for (const e of existsSync(store) ? readdirSync(store) : []) { | ||
if (e.startsWith('.')) continue | ||
try { | ||
const r = JSON.parse(readFileSync(join(store, e, 'result.json'), 'utf8')) | ||
if (r.covered && r.map) entries.push({ dir: join(store, e), ...r }) | ||
} catch {} | ||
} | ||
entries.sort((a, b) => (b.at > a.at ? 1 : -1)) | ||
const maps = new Map() | ||
const map = (id) => { | ||
if (!maps.has(id)) { | ||
try { | ||
maps.set(id, JSON.parse(readFileSync(join(store, '.maps', `${id}.json`), 'utf8'))) | ||
} catch { | ||
maps.set(id, null) | ||
} | ||
} | ||
return maps.get(id) | ||
} | ||
const diffs = new Map() | ||
/// What changed in a file since the map's build, null if it is gone. | ||
const diff = (m, path) => { | ||
const k = `${m.files[path]} ${path}` | ||
if (!diffs.has(k)) { | ||
let now | ||
try { | ||
now = sha(readFileSync(join(root, path))) | ||
} catch { | ||
now = null | ||
} | ||
diffs.set(k, now === null ? null : now === m.files[path] ? { changed: [], gaps: [] } : touched(root, store, m.files[path], path)) | ||
} | ||
return diffs.get(k) | ||
} | ||
const inFn = (m, path, n) => (m.inside[path] ?? []).some(([s, e]) => s <= n && n <= e) | ||
/// Why nothing of a map's build may be reused, if so: what the build | ||
/// reads besides Rust changed, a Rust file went, or a line outside | ||
/// every function changed (a const, a macro, a use), which may change | ||
/// what runs anywhere. Lines only put between functions add items, which | ||
/// a changed function must call to matter. | ||
const blocked = new Map() | ||
const blockedWhy = (m) => { | ||
if (blocked.has(m)) return blocked.get(m) | ||
let why | ||
if (m.inputs !== inputs) why = 'what the build reads besides Rust changed' | ||
else | ||
for (const path of Object.keys(m.files)) { | ||
const d = diff(m, path) | ||
if (d === null) why = `${path} is gone` | ||
else { | ||
const n = d.changed.find((n) => !inFn(m, path, n)) | ||
if (n !== undefined) why = `${path}:${n} changed, outside any function` | ||
} | ||
if (why) break | ||
} | ||
blocked.set(m, why) | ||
return why | ||
} | ||
return (net, name, ctx) => { | ||
const r = entries.find((e) => e.net === net && e.name === name && e.ctx === ctx) | ||
if (!r) return { why: 'none stored' } | ||
const m = map(r.map) | ||
if (!m) return { why: 'its map is gone' } | ||
const why = blockedWhy(m) | ||
if (why) return { why } | ||
for (const f of r.covered) { | ||
const at = m.fns[f] | ||
if (!at) continue | ||
const [path, s, e] = at | ||
const d = diff(m, path) | ||
const n = d.changed.find((n) => s <= n && n <= e) ?? d.gaps.find((n) => s <= n && n + 1 <= e) | ||
if (n !== undefined) return { why: `${path}:${n}, in a function it ran` } | ||
} | ||
return { run: r } | ||
} | ||
} | ||
| ||