chadtmiller.com/monolith
An atproto PDS
0
0
// Records checked against their lexicons (spec/VALIDATION.md). Records of
// the known types are drawn valid from their lexicons in spec/lexicons, and
// then sometimes broken in one way; some are of other types. They are
// written by createRecord, putRecord and applyWrites, with `validate` left
// out, true, false or not a boolean, and each answer held to the
// reference's, `validationStatus` and the record's CID among it. Follows,
// blocks, likes and reposts are of few subjects, so that createRecord often
// replaces duplicates: the collection is then listed on both sides, and the
// reference's held to what the harness knows of it. Some writes go to
// another account's repository.
import { readdirSync, readFileSync } from 'node:fs'
import { isDeepStrictEqual } from 'node:util'
import { CID } from 'multiformats/cid'
import { sha256 } from 'multiformats/hashes/sha2'
import { isValidAtUri, isValidDid } from '@atproto/syntax'
import { RKEYS, below, chance, pick, rand, value, word } from '../lib/random.mjs'
import { cand, ref } from '../lib/processes.mjs'
import { count, differ } from '../lib/report.mjs'
import { accounts, cids, nextBatch, noteCommit } from '../lib/state.mjs'
import { both, normalize, shown, xrpc } from '../lib/xrpc.mjs'
import { heldRef, noteRefs } from './blobs.mjs'
// ---- The lexicons and the syntax vectors ----------------------------------
const SPEC = new URL('../../spec/', import.meta.url)
/// Every lexicon in spec/lexicons, by its NSID (spec/VALIDATION.md, "Data").
const LEXICONS = new Map()
;(function load(dir) {
for (const e of readdirSync(dir, { withFileTypes: true })) {
if (e.isDirectory()) load(new URL(`${e.name}/`, dir))
else if (e.name.endsWith('.json')) {
const doc = JSON.parse(readFileSync(new URL(e.name, dir), 'utf8'))
LEXICONS.set(doc.id, doc)
}
}
})(new URL('lexicons/', SPEC))
/// The record types the server knows: those whose lexicon's `main` is a
/// record, but the spaces' (spec/VALIDATION.md, "Which records").
const SPACES = /^com\.atproto\.(space|memberspace|personalspace|managedspace)\./
const KNOWN = [...LEXICONS.values()].filter((d) => d.defs?.main?.type === 'record' && !SPACES.test(d.id)).map((d) => d.id).sort()
if (KNOWN.length !== 36) throw new Error(`spec/lexicons has ${KNOWN.length} record types, not the 36 spec/VALIDATION.md lists`)
const APP_BSKY = KNOWN.filter((t) => t.startsWith('app.bsky.'))
/// Types the server knows no lexicon for: in no known namespace; one in a
/// known namespace that names nothing; lexicons that are no record's; and
/// the spaces' records (spec/VALIDATION.md, "Which records").
const OTHER = ['com.example.validation.thing', 'app.bsky.feed.unknownthing', 'app.bsky.embed.images', 'app.bsky.feed.defs', 'app.bsky.feed.getTimeline', 'com.atproto.space.policy', 'com.atproto.memberspace.member']
/// The types whose records link to a subject (spec/VALIDATION.md,
/// "Duplicates").
const LINKED = ['app.bsky.graph.follow', 'app.bsky.graph.block', 'app.bsky.feed.like', 'app.bsky.feed.repost']
/// The definition `name` refers to from the lexicon `lex`: its lexicon's
/// NSID, the definition, and the `$type` that names it.
function resolve(name, lex) {
const [id, def = 'main'] = name.startsWith('#') ? [lex, name.slice(1)] : name.split('#')
const found = LEXICONS.get(id)?.defs?.[def]
if (!found) throw new Error(`spec/lexicons has no ${name} (named in ${lex})`)
return { id, def: found, type: def === 'main' ? id : `${id}#${def}` }
}
/// The strings of a file of atproto's syntax vectors, in spec/interop/syntax
/// (spec/VALIDATION.md, "Data", "String formats").
const vectors = (file) => {
try {
return readFileSync(new URL(`interop/syntax/${file}.txt`, SPEC), 'utf8').split('\n').filter((l) => l && !l.startsWith('#'))
} catch {
return []
}
}
/// Strings of each format beyond the vectors: the rules spec/VALIDATION.md
/// settles where the vectors say nothing (spec/VALIDATION.md, "String
/// formats").
const MORE = {
datetime: ['2023-02-30T00:00:00Z', '2023-04-31T00:00:00Z', '0000-01-01T00:00:00Z', '0000-01-01T00:00:00+01:00', '0000-01-01T01:00:00+01:00', '9999-12-31T23:59:59Z', '9999-12-31T23:59:59-01:00', '2023-06-30T23:59:60Z', '2023-01-01T00:00:00-00:00', '2023-01-01T00:00:00+00:00', '2023-01-01t00:00:00z', `2023-01-01T00:00:00.${'1'.repeat(43)}Z`, `2023-01-01T00:00:00.${'1'.repeat(44)}Z`],
language: ['JA', 'Ja', 'jA', 'x-fr-ch', 'i-default', 'I-default', 'i-Klingon', 'i-foo', 'en-GB-oed', 'en-gb-oed', 'zh-hakka', 'zh-Hakka', 'Zh-hakka', 'ZH-HAKKA', 'sgn-be-nl', 'art-lojban', 'abcde', 'abcdefgh', 'en-latn-us', 'EN-LATN-US', 'zh-YUE-hk', 'en-X-A', 'x', 'en-x', 'en-a-BB-A-cc', 'abc-def-ghi-jkl', 'abc-def-ghi-jkl-mno', 'en-1234', 'x-12345678', 'x-abcdefghi'],
cid: ['QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR', 'BAFYREIDS2FYDLXL7O67KIVYXXWXQABK5BZBC5ND5NY266XSBSW6YHFNJHM'],
uri: ['content-type:text/plan', 'a:b', 'a_b:c', 'a:/b', 'a: b', 'https://example.com/ x', `https://example.com/${'x'.repeat(9000)}`],
did: ['did:example:x', 'did:plc:abc', 'did:plc:aaaaaaaaaaaaaaaaaaaaaaab'],
'at-uri': ['at://bob.test', 'at://did:plc:aaaaaaaaaaaaaaaaaaaaaaab/app.bsky.feed.post', 'at://bob.test/app.bsky.feed.post/3kaaaaaaaaa22'],
}
/// Where the reference differs from the vectors (spec/VALIDATION.md, "String
/// formats"): a CID is one that is written the same way again, a URI one
/// this matches.
const TAKES = {
cid: (s) => {
try {
return CID.parse(s).toString() === s
} catch {
return false
}
},
uri: (s) => /^\w+:(?:\/\/)?[^\s/][^\s]*$/.test(s),
}
const FILES = { 'at-identifier': 'atidentifier', 'at-uri': 'aturi', cid: 'cid', datetime: 'datetime', did: 'did', handle: 'handle', language: 'language', nsid: 'nsid', 'record-key': 'recordkey', tid: 'tid', uri: 'uri' }
/// For each format, every string the harness draws, and those the
/// reference takes, to draw valid records from.
const FORMATS = Object.fromEntries(
Object.entries(FILES).map(([format, file]) => {
const all = [...new Set([...vectors(`${file}_syntax_valid`), ...vectors(`${file}_syntax_invalid`), ...vectors(`${file}_parse_invalid`), ...(MORE[format] ?? [])])]
const valid = TAKES[format] ? all.filter(TAKES[format]) : vectors(`${file}_syntax_valid`)
return [format, { all, valid }]
}),
)
// ---- Strings ----------------------------------------------------------------
const segmenter = new Intl.Segmenter()
const graphemes = (s) => [...segmenter.segment(s)].length
const bytes = (s) => Buffer.byteLength(s)
/// Text in pieces of one grapheme cluster each, of 1 to 25 bytes: ASCII,
/// accented letters precomposed and with combining accents, CJK, emoji, ZWJ
/// emoji, a flag and a skin tone. None is segmented differently by recent
/// Unicode versions.
const PIECES = ['a', 'Z', ' ', '7', 'é', 'e\u0301', 'n\u0303', '日', '🙂', '👨👩👧👦', '🏳️🌈', '🇯🇵', '👍🏽']
/// Larger than this, a string is not filled to its limit, to keep requests
/// well inside the reference's 1 MB body.
const FILL_LIMIT = 20000
const fits = (s, schema) =>
(schema.maxLength === undefined || bytes(s) <= schema.maxLength) &&
(schema.minLength === undefined || bytes(s) >= schema.minLength) &&
(schema.maxGraphemes === undefined || graphemes(s) <= schema.maxGraphemes) &&
(schema.minGraphemes === undefined || graphemes(s) >= schema.minGraphemes)
/// Text `schema` takes: short, or as long as it may be, of one piece over
/// and over or of pieces at random.
function text(schema) {
const maxB = schema.maxLength ?? Infinity
const maxG = schema.maxGraphemes ?? Infinity
let s = ''
if (chance(0.15) && Math.min(maxB, maxG) <= FILL_LIMIT) {
const p = pick(PIECES)
s = p.repeat(Math.min(Math.floor(maxB / bytes(p)), maxG))
} else {
for (let n = below(12); n > 0; n--) {
const next = s + pick(PIECES)
if (bytes(next) > maxB || graphemes(next) > maxG) break
s = next
}
}
while (bytes(s) < (schema.minLength ?? 0) || graphemes(s) < (schema.minGraphemes ?? 0)) s += 'a'
return s
}
/// A string `schema` takes.
function validString(schema) {
if (schema.format) {
const ok = FORMATS[schema.format].valid.filter((s) => fits(s, schema))
if (!ok.length) throw new Error(`no ${schema.format} the harness draws fits ${JSON.stringify(schema)}`)
return pick(ok)
}
if (schema.knownValues && chance(0.7)) {
const ok = schema.knownValues.filter((s) => fits(s, schema))
if (ok.length) return pick(ok)
}
return text(schema)
}
/// Strings `schema` refuses for their length, each as `[what, string]`.
function badLengths(schema) {
const out = []
const big = '👨👩👧👦'
const { maxLength: maxB, minLength: minB, maxGraphemes: maxG } = schema
if (maxB !== undefined && maxB <= 10 * FILL_LIMIT) {
out.push(['a string a byte too long', 'a'.repeat(maxB + 1)])
// Few graphemes, but too many bytes.
const n = Math.floor(maxB / bytes(big)) + 1
if (maxG === undefined || n <= maxG) out.push(['a string too long in bytes, not graphemes', big.repeat(n)])
out.push(['a string too long in bytes, of accented letters', 'é'.repeat(Math.floor(maxB / 2) + 1)])
}
if (maxG !== undefined && maxG <= FILL_LIMIT) {
const p = pick(PIECES)
out.push([`a string a grapheme too long, of ${JSON.stringify(p)}`, p.repeat(maxG + 1)])
}
if (minB !== undefined && minB > 0) out.push(['a string too short', 'a'.repeat(minB - 1)])
return out
}
// ---- Records ----------------------------------------------------------------
/// Subjects few enough that follows, blocks, likes and reposts often
/// duplicate each other: DIDs, and AT URIs, one naming the same post as
/// another by handle.
const SUBJECT_DIDS = ['did:plc:aaaaaaaaaaaaaaaaaaaaaaab', 'did:plc:aaaaaaaaaaaaaaaaaaaaaaac', 'did:web:example.com']
const SUBJECT_URIS = ['at://did:plc:aaaaaaaaaaaaaaaaaaaaaaab/app.bsky.feed.post/3kaaaaaaaaa22', 'at://bob.test/app.bsky.feed.post/3kaaaaaaaaa22', 'at://did:plc:aaaaaaaaaaaaaaaaaaaaaaac/app.bsky.feed.post/3kaaaaaaaaa23']
const SOME_CID = 'bafyreids2fydlxl7o67kivyxxwxqabk5bzbc5nd5ny266xsbsw6yhfnjhm'
/// A raw CID no one here uploads, for blob refs to a blob no account holds.
const NEVER = CID.create(1, 0x55, await sha256.digest(Buffer.from('spec-pds validation: never uploaded'))).toString()
/// Every `$type` a definition may have: what a nested object's `$type` or a
/// union's names instead of its own.
const DEF_TYPES = [...LEXICONS.values()].flatMap((d) => Object.keys(d.defs ?? {}).map((n) => (n === 'main' ? d.id : `${d.id}#${n}`)))
/// A value of each other type than `schema`'s, for a field of the wrong type.
function otherType(schema) {
const t = schema.type === 'ref' || schema.type === 'union' ? 'object' : schema.type
const all = { string: 'a string', integer: 5, boolean: true, object: { a: 1 }, array: ['x'], blob: 'blob', bytes: 'aGVsbG8' }
return pick(Object.entries(all).filter(([k]) => k !== t && !(t === 'blob' && k === 'object') && !(t === 'bytes' && k === 'object')))[1]
}
/// A record of `type`, valid by its lexicon, for `acct`, with the ways it
/// may be broken: `[family, what, apply]`, each changing it in place
/// (spec/VALIDATION.md, "The record").
function draw(type, acct) {
const breaks = []
const g = { acct, breaks }
const rec = object(resolve(type, type).def.record, type, g, 0)
rec.$type = type
// Few subjects, set in place so that the ways to break them still apply.
if (type === 'app.bsky.graph.follow' || type === 'app.bsky.graph.block') rec.subject = pick(SUBJECT_DIDS)
if (type === 'app.bsky.feed.like' || type === 'app.bsky.feed.repost') Object.assign(rec.subject, { uri: pick(SUBJECT_URIS), cid: chance(0.8) ? SOME_CID : rec.subject.cid })
return { rec, breaks }
}
/// A value of `schema`, valid, in the lexicon `lex`; `set` puts another in
/// its place, for the ways to break it (spec/VALIDATION.md, "The record",
/// "String formats").
function make(schema, lex, g, depth, set) {
switch (schema.type) {
case 'ref': {
const r = resolve(schema.ref, lex)
if (r.def.type !== 'object') return make(r.def, r.id, g, depth, set)
const v = object(r.def, r.id, g, depth)
if (chance(0.25)) v.$type = r.type
g.breaks.push(['nested $type', `${r.type} with the $type of another definition`, () => void (v.$type = pick(DEF_TYPES.filter((t) => t !== r.type)))])
return v
}
case 'union': {
// A union may name no definition at all: then any object with a
// `$type` is one of it.
const r = schema.refs.length ? resolve(pick(schema.refs), lex) : { type: pick(['com.example.validation.member', 'app.bsky.feed.post']) }
const v = r.def ? object(r.def.type === 'record' ? r.def.record : r.def, r.id, g, depth) : { n: below(9) }
v.$type = r.type
g.breaks.push(
['union', `a ${r.type} without $type`, () => void delete v.$type],
['union', `a ${r.type} whose $type is not a string`, () => void (v.$type = pick([5, null, true, { a: 1 }, ['x']]))],
['union', `a ${r.type} whose $type is a definition the union does not name`, () => {
const named = new Set(schema.refs.map((x) => resolve(x, lex).type))
v.$type = pick(DEF_TYPES.filter((t) => !named.has(t)))
}],
['union', `a ${r.type} whose $type names nothing`, () => void (v.$type = pick(['not a type!', 'com.example.nothing', '', `${r.type}x`]))],
)
return v
}
case 'object':
return object(schema, lex, g, depth)
case 'string': {
const v = validString(schema)
for (const [what, s] of badLengths(schema)) g.breaks.push(['length', what, () => set(s)])
if (schema.format) g.breaks.push(['format', `a ${schema.format} of the vectors`, () => set(pick(FORMATS[schema.format].all))])
return v
}
case 'integer': {
const min = schema.minimum ?? (schema.maximum !== undefined ? schema.maximum - 1000 : -1000)
const max = schema.maximum ?? min + 1000
if (schema.minimum !== undefined) g.breaks.push(['range', 'an integer below its minimum', () => set(schema.minimum - 1 - below(3))])
if (schema.maximum !== undefined) g.breaks.push(['range', 'an integer above its maximum', () => set(schema.maximum + 1 + below(3))])
return pick([min, max, min + below(max - min + 1)])
}
case 'boolean':
return chance(0.5)
case 'array': {
const lo = schema.minLength ?? 0
const hi = Math.min(schema.maxLength ?? Infinity, lo + 3)
const n = schema.maxLength !== undefined && schema.maxLength <= 50 && chance(0.1) ? schema.maxLength : lo + below(hi - lo + 1)
const arr = []
for (let i = 0; i < n; i++) arr.push(make(schema.items, lex, g, depth + 1, (x) => void (arr[i] = x)))
if (schema.maxLength !== undefined && schema.maxLength <= 1000) {
g.breaks.push(['items', 'an array longer than its maxLength', () => {
while (arr.length <= schema.maxLength) arr.push(arr.length ? arr[below(arr.length)] : make(schema.items, lex, { ...g, breaks: [] }, depth + 1, () => {}))
}])
}
if (lo > 0) g.breaks.push(['items', 'an array shorter than its minLength', () => void arr.splice(lo - 1)])
return arr
}
case 'blob': {
const ok = heldRef(g.acct, schema.accept, schema.maxSize)
const type = schema.accept ? pick(schema.accept).replace(/\*$/, 'png').replace(/^\*\//, 'image/') : 'image/png'
const v = ok ?? { $type: 'blob', ref: { $link: NEVER }, mimeType: type, size: 100 }
const refused = ['text/plain', 'IMAGE/PNG', 'imagex/png', 'image', 'video/mp4', 'image/png', 'application/octet-stream'].filter((t) => schema.accept && !schema.accept.some((a) => (a.endsWith('/*') ? t.startsWith(a.slice(0, -1)) : t === a)))
if (refused.length) g.breaks.push(['blob', 'a blob of a type its accept refuses', () => set({ ...v, mimeType: pick(refused) })])
if (schema.maxSize !== undefined) g.breaks.push(['blob', 'a blob larger than its maxSize', () => set({ ...v, size: schema.maxSize + 1 })])
g.breaks.push(
['blob', 'a blob of size -1', () => set({ ...v, size: -1 })],
['blob', 'a blob without $type', () => set({ ref: v.ref, mimeType: v.mimeType, size: v.size })],
['blob', 'a blob in the legacy shape', () => set({ cid: v.ref.$link, mimeType: v.mimeType })],
)
return v
}
case 'bytes':
g.breaks.push(['bytes', 'bytes that are a string', () => set(pick(['aGVsbG8', '', 'AAEC/w']))], ['bytes', 'bytes that are a link', () => set({ $link: SOME_CID })])
return { $bytes: pick(['', 'AA', 'aGVsbG8', 'AAEC/w']) }
case 'cid-link':
return { $link: SOME_CID }
default:
// unknown, or a type no record of a known type uses.
return {}
}
}
/// An object of `schema`: its required fields, some others, and sometimes
/// a field it does not name; with the ways to break each field
/// (spec/VALIDATION.md, "The record").
function object(schema, lex, g, depth) {
const obj = {}
const props = schema.properties ?? {}
const required = schema.required ?? []
for (const [f, fs] of Object.entries(props)) {
const req = required.includes(f)
if (!req && (depth > 4 || !chance(0.5 / (depth + 1)))) continue
obj[f] = make(fs, lex, g, depth + 1, (x) => void (obj[f] = x))
if (req) g.breaks.push(['required', `${f} left out`, () => void delete obj[f]])
g.breaks.push(['null', `${f} null`, () => void (obj[f] = null)], ['type', `${f} of another type`, () => void (obj[f] = otherType(fs))])
}
if (chance(0.15)) obj[pick(['extraField', 'zz', 'note2', 'x_y'].filter((k) => !(k in props)))] = value()
return obj
}
// ---- Record keys ------------------------------------------------------------
const TIDS = ['3kaaaaaaaaa22', '3kaaaaaaaaa23', '3kaaaaaaaaa24', '3kaaaaaaaaa25', '3kaaaaaaaaa26']
/// Keys of the right syntax for no type's `key`, or for another's
/// (spec/VALIDATION.md, "The record key").
const BAD_KEYS = {
tid: ['self', '3kaaaaaaaaa2', '3kaaaaaaaaa222', '3kaaaaaaaaa21', 'kkaaaaaaaaa22', '3KAAAAAAAAA22', 'a'],
'literal:self': ['Self', 'SELF', 'selfie', 'a', '3kaaaaaaaaa22'],
nsid: ['self', 'abc', 'com.example', 'com.example.a-b', '3kaaaaaaaaa22', 'com.example.1a'],
any: [],
}
const GOOD_KEYS = { tid: TIDS, 'literal:self': ['self'], nsid: ['com.example.thing', 'app.bsky.feed.post', 'a.b.c'], any: ['a', 'x-y', 'z~z', 'self', 'a.b', '3kaaaaaaaaa22'] }
const keyOf = (type) => (KNOWN.includes(type) ? LEXICONS.get(type).defs.main.key : 'any')
const keyKind = (type) => (keyOf(type).startsWith('literal:') ? 'literal:self' : keyOf(type))
/// What a write sends: `validate` left out (undefined), true, false, or not
/// a boolean (spec/VALIDATION.md, "validate").
function drawValidate() {
const r = rand()
if (r < 0.4) return undefined
if (r < 0.65) return true
if (r < 0.92) return false
return pick([null, 'true', 1, 0, {}])
}
/// A write's record and key: of a known type (mostly app.bsky) or another,
/// valid or broken in one way: `family` names the way, `what` says it.
/// `rkey` is undefined for none, which createRecord makes a TID
/// (spec/VALIDATION.md, "The record key").
function drawWrite(acct, { types, broken = 0.5, keyless = true } = {}) {
const type = types ? pick(types) : chance(0.88) ? (chance(0.8) ? pick(APP_BSKY) : pick(KNOWN)) : pick(OTHER)
const kind = keyKind(type)
let rkey = pick(KNOWN.includes(type) || chance(0.3) ? GOOD_KEYS[kind] : [...TIDS, ...RKEYS])
if (!KNOWN.includes(type)) {
const rec = { $type: type, text: word(), ...(chance(0.3) ? { extra: value() } : {}) }
return { type, rec, rkey, family: 'another type', what: 'another type' }
}
const { rec, breaks } = draw(type, acct)
let family = 'valid', what = 'valid'
if (keyless && chance(0.04)) {
rkey = undefined
family = what = 'no key'
} else if (chance(broken)) {
if (BAD_KEYS[kind].length && (chance(0.12) || !breaks.length)) {
rkey = pick(BAD_KEYS[kind])
family = 'key'
what = `a key not of its type's ${keyOf(type)}`
} else if (breaks.length) {
family = pick([...new Set(breaks.map(([f]) => f))])
const [, w, apply] = pick(breaks.filter(([f]) => f === family))
apply()
what = `${family}: ${w}`
}
}
return { type, rec, rkey, family, what }
}
// ---- What the harness knows -------------------------------------------------
/// What the steps know of an account's records of the known types and the
/// others they write: by `collection/rkey`, the link of each that links to
/// a subject, and the last record written there.
function known(acct) {
acct.validation ??= { links: new Map(), last: new Map() }
return acct.validation
}
/// The subject `rec`, of `type`, links to, or undefined (spec/VALIDATION.md,
/// "Duplicates").
function linkOf(type, rec) {
if (!rec || typeof rec !== 'object') return undefined
if (type === 'app.bsky.graph.follow' || type === 'app.bsky.graph.block') return typeof rec.subject === 'string' && isValidDid(rec.subject) ? rec.subject : undefined
if (type === 'app.bsky.feed.like' || type === 'app.bsky.feed.repost') return typeof rec.subject?.uri === 'string' && isValidAtUri(rec.subject.uri) ? rec.subject.uri : undefined
return undefined
}
/// Note a record written at `collection/rkey`, of CID `cid`, or deleted
/// (`rec` null).
function noteRecord(acct, collection, rkey, rec, cid) {
const k = `${collection}/${rkey}`
const m = known(acct)
if (rec === null) {
acct.records.delete(k)
m.links.delete(k)
m.last.delete(k)
return
}
acct.records.set(k, cid)
cids.push(cid)
m.last.set(k, rec)
const link = linkOf(collection, rec)
if (link === undefined) m.links.delete(k)
else m.links.set(k, link)
}
/// The records a createRecord of `rec` in `collection` replaces: every one
/// in the collection linking to the same subject, its own key's too
/// (spec/VALIDATION.md, "Duplicates").
function duplicatesOf(acct, collection, rec, validate) {
const link = validate === false ? undefined : linkOf(collection, rec)
if (link === undefined) return []
return [...known(acct).links].filter(([k, l]) => l === link && k.startsWith(`${collection}/`)).map(([k]) => k.slice(collection.length + 1))
}
/// The collection listed on both sides, compared, and the reference's held
/// to the keys the harness knows it holds.
async function listCollection(acct, collection, why) {
const params = { repo: acct.handle, collection, limit: 100 }
const [a] = await both('validation: listRecords after a write', 'com.atproto.repo.listRecords', { params })
if (a.status !== 200) return
const listed = a.body.records.map((r) => r.uri.split('/').pop()).sort()
const want = [...acct.records.keys()].filter((k) => k.startsWith(`${collection}/`)).map((k) => k.slice(collection.length + 1)).sort()
if (!isDeepStrictEqual(listed, want)) throw new Error(`after ${why}, the reference lists ${collection} of <did:${acct.n}> as ${JSON.stringify(listed)}, where the harness knows ${JSON.stringify(want)} (spec/VALIDATION.md)`)
}
// ---- Steps ------------------------------------------------------------------
const CREATE = 'com.atproto.repo.createRecord'
const PUT = 'com.atproto.repo.putRecord'
const APPLY = 'com.atproto.repo.applyWrites'
const DELETE = 'com.atproto.repo.deleteRecord'
/// A record of a known type or another, valid or broken in one way, by
/// createRecord or putRecord, with `validate` drawn; a quarter of them
/// follows, blocks, likes and reposts (spec/VALIDATION.md, "validate",
/// "The record key", "The record").
export async function validatedWrite(acct) {
await write(acct, drawWrite(acct, chance(0.25) ? { types: LINKED } : {}))
}
/// Duplicates: follows, blocks, likes and reposts of few subjects at few
/// keys, by every write, mostly valid, then the collection listed
/// (spec/VALIDATION.md, "Duplicates").
export async function duplicates(acct) {
const w = drawWrite(acct, { types: LINKED, broken: 0.1, keyless: false })
// The record at a key written again: exactly, or to the same subject.
const last = known(acct).last.get(`${w.type}/${w.rkey}`)
if (last && chance(0.3)) Object.assign(w, chance(0.5) ? { rec: last, same: true, what: 'the record there again' } : { rec: last, what: 'the subject there again' })
if (chance(0.25)) {
const [a] = await batch(acct, [{ action: 'create', ...w }], drawValidate())
if (a.status < 400) await listCollection(acct, w.type, 'an applyWrites')
return
}
await write(acct, w, chance(0.7) ? CREATE : PUT)
await listCollection(acct, w.type, 'a write of a duplicate')
}
/// One write by createRecord or putRecord, compared; then what the
/// harness knows of the repository brought up to date. Each record is one
/// no write sent before, but where `same` or sometimes by chance, exactly
/// the record written there last.
async function write(acct, { type, rec, rkey, what, family, same }, nsid = rkey === undefined || chance(0.5) ? CREATE : PUT) {
const k = `${type}/${rkey}`
const last = known(acct).last.get(k)
if (!same && last && chance(0.15)) {
rec = last
what = family = 'the record there again'
} else if (!same && rec && typeof rec === 'object') rec = { ...rec, batch: nextBatch() }
const validate = drawValidate()
const body = { repo: acct.handle, collection: type, record: rec }
if (rkey !== undefined) body.rkey = rkey
if (validate !== undefined) body.validate = validate
const kind = `validation: ${nsid === CREATE ? 'createRecord' : 'putRecord'}, ${family}${typeof validate === 'boolean' || validate === undefined ? '' : ', validate not a boolean'}`
if (rkey === undefined) return keyless(acct, body, kind, what)
const [a, b] = await both(kind, nsid, { body, token: acct, what })
noteCommit(ref, acct, a)
noteCommit(cand, acct, b)
const dups = nsid === CREATE && a.status < 400 ? duplicatesOf(acct, type, rec, validate) : []
noteRefs(acct, a, [...dups.map((d) => ({ rkey: `${type}/${d}`, value: null })), { rkey: k, value: rec }], [rec])
if (a.status >= 400) return
for (const d of dups) noteRecord(acct, type, d, null)
noteRecord(acct, type, rkey, rec, a.body.cid)
}
/// createRecord without a key: each side makes a TID of its own, so the
/// answers are compared with it taken out, and a record made is deleted
/// again on both, after its duplicates are gone.
async function keyless(acct, body, kind, what) {
const [a, b] = await Promise.all([ref, cand].map((side) => xrpc(side, CREATE, { body, token: acct })))
const strip = (side, res) => {
const n = normalize(side, res)
if (n.body?.uri) n.body.uri = n.body.uri.replace(/[^/]+$/, '<tid>')
return n
}
if (!isDeepStrictEqual(strip(ref, a), strip(cand, b))) await differ(CREATE, { ...shown({ body, token: acct }), what }, strip(ref, a), strip(cand, b), 'with no key')
count(`${kind}, no key`, a.status < 400)
noteCommit(ref, acct, a)
noteCommit(cand, acct, b)
const type = body.collection
if (a.status >= 400) return noteRefs(acct, a, [], [body.record])
const dups = duplicatesOf(acct, type, body.record, body.validate)
for (const d of dups) noteRecord(acct, type, d, null)
noteRefs(acct, a, [...dups.map((d) => ({ rkey: `${type}/${d}`, value: null })), { rkey: `${type}/<keyless>`, value: body.record }], [])
for (const [side, res] of [[ref, a], [cand, b]].filter(([, res]) => typeof res.body?.uri === 'string')) {
const del = await xrpc(side, DELETE, { body: { repo: acct.handle, collection: type, rkey: res.body.uri.split('/').pop() }, token: acct })
if (del.status >= 400) await differ('deleting a record made with no key', {}, 'ok', del, `on the ${side.name}`)
noteCommit(side, acct, del)
}
noteRefs(acct, { status: 200 }, [{ rkey: `${type}/<keyless>`, value: null }], [])
if (LINKED.includes(type)) await listCollection(acct, type, 'a createRecord with no key')
}
/// An applyWrites of 1 to 4 writes, each a create, update or delete of a
/// record of a known type or another, valid or broken; `validate` for all
/// (spec/VALIDATION.md, "validate").
export async function validatedBatch(acct) {
const used = new Set()
const writes = []
for (let i = 1 + below(4); i > 0; i--) {
const w = drawWrite(acct, { broken: 0.15, keyless: false })
const k = `${w.type}/${w.rkey}`
if (used.has(k)) continue
used.add(k)
const exists = acct.records.has(k)
const action = chance(0.9) ? (exists ? pick(['update', 'update', 'delete']) : 'create') : pick(['create', 'update', 'delete'])
writes.push({ action, ...w })
}
await batch(acct, writes, drawValidate())
}
/// Send an applyWrites of `writes`, compare it, and note what it did.
async function batch(acct, writes, validate) {
for (const w of writes) if (w.action !== 'delete' && w.rec && typeof w.rec === 'object') w.rec = { ...w.rec, batch: nextBatch() }
const body = {
repo: acct.handle,
writes: writes.map((w) => ({ $type: `com.atproto.repo.applyWrites#${w.action}`, collection: w.type, rkey: w.rkey, ...(w.action === 'delete' ? {} : { value: w.rec }) })),
}
if (validate !== undefined) body.validate = validate
const what = writes.map((w) => `${w.action} ${w.type}: ${w.what}`)
const broken = writes.some((w) => w.what !== 'valid')
const res = await both(`validation: applyWrites, ${broken ? 'one broken' : 'valid'}, validate ${JSON.stringify(validate) ?? 'left out'}`, APPLY, { body, token: acct, what })
const [a, b] = res
noteCommit(ref, acct, a)
noteCommit(cand, acct, b)
noteRefs(acct, a, writes.map((w) => ({ rkey: `${w.type}/${w.rkey}`, value: w.action === 'delete' ? null : w.rec })), writes.map((w) => w.rec).filter(Boolean))
if (a.status < 400) writes.forEach((w, i) => noteRecord(acct, w.type, w.rkey, w.action === 'delete' ? null : w.rec, a.body.results?.[i]?.cid))
return res
}
/// A write to another account's repository, with this one's token: of a
/// record of another type with `validate: true`, an invalid record, a
/// record key its type refuses, or `validate` not a boolean, refused for
/// the repository first, or for `validate` not a boolean, as input
/// (spec/VALIDATION.md, "Where in a write").
export async function validatedElsewhere(acct) {
const other = accounts.filter((x) => x !== acct)
if (!other.length) return
const w = drawWrite(acct, { broken: 0.6, keyless: false })
const validate = KNOWN.includes(w.type) ? drawValidate() : pick([true, true, undefined, null])
const nsid = pick([CREATE, PUT, APPLY])
const repo = pick(other).handle
const body =
nsid === APPLY
? { repo, writes: [{ $type: 'com.atproto.repo.applyWrites#create', collection: w.type, rkey: w.rkey, value: w.rec }] }
: { repo, collection: w.type, rkey: w.rkey, record: w.rec }
if (validate !== undefined) body.validate = validate
await both(`validation: to another account's repository, validate ${JSON.stringify(validate) ?? 'left out'}`, nsid, { body, token: acct, what: w.what })
}