import Database from 'better-sqlite3'; import { afterEach, describe, expect, it } from 'vitest'; import { backfill } from '../src/backfill.js'; import { createAppViewStore } from '../src/store.js'; const COLLECTION = 'dev.pdsjs.git.pull'; const REPO = 'dev.pdsjs.git.repo'; const OWNER = 'did:plc:owner'; const STRANGER = 'did:plc:stranger'; const PDS = 'https://pds.example.com'; /** * A fetch that plays the relay, the PLC directory and the PDS, each against * its own origin. * @param {Record} routes */ function fakeFetch(routes) { /** @type {import('../src/backfill.js').FetchLike} */ return async (input) => { const url = String(input); for (const [match, body] of Object.entries(routes)) { if (url.includes(match)) { return Response.json(typeof body === 'function' ? body(url) : body); } } return new Response('not found', { status: 404 }); }; } const didDoc = { alsoKnownAs: ['at://owner.example.com'], service: [ { id: '#atproto_pds', type: 'AtprotoPersonalDataServer', serviceEndpoint: PDS, }, ], }; describe('backfill', () => { /** @type {import('node:http').Server|undefined} */ let server; afterEach(() => { server?.close(); }); it('walks the relay listing and indexes what each account holds', async () => { const store = createAppViewStore(new Database(':memory:')); let listed = 0; await backfill({ store, collections: [COLLECTION], fetch: fakeFetch({ '/xrpc/com.atproto.sync.listReposByCollection': () => { listed += 1; return listed === 1 ? { repos: [{ did: STRANGER }, { did: OWNER }] } : { repos: [] }; }, '/did:plc:stranger/data': { service: [{ id: '#atproto_pds', serviceEndpoint: PDS }], }, '/did:plc:owner/data': didDoc, '/xrpc/com.atproto.repo.listRecords': (/** @type {string} */ url) => { // Only the stranger holds pull records; the owner holds none. const repo = new URL(url).searchParams.get('repo'); return repo === STRANGER ? { records: [ { uri: `at://${STRANGER}/${COLLECTION}/3kpull`, cid: 'bafypull', value: { subject: { uri: `at://${OWNER}/${REPO}/proj` }, }, }, ], } : { records: [] }; }, }), }); const found = store.about(`at://${OWNER}/${REPO}/proj`, COLLECTION); expect(found).toHaveLength(1); expect(found[0].did).toBe(STRANGER); // The handle from the DID document rode along. expect(store.handleOf(OWNER)).toBe('owner.example.com'); // The walk finished: no cursor is stored past the end. expect(store.textCursor(`backfill:${COLLECTION}`)).toBeUndefined(); }); it('resumes from the stored pagination cursor', async () => { const store = createAppViewStore(new Database(':memory:')); store.setTextCursor(`backfill:${COLLECTION}`, 'page-two'); /** @type {string[]} */ const cursors = []; await backfill({ store, collections: [COLLECTION], fetch: fakeFetch({ '/xrpc/com.atproto.sync.listReposByCollection': ( /** @type {string} */ url, ) => { cursors.push(new URL(url).searchParams.get('cursor') ?? ''); return { repos: [] }; }, }), }); expect(cursors).toEqual(['page-two']); }); it('skips an account whose DID document does not resolve', async () => { const store = createAppViewStore(new Database(':memory:')); await backfill({ store, collections: [COLLECTION], fetch: fakeFetch({ '/xrpc/com.atproto.sync.listReposByCollection': { repos: [{ did: 'did:plc:missing' }, { did: OWNER }], cursor: '', }, '/did:plc:owner/data': didDoc, '/xrpc/com.atproto.repo.listRecords': { records: [ { uri: `at://${OWNER}/${COLLECTION}/3kpull`, cid: 'bafyowner', value: {}, }, ], }, }), }); expect(store.getRecord(`at://${OWNER}/${COLLECTION}/3kpull`)).not.toBe( null, ); expect(store.getRecord('at://did:plc:missing/anything/1')).toBe(null); }); it('stops with a notice when the relay refuses to answer', async () => { const store = createAppViewStore(new Database(':memory:')); /** @type {string[]} */ const notices = []; await backfill({ store, collections: [COLLECTION], fetch: fakeFetch({}), onNotice: (message) => notices.push(message), }); expect(notices.some((message) => message.includes('answered 404'))).toBe( true, ); expect(store.listRecords(COLLECTION)).toEqual([]); }); });