chadtmiller.com/pds.js
An atproto PDS in JavaScript, for Cloudflare Workers and Node
0
0
CBOR integers of 32 bits or more are written, and what cannot be is refused
.changeset/cbor-large-integers.mdnew+19−0
| @@ -0,0 +1,19 @@ | ||
--- | ||
'@pdsjs/core': patch | ||
--- | ||
| ||
The CBOR encoders write an integer of 2^32 or more with an 8-byte head. They | ||
wrote nothing for it, which left a map key with no value: a record holding a | ||
time in milliseconds was stored cut short, under a CID of the cut bytes. | ||
knotless starts a ref's generation at such a time, so every new ref in a | ||
space was stored so, and knotless-serve could not sync any space that held one. | ||
| ||
An encoder now throws a TypeError for a value it cannot write: a number that | ||
is not a safe integer, a BigInt, or `undefined` in an array. It wrote nothing | ||
for those too. | ||
| ||
The decoder reads 8-byte integers, and throws for bytes that end inside an | ||
item, for floats and for reserved heads. It read a missing byte as 0, so a | ||
record cut short decoded as one with plausible values. | ||
| ||
A record already stored cut short stays so until it is written again. | ||
packages/core/src/repo.js+74−28
| @@ -71,9 +71,37 @@ function encodeHead(parts, majorType, length) { | ||
Math.floor(length / 0x100) & 0xff, | ||
length & 0xff, | ||
); | ||
} else if (length <= Number.MAX_SAFE_INTEGER) { | ||
const high = Math.floor(length / 0x100000000); | ||
const low = length % 0x100000000; | ||
parts.push(mt | 27); | ||
for (const word of [high, low]) { | ||
parts.push( | ||
Math.floor(word / 0x1000000) & 0xff, | ||
Math.floor(word / 0x10000) & 0xff, | ||
Math.floor(word / 0x100) & 0xff, | ||
word & 0xff, | ||
); | ||
} | ||
} else { | ||
throw new RangeError( | ||
`CBOR length ${length} is past Number.MAX_SAFE_INTEGER`, | ||
); | ||
} | ||
} | ||
| ||
/** | ||
* The error for a value an encoder cannot write. | ||
* @param {unknown} val | ||
* @returns {TypeError} | ||
*/ | ||
function unencodable(val) { | ||
const what = typeof val === 'number' ? String(val) : typeof val; | ||
return new TypeError( | ||
`CBOR cannot encode ${what}: only safe integers, strings, bytes, arrays, maps, booleans and null`, | ||
); | ||
} | ||
| ||
/** | ||
* Append bytes one at a time. A spread (`parts.push(...bytes)`) passes every | ||
* byte as a call argument, and past the engine's argument limit (about 125k) | ||
| @@ -104,6 +132,7 @@ export function cborEncode(value) { | ||
} else if (val === false) { | ||
parts.push(CBOR_FALSE); | ||
} else if (typeof val === 'number') { | ||
if (!Number.isSafeInteger(val)) throw unencodable(val); | ||
encodeInteger(val); | ||
} else if (typeof val === 'string') { | ||
const bytes = new TextEncoder().encode(val); | ||
| @@ -123,6 +152,8 @@ export function cborEncode(value) { | ||
encode(key); | ||
encode(val[key]); | ||
} | ||
} else { | ||
throw unencodable(val); | ||
} | ||
} | ||
| ||
| @@ -157,11 +188,9 @@ export function cborEncodeDagCbor(value) { | ||
} else if (val === false) { | ||
parts.push(CBOR_FALSE); | ||
} else if (typeof val === 'number') { | ||
if (Number.isInteger(val) && val >= 0) { | ||
encodeHead(parts, 0, val); | ||
} else if (Number.isInteger(val) && val < 0) { | ||
encodeHead(parts, 1, -val - 1); | ||
} | ||
if (!Number.isSafeInteger(val)) throw unencodable(val); | ||
if (val >= 0) encodeHead(parts, 0, val); | ||
else encodeHead(parts, 1, -val - 1); | ||
} else if (typeof val === 'string') { | ||
const bytes = new TextEncoder().encode(val); | ||
encodeHead(parts, 3, bytes.length); | ||
| @@ -195,6 +224,8 @@ export function cborEncodeDagCbor(value) { | ||
pushBytes(parts, keyBytes); | ||
encode(val[key]); | ||
} | ||
} else { | ||
throw unencodable(val); | ||
} | ||
} | ||
| ||
| @@ -232,23 +263,46 @@ export function cborDecodeLinks(bytes) { | ||
export function cborDecodeFirst(bytes, opts = {}) { | ||
let offset = 0; | ||
| ||
/** | ||
* The next `n` bytes, which must all be there. | ||
* @param {number} n | ||
*/ | ||
function take(n) { | ||
if (offset + n > bytes.length) { | ||
throw new RangeError('CBOR ends inside an item'); | ||
} | ||
const start = offset; | ||
offset += n; | ||
return bytes.subarray(start, offset); | ||
} | ||
| ||
/** @param {number} n */ | ||
function uint(n) { | ||
let value = 0; | ||
for (const byte of take(n)) value = value * 0x100 + byte; | ||
return value; | ||
} | ||
| ||
/** @returns {*} */ | ||
function read() { | ||
const initial = bytes[offset++]; | ||
const initial = take(1)[0]; | ||
const major = initial >> 5; | ||
const info = initial & 0x1f; | ||
| ||
if (major === 7 && info >= 25) { | ||
throw new TypeError( | ||
'CBOR holds a float or a reserved value, which atproto data does not', | ||
); | ||
} | ||
let length = info; | ||
if (info === 24) length = bytes[offset++]; | ||
else if (info === 25) { | ||
length = (bytes[offset++] << 8) | bytes[offset++]; | ||
} else if (info === 26) { | ||
// Use multiplication instead of bitshift to avoid 32-bit signed integer overflow | ||
length = | ||
bytes[offset++] * 0x1000000 + | ||
bytes[offset++] * 0x10000 + | ||
bytes[offset++] * 0x100 + | ||
bytes[offset++]; | ||
if (info === 24) length = uint(1); | ||
else if (info === 25) length = uint(2); | ||
else if (info === 26) length = uint(4); | ||
else if (info === 27) length = uint(8); | ||
else if (info > 27) | ||
throw new TypeError('CBOR item of indefinite or reserved length'); | ||
if (!Number.isSafeInteger(length)) { | ||
throw new RangeError('CBOR integer past Number.MAX_SAFE_INTEGER'); | ||
} | ||
| ||
switch (major) { | ||
| @@ -256,20 +310,12 @@ export function cborDecodeFirst(bytes, opts = {}) { | ||
return length; // unsigned int | ||
case 1: | ||
return -1 - length; // negative int | ||
case 2: { | ||
case 2: | ||
// byte string | ||
const data = bytes.slice(offset, offset + length); | ||
offset += length; | ||
return data; | ||
} | ||
case 3: { | ||
return take(length).slice(); | ||
case 3: | ||
// text string | ||
const data = new TextDecoder().decode( | ||
bytes.slice(offset, offset + length), | ||
); | ||
offset += length; | ||
return data; | ||
} | ||
return new TextDecoder().decode(take(length)); | ||
case 4: { | ||
// array | ||
const arr = []; | ||
packages/core/test/repo.test.js+55−0
| @@ -2,6 +2,7 @@ import { | ||
buildCarFile, | ||
cborDecode, | ||
cborEncode, | ||
cborEncodeDagCbor, | ||
cidToString, | ||
createBlobCid, | ||
createCid, | ||
| @@ -516,3 +517,57 @@ describe('Record Time Detection', () => { | ||
expect(recordTimeOf(['2026-05-07T14:32:34Z'])).toBe(null); | ||
}); | ||
}); | ||
| ||
describe('CBOR integers past 32 bits', () => { | ||
// knotless starts a ref's generation at the time in milliseconds. | ||
const ref = { | ||
$type: 'dev.knotless.ref', | ||
name: 'refs/heads/main', | ||
generation: 1793736300000, | ||
}; | ||
| ||
test('encode with an 8-byte head and read back', () => { | ||
for (const encode of [cborEncode, cborEncodeDagCbor]) { | ||
expect(encode(1793736300000)).toEqual( | ||
new Uint8Array([0x1b, 0x00, 0x00, 0x01, 0xa1, 0xa3, 0x03, 0xd5, 0xe0]), | ||
); | ||
expect(encode(2 ** 32)).toEqual( | ||
new Uint8Array([0x1b, 0, 0, 0, 1, 0, 0, 0, 0]), | ||
); | ||
expect(cborDecode(encode(-(2 ** 40)))).toBe(-(2 ** 40)); | ||
expect(cborDecode(encode(Number.MAX_SAFE_INTEGER))).toBe( | ||
Number.MAX_SAFE_INTEGER, | ||
); | ||
} | ||
expect(cborDecode(cborEncodeDagCbor(ref))).toEqual(ref); | ||
}); | ||
| ||
test('a value an encoder cannot write is refused, not left out', () => { | ||
for (const encode of [cborEncode, cborEncodeDagCbor]) { | ||
for (const bad of [1.5, Number.NaN, 2 ** 53, 10n, [undefined]]) { | ||
expect(() => encode({ a: bad })).toThrow(/CBOR cannot encode/); | ||
} | ||
} | ||
}); | ||
| ||
test('bytes cut short are refused, not read as zeros', () => { | ||
const whole = cborEncodeDagCbor(ref); | ||
expect(() => cborDecode(whole.subarray(0, whole.length - 1))).toThrow( | ||
'CBOR ends inside an item', | ||
); | ||
expect(() => cborDecode(new Uint8Array([0x62, 0x61]))).toThrow( | ||
'CBOR ends inside an item', | ||
); | ||
}); | ||
| ||
test('floats and reserved heads are refused', () => { | ||
expect(() => cborDecode(new Uint8Array([0xf9, 0x3c, 0x00]))).toThrow( | ||
/float/, | ||
); | ||
expect(() => cborDecode(new Uint8Array([0x9f]))).toThrow( | ||
/indefinite or reserved/, | ||
); | ||
const past = new Uint8Array([0x1b, 0x00, 0x20, 0, 0, 0, 0, 0, 0]); | ||
expect(() => cborDecode(past)).toThrow(/MAX_SAFE_INTEGER/); | ||
}); | ||
}); | ||
packages/git-host/test/writes.test.js+3−3
| @@ -6,7 +6,7 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; | ||
import { tmpdir } from 'node:os'; | ||
import { join } from 'node:path'; | ||
import { createAppViewStore } from '@pdsjs/appview/store'; | ||
import { cborDecode } from '@pdsjs/core/repo'; | ||
import { cborDecode, cborEncodeDagCbor } from '@pdsjs/core/repo'; | ||
import Database from 'better-sqlite3'; | ||
import { afterEach, describe, expect, it } from 'vitest'; | ||
import { createForgeAuth } from '../src/auth.js'; | ||
| @@ -52,7 +52,7 @@ describe('remote account writes', () => { | ||
`at://${DID}/dev.pdsjs.git.repo/proj`, | ||
'dev.pdsjs.git.repo', | ||
'proj', | ||
new TextEncoder().encode(JSON.stringify({ name: 'proj' })), | ||
cborEncodeDagCbor({ name: 'proj' }), | ||
); | ||
expect(seen).toHaveLength(1); | ||
expect(seen[0]).toContain('Bearer the-token'); | ||
| @@ -69,7 +69,7 @@ describe('remote account writes', () => { | ||
`at://${DID}/dev.pdsjs.git.repo/proj`, | ||
'dev.pdsjs.git.repo', | ||
'proj', | ||
new TextEncoder().encode('{}'), | ||
cborEncodeDagCbor({}), | ||
), | ||
).rejects.toThrow(/refused/); | ||
}); | ||