// @pdsjs/core/repo - Repository utilities for AT Protocol // CBOR encoding, CIDs, TIDs, CAR files, blob handling import { base32Decode, base32Encode } from './crypto.js'; // CBOR primitive markers (RFC 8949) const CBOR_FALSE = 0xf4; const CBOR_TRUE = 0xf5; const CBOR_NULL = 0xf6; // DAG-CBOR CID link tag const CBOR_TAG_CID = 42; // CID codec constants const CODEC_DAG_CBOR = 0x71; const CODEC_RAW = 0x55; // TID generation constants const TID_CHARS = '234567abcdefghijklmnopqrstuvwxyz'; let lastTimestamp = 0; const clockId = Math.floor(Math.random() * 1024); /** * Encode integer as unsigned varint * @param {number} n - Non-negative integer * @returns {Uint8Array} Varint-encoded bytes */ export function varint(n) { const bytes = []; while (n >= 0x80) { bytes.push((n & 0x7f) | 0x80); n >>>= 7; } bytes.push(n); return new Uint8Array(bytes); } // === CID WRAPPER === // Explicit CID type for DAG-CBOR encoding (avoids fragile heuristic detection) export class CID { /** @param {Uint8Array} bytes */ constructor(bytes) { if (!(bytes instanceof Uint8Array)) { throw new Error('CID must be constructed with Uint8Array'); } this.bytes = bytes; } } /** * Encode CBOR type header (major type + length) * @param {number[]} parts - Array to push bytes to * @param {number} majorType - CBOR major type (0-7) * @param {number} length - Value or length to encode */ function encodeHead(parts, majorType, length) { const mt = majorType << 5; if (length < 24) { parts.push(mt | length); } else if (length < 256) { parts.push(mt | 24, length); } else if (length < 65536) { parts.push(mt | 25, length >> 8, length & 0xff); } else if (length < 4294967296) { // Use Math.floor instead of bitshift to avoid 32-bit signed integer overflow parts.push( mt | 26, Math.floor(length / 0x1000000) & 0xff, Math.floor(length / 0x10000) & 0xff, Math.floor(length / 0x100) & 0xff, length & 0xff, ); } else if (length <= Number.MAX_SAFE_INTEGER) { const high = Math.floor(length / 0x100000000); const low = length % 0x100000000; parts.push(mt | 27); for (const word of [high, low]) { parts.push( Math.floor(word / 0x1000000) & 0xff, Math.floor(word / 0x10000) & 0xff, Math.floor(word / 0x100) & 0xff, word & 0xff, ); } } else { throw new RangeError( `CBOR length ${length} is past Number.MAX_SAFE_INTEGER`, ); } } /** * The error for a value an encoder cannot write. * @param {unknown} val * @returns {TypeError} */ function unencodable(val) { const what = typeof val === 'number' ? String(val) : typeof val; return new TypeError( `CBOR cannot encode ${what}: only safe integers, strings, bytes, arrays, maps, booleans and null`, ); } /** * Append bytes one at a time. A spread (`parts.push(...bytes)`) passes every * byte as a call argument, and past the engine's argument limit (about 125k) * it throws RangeError. A commit event carries a whole CAR file, so the * inputs here are routinely larger than that. * @param {number[]} parts * @param {Uint8Array|number[]} bytes */ function pushBytes(parts, bytes) { for (let i = 0; i < bytes.length; i++) parts.push(bytes[i]); } /** * Encode a value as CBOR bytes (RFC 8949 deterministic encoding) * @param {*} value - Value to encode (null, boolean, number, string, Uint8Array, array, or object) * @returns {Uint8Array} CBOR-encoded bytes */ export function cborEncode(value) { /** @type {number[]} */ const parts = []; /** @param {*} val */ function encode(val) { if (val === null) { parts.push(CBOR_NULL); } else if (val === true) { parts.push(CBOR_TRUE); } else if (val === false) { parts.push(CBOR_FALSE); } else if (typeof val === 'number') { if (!Number.isSafeInteger(val)) throw unencodable(val); encodeInteger(val); } else if (typeof val === 'string') { const bytes = new TextEncoder().encode(val); encodeHead(parts, 3, bytes.length); // major type 3 = text string pushBytes(parts, bytes); } else if (val instanceof Uint8Array) { encodeHead(parts, 2, val.length); // major type 2 = byte string pushBytes(parts, val); } else if (Array.isArray(val)) { encodeHead(parts, 4, val.length); // major type 4 = array for (const item of val) encode(item); } else if (typeof val === 'object') { // Sort keys for deterministic encoding const keys = Object.keys(val).sort(); encodeHead(parts, 5, keys.length); // major type 5 = map for (const key of keys) { encode(key); encode(val[key]); } } else { throw unencodable(val); } } /** @param {number} n */ function encodeInteger(n) { if (n >= 0) { encodeHead(parts, 0, n); // major type 0 = unsigned int } else { encodeHead(parts, 1, -n - 1); // major type 1 = negative int } } encode(value); return new Uint8Array(parts); } /** * DAG-CBOR encoder that handles CIDs with tag 42 * @param {*} value * @returns {Uint8Array} */ export function cborEncodeDagCbor(value) { /** @type {number[]} */ const parts = []; /** @param {*} val */ function encode(val) { if (val === null) { parts.push(CBOR_NULL); } else if (val === true) { parts.push(CBOR_TRUE); } else if (val === false) { parts.push(CBOR_FALSE); } else if (typeof val === 'number') { if (!Number.isSafeInteger(val)) throw unencodable(val); if (val >= 0) encodeHead(parts, 0, val); else encodeHead(parts, 1, -val - 1); } else if (typeof val === 'string') { const bytes = new TextEncoder().encode(val); encodeHead(parts, 3, bytes.length); pushBytes(parts, bytes); } else if (val instanceof CID) { // CID links in DAG-CBOR use tag 42 + 0x00 multibase prefix // The 0x00 prefix indicates "identity" multibase (raw bytes) parts.push(0xd8, CBOR_TAG_CID); encodeHead(parts, 2, val.bytes.length + 1); // +1 for 0x00 prefix parts.push(0x00); pushBytes(parts, val.bytes); } else if (val instanceof Uint8Array) { // Regular byte string encodeHead(parts, 2, val.length); pushBytes(parts, val); } else if (Array.isArray(val)) { encodeHead(parts, 4, val.length); for (const item of val) encode(item); } else if (typeof val === 'object') { // DAG-CBOR: sort keys by length first, then lexicographically // (differs from standard CBOR which sorts lexicographically only) const keys = Object.keys(val).filter((k) => val[k] !== undefined); keys.sort((a, b) => { if (a.length !== b.length) return a.length - b.length; return a < b ? -1 : a > b ? 1 : 0; }); encodeHead(parts, 5, keys.length); for (const key of keys) { const keyBytes = new TextEncoder().encode(key); encodeHead(parts, 3, keyBytes.length); pushBytes(parts, keyBytes); encode(val[key]); } } else { throw unencodable(val); } } encode(value); return new Uint8Array(parts); } /** * Decode CBOR bytes to a JavaScript value * @param {Uint8Array} bytes - CBOR-encoded bytes * @returns {*} Decoded value */ export function cborDecode(bytes) { return cborDecodeFirst(bytes)[0]; } /** * Decode CBOR bytes, keeping a CID link apart from a byte string: tag 42 comes * back as a CID rather than as its raw bytes. * @param {Uint8Array} bytes - CBOR-encoded bytes * @returns {*} Decoded value */ export function cborDecodeLinks(bytes) { return cborDecodeFirst(bytes, { links: true })[0]; } /** * Decode the first CBOR item in a byte stream, returning the value and the * number of bytes consumed. Useful for parsing firehose frames, which are a * header item followed by a body item. * @param {Uint8Array} bytes - CBOR-encoded bytes * @param {{links?: boolean}} [opts] - links: return tag-42 links as CIDs * @returns {[*, number]} Decoded value and bytes consumed */ export function cborDecodeFirst(bytes, opts = {}) { let offset = 0; /** @returns {*} */ function read() { const initial = bytes[offset++]; const major = initial >> 5; const info = initial & 0x1f; let length = info; if (info === 24) length = bytes[offset++]; else if (info === 25) { length = (bytes[offset++] << 8) | bytes[offset++]; } else if (info === 26) { // Use multiplication instead of bitshift to avoid 32-bit signed integer overflow length = bytes[offset++] * 0x1000000 + bytes[offset++] * 0x10000 + bytes[offset++] * 0x100 + bytes[offset++]; } else if (info === 27) { // The encoders write an integer of 2^32 or more with an 8-byte head. length = 0; for (let i = 0; i < 8; i++) length = length * 0x100 + bytes[offset++]; } switch (major) { case 0: return length; // unsigned int case 1: return -1 - length; // negative int case 2: { // byte string const data = bytes.slice(offset, offset + length); offset += length; return data; } case 3: { // text string const data = new TextDecoder().decode( bytes.slice(offset, offset + length), ); offset += length; return data; } case 4: { // array const arr = []; for (let i = 0; i < length; i++) arr.push(read()); return arr; } case 5: { // map /** @type {Record} */ const obj = {}; for (let i = 0; i < length; i++) { const key = /** @type {string} */ (read()); obj[key] = read(); } return obj; } case 6: { // tag // length is the tag number const taggedValue = read(); if (length === CBOR_TAG_CID) { // CID link: byte string with 0x00 multibase prefix const cid = taggedValue.slice(1); return opts.links ? new CID(cid) : cid; } return taggedValue; } case 7: { // special if (info === 20) return false; if (info === 21) return true; if (info === 22) return null; return undefined; } } } return [read(), offset]; } /** * Parse a (signed or unsigned) commit block. * @param {Uint8Array} bytes - DAG-CBOR encoded commit block * @returns {{did: string, rev: string, data: string|null, prev: string|null}} */ export function parseCommitBlock(bytes) { const commit = cborDecode(bytes); return { did: commit.did, rev: commit.rev, data: commit.data ? cidToString(commit.data) : null, prev: commit.prev ? cidToString(commit.prev) : null, }; } /** * @param {BufferSource} data * @returns {Promise} */ export async function sha256(data) { const hash = await crypto.subtle.digest('SHA-256', data); return new Uint8Array(hash); } /** * Create a CIDv1 with SHA-256 hash * @param {Uint8Array} bytes - Content to hash * @param {number} codec - Codec identifier (0x71 for dag-cbor, 0x55 for raw) * @returns {Promise} CID bytes (36 bytes: version + codec + multihash) */ async function createCidWithCodec(bytes, codec) { const hash = await crypto.subtle.digest( 'SHA-256', /** @type {BufferSource} */ (bytes), ); return cidFromDigest(new Uint8Array(hash), codec); } /** * @param {Uint8Array} digest - 32-byte SHA-256 digest * @param {number} codec - Codec identifier (0x71 for dag-cbor, 0x55 for raw) * @returns {Uint8Array} CID bytes (36 bytes: version + codec + multihash) */ function cidFromDigest(digest, codec) { // CIDv1: version(1) + codec + multihash(sha256) // Multihash: hash-type(0x12) + length(0x20=32) + digest const cid = new Uint8Array(2 + 2 + 32); cid[0] = 0x01; // CIDv1 cid[1] = codec; cid[2] = 0x12; // sha-256 cid[3] = 0x20; // 32 bytes cid.set(digest, 4); return cid; } /** * Whether a record URI addresses a space record. A space record's URI puts * the literal `space` marker where a public URI has its collection NSID, and * an NSID always contains a dot, so the second path segment separates them. * @param {string} uri * @returns {boolean} */ export function isSpaceRecordUri(uri) { if (!uri.startsWith('at://')) return false; return uri.slice('at://'.length).split('/')[1] === 'space'; } /** * Build the CID for raw blob content from an already computed SHA-256 digest. * The streamed upload path hashes bytes as they arrive, so it has no * whole-blob buffer to pass to createBlobCid. * @param {Uint8Array} digest - 32-byte SHA-256 digest * @returns {Uint8Array} CID bytes */ export function blobCidFromDigest(digest) { return cidFromDigest(digest, CODEC_RAW); } /** * Create CID for DAG-CBOR encoded data (records, commits) * @param {Uint8Array} bytes - DAG-CBOR encoded content * @returns {Promise} CID bytes */ export async function createCid(bytes) { return createCidWithCodec(bytes, CODEC_DAG_CBOR); } /** * Create CID for raw blob data (images, videos) * @param {Uint8Array} bytes - Raw binary content * @returns {Promise} CID bytes */ export async function createBlobCid(bytes) { return createCidWithCodec(bytes, CODEC_RAW); } /** * Convert CID bytes to base32lower string representation * @param {Uint8Array} cid - CID bytes * @returns {string} Base32lower-encoded CID with 'b' prefix */ export function cidToString(cid) { // base32lower encoding for CIDv1 return `b${base32Encode(cid)}`; } /** * Convert base32lower CID string to raw bytes * @param {string} cidStr - CID string with 'b' prefix * @returns {Uint8Array} CID bytes */ export function cidToBytes(cidStr) { // Decode base32lower CID string to bytes if (!cidStr.startsWith('b')) throw new Error('expected base32lower CID'); return base32Decode(cidStr.slice(1)); } /** * Generate a timestamp-based ID (TID) for record keys * Monotonic within a process, sortable by time * @returns {string} 13-character base32-sort encoded TID */ export function createTid() { let timestamp = Date.now() * 1000; // microseconds // Ensure monotonic if (timestamp <= lastTimestamp) { timestamp = lastTimestamp + 1; } lastTimestamp = timestamp; // 13 chars: 11 for timestamp (64 bits but only ~53 used), 2 for clock ID let tid = ''; // Encode timestamp (high bits first for sortability) let ts = timestamp; for (let i = 0; i < 11; i++) { tid = TID_CHARS[ts & 31] + tid; ts = Math.floor(ts / 32); } // Append clock ID (2 chars) tid += TID_CHARS[(clockId >> 5) & 31]; tid += TID_CHARS[clockId & 31]; return tid; } /** * Decode the timestamp a TID encodes, in epoch milliseconds, or null when the * string is not a 13-character TID. The first 11 characters hold a microsecond * timestamp, high bits first; the trailing two are the clock id and are ignored. * The inverse of the timestamp half of {@link createTid}. * @param {string} tid * @returns {number|null} */ export function tidToMs(tid) { if (typeof tid !== 'string' || tid.length !== 13) return null; let micros = 0; for (let i = 0; i < 11; i++) { const value = TID_CHARS.indexOf(tid[i]); if (value < 0) return null; micros = micros * 32 + value; } return Math.floor(micros / 1000); } // Cache for key depths (SHA-256 is expensive) const keyDepthCache = new Map(); /** * Get MST tree depth for a key based on leading zeros in SHA-256 hash * @param {string} key - Record key (collection/rkey) * @returns {Promise} Tree depth (leading zeros / 2) */ export async function getKeyDepth(key) { // Count leading zeros in SHA-256 hash, divide by 2 if (keyDepthCache.has(key)) return keyDepthCache.get(key); const keyBytes = new TextEncoder().encode(key); const hash = await sha256(keyBytes); let zeros = 0; for (const byte of hash) { if (byte === 0) { zeros += 8; } else { // Count leading zeros in this byte for (let i = 7; i >= 0; i--) { if ((byte >> i) & 1) break; zeros++; } break; } } // MST depth = leading zeros in SHA-256 hash / 2 // This creates a probabilistic tree where ~50% of keys are at depth 0, // ~25% at depth 1, etc., giving O(log n) lookups const depth = Math.floor(zeros / 2); keyDepthCache.set(key, depth); return depth; } /** * Build a CAR (Content Addressable aRchive) file * @param {string|string[]} rootCid - Root CID string, or an ordered list of them * @param {Array<{cid: string, data: Uint8Array}>} blocks - Blocks to include * @returns {Uint8Array} CAR file bytes */ export function buildCarFile(rootCid, blocks) { return concatBytes([carHeader(rootCid), ...blocks.map(carBlock)]); } /** * A CAR file's header, which a stream of {@link carBlock}s follows. * * Header: { version: 1, roots: [...] }. Most callers have a single root; * permissioned repos (proposal 0016) declare two — a signed commit and an * index — so an array is accepted as well. * @param {string|string[]} rootCid * @returns {Uint8Array} */ export function carHeader(rootCid) { const rootCids = Array.isArray(rootCid) ? rootCid : [rootCid]; const header = cborEncodeDagCbor({ version: 1, roots: rootCids.map((cid) => new CID(cidToBytes(cid))), }); return concatBytes([varint(header.length), header]); } /** * One block of a CAR file: varint(len) + cid + data. * @param {{cid: string, data: Uint8Array}} block * @returns {Uint8Array} */ export function carBlock(block) { const cidBytes = cidToBytes(block.cid); const blockLen = cidBytes.length + block.data.length; return concatBytes([varint(blockLen), cidBytes, block.data]); } /** * @param {Uint8Array[]} parts * @returns {Uint8Array} */ function concatBytes(parts) { const totalLen = parts.reduce((sum, p) => sum + p.length, 0); const out = new Uint8Array(totalLen); let offset = 0; for (const part of parts) { out.set(part, offset); offset += part.length; } return out; } /** * Sniff MIME type from file magic bytes * @param {Uint8Array|ArrayBuffer} bytes - File bytes (only first 12 needed) * @returns {string|null} Detected MIME type or null if unknown */ export function sniffMimeType(bytes) { const arr = new Uint8Array(bytes.slice(0, 12)); // JPEG: FF D8 FF if (arr[0] === 0xff && arr[1] === 0xd8 && arr[2] === 0xff) { return 'image/jpeg'; } // PNG: 89 50 4E 47 0D 0A 1A 0A if ( arr[0] === 0x89 && arr[1] === 0x50 && arr[2] === 0x4e && arr[3] === 0x47 && arr[4] === 0x0d && arr[5] === 0x0a && arr[6] === 0x1a && arr[7] === 0x0a ) { return 'image/png'; } // GIF: 47 49 46 38 (GIF8) if ( arr[0] === 0x47 && arr[1] === 0x49 && arr[2] === 0x46 && arr[3] === 0x38 ) { return 'image/gif'; } // WebP: RIFF....WEBP if ( arr[0] === 0x52 && arr[1] === 0x49 && arr[2] === 0x46 && arr[3] === 0x46 && arr[8] === 0x57 && arr[9] === 0x45 && arr[10] === 0x42 && arr[11] === 0x50 ) { return 'image/webp'; } // ISOBMFF container: ....ftyp at byte 4 (MP4, AVIF, HEIC, etc.) if ( arr[4] === 0x66 && arr[5] === 0x74 && arr[6] === 0x79 && arr[7] === 0x70 ) { // Check brand code at bytes 8-11 const brand = String.fromCharCode(arr[8], arr[9], arr[10], arr[11]); if (brand === 'avif') { return 'image/avif'; } if (brand === 'heic' || brand === 'heix' || brand === 'mif1') { return 'image/heic'; } return 'video/mp4'; } return null; } // A field that carries a moment, by naming convention across lexicons: // createdAt, postedAt, playedTime, publishDate, and the like. const TIME_KEY = /(at|time|date|timestamp)$/i; // Only moments inside this window date a record: nothing before the protocol // existed (a profile's birthDate must not date an avatar), nothing from the // future (a scheduled event must not pin its poster to that day). const RECORD_TIME_FLOOR = Date.parse('2020-01-01T00:00:00Z'); /** * The moment a record says its content was made, or null when it carries * none. Lexicons name their timestamps freely (`createdAt`, `postedAt`, * `playedTime`, …), so any top-level field named like a timestamp whose value * is an ISO datetime counts, and the earliest wins — an update time can only * be later than creation, so the minimum is the creation-most moment. Used to * date the blobs a record references by the record's own clock, which unlike * server arrival time survives a migration. * @param {*} value - Record value * @returns {number|null} */ export function recordTimeOf(value) { if (!value || typeof value !== 'object' || Array.isArray(value)) return null; let earliest = null; const ceiling = Date.now() + 24 * 60 * 60 * 1000; for (const [key, raw] of Object.entries(value)) { if (!TIME_KEY.test(key)) continue; if (typeof raw !== 'string' || !/^\d{4}-\d{2}-\d{2}T/.test(raw)) continue; const parsed = Date.parse(raw); if (!Number.isFinite(parsed)) continue; if (parsed < RECORD_TIME_FLOOR || parsed > ceiling) continue; if (earliest === null || parsed < earliest) earliest = parsed; } return earliest; } /** * Find all blob CID references in a record * @param {*} obj - Record value to scan * @param {string[]} refs - Accumulator array (internal) * @returns {string[]} Array of blob CID strings */ export function findBlobRefs(obj, refs = []) { if (!obj || typeof obj !== 'object') { return refs; } // Check if this object is a blob ref. A ref written over XRPC is `{$link}`; // one decoded from a CAR (an imported repo) is the raw CID bytes, since // cborDecode hands back tag-42 links as bytes. Both must be found, or an // imported account's blobs go unseen and never transfer. if (obj.$type === 'blob') { if (obj.ref?.$link) { refs.push(obj.ref.$link); } else if (obj.ref instanceof Uint8Array) { refs.push(cidToString(obj.ref)); } } // Legacy (pre-$type) blob ref: exactly {cid, mimeType} const keys = Object.keys(obj); if ( keys.length === 2 && typeof obj.cid === 'string' && typeof obj.mimeType === 'string' ) { refs.push(obj.cid); } // Recurse into arrays and objects if (Array.isArray(obj)) { for (const item of obj) { findBlobRefs(item, refs); } } else { for (const value of Object.values(obj)) { findBlobRefs(value, refs); } } return refs; } /** * The DAG-CBOR data model of a record in its JSON form: `{$link}` becomes a * CID link and `{$bytes}` a byte string, so the record encodes to the CID any * other implementation computes from the same JSON. * @param {unknown} value * @returns {unknown} */ export function jsonToLex(value) { if (Array.isArray(value)) return value.map(jsonToLex); if (value && typeof value === 'object' && !(value instanceof CID)) { const obj = /** @type {Record} */ (value); if (obj instanceof Uint8Array) return obj; const keys = Object.keys(obj); if (keys.length === 1 && typeof obj.$link === 'string') { return new CID(cidToBytes(obj.$link)); } if (keys.length === 1 && typeof obj.$bytes === 'string') { return Uint8Array.from(atob(obj.$bytes), (c) => c.charCodeAt(0)); } return Object.fromEntries( Object.entries(obj).map(([key, inner]) => [key, jsonToLex(inner)]), ); } return value; } /** * The JSON form of a record decoded by cborDecodeLinks. * @param {unknown} value * @returns {unknown} */ export function lexToJson(value) { if (value instanceof CID) return { $link: cidToString(value.bytes) }; if (value instanceof Uint8Array) { let binary = ''; for (const b of value) binary += String.fromCharCode(b); return { $bytes: btoa(binary).replace(/=+$/, '') }; } if (Array.isArray(value)) return value.map(lexToJson); if (value && typeof value === 'object') { return Object.fromEntries( Object.entries(value).map(([key, inner]) => [key, lexToJson(inner)]), ); } return value; } /** * A stored record, in its JSON form. * @param {Uint8Array} bytes - DAG-CBOR * @returns {any} */ export function cborDecodeJson(bytes) { return lexToJson(cborDecodeLinks(bytes)); } /** * Make a decoded record safe to render as JSON. DAG-CBOR gives back raw bytes * for a CID link, which stringifies as an object full of numbers, and * everything else in a record is already plain JSON. * @param {unknown} value * @returns {unknown} */ export function toDisplayValue(value) { if (value instanceof Uint8Array) return { $link: cidToString(value) }; if (Array.isArray(value)) return value.map(toDisplayValue); if (value && typeof value === 'object') { return Object.fromEntries( Object.entries(value).map(([key, inner]) => [key, toDisplayValue(inner)]), ); } return value; }