chadtmiller.com/pds.js

Switch repository

Loading…

// @pdsjs/core/car - CAR (Content Addressable aRchive) file parser
// Parses CAR v1 files for importing AT Protocol repositories

import { cborDecode, cidToString } from './repo.js';

/**
 * Read an unsigned varint from bytes at the given offset
 * @param {Uint8Array} bytes - Input bytes
 * @param {number} offset - Starting offset
 * @returns {[number, number]} - [decoded value, new offset after varint]
 */
export function readVarint(bytes, offset) {
  let value = 0;
  let shift = 0;
  let pos = offset;

  while (pos < bytes.length) {
    const byte = bytes[pos++];
    value |= (byte & 0x7f) << shift;
    if ((byte & 0x80) === 0) {
      return [value, pos];
    }
    shift += 7;
    if (shift > 35) {
      throw new Error('Varint too long');
    }
  }

  throw new Error('Unexpected end of varint');
}

/**
 * Parse a CID from raw bytes
 * Handles CIDv0 (starts with 0x12) and CIDv1 (starts with 0x01)
 * @param {Uint8Array} bytes - CID bytes
 * @param {number} offset - Starting offset
 * @returns {[Uint8Array, number]} - [CID bytes, new offset]
 */
function parseCidFromBytes(bytes, offset) {
  const firstByte = bytes[offset];

  if (firstByte === 0x12) {
    // CIDv0: SHA-256 multihash (0x12 = sha2-256, 0x20 = 32 bytes)
    // Total: 34 bytes
    const cidLen = 34;
    return [bytes.slice(offset, offset + cidLen), offset + cidLen];
  }

  // CIDv1: version + codec + multihash
  let pos = offset;

  // Read version (should be 1)
  const [version, afterVersion] = readVarint(bytes, pos);
  if (version !== 1) {
    throw new Error(`Unsupported CID version: ${version}`);
  }
  pos = afterVersion;

  // Read codec
  const [_codec, afterCodec] = readVarint(bytes, pos);
  pos = afterCodec;

  // Read multihash: hash function + digest length + digest
  const [_hashFn, afterHashFn] = readVarint(bytes, pos);
  pos = afterHashFn;

  const [digestLen, afterDigestLen] = readVarint(bytes, pos);
  pos = afterDigestLen + digestLen;

  return [bytes.slice(offset, pos), pos];
}

/**
 * @typedef {Object} CarBlock
 * @property {string} cid - Block CID as base32lower string
 * @property {Uint8Array} data - Block data bytes
 */

/**
 * @typedef {Object} ParsedCar
 * @property {string[]} roots - Root CID strings
 * @property {Map<string, Uint8Array>} blocks - Map of CID string to block data
 */

/**
 * Parse a complete CAR file
 * @param {Uint8Array} carBytes - Complete CAR file bytes
 * @returns {ParsedCar} - Parsed CAR with roots and blocks
 */
export function parseCarFile(carBytes) {
  let offset = 0;

  // Read header length (varint)
  const [headerLen, afterHeaderLen] = readVarint(carBytes, offset);
  offset = afterHeaderLen;

  // Decode header (CBOR)
  const headerBytes = carBytes.slice(offset, offset + headerLen);
  const header = cborDecode(headerBytes);
  offset += headerLen;

  // Validate header
  if (header.version !== 1) {
    throw new Error(`Unsupported CAR version: ${header.version}`);
  }

  // Extract roots (CIDs stored as Uint8Array from DAG-CBOR tag 42)
  const roots = (header.roots || []).map(
    (/** @type {Uint8Array} */ cidBytes) => {
      return cidToString(cidBytes);
    },
  );

  // Parse blocks
  /** @type {Map<string, Uint8Array>} */
  const blocks = new Map();

  while (offset < carBytes.length) {
    // Read block length (varint)
    const [blockLen, afterBlockLen] = readVarint(carBytes, offset);
    offset = afterBlockLen;

    if (blockLen === 0) break;

    // Parse CID
    const [cidBytes, afterCid] = parseCidFromBytes(carBytes, offset);
    const cid = cidToString(cidBytes);

    // Extract block data
    const dataLen = blockLen - (afterCid - offset);
    const data = carBytes.slice(afterCid, afterCid + dataLen);

    blocks.set(cid, data);
    offset = afterCid + dataLen;
  }

  return { roots, blocks };
}

/**
 * Iterate over CAR file blocks (memory-efficient streaming)
 * @param {Uint8Array} carBytes - Complete CAR file bytes
 * @yields {CarBlock} - Block with CID and data
 * @returns {Generator<CarBlock, {roots: string[]}, unknown>}
 */
export function* iterateCarBlocks(carBytes) {
  let offset = 0;

  // Read header length (varint)
  const [headerLen, afterHeaderLen] = readVarint(carBytes, offset);
  offset = afterHeaderLen;

  // Decode header (CBOR)
  const headerBytes = carBytes.slice(offset, offset + headerLen);
  const header = cborDecode(headerBytes);
  offset += headerLen;

  // Validate header
  if (header.version !== 1) {
    throw new Error(`Unsupported CAR version: ${header.version}`);
  }

  // Extract roots for return value
  const roots = (header.roots || []).map(
    (/** @type {Uint8Array} */ cidBytes) => {
      return cidToString(cidBytes);
    },
  );

  // Iterate blocks
  while (offset < carBytes.length) {
    // Read block length (varint)
    const [blockLen, afterBlockLen] = readVarint(carBytes, offset);
    offset = afterBlockLen;

    if (blockLen === 0) break;

    // Parse CID
    const [cidBytes, afterCid] = parseCidFromBytes(carBytes, offset);
    const cid = cidToString(cidBytes);

    // Extract block data
    const dataLen = blockLen - (afterCid - offset);
    const data = carBytes.slice(afterCid, afterCid + dataLen);

    yield { cid, data };
    offset = afterCid + dataLen;
  }

  return { roots };
}

/**
 * Get CAR file header (roots) without parsing all blocks
 * @param {Uint8Array} carBytes - CAR file bytes
 * @returns {{ version: number, roots: string[] }}
 */
export function getCarHeader(carBytes) {
  let offset = 0;

  // Read header length (varint)
  const [headerLen, afterHeaderLen] = readVarint(carBytes, offset);
  offset = afterHeaderLen;

  // Decode header (CBOR)
  const headerBytes = carBytes.slice(offset, offset + headerLen);
  const header = cborDecode(headerBytes);

  // Extract roots
  const roots = (header.roots || []).map(
    (/** @type {Uint8Array} */ cidBytes) => {
      return cidToString(cidBytes);
    },
  );

  return { version: header.version, roots };
}