~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
82 lines · 3.2 KB Raw History
import { CopyObjectCommand, HeadObjectCommand, ListObjectsV2Command, S3Client } from "npm:@aws-sdk/client-s3@3.700.0";
import { parseArgs } from "jsr:@std/cli@1/parse-args";

const args = parseArgs(Deno.args, { string: ["from", "to"], collect: ["to"] });
const need = (name: string): string => {
  const v = Deno.env.get(name);
  if (!v) {
    console.error(`copy-spaces: ${name} is not set`);
    Deno.exit(2);
  }
  return v;
};
const to = (args.to as string[]) ?? [];
if (!args.from || to.length === 0) {
  console.error("usage: copy-spaces.ts --from <bucket prefix> --to <bucket prefix> [--to <bucket prefix> ...]");
  Deno.exit(2);
}
const bucket = need("SPACES_BUCKET");
const s3 = new S3Client({
  region: "us-east-1",
  endpoint: `https://${need("SPACES_REGION")}.digitaloceanspaces.com`,
  credentials: { accessKeyId: need("SPACES_ACCESS_KEY_ID"), secretAccessKey: need("SPACES_SECRET_ACCESS_KEY") },
});

const attempt = async <T>(key: string, fn: () => Promise<T>): Promise<T> => {
  for (let n = 1; ; n++) {
    try {
      return await fn();
    } catch (err) {
      console.error(`copy-spaces: ${key} attempt ${n} failed: ${err instanceof Error ? err.message : err}`);
      if (n >= 4) throw err;
      await new Promise((r) => setTimeout(r, 2000 * n));
    }
  }
};

const head = async (key: string): Promise<{ sha256?: string; size?: number } | undefined> => {
  try {
    const h = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
    return { sha256: h.Metadata?.sha256, size: h.ContentLength };
  } catch (err) {
    if ((err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode === 404) return undefined;
    throw err;
  }
};

const from = args.from.replace(/\/+$/, "");
const keys: Array<{ rel: string; size: number }> = [];
for (let token: string | undefined;;) {
  const page = await attempt(`${from} list`, () =>
    s3.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: `${from}/`, ContinuationToken: token })));
  for (const o of page.Contents ?? []) keys.push({ rel: o.Key!.slice(from.length + 1), size: o.Size ?? 0 });
  if (!page.IsTruncated) break;
  token = page.NextContinuationToken;
}
if (keys.length === 0) {
  console.error(`copy-spaces: nothing under ${from}/`);
  Deno.exit(1);
}

const queue = keys.flatMap((k) => to.map((t) => ({ ...k, dest: `${t.replace(/\/+$/, "")}/${k.rel}` })));
const workers = Array.from({ length: Math.min(4, queue.length) }, async () => {
  for (let c = queue.shift(); c; c = queue.shift()) {
    const started = Date.now();
    const source = await attempt(`${from}/${c.rel} head`, () => head(`${from}/${c.rel}`));
    const held = await attempt(`${c.dest} head`, () => head(c.dest));
    if (source?.sha256 && held?.sha256 === source.sha256 && held.size === source.size) {
      console.log(`${c.dest} ${c.size} bytes sha256 ${held.sha256} already there, not copied`);
      continue;
    }
    await attempt(c.dest, () =>
      s3.send(new CopyObjectCommand({
        Bucket: bucket,
        Key: c.dest,
        CopySource: encodeURI(`${bucket}/${from}/${c.rel}`),
        MetadataDirective: "COPY",
        ACL: "public-read",
      })));
    console.log(`${c.dest} ${c.size} bytes copied from ${from}/${c.rel} (${((Date.now() - started) / 1000).toFixed(1)}s)`);
  }
});
await Promise.all(workers);