~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
301 lines · 10.1 KB Raw History
package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"math"
	"net/http"
	"net/url"
	"os"
	"slices"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/bluesky-social/indigo/atproto/atdata"
	"github.com/bluesky-social/indigo/atproto/identity"
	"github.com/bluesky-social/indigo/atproto/syntax"
	"github.com/ipfs/go-cid"
	"github.com/multiformats/go-multihash"

	"github.com/johnandersen777/socialweb-computer-kcp/internal/atproto/market"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/atproto/recordcid"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/identlink"
)

type tally struct {
	cid, cidBad, roundBad, signed, sigOK, sigDIDKey, sigBound, sigBad, sigErr, foreignEntries, foreignOnly int
	examples                                                                                               []string
}

type run struct {
	mu      sync.Mutex
	by      map[string]*tally
	dir     identity.Directory
	records *identlink.Remote
	client  *http.Client
}

func (r *run) t(collection string) *tally {
	r.mu.Lock()
	defer r.mu.Unlock()
	t := r.by[collection]
	if t == nil {
		t = &tally{}
		r.by[collection] = t
	}
	return t
}

func (r *run) note(collection, example string, f func(*tally)) {
	t := r.t(collection)
	r.mu.Lock()
	defer r.mu.Unlock()
	f(t)
	if example != "" && len(t.examples) < 3 {
		t.examples = append(t.examples, example)
	}
}

func (r *run) getJSON(ctx context.Context, u string, out any) error {
	for attempt := range 4 {
		req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
		if err != nil {
			return err
		}
		resp, err := r.client.Do(req)
		if err != nil {
			time.Sleep(time.Duration(attempt+1) * time.Second)
			continue
		}
		data, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
		resp.Body.Close()
		if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
			time.Sleep(time.Duration(attempt+1) * 2 * time.Second)
			continue
		}
		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf("%s: HTTP %d", u, resp.StatusCode)
		}
		return json.Unmarshal(data, out)
	}
	return fmt.Errorf("%s: gave up", u)
}

func (r *run) bound(ctx context.Context, did, key string) (bool, error) {
	value, _, err := r.records.GetVerifiedRecord(ctx, did, market.BadgeBlueKeysNSID, market.BadgeBlueKeysRkey(did, key))
	if err != nil {
		return false, err
	}
	keyID, _ := value["keyId"].(string)
	challenge, _ := value["challenge"].(string)
	return value != nil && keyID == key && challenge == did, nil
}

func (r *run) checkRecord(ctx context.Context, did, collection, uri, reported string, raw json.RawMessage) {
	if err := recordcid.Check(raw, reported); err != nil {
		r.note(collection, uri+": "+err.Error(), func(t *tally) { t.cidBad++ })
		return
	}
	r.note(collection, "", func(t *tally) { t.cid++ })
	obj, err := atdata.UnmarshalJSON(raw)
	if err == nil {
		var data []byte
		if data, err = atdata.MarshalCBOR(obj); err == nil {
			var back map[string]any
			if back, err = atdata.UnmarshalCBOR(data); err == nil {
				var again []byte
				if again, err = json.Marshal(back); err == nil {
					err = recordcid.Check(again, reported)
				}
			}
		}
	}
	if err != nil {
		r.note(collection, uri+" (cbor round trip): "+err.Error(), func(t *tally) { t.roundBad++ })
	}
	var value map[string]any
	if json.Unmarshal(raw, &value) != nil || len(market.InlineEntries(value)) == 0 {
		return
	}
	directKeys, _ := market.NewDirectoryKeyLister(r.dir).KeysForDID(ctx, did)
	opts := market.VerifyRecordOptions{KeysForDID: market.NewDirectoryKeyLister(r.dir), KeyResolver: market.DefaultKeyResolver, KeyBoundByBadgeBlueKey: r.bound}
	err = market.CheckRecordSignatures(ctx, value, did, opts)
	direct := false
	for _, entry := range market.InlineEntries(value) {
		for _, k := range directKeys {
			direct = direct || entry.Key == k
		}
	}
	foreignEntries, foreignOnly := 0, false
	for _, entry := range market.InlineEntries(value) {
		issuer, _ := entry.Fields["issuer"].(string)
		if issuer == "" || issuer == did {
			continue
		}
		foreignEntries++
		if err != nil && market.VerifyInlineAttestation(ctx, value, entry, did, market.DefaultKeyResolver) {
			if keys, kerr := market.NewDirectoryKeyLister(r.dir).KeysForDID(ctx, issuer); kerr == nil && slices.Contains(keys, entry.Key) {
				foreignOnly = true
			}
			if strings.HasPrefix(issuer, "did:key:") && issuer == entry.Key {
				foreignOnly = true
			}
		}
	}
	if foreignEntries > 0 {
		r.note(collection, fmt.Sprintf("%s: %d signature entries name another issuer", uri, foreignEntries), func(t *tally) { t.foreignEntries += foreignEntries })
	}
	if foreignOnly {
		r.note(collection, uri+": passes only by a foreign issuer", func(t *tally) { t.foreignOnly++ })
	}
	switch {
	case err == nil && direct:
		r.note(collection, "", func(t *tally) { t.signed++; t.sigOK++; t.sigDIDKey++ })
	case err == nil:
		r.note(collection, "", func(t *tally) { t.signed++; t.sigOK++; t.sigBound++ })
	case strings.Contains(err.Error(), market.ErrUnsignedRecord.Error()):
		r.note(collection, uri+": "+err.Error(), func(t *tally) { t.signed++; t.sigBad++ })
	default:
		r.note(collection, uri+": "+err.Error(), func(t *tally) { t.signed++; t.sigErr++ })
	}
}

func synthetic() []string {
	out := []string{}
	cases := map[string]int64{"max int64": math.MaxInt64, "2^53+1": 1<<53 + 1, "negative": -42, "min int64": math.MinInt64, "zero": 0}
	names := make([]string, 0, len(cases))
	for name := range cases {
		names = append(names, name)
	}
	sort.Strings(names)
	for _, name := range names {
		n := cases[name]
		record := map[string]any{"$type": "computer.socialweb.test.number", "n": n, "nested": map[string]any{"list": []any{n, "x"}}}
		data, err := atdata.MarshalCBOR(record)
		if err != nil {
			out = append(out, name+": encode: "+err.Error())
			continue
		}
		want, _ := cid.NewPrefixV1(cid.DagCBOR, multihash.SHA2_256).Sum(data)
		raw := fmt.Sprintf(`{"$type":"computer.socialweb.test.number","n":%d,"nested":{"list":[%d,"x"]}}`, n, n)
		got, err := recordcid.Of([]byte(raw))
		switch {
		case err != nil:
			out = append(out, fmt.Sprintf("synthetic %s: FAIL %v", name, err))
		case !got.Equals(want):
			out = append(out, fmt.Sprintf("synthetic %s: FAIL %s != %s", name, got, want))
		default:
			out = append(out, fmt.Sprintf("synthetic %s: ok %s", name, got))
		}
	}
	if _, err := recordcid.Of([]byte(`{"$type":"computer.socialweb.test.number","n":1.5}`)); err == nil {
		out = append(out, "synthetic non-integer 1.5: FAIL accepted")
	} else {
		out = append(out, "synthetic non-integer 1.5: ok refused")
	}
	return out
}

func main() {
	perCollection := flag.Int("n", 60, "records per collection and repo")
	only := flag.String("collections", "", "comma list of collections (empty: every collection of the repo)")
	flag.Parse()
	ctx := context.Background()
	client := &http.Client{Timeout: 30 * time.Second}
	dir := &identity.BaseDirectory{PLCURL: identity.DefaultPLCURL, HTTPClient: *client, SkipHandleVerification: true}
	r := &run{by: map[string]*tally{}, dir: dir, records: identlink.NewRemote(dir, client), client: client}
	for _, line := range synthetic() {
		fmt.Println(line)
	}
	var wg sync.WaitGroup
	sem := make(chan struct{}, 6)
	for _, arg := range flag.Args() {
		ident, err := dir.Lookup(ctx, mustIdent(arg))
		if err != nil {
			fmt.Fprintln(os.Stderr, "resolve", arg, err)
			continue
		}
		did, pds := ident.DID.String(), strings.TrimRight(ident.PDSEndpoint(), "/")
		var desc struct {
			Collections []string `json:"collections"`
		}
		if err := r.getJSON(ctx, pds+"/xrpc/com.atproto.repo.describeRepo?repo="+url.QueryEscape(did), &desc); err != nil {
			fmt.Fprintln(os.Stderr, "describe", arg, err)
			continue
		}
		collections := desc.Collections
		if *only != "" {
			collections = strings.Split(*only, ",")
		}
		for _, collection := range collections {
			var list struct {
				Records []struct {
					URI string `json:"uri"`
				} `json:"records"`
			}
			if err := r.getJSON(ctx, pds+"/xrpc/com.atproto.repo.listRecords?repo="+url.QueryEscape(did)+"&collection="+url.QueryEscape(collection)+fmt.Sprintf("&limit=%d", *perCollection), &list); err != nil {
				continue
			}
			for _, rec := range list.Records {
				rkey := rec.URI[strings.LastIndex(rec.URI, "/")+1:]
				wg.Add(1)
				sem <- struct{}{}
				go func(collection, rkey string) {
					defer wg.Done()
					defer func() { <-sem }()
					var got struct {
						URI   string          `json:"uri"`
						CID   string          `json:"cid"`
						Value json.RawMessage `json:"value"`
					}
					u := pds + "/xrpc/com.atproto.repo.getRecord?repo=" + url.QueryEscape(did) + "&collection=" + url.QueryEscape(collection) + "&rkey=" + url.QueryEscape(rkey)
					if err := r.getJSON(ctx, u, &got); err != nil {
						r.note(collection, rkey+": "+err.Error(), func(t *tally) { t.sigErr++ })
						return
					}
					r.checkRecord(ctx, did, collection, got.URI, got.CID, got.Value)
				}(collection, rkey)
			}
		}
	}
	wg.Wait()
	names := make([]string, 0, len(r.by))
	for name := range r.by {
		names = append(names, name)
	}
	sort.Strings(names)
	var total tally
	for _, name := range names {
		t := r.by[name]
		fmt.Printf("%-64s cid=%d cidBad=%d roundBad=%d signed=%d sigOk=%d (didKey=%d bound=%d) sigBad=%d sigErr=%d foreignIssuerEntries=%d passOnlyByForeignIssuer=%d\n", name, t.cid, t.cidBad, t.roundBad, t.signed, t.sigOK, t.sigDIDKey, t.sigBound, t.sigBad, t.sigErr, t.foreignEntries, t.foreignOnly)
		for _, e := range t.examples {
			fmt.Printf("    %s\n", e)
		}
		total.cid += t.cid
		total.cidBad += t.cidBad
		total.roundBad += t.roundBad
		total.signed += t.signed
		total.sigOK += t.sigOK
		total.sigBad += t.sigBad
		total.sigErr += t.sigErr
		total.sigDIDKey += t.sigDIDKey
		total.sigBound += t.sigBound
		total.foreignEntries += t.foreignEntries
		total.foreignOnly += t.foreignOnly
	}
	fmt.Printf("TOTAL cid=%d cidBad=%d roundBad=%d signed=%d sigOk=%d (didKey=%d bound=%d) sigBad=%d sigErr=%d foreignIssuerEntries=%d passOnlyByForeignIssuer=%d\n", total.cid, total.cidBad, total.roundBad, total.signed, total.sigOK, total.sigDIDKey, total.sigBound, total.sigBad, total.sigErr, total.foreignEntries, total.foreignOnly)
}

func mustIdent(arg string) syntax.AtIdentifier {
	id, err := syntax.ParseAtIdentifier(arg)
	if err != nil {
		fmt.Fprintln(os.Stderr, arg, err)
		os.Exit(2)
	}
	return id
}