package main import ( "context" "encoding/json" "flag" "fmt" "io" "math" "net/http" "net/url" "os" "slices" "sort" "strings" "sync" "time" "github.com/bluesky-social/indigo/atproto/atdata" "github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" "github.com/johnandersen777/socialweb-computer-kcp/internal/atproto/market" "github.com/johnandersen777/socialweb-computer-kcp/internal/atproto/recordcid" "github.com/johnandersen777/socialweb-computer-kcp/internal/identlink" ) type tally struct { cid, cidBad, roundBad, signed, sigOK, sigDIDKey, sigBound, sigBad, sigErr, foreignEntries, foreignOnly int examples []string } type run struct { mu sync.Mutex by map[string]*tally dir identity.Directory records *identlink.Remote client *http.Client } func (r *run) t(collection string) *tally { r.mu.Lock() defer r.mu.Unlock() t := r.by[collection] if t == nil { t = &tally{} r.by[collection] = t } return t } func (r *run) note(collection, example string, f func(*tally)) { t := r.t(collection) r.mu.Lock() defer r.mu.Unlock() f(t) if example != "" && len(t.examples) < 3 { t.examples = append(t.examples, example) } } func (r *run) getJSON(ctx context.Context, u string, out any) error { for attempt := range 4 { req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return err } resp, err := r.client.Do(req) if err != nil { time.Sleep(time.Duration(attempt+1) * time.Second) continue } data, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) resp.Body.Close() if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 { time.Sleep(time.Duration(attempt+1) * 2 * time.Second) continue } if resp.StatusCode != http.StatusOK { return fmt.Errorf("%s: HTTP %d", u, resp.StatusCode) } return json.Unmarshal(data, out) } return fmt.Errorf("%s: gave up", u) } func (r *run) bound(ctx context.Context, did, key string) (bool, error) { value, _, err := r.records.GetVerifiedRecord(ctx, did, market.BadgeBlueKeysNSID, market.BadgeBlueKeysRkey(did, key)) if err != nil { return false, err } keyID, _ := value["keyId"].(string) challenge, _ := value["challenge"].(string) return value != nil && keyID == key && challenge == did, nil } func (r *run) checkRecord(ctx context.Context, did, collection, uri, reported string, raw json.RawMessage) { if err := recordcid.Check(raw, reported); err != nil { r.note(collection, uri+": "+err.Error(), func(t *tally) { t.cidBad++ }) return } r.note(collection, "", func(t *tally) { t.cid++ }) obj, err := atdata.UnmarshalJSON(raw) if err == nil { var data []byte if data, err = atdata.MarshalCBOR(obj); err == nil { var back map[string]any if back, err = atdata.UnmarshalCBOR(data); err == nil { var again []byte if again, err = json.Marshal(back); err == nil { err = recordcid.Check(again, reported) } } } } if err != nil { r.note(collection, uri+" (cbor round trip): "+err.Error(), func(t *tally) { t.roundBad++ }) } var value map[string]any if json.Unmarshal(raw, &value) != nil || len(market.InlineEntries(value)) == 0 { return } directKeys, _ := market.NewDirectoryKeyLister(r.dir).KeysForDID(ctx, did) opts := market.VerifyRecordOptions{KeysForDID: market.NewDirectoryKeyLister(r.dir), KeyResolver: market.DefaultKeyResolver, KeyBoundByBadgeBlueKey: r.bound} err = market.CheckRecordSignatures(ctx, value, did, opts) direct := false for _, entry := range market.InlineEntries(value) { for _, k := range directKeys { direct = direct || entry.Key == k } } foreignEntries, foreignOnly := 0, false for _, entry := range market.InlineEntries(value) { issuer, _ := entry.Fields["issuer"].(string) if issuer == "" || issuer == did { continue } foreignEntries++ if err != nil && market.VerifyInlineAttestation(ctx, value, entry, did, market.DefaultKeyResolver) { if keys, kerr := market.NewDirectoryKeyLister(r.dir).KeysForDID(ctx, issuer); kerr == nil && slices.Contains(keys, entry.Key) { foreignOnly = true } if strings.HasPrefix(issuer, "did:key:") && issuer == entry.Key { foreignOnly = true } } } if foreignEntries > 0 { r.note(collection, fmt.Sprintf("%s: %d signature entries name another issuer", uri, foreignEntries), func(t *tally) { t.foreignEntries += foreignEntries }) } if foreignOnly { r.note(collection, uri+": passes only by a foreign issuer", func(t *tally) { t.foreignOnly++ }) } switch { case err == nil && direct: r.note(collection, "", func(t *tally) { t.signed++; t.sigOK++; t.sigDIDKey++ }) case err == nil: r.note(collection, "", func(t *tally) { t.signed++; t.sigOK++; t.sigBound++ }) case strings.Contains(err.Error(), market.ErrUnsignedRecord.Error()): r.note(collection, uri+": "+err.Error(), func(t *tally) { t.signed++; t.sigBad++ }) default: r.note(collection, uri+": "+err.Error(), func(t *tally) { t.signed++; t.sigErr++ }) } } func synthetic() []string { out := []string{} cases := map[string]int64{"max int64": math.MaxInt64, "2^53+1": 1<<53 + 1, "negative": -42, "min int64": math.MinInt64, "zero": 0} names := make([]string, 0, len(cases)) for name := range cases { names = append(names, name) } sort.Strings(names) for _, name := range names { n := cases[name] record := map[string]any{"$type": "computer.socialweb.test.number", "n": n, "nested": map[string]any{"list": []any{n, "x"}}} data, err := atdata.MarshalCBOR(record) if err != nil { out = append(out, name+": encode: "+err.Error()) continue } want, _ := cid.NewPrefixV1(cid.DagCBOR, multihash.SHA2_256).Sum(data) raw := fmt.Sprintf(`{"$type":"computer.socialweb.test.number","n":%d,"nested":{"list":[%d,"x"]}}`, n, n) got, err := recordcid.Of([]byte(raw)) switch { case err != nil: out = append(out, fmt.Sprintf("synthetic %s: FAIL %v", name, err)) case !got.Equals(want): out = append(out, fmt.Sprintf("synthetic %s: FAIL %s != %s", name, got, want)) default: out = append(out, fmt.Sprintf("synthetic %s: ok %s", name, got)) } } if _, err := recordcid.Of([]byte(`{"$type":"computer.socialweb.test.number","n":1.5}`)); err == nil { out = append(out, "synthetic non-integer 1.5: FAIL accepted") } else { out = append(out, "synthetic non-integer 1.5: ok refused") } return out } func main() { perCollection := flag.Int("n", 60, "records per collection and repo") only := flag.String("collections", "", "comma list of collections (empty: every collection of the repo)") flag.Parse() ctx := context.Background() client := &http.Client{Timeout: 30 * time.Second} dir := &identity.BaseDirectory{PLCURL: identity.DefaultPLCURL, HTTPClient: *client, SkipHandleVerification: true} r := &run{by: map[string]*tally{}, dir: dir, records: identlink.NewRemote(dir, client), client: client} for _, line := range synthetic() { fmt.Println(line) } var wg sync.WaitGroup sem := make(chan struct{}, 6) for _, arg := range flag.Args() { ident, err := dir.Lookup(ctx, mustIdent(arg)) if err != nil { fmt.Fprintln(os.Stderr, "resolve", arg, err) continue } did, pds := ident.DID.String(), strings.TrimRight(ident.PDSEndpoint(), "/") var desc struct { Collections []string `json:"collections"` } if err := r.getJSON(ctx, pds+"/xrpc/com.atproto.repo.describeRepo?repo="+url.QueryEscape(did), &desc); err != nil { fmt.Fprintln(os.Stderr, "describe", arg, err) continue } collections := desc.Collections if *only != "" { collections = strings.Split(*only, ",") } for _, collection := range collections { var list struct { Records []struct { URI string `json:"uri"` } `json:"records"` } if err := r.getJSON(ctx, pds+"/xrpc/com.atproto.repo.listRecords?repo="+url.QueryEscape(did)+"&collection="+url.QueryEscape(collection)+fmt.Sprintf("&limit=%d", *perCollection), &list); err != nil { continue } for _, rec := range list.Records { rkey := rec.URI[strings.LastIndex(rec.URI, "/")+1:] wg.Add(1) sem <- struct{}{} go func(collection, rkey string) { defer wg.Done() defer func() { <-sem }() var got struct { URI string `json:"uri"` CID string `json:"cid"` Value json.RawMessage `json:"value"` } u := pds + "/xrpc/com.atproto.repo.getRecord?repo=" + url.QueryEscape(did) + "&collection=" + url.QueryEscape(collection) + "&rkey=" + url.QueryEscape(rkey) if err := r.getJSON(ctx, u, &got); err != nil { r.note(collection, rkey+": "+err.Error(), func(t *tally) { t.sigErr++ }) return } r.checkRecord(ctx, did, collection, got.URI, got.CID, got.Value) }(collection, rkey) } } } wg.Wait() names := make([]string, 0, len(r.by)) for name := range r.by { names = append(names, name) } sort.Strings(names) var total tally for _, name := range names { t := r.by[name] fmt.Printf("%-64s cid=%d cidBad=%d roundBad=%d signed=%d sigOk=%d (didKey=%d bound=%d) sigBad=%d sigErr=%d foreignIssuerEntries=%d passOnlyByForeignIssuer=%d\n", name, t.cid, t.cidBad, t.roundBad, t.signed, t.sigOK, t.sigDIDKey, t.sigBound, t.sigBad, t.sigErr, t.foreignEntries, t.foreignOnly) for _, e := range t.examples { fmt.Printf(" %s\n", e) } total.cid += t.cid total.cidBad += t.cidBad total.roundBad += t.roundBad total.signed += t.signed total.sigOK += t.sigOK total.sigBad += t.sigBad total.sigErr += t.sigErr total.sigDIDKey += t.sigDIDKey total.sigBound += t.sigBound total.foreignEntries += t.foreignEntries total.foreignOnly += t.foreignOnly } fmt.Printf("TOTAL cid=%d cidBad=%d roundBad=%d signed=%d sigOk=%d (didKey=%d bound=%d) sigBad=%d sigErr=%d foreignIssuerEntries=%d passOnlyByForeignIssuer=%d\n", total.cid, total.cidBad, total.roundBad, total.signed, total.sigOK, total.sigDIDKey, total.sigBound, total.sigBad, total.sigErr, total.foreignEntries, total.foreignOnly) } func mustIdent(arg string) syntax.AtIdentifier { id, err := syntax.ParseAtIdentifier(arg) if err != nil { fmt.Fprintln(os.Stderr, arg, err) os.Exit(2) } return id }