johnandersen777.bsky.social/socialweb-computer-watch
The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay
0
0
# Capture lines ({src, recv, frame}) in recv order -> only the lines that deliver an op not seen before
# (repo/collection/rkey@rev). Jetstream frames, relay frames (websocat JSON, goat firehose) and trimmed ones.
# jq -sc 'sort_by(.recv)[]' a.ndjson b.ndjson | jq -nc -f dedup.jq > capture.ndjson
def op_keys:
.frame as $f
| if $f.kind == "commit" then ["\($f.did)/\($f.commit.collection)/\($f.commit.rkey)@\($f.commit.rev)"]
else ($f.payload // $f) as $c | [$c.ops[]? | "\($c.repo)/\(.path)@\($c.rev)"] end;
foreach inputs as $l ({seen: {}};
($l | op_keys) as $k
| .seen as $s
| [$k[] | select($s[.] | not)] as $new
| .emit = (if ($new | length) > 0 then $l else null end)
| reduce $new[] as $x (.; .seen[$x] = true);
.emit // empty)