# Capture lines ({src, recv, frame}) in recv order -> only the lines that deliver an op not seen before # (repo/collection/rkey@rev). Jetstream frames, relay frames (websocat JSON, goat firehose) and trimmed ones. # jq -sc 'sort_by(.recv)[]' a.ndjson b.ndjson | jq -nc -f dedup.jq > capture.ndjson def op_keys: .frame as $f | if $f.kind == "commit" then ["\($f.did)/\($f.commit.collection)/\($f.commit.rkey)@\($f.commit.rev)"] else ($f.payload // $f) as $c | [$c.ops[]? | "\($c.repo)/\(.path)@\($c.rev)"] end; foreach inputs as $l ({seen: {}}; ($l | op_keys) as $k | .seen as $s | [$k[] | select($s[.] | not)] as $new | .emit = (if ($new | length) > 0 then $l else null end) | reduce $new[] as $x (.; .seen[$x] = true); .emit // empty)