chadtmiller.com/knotless
Git hosted in your atproto account, on any stock PDS, fast enough for agents.
git clone knotless://chadtmiller.com/knotlessneeds the knotless git helper. set it up
knotless: git repositories stored natively on atproto
A repository is a handful of record types in its owner's account and works against any stock PDS: per-ref compare-and-swap claims, immutable packs, redirects for renames, and private repositories in permissioned spaces. Includes the git remote helper and CLI, a forge (pulls, issues, comments, reviews, merges, transfers, retargeting), atproto-aligned moderation (owner gates and dev.knotless.block records, createReport, admin takedowns and !takedown labels), the knotless-serve appview with web pages and HTTP clones, and the knotless-mcp server. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Chad Miller committed on Oct 21, 2026 commit 4de1da8
Showing 263 changed files with 35930 additions and 0 deletions
.gitignorenew+4−0
| @@ -0,0 +1,4 @@ | ||
/target | ||
/harness/node_modules | ||
*.profraw | ||
/lcov.info | ||
CLAUDE.mdnew+25−0
| @@ -0,0 +1,25 @@ | ||
# Working on knotless | ||
| ||
Use `rustup run stable cargo` (the Justfile's `cargo`), not a system cargo. | ||
Tests run against the npm reference PDS through `harness/`, never pds.js. | ||
| ||
## Running tests | ||
| ||
Run what a change needs while working, and the full gates once, when the | ||
work is done. | ||
| ||
- `just pds` in the background first: test binaries then share that PDS | ||
instead of each starting one. | ||
- `just changed` names the files changed since tests last passed and the | ||
tests that cover them; `just t-changed` runs those and marks the pass. | ||
`just t <cargo test args>` runs any selection on the shared PDS. | ||
- For the web pages: `just shots-up` in the background once, then | ||
`just shots`, which retakes only pages whose HTML changed. Look at the | ||
PNGs it writes under `harness/shots/baseline/<platform>/`. | ||
- Done: `just lint`, `just cov` (100% lines and functions over the stock and | ||
spaces-alpha runs; slow, so once) and, if pages changed, `just | ||
shots-check`. | ||
| ||
`cargo fmt` can put a `?` on a line of its own, which coverage then counts | ||
as a line never run: bind the value first, or use a named function for a | ||
row mapper, rather than lowering the gate. | ||
Cargo.locknew+4024−0
A large diff, not shown.
Cargo.tomlnew+36−0
| @@ -0,0 +1,36 @@ | ||
[workspace] | ||
members = ["crates/knotless", "crates/knotless-serve", "crates/knotless-mcp"] | ||
resolver = "3" | ||
| ||
[workspace.package] | ||
version = "0.1.0" | ||
edition = "2024" | ||
license = "MIT OR Apache-2.0" | ||
rust-version = "1.88" | ||
| ||
[workspace.dependencies] | ||
knotless = { path = "crates/knotless" } | ||
anyhow = "1" | ||
base64 = "0.22" | ||
futures = "0.3" | ||
humantime = "2" | ||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "http2", "gzip"] } | ||
serde = { version = "1", features = ["derive"] } | ||
serde_json = "1" | ||
sha2 = "0.10" | ||
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "process", "fs", "io-util", "time", "sync", "net", "io-std", "signal"] } | ||
axum = "0.8" | ||
ciborium = "0.2" | ||
tokio-tungstenite = { version = "0.29", features = ["rustls-tls-webpki-roots"] } | ||
flate2 = "1" | ||
tokio-util = { version = "0.7", features = ["io"] } | ||
tempfile = "3" | ||
k256 = { version = "0.13", features = ["ecdsa"] } | ||
p256 = { version = "0.13", features = ["ecdsa"] } | ||
rand_core = { version = "0.6", features = ["getrandom"] } | ||
bs58 = "0.5" | ||
| ||
[profile.release] | ||
lto = "thin" | ||
codegen-units = 1 | ||
strip = true | ||
Justfilenew+85−0
| @@ -0,0 +1,85 @@ | ||
cargo := "rustup run stable cargo" | ||
ignore := "(tests/|harness/|examples/|rustlib/)" | ||
| ||
# Install the reference PDS harness and coverage tools. | ||
setup: | ||
cd harness && npm install --no-audit --no-fund | ||
rustup component add llvm-tools-preview --toolchain stable | ||
command -v cargo-llvm-cov >/dev/null || cargo install cargo-llvm-cov | ||
| ||
test: | ||
{{cargo}} test --workspace | ||
| ||
# Tests while working: only those `args` pick, as cargo test takes them, | ||
# against the harness `just pds` left running if there is one. For a change | ||
# to the web pages, `just t -p knotless-serve --test web`. | ||
t *args: | ||
KNOTLESS_TEST_NET={{justfile_directory()}}/target/test-net.json {{cargo}} test {{args}} | ||
| ||
# The source files changed since tests last passed with `t-changed`, and | ||
# the tests that cover them. | ||
changed: | ||
@node harness/changed.mjs | ||
| ||
# Runs the tests `changed` names, and marks them passed. | ||
t-changed: | ||
@node harness/changed.mjs --run | ||
| ||
# A reference PDS left running for tests to share, so each test binary | ||
# need not start its own. Stop it with ctrl-c. | ||
pds: | ||
mkdir -p target | ||
tail -f /dev/null | node harness/pds.mjs --accounts seed > target/test-net.json | ||
| ||
# The same, against the spaces alpha. | ||
test-spaces: | ||
KNOTLESS_DEV_ENV={{spaces}} {{cargo}} test --workspace | ||
| ||
# The spaces alpha build of the reference PDS, for private repositories. | ||
spaces := env_var_or_default("KNOTLESS_DEV_ENV", "../atproto-space-ref/packages/dev-env") | ||
| ||
# Fails under 100% of lines or functions, over runs against the stock | ||
# reference PDS and the spaces alpha together. | ||
cov: | ||
{{cargo}} llvm-cov clean --workspace | ||
KNOTLESS_DEV_ENV= {{cargo}} llvm-cov --workspace --no-report | ||
KNOTLESS_DEV_ENV={{spaces}} {{cargo}} llvm-cov --workspace --no-report | ||
{{cargo}} llvm-cov report --ignore-filename-regex '{{ignore}}' --fail-under-lines 100 --fail-under-functions 100 --show-missing-lines | ||
| ||
# Coverage over the stock run alone, failing on nothing: quicker than | ||
# `cov`, but what only the spaces alpha reaches shows as missing. | ||
cov-fast: | ||
KNOTLESS_DEV_ENV= {{cargo}} llvm-cov --workspace --ignore-filename-regex '{{ignore}}' --show-missing-lines | ||
| ||
cov-html: | ||
{{cargo}} llvm-cov --workspace --ignore-filename-regex '{{ignore}}' --html --open | ||
| ||
lint: | ||
{{cargo}} fmt --all --check | ||
{{cargo}} clippy --workspace --all-targets -- -D warnings | ||
| ||
# Latency of agent operations at a given round-trip time, in milliseconds. | ||
bench rtt="50": | ||
{{cargo}} build --release | ||
{{cargo}} run --release --example bench -- --rtt {{rtt}} | ||
| ||
serve *args: | ||
{{cargo}} run --release -p knotless-serve -- {{args}} | ||
| ||
# Screenshots of the pages whose HTML changed since their baselines were | ||
# taken, taken again as the baselines in harness/shots/baseline. Arguments | ||
# go to Playwright: a page name to take only it, or --project=desktop-light. | ||
shots *args: | ||
{{cargo}} build --bins | ||
cd harness && npx playwright test -c shots/playwright.config.mjs --update-snapshots=all {{args}} | ||
| ||
# Every screenshot, failing on any pixel that differs from the baselines. | ||
shots-check *args: | ||
{{cargo}} build --bins | ||
cd harness && SHOTS_ALL=1 npx playwright test -c shots/playwright.config.mjs {{args}} | ||
| ||
# The screenshots' PDS, seeded once and left running, so `shots` need only | ||
# start the server. Stop it with ctrl-c. | ||
shots-up: | ||
{{cargo}} build --bins | ||
tail -f /dev/null | node harness/shots/world.mjs | ||
LICENSE-APACHEnew+202−0
| @@ -0,0 +1,202 @@ | ||
| ||
Apache License | ||
Version 2.0, January 2004 | ||
http://www.apache.org/licenses/ | ||
| ||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION | ||
| ||
1. Definitions. | ||
| ||
"License" shall mean the terms and conditions for use, reproduction, | ||
and distribution as defined by Sections 1 through 9 of this document. | ||
| ||
"Licensor" shall mean the copyright owner or entity authorized by | ||
the copyright owner that is granting the License. | ||
| ||
"Legal Entity" shall mean the union of the acting entity and all | ||
other entities that control, are controlled by, or are under common | ||
control with that entity. For the purposes of this definition, | ||
"control" means (i) the power, direct or indirect, to cause the | ||
direction or management of such entity, whether by contract or | ||
otherwise, or (ii) ownership of fifty percent (50%) or more of the | ||
outstanding shares, or (iii) beneficial ownership of such entity. | ||
| ||
"You" (or "Your") shall mean an individual or Legal Entity | ||
exercising permissions granted by this License. | ||
| ||
"Source" form shall mean the preferred form for making modifications, | ||
including but not limited to software source code, documentation | ||
source, and configuration files. | ||
| ||
"Object" form shall mean any form resulting from mechanical | ||
transformation or translation of a Source form, including but | ||
not limited to compiled object code, generated documentation, | ||
and conversions to other media types. | ||
| ||
"Work" shall mean the work of authorship, whether in Source or | ||
Object form, made available under the License, as indicated by a | ||
copyright notice that is included in or attached to the work | ||
(an example is provided in the Appendix below). | ||
| ||
"Derivative Works" shall mean any work, whether in Source or Object | ||
form, that is based on (or derived from) the Work and for which the | ||
editorial revisions, annotations, elaborations, or other modifications | ||
represent, as a whole, an original work of authorship. For the purposes | ||
of this License, Derivative Works shall not include works that remain | ||
separable from, or merely link (or bind by name) to the interfaces of, | ||
the Work and Derivative Works thereof. | ||
| ||
"Contribution" shall mean any work of authorship, including | ||
the original version of the Work and any modifications or additions | ||
to that Work or Derivative Works thereof, that is intentionally | ||
submitted to Licensor for inclusion in the Work by the copyright owner | ||
or by an individual or Legal Entity authorized to submit on behalf of | ||
the copyright owner. For the purposes of this definition, "submitted" | ||
means any form of electronic, verbal, or written communication sent | ||
to the Licensor or its representatives, including but not limited to | ||
communication on electronic mailing lists, source code control systems, | ||
and issue tracking systems that are managed by, or on behalf of, the | ||
Licensor for the purpose of discussing and improving the Work, but | ||
excluding communication that is conspicuously marked or otherwise | ||
designated in writing by the copyright owner as "Not a Contribution." | ||
| ||
"Contributor" shall mean Licensor and any individual or Legal Entity | ||
on behalf of whom a Contribution has been received by Licensor and | ||
subsequently incorporated within the Work. | ||
| ||
2. Grant of Copyright License. Subject to the terms and conditions of | ||
this License, each Contributor hereby grants to You a perpetual, | ||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable | ||
copyright license to reproduce, prepare Derivative Works of, | ||
publicly display, publicly perform, sublicense, and distribute the | ||
Work and such Derivative Works in Source or Object form. | ||
| ||
3. Grant of Patent License. Subject to the terms and conditions of | ||
this License, each Contributor hereby grants to You a perpetual, | ||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable | ||
(except as stated in this section) patent license to make, have made, | ||
use, offer to sell, sell, import, and otherwise transfer the Work, | ||
where such license applies only to those patent claims licensable | ||
by such Contributor that are necessarily infringed by their | ||
Contribution(s) alone or by combination of their Contribution(s) | ||
with the Work to which such Contribution(s) was submitted. If You | ||
institute patent litigation against any entity (including a | ||
cross-claim or counterclaim in a lawsuit) alleging that the Work | ||
or a Contribution incorporated within the Work constitutes direct | ||
or contributory patent infringement, then any patent licenses | ||
granted to You under this License for that Work shall terminate | ||
as of the date such litigation is filed. | ||
| ||
4. Redistribution. You may reproduce and distribute copies of the | ||
Work or Derivative Works thereof in any medium, with or without | ||
modifications, and in Source or Object form, provided that You | ||
meet the following conditions: | ||
| ||
(a) You must give any other recipients of the Work or | ||
Derivative Works a copy of this License; and | ||
| ||
(b) You must cause any modified files to carry prominent notices | ||
stating that You changed the files; and | ||
| ||
(c) You must retain, in the Source form of any Derivative Works | ||
that You distribute, all copyright, patent, trademark, and | ||
attribution notices from the Source form of the Work, | ||
excluding those notices that do not pertain to any part of | ||
the Derivative Works; and | ||
| ||
(d) If the Work includes a "NOTICE" text file as part of its | ||
distribution, then any Derivative Works that You distribute must | ||
include a readable copy of the attribution notices contained | ||
within such NOTICE file, excluding those notices that do not | ||
pertain to any part of the Derivative Works, in at least one | ||
of the following places: within a NOTICE text file distributed | ||
as part of the Derivative Works; within the Source form or | ||
documentation, if provided along with the Derivative Works; or, | ||
within a display generated by the Derivative Works, if and | ||
wherever such third-party notices normally appear. The contents | ||
of the NOTICE file are for informational purposes only and | ||
do not modify the License. You may add Your own attribution | ||
notices within Derivative Works that You distribute, alongside | ||
or as an addendum to the NOTICE text from the Work, provided | ||
that such additional attribution notices cannot be construed | ||
as modifying the License. | ||
| ||
You may add Your own copyright statement to Your modifications and | ||
may provide additional or different license terms and conditions | ||
for use, reproduction, or distribution of Your modifications, or | ||
for any such Derivative Works as a whole, provided Your use, | ||
reproduction, and distribution of the Work otherwise complies with | ||
the conditions stated in this License. | ||
| ||
5. Submission of Contributions. Unless You explicitly state otherwise, | ||
any Contribution intentionally submitted for inclusion in the Work | ||
by You to the Licensor shall be under the terms and conditions of | ||
this License, without any additional terms or conditions. | ||
Notwithstanding the above, nothing herein shall supersede or modify | ||
the terms of any separate license agreement you may have executed | ||
with Licensor regarding such Contributions. | ||
| ||
6. Trademarks. This License does not grant permission to use the trade | ||
names, trademarks, service marks, or product names of the Licensor, | ||
except as required for reasonable and customary use in describing the | ||
origin of the Work and reproducing the content of the NOTICE file. | ||
| ||
7. Disclaimer of Warranty. Unless required by applicable law or | ||
agreed to in writing, Licensor provides the Work (and each | ||
Contributor provides its Contributions) on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
implied, including, without limitation, any warranties or conditions | ||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A | ||
PARTICULAR PURPOSE. You are solely responsible for determining the | ||
appropriateness of using or redistributing the Work and assume any | ||
risks associated with Your exercise of permissions under this License. | ||
| ||
8. Limitation of Liability. In no event and under no legal theory, | ||
whether in tort (including negligence), contract, or otherwise, | ||
unless required by applicable law (such as deliberate and grossly | ||
negligent acts) or agreed to in writing, shall any Contributor be | ||
liable to You for damages, including any direct, indirect, special, | ||
incidental, or consequential damages of any character arising as a | ||
result of this License or out of the use or inability to use the | ||
Work (including but not limited to damages for loss of goodwill, | ||
work stoppage, computer failure or malfunction, or any and all | ||
other commercial damages or losses), even if such Contributor | ||
has been advised of the possibility of such damages. | ||
| ||
9. Accepting Warranty or Additional Liability. While redistributing | ||
the Work or Derivative Works thereof, You may choose to offer, | ||
and charge a fee for, acceptance of support, warranty, indemnity, | ||
or other liability obligations and/or rights consistent with this | ||
License. However, in accepting such obligations, You may act only | ||
on Your own behalf and on Your sole responsibility, not on behalf | ||
of any other Contributor, and only if You agree to indemnify, | ||
defend, and hold each Contributor harmless for any liability | ||
incurred by, or claims asserted against, such Contributor by reason | ||
of your accepting any such warranty or additional liability. | ||
| ||
END OF TERMS AND CONDITIONS | ||
| ||
APPENDIX: How to apply the Apache License to your work. | ||
| ||
To apply the Apache License to your work, attach the following | ||
boilerplate notice, with the fields enclosed by brackets "[]" | ||
replaced with your own identifying information. (Don't include | ||
the brackets!) The text should be enclosed in the appropriate | ||
comment syntax for the file format. We also recommend that a | ||
file or class name and description of purpose be included on the | ||
same "printed page" as the copyright notice for easier | ||
identification within third-party archives. | ||
| ||
Copyright [yyyy] [name of copyright owner] | ||
| ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
| ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
| ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
LICENSE-MITnew+21−0
| @@ -0,0 +1,21 @@ | ||
MIT License | ||
| ||
Copyright (c) 2026 knotless contributors | ||
| ||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
| ||
The above copyright notice and this permission notice shall be included in all | ||
copies or substantial portions of the Software. | ||
| ||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
SOFTWARE. | ||
README.mdnew+358−0
| @@ -0,0 +1,358 @@ | ||
# knotless | ||
| ||
Git hosted in your atproto account, on any stock PDS, fast enough for agents. | ||
| ||
A repository is a handful of records in your own repo: no git server, no | ||
knot, nothing installed on the PDS. It moves with your account, rides in | ||
your CAR backups, and anyone can clone it from your PDS directly. Private | ||
repositories live in a permissioned space; collaborators push through a | ||
gateway you delegate to; forks, pulls, issues and reviews are records in | ||
each participant's own account. Agents work through MCP, people read through | ||
a small web interface, and both are served by one binary that is cheap to | ||
run. | ||
| ||
```sh | ||
cargo install --path crates/knotless # git-remote-knotless + knotless | ||
knotless login alice.bsky.social # app password, or KNOTLESS_PASSWORD | ||
| ||
git push knotless://alice.bsky.social/my-project main | ||
git clone knotless://alice.bsky.social/my-project | ||
``` | ||
| ||
## Why it is fast | ||
| ||
Every operation is planned around round trips to the PDS, because on a real | ||
network those are what an agent waits for. | ||
| ||
| Operation | Round trips (warm) | | ||
| --- | --- | | ||
| `git push` | 2: read the refs and newest packs (three reads in parallel), then one `applyWrites` | | ||
| `git fetch` | 1 for pushes under 192 KiB, whose packs arrive inside the listing; plus parallel blob reads for larger ones | | ||
| `knotless commit` (no checkout) | 2, plus 3 per lost race | | ||
| Ref compare-and-swap | 0 extra: a claim record in the same `applyWrites` | | ||
| ||
The identity, session and repository key are cached on disk after the first | ||
call, and every request in a process shares one connection. | ||
| ||
Measured with `cargo run --release --example bench -- --rtt 50`: the npm | ||
reference PDS (`@atproto/pds` 0.5.43, SQLite) on the same machine, behind a | ||
proxy adding 50 ms of round-trip time, on an Apple-silicon laptop. | ||
| ||
| | p50 | p95 | | ||
| --- | --- | --- | | ||
| `git push`, one small commit | 223 ms | 276 ms | | ||
| `git fetch`, one new commit | 177 ms | 211 ms | | ||
| `knotless commit`, no checkout | 366 ms | 596 ms | | ||
| `git clone`, 21 packs | 240 ms | 253 ms | | ||
| `git clone`, compacted | 158 ms | 172 ms | | ||
| `git clone --depth 1` via `knotless-serve` | 208 ms | 299 ms | | ||
| One file via the `knotless-serve` API, warm | 45 ms | 57 ms | | ||
| 16 agents pushing 16 branches at once | 443 ms total | | | ||
| 16 agents committing to one branch at once | 10.9 s total, 97 pushes | | | ||
| ||
A branch is one compare-and-swap, so direct writers to one branch land one | ||
per round, and the reference PDS serializes an account's writes. Give each | ||
agent its own branch and merge, or send commits through a gateway, which | ||
queues them per branch and lands each queue as one push. | ||
| ||
The web pages read a mirror's objects in process and render in about a | ||
millisecond warm; a highlighted 40 KB source file takes a few more. | ||
| ||
## How it works | ||
| ||
Four record types hold a repository (schemas in | ||
[`lexicons/`](lexicons/dev/knotless)): | ||
| ||
- **`dev.knotless.repo`**, keyed by name: the repository's `key` (a TID), | ||
description, default branch, collaborators, the gateway they push | ||
through, and the repository it was forked from. | ||
- **`dev.knotless.ref`**, keyed `<key>.<hash of ref name>`: one per ref, | ||
with its target and a generation counter. | ||
- **`dev.knotless.claim`**, keyed `<key>.<hash>.<generation>`: the lock. | ||
A push creates the claim for the generation it read, in the same | ||
`applyWrites` as the ref update. Two pushes from the same generation both | ||
create the same key, and the PDS accepts only the first, so every ref has | ||
its own compare-and-swap, with no `swapCommit` contention against the rest | ||
of the account. The claims double as the reflog. | ||
- **`dev.knotless.pack`**, keyed `<key>.<tid>`: an immutable, | ||
self-contained git pack, inline as bytes when small and split over blobs | ||
when not. | ||
| ||
Because the keys share a prefix, a `listRecords` from the cursor `<key>/` | ||
walks one repository's records newest first and stops at the first key | ||
outside it. A fetch reads the newest page of packs alongside the refs and | ||
downloads only the packs whose tips it lacks. | ||
| ||
Compaction folds a repository's packs into one once it holds more than 32, | ||
so a clone stays a few reads however long the history. It runs detached | ||
after a push, and keeps packs younger than ten minutes so a reader halfway | ||
through a walk still finds them. | ||
| ||
Four more hold the conversation around it, each in its author's account and | ||
naming the repository by its `at://` URI: **`dev.knotless.pull`**, | ||
**`dev.knotless.issue`**, **`dev.knotless.comment`** (a review, when it | ||
carries a verdict and the head it saw) and **`dev.knotless.state`** (open, | ||
closed or merged). An item's state is the latest one set by someone allowed | ||
to: its author, the owner, or a collaborator, and only the latter two merge. | ||
| ||
[`docs/design.md`](docs/design.md) has the details: the failure cases, the | ||
limits of a stock PDS, and the prior art. | ||
| ||
## Private repositories | ||
| ||
```sh | ||
knotless create knotless://alice.test/secret --private | ||
git push knotless://alice.test/secret main | ||
knotless share knotless://alice.test/secret bob.test # bob may clone | ||
knotless members knotless://alice.test/secret | ||
``` | ||
| ||
A private repository is a permissioned space, | ||
`at://<did>/space/dev.knotless.space/<name>`, holding the same records. Its | ||
members read with a space credential, which needs a full login rather than | ||
an app password, and which is cached until shortly before it expires. Each | ||
member writes their issues and comments into their own repo in the space, | ||
so the conversation stays as private as the code. Spaces are in the | ||
reference PDS's permissioned-data alpha; on a PDS without them, every | ||
repository is public. | ||
| ||
## Collaborating | ||
| ||
Only an account can write to its own repo, so collaborators push through a | ||
gateway: a `knotless-serve` the owner has delegated to. | ||
| ||
```sh | ||
knotless delegate alice.test did:web:git.example.com # once, as the owner | ||
knotless share knotless://alice.test/proj bob.test --write | ||
| ||
# then, as bob, plain git: | ||
git push knotless://alice.test/proj feature | ||
``` | ||
| ||
A collaborator's helper signs each request with a service token from their | ||
own PDS, and the gateway checks it against the repository's collaborators | ||
before writing as the owner. Commits sent to one branch at once are queued | ||
and landed together as one push, rebuilt on the new tip if the branch moved | ||
underneath. | ||
| ||
Anyone can fork and offer changes back: | ||
| ||
```sh | ||
knotless fork knotless://alice.test/proj | ||
git push knotless://bob.test/proj topic | ||
knotless pull knotless://alice.test/proj main knotless://bob.test/proj topic "Fix the parser" -m "Details." | ||
knotless comment knotless://alice.test/proj <pull at-uri> "Looks right" --approve | ||
knotless merge knotless://alice.test/proj <pull at-uri> # fast-forward, or a merge commit | ||
``` | ||
| ||
Pulls and issues on public repositories are listed by an appview | ||
(`KNOTLESS_APPVIEW`), which indexes them from Jetstream or the firehose; on private ones, | ||
from the members' repos in the space. A review reads the pull from its | ||
author's repo directly, so it never waits for an index. | ||
| ||
Moderation is atproto's own. An owner hides what is said about their | ||
repository and blocks accounts from it or from all of theirs; anyone reports to the | ||
appview's moderation service, whose operator takes accounts and records | ||
down, and other appviews can follow its takedowns as labels: | ||
| ||
```sh | ||
knotless hide knotless://alice.test/proj <issue at-uri> | ||
knotless block troll.test knotless://alice.test/proj | ||
knotless report <issue at-uri> spam -m "the same link in every issue" | ||
KNOTLESS_ADMIN_PASSWORD=… knotless admin reports | ||
KNOTLESS_ADMIN_PASSWORD=… knotless admin takedown troll.test | ||
``` | ||
| ||
## For agents | ||
| ||
`knotless-mcp` is an MCP server over stdio. Logged in as the account it acts | ||
for, an agent reads and changes repositories with no clone and no checkout. | ||
| ||
To connect it: | ||
| ||
```sh | ||
cargo install --path crates/knotless-mcp | ||
knotless login you.bsky.social # the account the agent acts as | ||
| ||
claude mcp add knotless -- knotless-mcp # Claude Code | ||
``` | ||
| ||
Other clients, such as Claude Desktop (`claude_desktop_config.json`) and | ||
Cursor (`.cursor/mcp.json`), take the same command in their MCP settings: | ||
| ||
```json | ||
{ "mcpServers": { "knotless": { "command": "knotless-mcp" } } } | ||
``` | ||
| ||
It acts as the account `knotless login` last stored. To pin another, set | ||
`KNOTLESS_IDENTIFIER` in the server's `env`; set `KNOTLESS_APPVIEW` to a | ||
`knotless-serve` index for pulls and issues on public repositories. | ||
| ||
| Tools | | | ||
| --- | --- | | ||
| `list_repos`, `refs`, `list_tree`, `read_file`, `search`, `log`, `diff` | read, by `handle/name` | | ||
| `commit` | change files on a branch, optionally from a base and through the gateway | | ||
| `create_repo`, `rename_repo`, `share`, `members`, `transfer_repo`, `accept_transfer` | manage repositories and access | | ||
| `fork`, `open_pull`, `open_issue`, `list_items`, `show_item`, `comment`, `set_state`, `merge`, `retarget_pull` | collaborate | | ||
| `hide_item`, `block_account`, `report` | moderate your repositories, report to the appview | | ||
| `takedown`, `moderation_status`, `list_reports` | as the appview's operator (`KNOTLESS_ADMIN_PASSWORD`) | | ||
| ||
The same is on the command line: | ||
| ||
```sh | ||
# Change files on a branch without a checkout. If the branch moved, the same | ||
# changes are replayed onto the new tip and pushed again. | ||
knotless commit knotless://alice.test/proj -b main -m "fix typo" \ | ||
--put README.md ./README.md --put-exec run.sh - --delete old.txt --json < run.sh | ||
| ||
knotless refs knotless://alice.test/proj # one round trip | ||
``` | ||
| ||
## The server | ||
| ||
`knotless-serve` is one binary with four jobs, each optional: | ||
| ||
```sh | ||
knotless-serve --listen 0.0.0.0:2590 \ | ||
--jetstream wss://jetstream.us-east.bsky.network \ # keep mirrors warm; feed the index | ||
--index /var/lib/knotless/index.db \ # appview: repositories, pulls, issues | ||
--backfill https://bsky.network \ # index everything already there | ||
--public-url https://git.example.com \ # be a gateway, as did:web:git.example.com | ||
--labeler did:plc:… # honour a labeler's takedowns | ||
``` | ||
| ||
Events come from a Jetstream v2 host or, with `--firehose wss://…`, from a | ||
relay or PDS directly. Jetstream is the cheaper of the two: it sends only | ||
`dev.knotless.*` commits, plus identity and account changes, as small JSON | ||
frames carrying each record, so the server neither downloads and decodes | ||
every commit on the network nor fetches each record from its author's PDS. | ||
Its frames are unsigned, so the index trusts the Jetstream operator for | ||
what a record says; mirrors still sync from the PDS, and `--backfill` | ||
rebuilds the index. | ||
| ||
- **A read accelerator**, a cache and never a source of truth. It speaks git | ||
smart HTTP (protocol v0 and v2, partial and shallow clones), read only, | ||
and an object API: | ||
| ||
```sh | ||
git clone --filter=blob:none http://localhost:2590/alice.test/proj | ||
curl 'localhost:2590/api/alice.test/proj/blob?rev=main&path=src/lib.rs' | ||
curl 'localhost:2590/api/alice.test/proj/tree?rev=main&path=src' | ||
curl 'localhost:2590/api/alice.test/proj/log?rev=main&path=src&limit=20' | ||
curl 'localhost:2590/api/alice.test/proj/diff?base=main&head=feature' | ||
curl 'localhost:2590/api/alice.test/proj/object/<sha>' # immutable, cacheable | ||
``` | ||
| ||
- **A web interface**, read only: what knotless is and how to connect, | ||
repositories and search, each account's profile and public repositories, | ||
trees, files, history, commits, pulls and issues, laid out for a phone as well as a desktop. A repository's page is | ||
at its clone URL. Commits show each file's diff with old and new line | ||
numbers, syntax highlighting, and the changed part of each edited line. | ||
Code is sent as plain text and highlighted with tree-sitter ranges applied | ||
by the CSS Custom Highlight API; links prerender on hover; pages move with | ||
view transitions; anything named by a commit hash is cached as immutable. | ||
Writing is for git, the CLI and agents. | ||
- **An appview**: `dev.knotless.listRepos` and `dev.knotless.getForge`, | ||
from a SQLite index of what Jetstream or the firehose reports. | ||
- **A gateway**: `dev.knotless.pushRefs`, `dev.knotless.createCommit` and | ||
`dev.knotless.putDelegation`, for collaborators. | ||
- **A moderation service**, as an appview with a public URL: | ||
`com.atproto.moderation.createReport` for anyone, | ||
`com.atproto.admin.updateSubjectStatus` and `dev.knotless.listReports` | ||
for its operator (`KNOTLESS_ADMIN_PASSWORD`), and its takedowns as | ||
labels on `com.atproto.label.subscribeLabels`. | ||
| ||
State is a directory of bare mirrors and one SQLite file, so all four fit | ||
on one small VPS. | ||
| ||
## Commands | ||
| ||
| | | | ||
| --- | --- | | ||
| `knotless login <handle> [--password-stdin]` / `logout <handle>` | store or drop a session | | ||
| `knotless create <url> [--private] [--description] [--default-branch]` | create an empty repository | | ||
| `knotless ls <handle>` | list an account's repositories | | ||
| `knotless refs <url>` | print refs | | ||
| `knotless commit <url> -b <branch> -m <msg> ...` | commit without a checkout | | ||
| `knotless delegate <handle> <gateway did>` | let a gateway write for you | | ||
| `knotless share <url> <handle> [--write]` / `unshare` / `members` | access | | ||
| `knotless fork <url> [<name>]` | copy a repository into your account | | ||
| `knotless pull <url> <base> <head url> <branch> <title> [-m <body>]` | open a pull | | ||
| `knotless issue <url> <title> [-m <body>]` | open an issue | | ||
| `knotless list <url> [--pulls \| --issues]` / `show <url> <at-uri>` | read them | | ||
| `knotless comment <url> <at-uri> <text> [--approve \| --changes]` | comment or review | | ||
| `knotless close` / `reopen` / `merge <url> <at-uri>` | change state | | ||
| `knotless retarget <url> <at-uri> [--base <b>] [--head <b>]` | point a pull at other branches | | ||
| `knotless hide` / `unhide <url> <at-uri>` | as owner, hide a pull, issue or comment | | ||
| `knotless block` / `unblock <handle> [<url>]` | keep an account out of your repositories, or out of one | | ||
| `knotless report <handle, url or at-uri> <reason> [-m <text>]` | report to the moderation service | | ||
| `knotless admin takedown <subject> [--undo]` / `status` / `reports` | as the appview's operator | | ||
| `knotless rename <url> <name>` | rename; the old name redirects | | ||
| `knotless transfer <url> <handle or url>` / `--cancel`, `accept <url>` | move to another account | | ||
| `knotless delete <url> --yes` | delete a repository and its history | | ||
| `knotless compact <url>` | fold packs into one | | ||
| `knotless mirror <url>` | sync the cache mirror and print its path | | ||
| ||
Environment: `KNOTLESS_PASSWORD`, `KNOTLESS_IDENTIFIER`, `KNOTLESS_SERVICE` | ||
(skip DID resolution), `KNOTLESS_PLC_URL`, `KNOTLESS_HANDLE_RESOLVER`, | ||
`KNOTLESS_APPVIEW`, `KNOTLESS_MODERATOR` (the appview's), | ||
`KNOTLESS_ADMIN_PASSWORD`, `KNOTLESS_DID_WEB_SCHEME` (`https`), | ||
`KNOTLESS_INLINE_MAX` (192 KiB), `KNOTLESS_CHUNK_SIZE` (4 MB), | ||
`KNOTLESS_COMPACT` (`background`, `inline`, `off`), | ||
`KNOTLESS_COMPACT_THRESHOLD` (32), `KNOTLESS_COMPACT_GRACE_SECS` (600), | ||
`KNOTLESS_CLAIM_GRACE_SECS` (86400), | ||
`KNOTLESS_IDENTITY_TTL_SECS` (3600), `KNOTLESS_TIMEOUT_SECS` (60), | ||
`KNOTLESS_CONFIG_DIR`, `KNOTLESS_CACHE_DIR`. | ||
| ||
## Development | ||
| ||
Tests run against the reference PDS from npm, started by | ||
[`harness/pds.mjs`](harness/pds.mjs), and again against the permissioned-data | ||
alpha build of it for private repositories. Every line and function is | ||
covered across the two runs, and CI fails below that. | ||
| ||
```sh | ||
just setup # npm install in harness/, llvm-tools for coverage | ||
just test # every test | ||
just test-spaces # against the alpha: KNOTLESS_DEV_ENV names its dev-env | ||
just cov # both runs, failing under 100% lines or functions | ||
just lint # rustfmt and clippy | ||
just bench 50 # the table above, at 50 ms RTT | ||
``` | ||
| ||
While working, run only what a change needs: | ||
| ||
```sh | ||
just pds # leave a reference PDS up for test binaries to share | ||
just changed # files changed since tests last passed, and their tests | ||
just t-changed # run those, and mark them passed | ||
just t -p knotless-serve --test web # or any cargo test arguments | ||
``` | ||
| ||
Each test binary otherwise starts a PDS of its own, which takes most of a | ||
short run. On a shared one, tests make accounts as they need them. | ||
| ||
The screenshots ([`harness/shots`](harness/shots)) fill a reference PDS | ||
with the same repositories, pulls and issues each time, with fixed commit | ||
dates and record times, and take every page in light and dark, at desktop | ||
and phone widths, with Playwright's Chromium: | ||
| ||
```sh | ||
just shots-up # seed once, and keep it up for the runs below | ||
just shots # retake the pages whose HTML changed, as the baselines | ||
just shots profile --project=desktop-light # or only some | ||
just shots-check # take every page, failing on any pixel that differs | ||
``` | ||
| ||
Each baseline keeps a digest of the HTML it was taken from, and a page | ||
whose HTML still matches is skipped as unchanged before a browser opens | ||
it. Without `shots-up`, each run seeds a world of its own. Baselines are | ||
kept per platform, under `harness/shots/baseline/<platform>`; DIDs, which | ||
the PLC directory makes new for each world, are masked. | ||
| ||
## Not yet | ||
| ||
- OAuth: logins are app passwords, and reading a private repository takes a | ||
full login. | ||
- Verifying refs against signed repository commits. | ||
- The `dev.knotless` NSIDs assume control of `knotless.dev`. | ||
crates/knotless-mcp/Cargo.tomlnew+24−0
| @@ -0,0 +1,24 @@ | ||
[package] | ||
name = "knotless-mcp" | ||
description = "An MCP server for knotless repositories: read, commit, collaborate" | ||
version.workspace = true | ||
edition.workspace = true | ||
license.workspace = true | ||
rust-version.workspace = true | ||
| ||
[dependencies] | ||
knotless.workspace = true | ||
anyhow.workspace = true | ||
base64.workspace = true | ||
rmcp = { version = "3.6", features = ["server", "transport-io", "macros"] } | ||
schemars = "1" | ||
serde.workspace = true | ||
serde_json.workspace = true | ||
tokio.workspace = true | ||
| ||
[dev-dependencies] | ||
rmcp = { version = "3.6", features = ["client"] } | ||
tempfile.workspace = true | ||
reqwest = { workspace = true, features = ["blocking"] } | ||
knotless-serve = { path = "../knotless-serve" } | ||
axum.workspace = true | ||
crates/knotless-mcp/src/lib.rsnew+866−0
| @@ -0,0 +1,866 @@ | ||
//! An MCP server for knotless repositories. | ||
//! | ||
//! Agents read files, trees, history, diffs and search results straight from | ||
//! a repository's records, with no clone, and commit with no checkout. The | ||
//! same tools create repositories and decide who may read and push them. | ||
//! Everything runs as the account `knotless login` last logged in, or as | ||
//! `KNOTLESS_IDENTIFIER`. | ||
| ||
use knotless::commit::{self, Change, Spec}; | ||
use knotless::env::Ctx; | ||
use knotless::forge; | ||
use knotless::git::Git; | ||
use knotless::manage::{self, NewRepo}; | ||
use knotless::mirror::Mirror; | ||
use knotless::moderation::{self, Admin}; | ||
use knotless::read; | ||
use knotless::remote::Remote; | ||
use knotless::url::RepoUrl; | ||
use rmcp::handler::server::wrapper::Parameters; | ||
use rmcp::model::{CallToolResult, ContentBlock}; | ||
use rmcp::{ErrorData as McpError, ServerHandler, schemars, tool, tool_handler, tool_router}; | ||
use serde::{Deserialize, Serialize}; | ||
use serde_json::{Value, json}; | ||
| ||
#[derive(Clone)] | ||
pub struct Server { | ||
ctx: Ctx, | ||
} | ||
| ||
/// A repository as `handle/name`, `did/name`, or a `knotless://` URL. | ||
fn repo_url(repo: &str) -> anyhow::Result<RepoUrl> { | ||
let repo = repo.trim(); | ||
if repo.starts_with("knotless://") { | ||
RepoUrl::parse(repo) | ||
} else { | ||
RepoUrl::parse(&format!("knotless://{repo}")) | ||
} | ||
} | ||
| ||
/// What a report or takedown is about: an account as `handle` or DID, a | ||
/// repository as `handle/name` or a `knotless://` URL, or a record by its | ||
/// `at://` URI. | ||
fn target(subject: &str) -> String { | ||
let subject = subject.trim(); | ||
match subject.contains('/') && !subject.contains("://") { | ||
true => format!("knotless://{subject}"), | ||
false => subject.to_string(), | ||
} | ||
} | ||
| ||
/// A tool's answer: JSON, or an error the model can read and act on. | ||
fn answer(res: anyhow::Result<Value>) -> Result<CallToolResult, McpError> { | ||
Ok(match res { | ||
Ok(Value::String(text)) => CallToolResult::success(vec![ContentBlock::text(text)]), | ||
Ok(value) => CallToolResult::success(vec![ContentBlock::text(value.to_string())]), | ||
Err(err) => CallToolResult::error(vec![ContentBlock::text(format!("{err:#}"))]), | ||
}) | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Owner { | ||
/// A handle or DID. | ||
pub owner: String, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Repo { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct At { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
/// A branch, tag or commit. Defaults to the default branch. | ||
pub rev: Option<String>, | ||
/// A path in the repository. Defaults to the root. | ||
pub path: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct File { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
pub path: String, | ||
/// A branch, tag or commit. Defaults to the default branch. | ||
pub rev: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Search { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
/// A fixed string, matched case-insensitively. | ||
pub pattern: String, | ||
pub rev: Option<String>, | ||
/// At most this many matches. Defaults to 100. | ||
pub limit: Option<usize>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct History { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
pub rev: Option<String>, | ||
/// Only commits that touch this path. | ||
pub path: Option<String>, | ||
/// At most this many commits. Defaults to 20. | ||
pub limit: Option<usize>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Compare { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
pub base: String, | ||
pub head: String, | ||
} | ||
| ||
#[derive(Deserialize, Serialize, schemars::JsonSchema)] | ||
pub struct FileChange { | ||
pub path: String, | ||
/// The new content, as text. | ||
pub content: Option<String>, | ||
/// The new content, base64, for binary files. | ||
pub base64: Option<String>, | ||
/// Delete the file rather than write it. | ||
#[serde(default)] | ||
pub delete: bool, | ||
#[serde(default)] | ||
pub executable: bool, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct CommitParams { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
pub branch: String, | ||
pub message: String, | ||
pub changes: Vec<FileChange>, | ||
/// The commit these changes were made against. If the branch has moved | ||
/// on and changed a path these touch, the commit fails as a conflict | ||
/// instead of overwriting it. | ||
pub base: Option<String>, | ||
/// As the owner, go through the repository's gateway so concurrent | ||
/// commits to the branch are batched. Collaborators always do. | ||
#[serde(default)] | ||
pub batch: bool, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Create { | ||
/// The new repository's name, in your account. | ||
pub name: String, | ||
/// Readable only by the accounts it is shared with. | ||
#[serde(default)] | ||
pub private: bool, | ||
pub description: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Rename { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
/// The new name. | ||
pub name: String, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Transfer { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
/// The account to offer it to, as `handle` (keeping its name) or | ||
/// `handle/name`. Leave out to withdraw an offer. | ||
pub to: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Hide { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
/// The `at://` URI of a pull, issue or comment on it. | ||
pub uri: String, | ||
/// `false` shows it again. Hides by default. | ||
pub hidden: Option<bool>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Block { | ||
/// The handle or DID to block. | ||
pub account: String, | ||
/// Only from this repository (`handle/name`); from all of yours when | ||
/// left out. | ||
pub repo: Option<String>, | ||
/// `false` unblocks. Blocks by default. | ||
pub blocked: Option<bool>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Report { | ||
/// An account (`handle`), a repository (`handle/name`) or a pull, issue | ||
/// or comment (`at://` URI). | ||
pub subject: String, | ||
/// spam, violation, misleading, sexual, rude or other. | ||
pub reason: String, | ||
/// What the moderators should know. | ||
pub text: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Takedown { | ||
/// An account (`handle`), a repository (`handle/name`) or a pull, issue | ||
/// or comment (`at://` URI). | ||
pub subject: String, | ||
/// `false` puts it back. Takes it down by default. | ||
pub applied: Option<bool>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Moderated { | ||
/// An account (`handle`), a repository (`handle/name`) or a pull, issue | ||
/// or comment (`at://` URI). | ||
pub subject: String, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Reports { | ||
/// How many, newest first. 50 by default. | ||
pub limit: Option<usize>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Share { | ||
/// `handle/name`, `did/name` or `knotless://handle/name`. | ||
pub repo: String, | ||
/// The handle or DID to share with. | ||
pub with: String, | ||
/// Let them push, not just read. Pushes go through the gateway the | ||
/// owner has delegated writes to. | ||
#[serde(default)] | ||
pub write: bool, | ||
/// Take access away instead. | ||
#[serde(default)] | ||
pub revoke: bool, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Fork { | ||
/// The repository to fork: `handle/name`, `did/name` or a `knotless://` URL. | ||
pub repo: String, | ||
/// The fork's name in your account. Defaults to the repository's. | ||
pub name: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct OpenPull { | ||
/// The repository to merge into. | ||
pub repo: String, | ||
/// The branch to merge into. | ||
pub base: String, | ||
/// The repository the branch is in: the same one, or a fork. | ||
pub head_repo: String, | ||
pub head_branch: String, | ||
pub title: String, | ||
pub body: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct OpenIssue { | ||
pub repo: String, | ||
pub title: String, | ||
pub body: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Items { | ||
pub repo: String, | ||
/// `pull` or `issue`. Both when absent. | ||
pub kind: Option<String>, | ||
/// `open`, `closed` or `merged`. Every state when absent. | ||
pub state: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Subject { | ||
pub repo: String, | ||
/// The pull's or issue's at:// URI. | ||
pub uri: String, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Retarget { | ||
pub repo: String, | ||
/// The pull's at:// URI. | ||
pub uri: String, | ||
/// The branch to merge into from now on. | ||
pub base: Option<String>, | ||
/// The branch to merge from, in the same repository as before; the | ||
/// pull's author only. | ||
pub head: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct Remark { | ||
pub repo: String, | ||
/// The pull's or issue's at:// URI. | ||
pub uri: String, | ||
pub body: String, | ||
/// `approve` or `changes`, to review a pull. | ||
pub verdict: Option<String>, | ||
} | ||
| ||
#[derive(Deserialize, schemars::JsonSchema)] | ||
pub struct SetState { | ||
pub repo: String, | ||
/// The pull's or issue's at:// URI. | ||
pub uri: String, | ||
/// `open` or `closed`. | ||
pub state: String, | ||
} | ||
| ||
impl Server { | ||
pub fn new(ctx: Ctx) -> Server { | ||
Server { ctx } | ||
} | ||
| ||
/// A synced mirror of `repo`, to read from. | ||
async fn mirror(&self, repo: &str) -> anyhow::Result<Git> { | ||
let url = repo_url(repo)?; | ||
let (remote, snap) = Remote::locate(&self.ctx, &url).await?; | ||
let Some(key) = snap.key() else { | ||
anyhow::bail!("{url}: no such repository"); | ||
}; | ||
let mirror = Mirror::open(&self.ctx, remote.did(), key).await?; | ||
mirror.sync(&remote, &snap).await?; | ||
Ok(mirror.git) | ||
} | ||
} | ||
| ||
#[tool_router] | ||
impl Server { | ||
#[tool(description = "List an account's public repositories.")] | ||
async fn list_repos( | ||
&self, | ||
Parameters(p): Parameters<Owner>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer(async { | ||
let url = RepoUrl::parse(&format!("knotless://{}/_", p.owner))?; | ||
let remote = Remote::connect(&self.ctx, &url).await?; | ||
let repos = remote.list_repos().await?; | ||
Ok(json!(repos | ||
.into_iter() | ||
.map(|r| json!({"name": r.value.name, "description": r.value.description, "defaultBranch": r.value.default_branch})) | ||
.collect::<Vec<_>>())) | ||
} | ||
.await) | ||
} | ||
| ||
#[tool(description = "A repository's branches and tags, with the commit each points at.")] | ||
async fn refs(&self, Parameters(p): Parameters<Repo>) -> Result<CallToolResult, McpError> { | ||
answer(async { | ||
let url = repo_url(&p.repo)?; | ||
let (remote, snap) = Remote::locate(&self.ctx, &url).await?; | ||
let Some(repo) = &snap.repo else { | ||
anyhow::bail!("{url}: no such repository"); | ||
}; | ||
Ok(json!({ | ||
"defaultBranch": repo.value.default_branch, | ||
"private": remote.space.is_some(), | ||
"refs": snap.live_refs().into_iter().map(|(name, sha)| json!({"name": name, "sha": sha})).collect::<Vec<_>>(), | ||
})) | ||
} | ||
.await) | ||
} | ||
| ||
#[tool(description = "List a directory: names, kinds (blob or tree), modes and sizes.")] | ||
async fn list_tree(&self, Parameters(p): Parameters<At>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let git = self.mirror(&p.repo).await?; | ||
let rev = read::rev(p.rev.as_deref())?; | ||
let value = read::tree(&git, &rev, &read::path(p.path.as_deref())).await?; | ||
Ok(serde_json::to_value(value)?) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "Read a file. Text comes back as is; binary files as base64.")] | ||
async fn read_file(&self, Parameters(p): Parameters<File>) -> Result<CallToolResult, McpError> { | ||
answer(async { | ||
let git = self.mirror(&p.repo).await?; | ||
let rev = read::rev(p.rev.as_deref())?; | ||
let (_, bytes) = read::blob(&git, &rev, &read::path(Some(&p.path))).await?; | ||
Ok(match String::from_utf8(bytes) { | ||
Ok(text) => Value::String(text), | ||
Err(err) => { | ||
use base64::Engine; | ||
json!({"base64": base64::engine::general_purpose::STANDARD.encode(err.into_bytes())}) | ||
} | ||
}) | ||
} | ||
.await) | ||
} | ||
| ||
#[tool( | ||
description = "Search the files at a revision for a fixed string, case-insensitively. Returns path, line number and line." | ||
)] | ||
async fn search(&self, Parameters(p): Parameters<Search>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let git = self.mirror(&p.repo).await?; | ||
let rev = read::rev(p.rev.as_deref())?; | ||
let value = read::grep(&git, &rev, &p.pattern, p.limit.unwrap_or(100)).await?; | ||
Ok(serde_json::to_value(value)?) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "Commit history, newest first, optionally only commits touching a path.")] | ||
async fn log(&self, Parameters(p): Parameters<History>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let git = self.mirror(&p.repo).await?; | ||
let rev = read::rev(p.rev.as_deref())?; | ||
let path = read::path(p.path.as_deref()); | ||
let value = read::log(&git, &rev, &path, p.limit.unwrap_or(20)).await?; | ||
Ok(serde_json::to_value(value)?) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "A unified diff of what head changed since it forked from base (base...head)." | ||
)] | ||
async fn diff(&self, Parameters(p): Parameters<Compare>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let git = self.mirror(&p.repo).await?; | ||
let (base, head) = (read::rev(Some(&p.base))?, read::rev(Some(&p.head))?); | ||
Ok(Value::String(read::diff(&git, &base, &head).await?)) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Commit file changes to a branch without a checkout, creating the branch or repository if needed. If the branch moved meanwhile, the changes are replayed onto its new tip, unless `base` says what they were made against and a path they touch changed since." | ||
)] | ||
async fn commit( | ||
&self, | ||
Parameters(p): Parameters<CommitParams>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
use base64::Engine; | ||
let mut changes = Vec::new(); | ||
for c in p.changes { | ||
changes.push(match (c.delete, c.content, c.base64) { | ||
(true, None, None) => Change::Delete { path: c.path }, | ||
(false, Some(text), None) => Change::Put { | ||
path: c.path, | ||
data: text.into_bytes(), | ||
executable: c.executable, | ||
}, | ||
(false, None, Some(b64)) => Change::Put { | ||
path: c.path, | ||
data: base64::engine::general_purpose::STANDARD.decode(b64)?, | ||
executable: c.executable, | ||
}, | ||
_ => anyhow::bail!( | ||
"{}: give exactly one of content, base64 or delete", | ||
c.path | ||
), | ||
}); | ||
} | ||
let spec = Spec { | ||
branch: p.branch, | ||
message: p.message, | ||
changes, | ||
base: p.base, | ||
retries: 5, | ||
via_gateway: p.batch, | ||
..Default::default() | ||
}; | ||
let c = commit::commit(&self.ctx, &repo_url(&p.repo)?, &spec).await?; | ||
Ok(json!({"sha": c.sha, "parent": c.parent, "changed": c.changed})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "Create a repository in your account, public or private.")] | ||
async fn create_repo( | ||
&self, | ||
Parameters(p): Parameters<Create>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer(async { | ||
let me = knotless::session::me(&self.ctx).await?; | ||
let url = RepoUrl::parse(&format!("knotless://{}/{}", me.did, p.name))?; | ||
let new = NewRepo { | ||
private: p.private, | ||
description: p.description, | ||
default_branch: None, | ||
..Default::default() | ||
}; | ||
manage::create(&self.ctx, &url, &new).await?; | ||
Ok(json!({"repo": format!("{}/{}", me.handle, p.name), "url": format!("knotless://{}/{}", me.handle, p.name)})) | ||
} | ||
.await) | ||
} | ||
| ||
#[tool( | ||
description = "Rename a public repository you own. The old name keeps redirecting to it, and pulls, issues and forks follow it." | ||
)] | ||
async fn rename_repo( | ||
&self, | ||
Parameters(p): Parameters<Rename>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let to = manage::rename(&self.ctx, &repo_url(&p.repo)?, &p.name).await?; | ||
Ok(json!({"url": to.to_string()})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Offer a public repository you own to another account, or withdraw the offer. It moves, with its pulls, issues and forks, when that account accepts." | ||
)] | ||
async fn transfer_repo( | ||
&self, | ||
Parameters(p): Parameters<Transfer>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let url = repo_url(&p.repo)?; | ||
let to = match p.to.as_deref() { | ||
Some(to) if to.contains('/') => Some(repo_url(to)?), | ||
Some(to) => Some(repo_url(&format!("{to}/{}", url.name))?), | ||
None => None, | ||
}; | ||
manage::transfer(&self.ctx, &url, to.as_ref()).await?; | ||
Ok(json!({"offeredTo": to.map(|u| u.to_string())})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Take up a repository offered to your account: it is copied in and moves here, with its pulls, issues and forks." | ||
)] | ||
async fn accept_transfer( | ||
&self, | ||
Parameters(p): Parameters<Repo>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let to = forge::accept(&self.ctx, &repo_url(&p.repo)?).await?; | ||
Ok(json!({"url": to.to_string()})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Share a repository you own: let an account read it (private repositories), push to it, or with revoke, take that back." | ||
)] | ||
async fn share(&self, Parameters(p): Parameters<Share>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let url = repo_url(&p.repo)?; | ||
if p.revoke { | ||
manage::unshare(&self.ctx, &url, &p.with).await?; | ||
} else { | ||
manage::share(&self.ctx, &url, &p.with, p.write).await?; | ||
} | ||
let value = manage::members(&self.ctx, &url).await?; | ||
Ok(serde_json::to_value(value)?) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "Fork a repository into your account, with all its branches and tags.")] | ||
async fn fork(&self, Parameters(p): Parameters<Fork>) -> Result<CallToolResult, McpError> { | ||
answer(async { | ||
let url = forge::fork(&self.ctx, &repo_url(&p.repo)?, p.name.as_deref()).await?; | ||
let me = knotless::session::me(&self.ctx).await?; | ||
Ok(json!({"repo": format!("{}/{}", me.handle, url.name), "url": format!("knotless://{}/{}", me.handle, url.name)})) | ||
} | ||
.await) | ||
} | ||
| ||
#[tool( | ||
description = "Open a pull request: ask to merge head_branch of head_repo (this repository or a fork) into base." | ||
)] | ||
async fn open_pull( | ||
&self, | ||
Parameters(p): Parameters<OpenPull>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let uri = forge::open_pull( | ||
&self.ctx, | ||
&repo_url(&p.repo)?, | ||
&p.base, | ||
&repo_url(&p.head_repo)?, | ||
&p.head_branch, | ||
&p.title, | ||
p.body.as_deref(), | ||
) | ||
.await?; | ||
Ok(json!({"uri": uri})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "Open an issue on a repository.")] | ||
async fn open_issue( | ||
&self, | ||
Parameters(p): Parameters<OpenIssue>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let uri = | ||
forge::open_issue(&self.ctx, &repo_url(&p.repo)?, &p.title, p.body.as_deref()) | ||
.await?; | ||
Ok(json!({"uri": uri})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Pull requests and issues on a repository, newest first, with their state and comment counts." | ||
)] | ||
async fn list_items( | ||
&self, | ||
Parameters(p): Parameters<Items>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let items: Vec<forge::Item> = forge::list(&self.ctx, &repo_url(&p.repo)?) | ||
.await? | ||
.into_iter() | ||
.filter(|i| p.kind.as_ref().is_none_or(|k| *k == i.kind)) | ||
.filter(|i| p.state.as_ref().is_none_or(|s| *s == i.state)) | ||
.collect(); | ||
Ok(serde_json::to_value(items)?) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "A pull request or issue with its comments and reviews.")] | ||
async fn show_item( | ||
&self, | ||
Parameters(p): Parameters<Subject>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let value = forge::show(&self.ctx, &repo_url(&p.repo)?, &p.uri).await?; | ||
Ok(serde_json::to_value(value)?) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Comment on a pull request or issue. With a verdict of approve or changes, review a pull at its current head." | ||
)] | ||
async fn comment(&self, Parameters(p): Parameters<Remark>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let uri = forge::comment( | ||
&self.ctx, | ||
&repo_url(&p.repo)?, | ||
&p.uri, | ||
&p.body, | ||
p.verdict.as_deref(), | ||
) | ||
.await?; | ||
Ok(json!({"uri": uri})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Close or reopen a pull request or issue. Its author, the owner and collaborators may." | ||
)] | ||
async fn set_state( | ||
&self, | ||
Parameters(p): Parameters<SetState>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
forge::set_state(&self.ctx, &repo_url(&p.repo)?, &p.uri, &p.state).await?; | ||
Ok(json!({"state": p.state})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Merge an open pull request: a fast-forward when possible, else a merge commit. Fails on conflicts. The owner and collaborators may." | ||
)] | ||
async fn merge(&self, Parameters(p): Parameters<Subject>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let sha = forge::merge(&self.ctx, &repo_url(&p.repo)?, &p.uri).await?; | ||
Ok(json!({"sha": sha})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Point an open pull request at other branches, as after a branch was renamed. Anyone who may close it can change its target branch; only its author its head branch." | ||
)] | ||
async fn retarget_pull( | ||
&self, | ||
Parameters(p): Parameters<Retarget>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let url = repo_url(&p.repo)?; | ||
forge::retarget( | ||
&self.ctx, | ||
&url, | ||
&p.uri, | ||
p.base.as_deref(), | ||
p.head.as_deref(), | ||
) | ||
.await?; | ||
Ok(json!({"retargeted": p.uri})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "As a repository's owner, hide a pull, issue or comment on it from everyone, with what was said about it, or show it again." | ||
)] | ||
async fn hide_item(&self, Parameters(p): Parameters<Hide>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let hidden = p.hidden.unwrap_or(true); | ||
forge::hide(&self.ctx, &repo_url(&p.repo)?, &p.uri, hidden).await?; | ||
Ok(json!({"uri": p.uri, "hidden": hidden})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Block an account, or unblock it: nothing it writes about your repositories, or about the one repository named, shows." | ||
)] | ||
async fn block_account( | ||
&self, | ||
Parameters(p): Parameters<Block>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let blocked = p.blocked.unwrap_or(true); | ||
let repo = p.repo.as_deref().map(repo_url).transpose()?; | ||
let did = moderation::block(&self.ctx, &p.account, repo.as_ref(), blocked).await?; | ||
Ok(json!({"did": did, "blocked": blocked})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "Report an account, a repository, or a pull, issue or comment to the moderation service." | ||
)] | ||
async fn report(&self, Parameters(p): Parameters<Report>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let subject = target(&p.subject); | ||
moderation::report(&self.ctx, &subject, &p.reason, p.text.as_deref()).await | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool( | ||
description = "As the appview's operator (KNOTLESS_ADMIN_PASSWORD), take an account, repository, pull, issue or comment down, or put it back." | ||
)] | ||
async fn takedown( | ||
&self, | ||
Parameters(p): Parameters<Takedown>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let subject = target(&p.subject); | ||
let applied = p.applied.unwrap_or(true); | ||
Admin::new(&self.ctx)? | ||
.takedown(&self.ctx, &subject, applied) | ||
.await | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "As the appview's operator, whether something is taken down.")] | ||
async fn moderation_status( | ||
&self, | ||
Parameters(p): Parameters<Moderated>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let subject = target(&p.subject); | ||
Admin::new(&self.ctx)?.status(&self.ctx, &subject).await | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "As the appview's operator, the reports it was sent, newest first.")] | ||
async fn list_reports( | ||
&self, | ||
Parameters(p): Parameters<Reports>, | ||
) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let reports = Admin::new(&self.ctx)? | ||
.reports(p.limit.unwrap_or(50)) | ||
.await?; | ||
Ok(json!({"reports": reports})) | ||
} | ||
.await, | ||
) | ||
} | ||
| ||
#[tool(description = "Who may read and push a repository.")] | ||
async fn members(&self, Parameters(p): Parameters<Repo>) -> Result<CallToolResult, McpError> { | ||
answer( | ||
async { | ||
let value = manage::members(&self.ctx, &repo_url(&p.repo)?).await?; | ||
Ok(serde_json::to_value(value)?) | ||
} | ||
.await, | ||
) | ||
} | ||
} | ||
| ||
#[tool_handler( | ||
name = "knotless", | ||
instructions = "Git repositories stored in atproto accounts. Name a repository as handle/name. Read with list_tree, read_file, search, log and diff: no clone needed. Change files with commit: no checkout needed. Collaborate with fork, open_pull, open_issue, list_items, show_item, comment, set_state and merge. Moderate your repositories with hide_item and block_account, and report with report." | ||
)] | ||
impl ServerHandler for Server {} | ||
| ||
/// Serves MCP over stdin and stdout until the client hangs up. | ||
pub async fn serve_stdio(ctx: Ctx) -> anyhow::Result<()> { | ||
use rmcp::ServiceExt; | ||
let running = Server::new(ctx).serve(rmcp::transport::stdio()).await?; | ||
running.waiting().await?; | ||
Ok(()) | ||
} | ||
crates/knotless-mcp/src/main.rsnew+11−0
| @@ -0,0 +1,11 @@ | ||
fn main() -> std::process::ExitCode { | ||
let ctx = knotless::env::Ctx::new(knotless::env::Env::from_process()); | ||
let rt = tokio::runtime::Runtime::new().expect("a runtime builds"); | ||
match rt.block_on(knotless_mcp::serve_stdio(ctx)) { | ||
Ok(()) => std::process::ExitCode::SUCCESS, | ||
Err(err) => { | ||
eprintln!("knotless-mcp: {err:#}"); | ||
std::process::ExitCode::FAILURE | ||
} | ||
} | ||
} | ||
crates/knotless-mcp/tests/mcp.rsnew+517−0
| @@ -0,0 +1,517 @@ | ||
//! The MCP server against a reference PDS, driven by an MCP client. | ||
| ||
#[path = "../../knotless/tests/common/mod.rs"] | ||
mod common; | ||
use common::*; | ||
| ||
use rmcp::ServiceExt; | ||
use rmcp::model::CallToolRequestParams; | ||
use serde_json::{Value, json}; | ||
| ||
struct Client { | ||
rt: tokio::runtime::Runtime, | ||
peer: rmcp::service::RunningService<rmcp::RoleClient, ()>, | ||
} | ||
| ||
impl Client { | ||
fn new(s: &Sandbox) -> Client { | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let ctx = ctx(s); | ||
let peer = rt.block_on(async { | ||
let (server_io, client_io) = tokio::io::duplex(1 << 20); | ||
tokio::spawn(async move { | ||
let running = knotless_mcp::Server::new(ctx) | ||
.serve(server_io) | ||
.await | ||
.unwrap(); | ||
let _ = running.waiting().await; | ||
}); | ||
().serve(client_io).await.unwrap() | ||
}); | ||
Client { rt, peer } | ||
} | ||
| ||
/// The tool's text, and whether it was an error. | ||
fn call(&self, tool: &str, args: Value) -> (String, bool) { | ||
let params = CallToolRequestParams::new(tool.to_string()) | ||
.with_arguments(args.as_object().unwrap().clone()); | ||
let res = self.rt.block_on(self.peer.call_tool(params)).unwrap(); | ||
let text = res.content[0].as_text().unwrap().text.clone(); | ||
(text, res.is_error == Some(true)) | ||
} | ||
| ||
fn ok(&self, tool: &str, args: Value) -> String { | ||
let (text, err) = self.call(tool, args); | ||
assert!(!err, "{tool}: {text}"); | ||
text | ||
} | ||
| ||
fn json(&self, tool: &str, args: Value) -> Value { | ||
serde_json::from_str(&self.ok(tool, args)).unwrap() | ||
} | ||
} | ||
| ||
/// Logged in as its account, as `knotless login` leaves a machine. | ||
fn logged_in() -> Sandbox { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
let password = s.account.password.clone(); | ||
let handle = s.account.handle.clone(); | ||
assert_eq!( | ||
s.cli(&["login", &handle, "--password-stdin"], password.as_bytes()) | ||
.0, | ||
0 | ||
); | ||
s | ||
} | ||
| ||
#[test] | ||
fn an_agent_works_on_a_repository() { | ||
let s = logged_in(); | ||
let c = Client::new(&s); | ||
let tools = c.rt.block_on(c.peer.list_all_tools()).unwrap(); | ||
let names: Vec<&str> = tools.iter().map(|t| t.name.as_ref()).collect(); | ||
for tool in ["read_file", "commit", "search", "create_repo", "share"] { | ||
assert!(names.contains(&tool), "{names:?}"); | ||
} | ||
| ||
let created = c.json( | ||
"create_repo", | ||
json!({"name": "agentic", "description": "made by a tool"}), | ||
); | ||
let repo = created["repo"].as_str().unwrap().to_string(); | ||
assert_eq!(repo, format!("{}/agentic", s.account.handle)); | ||
let (text, err) = c.call("create_repo", json!({"name": "agentic"})); | ||
assert!(err && text.contains("already exists"), "{text}"); | ||
| ||
let first = c.json( | ||
"commit", | ||
json!({"repo": repo, "branch": "main", "message": "start", "changes": [ | ||
{"path": "README.md", "content": "# agentic\n\nhello world\n"}, | ||
{"path": "src/main.rs", "content": "fn main() {}\n"}, | ||
{"path": "bin/run", "content": "#!/bin/sh\n", "executable": true}, | ||
{"path": "logo.bin", "base64": "AAEC/w=="}, | ||
]}), | ||
); | ||
assert_eq!(first["changed"], true); | ||
let second = c.json( | ||
"commit", | ||
json!({"repo": format!("knotless://{repo}"), "branch": "main", "message": "tidy", "changes": [ | ||
{"path": "bin/run", "delete": true}, | ||
{"path": "README.md", "content": "# agentic\n\nhello there\n"}, | ||
], "base": first["sha"]}), | ||
); | ||
let (text, err) = c.call( | ||
"commit", | ||
json!({"repo": repo, "branch": "main", "message": "x", "changes": [{"path": "a", "content": "x", "delete": true}]}), | ||
); | ||
assert!(err && text.contains("exactly one"), "{text}"); | ||
let (text, err) = c.call( | ||
"commit", | ||
json!({"repo": repo, "branch": "main", "message": "x", "changes": [{"path": "a", "base64": "!!"}]}), | ||
); | ||
assert!(err, "{text}"); | ||
| ||
let refs = c.json("refs", json!({"repo": repo})); | ||
assert_eq!(refs["refs"][0]["sha"], second["sha"]); | ||
assert_eq!(refs["private"], false); | ||
let tree = c.json("list_tree", json!({"repo": repo})); | ||
let names: Vec<&str> = tree | ||
.as_array() | ||
.unwrap() | ||
.iter() | ||
.map(|e| e["name"].as_str().unwrap()) | ||
.collect(); | ||
assert_eq!(names, ["README.md", "logo.bin", "src"]); | ||
let sub = c.json( | ||
"list_tree", | ||
json!({"repo": repo, "path": "src", "rev": first["sha"]}), | ||
); | ||
assert_eq!(sub[0]["name"], "main.rs"); | ||
assert_eq!( | ||
c.ok("read_file", json!({"repo": repo, "path": "README.md"})), | ||
"# agentic\n\nhello there\n" | ||
); | ||
assert_eq!( | ||
c.json("read_file", json!({"repo": repo, "path": "logo.bin"}))["base64"], | ||
"AAEC/w==" | ||
); | ||
let hits = c.json("search", json!({"repo": repo, "pattern": "HELLO"})); | ||
assert_eq!(hits[0]["path"], "README.md"); | ||
assert_eq!(hits[0]["line"], 3); | ||
let log = c.json("log", json!({"repo": repo, "limit": 5})); | ||
assert_eq!(log.as_array().unwrap().len(), 2); | ||
assert_eq!(log[0]["message"], "tidy"); | ||
let touched = c.json("log", json!({"repo": repo, "path": "src"})); | ||
assert_eq!(touched.as_array().unwrap().len(), 1); | ||
let diff = c.ok( | ||
"diff", | ||
json!({"repo": repo, "base": first["sha"], "head": "main"}), | ||
); | ||
assert!( | ||
diff.contains("-hello world") && diff.contains("+hello there"), | ||
"{diff}" | ||
); | ||
| ||
let repos = c.json("list_repos", json!({"owner": s.account.handle})); | ||
assert_eq!(repos[0]["name"], "agentic"); | ||
assert_eq!(repos[0]["description"], "made by a tool"); | ||
let members = c.json("members", json!({"repo": repo})); | ||
assert_eq!(members[0]["access"], "owner"); | ||
| ||
// Renamed, it answers to both names. | ||
let renamed = c.json("rename_repo", json!({"repo": repo, "name": "agentic2"})); | ||
let moved = format!("knotless://{}/agentic2", s.account.handle); | ||
assert_eq!(renamed["url"], moved); | ||
assert_eq!(c.json("refs", json!({"repo": repo}))["refs"], refs["refs"]); | ||
assert_eq!(c.json("refs", json!({"repo": moved}))["refs"], refs["refs"]); | ||
| ||
// Errors come back as tool errors, for the agent to read. | ||
for (tool, args) in [ | ||
("read_file", json!({"repo": repo, "path": "nope"})), | ||
( | ||
"read_file", | ||
json!({"repo": repo, "path": "README.md", "rev": "--all"}), | ||
), | ||
("list_tree", json!({"repo": "not a repo"})), | ||
( | ||
"list_tree", | ||
json!({"repo": format!("{}/missing", s.account.handle)}), | ||
), | ||
( | ||
"refs", | ||
json!({"repo": format!("{}/missing", s.account.handle)}), | ||
), | ||
("search", json!({"repo": repo, "pattern": ""})), | ||
("log", json!({"repo": repo, "rev": "-x"})), | ||
("diff", json!({"repo": repo, "base": "-x", "head": "main"})), | ||
("list_repos", json!({"owner": "no-such-handle.test"})), | ||
( | ||
"members", | ||
json!({"repo": format!("{}/missing", s.account.handle)}), | ||
), | ||
("share", json!({"repo": repo, "with": s.account.handle})), | ||
("rename_repo", json!({"repo": "not a repo", "name": "x"})), | ||
] { | ||
let (text, err) = c.call(tool, args.clone()); | ||
assert!(err, "{tool} {args}: {text}"); | ||
} | ||
} | ||
| ||
#[test] | ||
fn sharing_from_an_agent() { | ||
if !net().spaces { | ||
eprintln!("skipped: the PDS is not on the spaces alpha"); | ||
return; | ||
} | ||
let s = logged_in(); | ||
let c = Client::new(&s); | ||
let created = c.json("create_repo", json!({"name": "secret", "private": true})); | ||
let repo = created["repo"].as_str().unwrap().to_string(); | ||
c.json("commit", json!({"repo": repo, "branch": "main", "message": "m", "changes": [{"path": "a", "content": "a"}]})); | ||
assert_eq!(c.json("refs", json!({"repo": repo}))["private"], true); | ||
let other = account(); | ||
let members = c.json("share", json!({"repo": repo, "with": other.handle})); | ||
assert_eq!(members[1], json!({"did": other.did, "access": "read"})); | ||
let members = c.json( | ||
"share", | ||
json!({"repo": repo, "with": other.handle, "revoke": true}), | ||
); | ||
assert_eq!(members.as_array().unwrap().len(), 1); | ||
} | ||
| ||
#[test] | ||
fn nobody_logged_in() { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
let c = Client::new(&s); | ||
let (text, err) = c.call("create_repo", json!({"name": "x"})); | ||
assert!(err && text.contains("not logged in"), "{text}"); | ||
| ||
// An identifier and password in the environment act as that account. | ||
let mut s = Sandbox::new(); | ||
let handle = s.account.handle.clone(); | ||
s.set("KNOTLESS_IDENTIFIER", &handle); | ||
let c = Client::new(&s); | ||
let created = c.json("create_repo", json!({"name": "from-env"})); | ||
assert_eq!(created["repo"], format!("{handle}/from-env")); | ||
} | ||
| ||
#[test] | ||
fn the_binary_speaks_mcp_on_stdio() { | ||
use std::io::{BufRead, Write}; | ||
let s = Sandbox::new(); | ||
let mut child = s | ||
.command(env!("CARGO_BIN_EXE_knotless-mcp"), s.dir.path()) | ||
.stdin(std::process::Stdio::piped()) | ||
.stdout(std::process::Stdio::piped()) | ||
.spawn() | ||
.unwrap(); | ||
let mut stdin = child.stdin.take().unwrap(); | ||
writeln!( | ||
stdin, | ||
"{}", | ||
json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { | ||
"protocolVersion": "2025-06-18", "capabilities": {}, | ||
"clientInfo": {"name": "t", "version": "0"}}}) | ||
) | ||
.unwrap(); | ||
let mut line = String::new(); | ||
std::io::BufReader::new(child.stdout.take().unwrap()) | ||
.read_line(&mut line) | ||
.unwrap(); | ||
let res: Value = serde_json::from_str(&line).unwrap(); | ||
assert_eq!(res["result"]["serverInfo"]["name"], "knotless"); | ||
drop(stdin); | ||
assert!(child.wait().unwrap().success()); | ||
| ||
// A client that leaves before it says hello is an error. | ||
let out = s | ||
.command(env!("CARGO_BIN_EXE_knotless-mcp"), s.dir.path()) | ||
.stdin(std::process::Stdio::null()) | ||
.output() | ||
.unwrap(); | ||
assert!(!out.status.success()); | ||
} | ||
| ||
/// An appview following the PDS, for listing public repositories' pulls | ||
/// and issues. | ||
/// It takes reports too, and its admin password is `hunter2`. | ||
fn appview(rt: &tokio::runtime::Runtime) -> (String, Sandbox) { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_ADMIN_PASSWORD", "hunter2"); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let base = format!("http://localhost:{}", listener.local_addr().unwrap().port()); | ||
let index = knotless_serve::index::Index::open(&s.path("index.db")).unwrap(); | ||
let gateway = knotless_serve::gateway::Gateway::new(&base).unwrap(); | ||
let app = knotless_serve::App::build( | ||
ctx(&s), | ||
std::time::Duration::ZERO, | ||
Some(gateway), | ||
Some(index), | ||
); | ||
let router = knotless_serve::router(app.clone()); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
rt.spawn(knotless_serve::firehose::watch( | ||
app, | ||
net().pds.replace("http://", "ws://"), | ||
)); | ||
(base, s) | ||
} | ||
| ||
#[test] | ||
fn agents_fork_review_and_merge() { | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let (base, _appview) = appview(&rt); | ||
let mut owner = logged_in(); | ||
owner.set("KNOTLESS_APPVIEW", &base); | ||
let mut contrib = logged_in(); | ||
contrib.set("KNOTLESS_APPVIEW", &base); | ||
let (o, c) = (Client::new(&owner), Client::new(&contrib)); | ||
| ||
let repo = c_repo(&o.json("create_repo", json!({"name": "reviewed"}))); | ||
o.json("commit", json!({"repo": repo, "branch": "main", "message": "start", "changes": [{"path": "a", "content": "a\n"}]})); | ||
let fork = c_repo(&c.json("fork", json!({"repo": repo}))); | ||
assert_eq!(fork, format!("{}/reviewed", contrib.account.handle)); | ||
let change = c.json("commit", json!({"repo": fork, "branch": "main", "message": "b", "changes": [{"path": "b", "content": "b\n"}]})); | ||
| ||
let pull = c.json("open_pull", json!({"repo": repo, "base": "main", "head_repo": fork, "head_branch": "main", "title": "Add b", "body": "small"})); | ||
let pull = pull["uri"].as_str().unwrap().to_string(); | ||
let issue = c.json("open_issue", json!({"repo": repo, "title": "Missing c"}))["uri"] | ||
.as_str() | ||
.unwrap() | ||
.to_string(); | ||
let listed = |state: &str| -> Vec<Value> { | ||
o.json( | ||
"list_items", | ||
json!({"repo": repo, "kind": "pull", "state": state}), | ||
) | ||
.as_array() | ||
.unwrap() | ||
.clone() | ||
}; | ||
wait_until(|| listed("open").len() == 1); | ||
wait_until(|| { | ||
o.json("list_items", json!({"repo": repo})) | ||
.as_array() | ||
.unwrap() | ||
.len() | ||
== 2 | ||
}); | ||
| ||
o.json( | ||
"comment", | ||
json!({"repo": repo, "uri": pull, "body": "good", "verdict": "approve"}), | ||
); | ||
let (text, err) = o.call( | ||
"comment", | ||
json!({"repo": repo, "uri": pull, "body": "?", "verdict": "maybe"}), | ||
); | ||
assert!(err, "{text}"); | ||
wait_until(|| { | ||
o.json("show_item", json!({"repo": repo, "uri": pull}))["comments"] | ||
.as_array() | ||
.unwrap() | ||
.len() | ||
== 1 | ||
}); | ||
let shown = o.json("show_item", json!({"repo": repo, "uri": pull})); | ||
assert_eq!(shown["item"]["approvals"], 1); | ||
| ||
// The pull's author points it at the head branch it already names. | ||
assert_eq!( | ||
c.json( | ||
"retarget_pull", | ||
json!({"repo": repo, "uri": pull, "head": "main"}) | ||
)["retargeted"], | ||
pull | ||
); | ||
let (text, err) = o.call( | ||
"retarget_pull", | ||
json!({"repo": repo, "uri": pull, "base": "nope"}), | ||
); | ||
assert!(err, "{text}"); | ||
| ||
assert_eq!( | ||
o.json("merge", json!({"repo": repo, "uri": pull}))["sha"], | ||
change["sha"] | ||
); | ||
wait_until(|| listed("merged").len() == 1); | ||
let (text, err) = o.call("merge", json!({"repo": repo, "uri": pull})); | ||
assert!(err, "{text}"); | ||
| ||
assert_eq!( | ||
o.json( | ||
"set_state", | ||
json!({"repo": repo, "uri": issue, "state": "closed"}) | ||
)["state"], | ||
"closed" | ||
); | ||
let (text, err) = o.call( | ||
"set_state", | ||
json!({"repo": repo, "uri": issue, "state": "merged"}), | ||
); | ||
assert!(err, "{text}"); | ||
// Offered to the contributor, withdrawn, offered again, and taken up. | ||
let them = contrib.account.handle.clone(); | ||
o.json("transfer_repo", json!({"repo": repo, "to": them})); | ||
o.json("transfer_repo", json!({"repo": repo})); | ||
let offered = o.json( | ||
"transfer_repo", | ||
json!({"repo": repo, "to": format!("{them}/kept")}), | ||
); | ||
assert_eq!(offered["offeredTo"], format!("knotless://{them}/kept")); | ||
let moved = c.json("accept_transfer", json!({"repo": repo})); | ||
assert_eq!( | ||
moved["url"], | ||
format!("knotless://{}/kept", contrib.account.did) | ||
); | ||
for (tool, args) in [ | ||
("transfer_repo", json!({"repo": "nobody.test/x", "to": "x"})), | ||
("accept_transfer", json!({"repo": "nobody.test/x"})), | ||
("fork", json!({"repo": "nobody.test/x"})), | ||
( | ||
"open_pull", | ||
json!({"repo": repo, "base": "main", "head_repo": fork, "head_branch": "none", "title": "t"}), | ||
), | ||
("open_issue", json!({"repo": "nobody.test/x", "title": "t"})), | ||
("list_items", json!({"repo": "nobody.test/x"})), | ||
( | ||
"show_item", | ||
json!({"repo": repo, "uri": "at://did:plc:nobody/dev.knotless.issue/x"}), | ||
), | ||
] { | ||
let (text, err) = o.call(tool, args.clone()); | ||
assert!(err, "{tool} {args}: {text}"); | ||
} | ||
} | ||
| ||
#[test] | ||
fn agents_moderate() { | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let (base, _appview) = appview(&rt); | ||
let mut owner = logged_in(); | ||
owner.set("KNOTLESS_APPVIEW", &base); | ||
let mut operator = logged_in(); | ||
operator.set("KNOTLESS_APPVIEW", &base); | ||
operator.set("KNOTLESS_ADMIN_PASSWORD", "hunter2"); | ||
let (o, c) = (Client::new(&owner), Client::new(&operator)); | ||
let repo = c_repo(&o.json("create_repo", json!({"name": "moderated"}))); | ||
o.json("commit", json!({"repo": repo, "branch": "main", "message": "start", "changes": [{"path": "a", "content": "a\n"}]})); | ||
let issue = c.json("open_issue", json!({"repo": repo, "title": "Spam"}))["uri"] | ||
.as_str() | ||
.unwrap() | ||
.to_string(); | ||
let items = || { | ||
o.json("list_items", json!({"repo": repo})) | ||
.as_array() | ||
.unwrap() | ||
.len() | ||
}; | ||
wait_until(|| items() == 1); | ||
| ||
let hidden = o.json("hide_item", json!({"repo": repo, "uri": issue})); | ||
assert_eq!(hidden["hidden"], true); | ||
wait_until(|| items() == 0); | ||
o.json( | ||
"hide_item", | ||
json!({"repo": repo, "uri": issue, "hidden": false}), | ||
); | ||
wait_until(|| items() == 1); | ||
let them = operator.account.handle.clone(); | ||
assert_eq!( | ||
o.json("block_account", json!({"account": them}))["blocked"], | ||
true | ||
); | ||
wait_until(|| items() == 0); | ||
o.json("block_account", json!({"account": them, "blocked": false})); | ||
wait_until(|| items() == 1); | ||
o.json("block_account", json!({"account": them, "repo": repo})); | ||
wait_until(|| items() == 0); | ||
o.json( | ||
"block_account", | ||
json!({"account": them, "repo": repo, "blocked": false}), | ||
); | ||
wait_until(|| items() == 1); | ||
| ||
let report = c.json( | ||
"report", | ||
json!({"subject": repo, "reason": "spam", "text": "all of it"}), | ||
); | ||
assert_eq!(report["id"], 1); | ||
assert_eq!( | ||
report["subject"]["uri"], | ||
format!("at://{}/dev.knotless.repo/moderated", owner.account.did) | ||
); | ||
c.json( | ||
"report", | ||
json!({"subject": owner.account.handle, "reason": "rude"}), | ||
); | ||
let reports = c.json("list_reports", json!({})); | ||
assert_eq!(reports["reports"].as_array().unwrap().len(), 2); | ||
let down = c.json("takedown", json!({"subject": issue})); | ||
assert_eq!(down["takedown"]["applied"], true); | ||
wait_until(|| items() == 0); | ||
let status = c.json("moderation_status", json!({"subject": issue})); | ||
assert_eq!(status["takedown"]["applied"], true); | ||
c.json("takedown", json!({"subject": issue, "applied": false})); | ||
wait_until(|| items() == 1); | ||
| ||
for (who, tool, args) in [ | ||
(&c, "hide_item", json!({"repo": repo, "uri": issue})), | ||
(&o, "block_account", json!({"account": "nobody.test"})), | ||
(&c, "report", json!({"subject": repo, "reason": "boring"})), | ||
(&o, "takedown", json!({"subject": issue})), | ||
(&o, "moderation_status", json!({"subject": issue})), | ||
(&o, "list_reports", json!({})), | ||
] { | ||
let (text, err) = who.call(tool, args.clone()); | ||
assert!(err, "{tool} {args}: {text}"); | ||
} | ||
} | ||
| ||
fn c_repo(v: &Value) -> String { | ||
v["repo"].as_str().unwrap().to_string() | ||
} | ||
crates/knotless-serve/Cargo.tomlnew+52−0
| @@ -0,0 +1,52 @@ | ||
[package] | ||
name = "knotless-serve" | ||
description = "A read accelerator for knotless repositories: smart HTTP, an object API, and a firehose-warmed cache" | ||
version.workspace = true | ||
edition.workspace = true | ||
license.workspace = true | ||
rust-version.workspace = true | ||
| ||
[dependencies] | ||
knotless.workspace = true | ||
anyhow.workspace = true | ||
axum = { workspace = true, features = ["ws"] } | ||
ciborium.workspace = true | ||
rusqlite = { version = "0.40", features = ["bundled"] } | ||
flate2.workspace = true | ||
futures.workspace = true | ||
serde.workspace = true | ||
serde_json.workspace = true | ||
tokio.workspace = true | ||
tokio-tungstenite.workspace = true | ||
tokio-util.workspace = true | ||
pulldown-cmark = { version = "0.13.4", default-features = false, features = ["html"] } | ||
tower-http = { version = "0.7.1", features = ["compression-gzip", "compression-br"] } | ||
sha2.workspace = true | ||
base64.workspace = true | ||
gix = { version = "0.89", default-features = false, features = ["sha1", "max-performance-safe", "revision"] } | ||
tree-sitter = "0.27" | ||
tree-sitter-highlight = "0.27" | ||
tree-sitter-rust = "0.24.2" | ||
tree-sitter-javascript = "0.25.0" | ||
tree-sitter-typescript = "0.23.2" | ||
tree-sitter-python = "0.25.0" | ||
tree-sitter-go = "0.25.0" | ||
tree-sitter-c = "0.24.2" | ||
tree-sitter-cpp = "0.23.4" | ||
tree-sitter-java = "0.23.5" | ||
tree-sitter-ruby = "0.23.1" | ||
tree-sitter-bash = "0.25.1" | ||
tree-sitter-json = "0.24.8" | ||
tree-sitter-toml-ng = "0.7.0" | ||
tree-sitter-yaml = "0.7.2" | ||
tree-sitter-html = "0.23.2" | ||
tree-sitter-css = "0.25.0" | ||
tree-sitter-lua = "0.5.0" | ||
tree-sitter-zig = "1.1.2" | ||
tree-sitter-language = "0.1.7" | ||
| ||
[dev-dependencies] | ||
futures.workspace = true | ||
reqwest = { workspace = true, features = ["blocking"] } | ||
flate2.workspace = true | ||
tempfile.workspace = true | ||
crates/knotless-serve/src/api.rsnew+232−0
| @@ -0,0 +1,232 @@ | ||
//! Files, trees, history and diffs without a clone. | ||
| ||
use crate::{App, AppError, repo_name}; | ||
use axum::Json; | ||
use axum::extract::{Path, Query, State}; | ||
use axum::http::{StatusCode, header}; | ||
use axum::response::{IntoResponse, Response}; | ||
use knotless::git::Git; | ||
use serde::Deserialize; | ||
use serde_json::{Value, json}; | ||
use std::sync::Arc; | ||
| ||
#[derive(Deserialize, Default)] | ||
pub struct Params { | ||
rev: Option<String>, | ||
path: Option<String>, | ||
limit: Option<usize>, | ||
base: Option<String>, | ||
head: Option<String>, | ||
} | ||
| ||
type Repo = Path<(String, String)>; | ||
| ||
async fn open(app: &App, (authority, repo): (String, String)) -> Result<Git, AppError> { | ||
app.mirror(&authority, repo_name(&repo)).await | ||
} | ||
| ||
/// A revision from the query, refusing anything git could read as an option. | ||
fn rev(value: Option<&str>) -> Result<String, AppError> { | ||
let rev = value.unwrap_or("HEAD"); | ||
if rev.is_empty() || rev.starts_with('-') || rev.contains(['\0', '\n', ' ']) { | ||
return Err(AppError( | ||
StatusCode::BAD_REQUEST, | ||
format!("bad revision {rev:?}"), | ||
)); | ||
} | ||
Ok(rev.to_string()) | ||
} | ||
| ||
fn path(value: Option<&str>) -> String { | ||
value.unwrap_or_default().trim_matches('/').to_string() | ||
} | ||
| ||
async fn git(git: &Git, args: &[&str]) -> Result<Vec<u8>, AppError> { | ||
git.run(args, None) | ||
.await | ||
.map_err(|e| AppError(StatusCode::NOT_FOUND, format!("{e:#}"))) | ||
} | ||
| ||
/// Caching for a response about `rev`: forever when it names a commit by | ||
/// its hash, briefly when it names a ref that can move. | ||
fn cache_for(rev: &str) -> &'static str { | ||
if rev.len() == 40 && rev.bytes().all(|b| b.is_ascii_hexdigit()) { | ||
"public, max-age=31536000, immutable" | ||
} else { | ||
"no-cache" | ||
} | ||
} | ||
| ||
pub async fn refs(State(app): State<Arc<App>>, Path(repo): Repo) -> Result<Json<Value>, AppError> { | ||
let g = open(&app, repo).await?; | ||
let refs: Vec<Value> = g | ||
.refs() | ||
.await | ||
.map_err(AppError::internal)? | ||
.into_iter() | ||
.map(|(name, target)| json!({"name": name, "target": target})) | ||
.collect(); | ||
let head = g.text(&["symbolic-ref", "-q", "HEAD"], None).await.ok(); | ||
Ok(Json(json!({"refs": refs, "head": head}))) | ||
} | ||
| ||
pub async fn tree( | ||
State(app): State<Arc<App>>, | ||
Path(repo): Repo, | ||
Query(q): Query<Params>, | ||
) -> Result<Response, AppError> { | ||
let g = open(&app, repo).await?; | ||
let rev = rev(q.rev.as_deref())?; | ||
let path = path(q.path.as_deref()); | ||
let out = git(&g, &["ls-tree", "-l", "-z", &format!("{rev}:{path}")]).await?; | ||
let entries: Vec<Value> = String::from_utf8_lossy(&out) | ||
.split('\0') | ||
.filter_map(|line| { | ||
let (meta, name) = line.split_once('\t')?; | ||
let mut f = meta.split_whitespace(); | ||
let (mode, kind, sha, size) = (f.next()?, f.next()?, f.next()?, f.next()?); | ||
Some(json!({"name": name, "mode": mode, "type": kind, "sha": sha, "size": size.parse::<u64>().ok()})) | ||
}) | ||
.collect(); | ||
Ok(( | ||
[(header::CACHE_CONTROL, cache_for(&rev))], | ||
Json(json!({"rev": rev, "path": path, "entries": entries})), | ||
) | ||
.into_response()) | ||
} | ||
| ||
pub async fn blob( | ||
State(app): State<Arc<App>>, | ||
Path(repo): Repo, | ||
Query(q): Query<Params>, | ||
) -> Result<Response, AppError> { | ||
let g = open(&app, repo).await?; | ||
let rev = rev(q.rev.as_deref())?; | ||
let spec = format!("{rev}:{}", path(q.path.as_deref())); | ||
let sha = String::from_utf8_lossy(&git(&g, &["rev-parse", "--verify", &spec]).await?) | ||
.trim() | ||
.to_string(); | ||
let bytes = git(&g, &["cat-file", "blob", &sha]).await?; | ||
Ok(( | ||
[ | ||
(header::CONTENT_TYPE, "application/octet-stream".to_string()), | ||
(header::ETAG, format!("\"{sha}\"")), | ||
(header::CACHE_CONTROL, cache_for(&rev).to_string()), | ||
], | ||
bytes, | ||
) | ||
.into_response()) | ||
} | ||
| ||
pub async fn log( | ||
State(app): State<Arc<App>>, | ||
Path(repo): Repo, | ||
Query(q): Query<Params>, | ||
) -> Result<Json<Value>, AppError> { | ||
let g = open(&app, repo).await?; | ||
let rev = rev(q.rev.as_deref())?; | ||
let limit = q.limit.unwrap_or(50).min(1000).to_string(); | ||
let mut args = vec![ | ||
"log", | ||
"-z", | ||
"--format=%H%x1f%P%x1f%an%x1f%ae%x1f%aI%x1f%B", | ||
"-n", | ||
&limit, | ||
&rev, | ||
"--", | ||
]; | ||
let path = path(q.path.as_deref()); | ||
if !path.is_empty() { | ||
args.push(&path); | ||
} | ||
let out = git(&g, &args).await?; | ||
let commits: Vec<Value> = String::from_utf8_lossy(&out) | ||
.split('\0') | ||
.filter(|s| !s.is_empty()) | ||
.map(|record| { | ||
let mut f = record.splitn(6, '\x1f'); | ||
let mut field = || f.next().unwrap_or_default(); | ||
let (sha, parents, name, email, date, message) = | ||
(field(), field(), field(), field(), field(), field()); | ||
json!({ | ||
"sha": sha, | ||
"parents": parents.split_whitespace().collect::<Vec<_>>(), | ||
"author": {"name": name, "email": email, "date": date}, | ||
"message": message.trim_end(), | ||
}) | ||
}) | ||
.collect(); | ||
Ok(Json(json!({"rev": rev, "commits": commits}))) | ||
} | ||
| ||
pub async fn diff( | ||
State(app): State<Arc<App>>, | ||
Path(repo): Repo, | ||
Query(q): Query<Params>, | ||
) -> Result<Response, AppError> { | ||
let g = open(&app, repo).await?; | ||
let base = rev(q.base.as_deref())?; | ||
let head = rev(q.head.as_deref())?; | ||
let out = git( | ||
&g, | ||
&[ | ||
"diff", | ||
"--no-color", | ||
"--no-ext-diff", | ||
&format!("{base}...{head}"), | ||
"--", | ||
], | ||
) | ||
.await?; | ||
Ok(([(header::CONTENT_TYPE, "text/x-diff; charset=utf-8")], out).into_response()) | ||
} | ||
| ||
pub async fn object( | ||
State(app): State<Arc<App>>, | ||
Path((authority, repo, sha)): Path<(String, String, String)>, | ||
) -> Result<Response, AppError> { | ||
if !(sha.len() == 40 || sha.len() == 64) || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { | ||
return Err(AppError( | ||
StatusCode::BAD_REQUEST, | ||
format!("not an object id: {sha}"), | ||
)); | ||
} | ||
let g = open(&app, (authority, repo)).await?; | ||
let kind = String::from_utf8_lossy(&git(&g, &["cat-file", "-t", &sha]).await?) | ||
.trim() | ||
.to_string(); | ||
let bytes = git(&g, &["cat-file", &kind, &sha]).await?; | ||
Ok(( | ||
[ | ||
(header::CONTENT_TYPE, "application/octet-stream".to_string()), | ||
(header::HeaderName::from_static("x-git-type"), kind), | ||
(header::ETAG, format!("\"{sha}\"")), | ||
( | ||
header::CACHE_CONTROL, | ||
"public, max-age=31536000, immutable".to_string(), | ||
), | ||
], | ||
bytes, | ||
) | ||
.into_response()) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn revisions_and_caching() { | ||
assert_eq!(rev(None).unwrap(), "HEAD"); | ||
assert!(rev(Some("--output=/x")).is_err()); | ||
assert!(rev(Some("")).is_err()); | ||
assert!(rev(Some("a b")).is_err()); | ||
assert_eq!(path(Some("/src/")), "src"); | ||
assert_eq!( | ||
cache_for(&"a".repeat(40)), | ||
"public, max-age=31536000, immutable" | ||
); | ||
assert_eq!(cache_for("main"), "no-cache"); | ||
let _ = Params::default(); | ||
} | ||
} | ||
crates/knotless-serve/src/appview.rsnew+66−0
| @@ -0,0 +1,66 @@ | ||
//! What the appview answers from its index. | ||
| ||
use crate::{App, AppError}; | ||
use axum::Json; | ||
use axum::extract::{Query, State}; | ||
use axum::http::StatusCode; | ||
use serde::Deserialize; | ||
use serde_json::{Value, json}; | ||
use std::sync::Arc; | ||
| ||
fn index(app: &App) -> Result<&crate::index::Index, AppError> { | ||
app.index | ||
.as_deref() | ||
.ok_or_else(|| AppError(StatusCode::NOT_FOUND, "this server keeps no index".into())) | ||
} | ||
| ||
#[derive(Deserialize)] | ||
pub struct ForgeParams { | ||
repo: String, | ||
key: String, | ||
} | ||
| ||
/// Every pull, issue, comment and state change about a repository: the | ||
/// one its owner made with `key`, whatever it has been named. | ||
pub async fn get_forge( | ||
State(app): State<Arc<App>>, | ||
Query(p): Query<ForgeParams>, | ||
) -> Result<Json<Value>, AppError> { | ||
let did = knotless::forge::parse_uri(&p.repo) | ||
.map_err(|e| AppError(StatusCode::BAD_REQUEST, e.to_string()))? | ||
.0; | ||
let entries = index(&app)? | ||
.forge(did, &p.key) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
Ok(Json(json!({ "entries": entries }))) | ||
} | ||
| ||
#[derive(Deserialize)] | ||
pub struct ListParams { | ||
q: Option<String>, | ||
limit: Option<usize>, | ||
cursor: Option<String>, | ||
} | ||
| ||
/// Repositories, newest first, matching `q` by name or description, a | ||
/// page at a time. | ||
pub async fn list_repos( | ||
State(app): State<Arc<App>>, | ||
Query(p): Query<ListParams>, | ||
) -> Result<Json<Value>, AppError> { | ||
let limit = p.limit.unwrap_or(50).clamp(1, 500); | ||
let (repos, cursor) = index(&app)? | ||
.repos_page( | ||
p.q.as_deref().unwrap_or_default(), | ||
limit, | ||
p.cursor.as_deref(), | ||
) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
let mut out = json!({ "repos": repos }); | ||
if let Some(cursor) = cursor { | ||
out["cursor"] = cursor.into(); | ||
} | ||
Ok(Json(out)) | ||
} | ||
crates/knotless-serve/src/diff.rsnew+257−0
| @@ -0,0 +1,257 @@ | ||
//! A patch read into files and rows, so a page can show each line with its | ||
//! old and new numbers rather than as the text git prints. | ||
| ||
/// One file's part of a patch. | ||
#[derive(Debug, Default, PartialEq)] | ||
pub struct File { | ||
/// Its path before, and after; the same unless it moved. | ||
pub old: String, | ||
pub new: String, | ||
pub status: Status, | ||
pub binary: bool, | ||
pub rows: Vec<Row>, | ||
pub added: usize, | ||
pub removed: usize, | ||
} | ||
| ||
#[derive(Debug, Default, Clone, Copy, PartialEq)] | ||
pub enum Status { | ||
Added, | ||
Deleted, | ||
Renamed, | ||
#[default] | ||
Modified, | ||
} | ||
| ||
#[derive(Debug, PartialEq)] | ||
pub struct Row { | ||
pub kind: Kind, | ||
pub old: Option<u32>, | ||
pub new: Option<u32>, | ||
/// The line without its `+`, `-` or space; a hunk's or note's whole line. | ||
pub text: String, | ||
} | ||
| ||
#[derive(Debug, Clone, Copy, PartialEq)] | ||
pub enum Kind { | ||
Hunk, | ||
Context, | ||
Add, | ||
Del, | ||
/// `\ No newline at end of file`. | ||
Note, | ||
} | ||
| ||
impl Kind { | ||
/// Whether the row is a line of the file, rather than about it. | ||
pub fn is_line(self) -> bool { | ||
matches!(self, Kind::Context | Kind::Add | Kind::Del) | ||
} | ||
} | ||
| ||
/// `-12,3` or `+12`: where a hunk's side starts, and how many lines it has. | ||
fn range(s: &str) -> Option<(u32, u32)> { | ||
let (start, len) = s[1..].split_once(',').unwrap_or((&s[1..], "1")); | ||
Some((start.parse().ok()?, len.parse().ok()?)) | ||
} | ||
| ||
fn path(s: &str) -> String { | ||
s.split_once('/') | ||
.map_or(s, |(_, p)| p) | ||
.trim_end_matches('\t') | ||
.to_string() | ||
} | ||
| ||
/// Reads `git show --patch` or `git diff` output. A hunk's lines are counted | ||
/// from its header, so a removed line that reads `-- x` is not taken for a | ||
/// file header. | ||
pub fn parse(patch: &str) -> Vec<File> { | ||
let mut files: Vec<File> = Vec::new(); | ||
// Lines left in the current hunk, old and new. | ||
let (mut old_left, mut new_left) = (0u32, 0u32); | ||
let (mut old_at, mut new_at) = (0u32, 0u32); | ||
for line in patch.lines() { | ||
if old_left > 0 || new_left > 0 { | ||
let Some(file) = files.last_mut() else { break }; | ||
let (kind, text) = match line.split_at_checked(1).unwrap_or((" ", "")) { | ||
("+", text) => (Kind::Add, text), | ||
("-", text) => (Kind::Del, text), | ||
("\\", _) => (Kind::Note, line), | ||
(_, text) => (Kind::Context, text), | ||
}; | ||
let (old, new) = match kind { | ||
Kind::Add => (None, Some(new_at)), | ||
Kind::Del => (Some(old_at), None), | ||
Kind::Context => (Some(old_at), Some(new_at)), | ||
_ => (None, None), | ||
}; | ||
if old.is_some() { | ||
(old_at, old_left) = (old_at + 1, old_left.saturating_sub(1)); | ||
} | ||
if new.is_some() { | ||
(new_at, new_left) = (new_at + 1, new_left.saturating_sub(1)); | ||
} | ||
file.added += (kind == Kind::Add) as usize; | ||
file.removed += (kind == Kind::Del) as usize; | ||
file.rows.push(Row { | ||
kind, | ||
old, | ||
new, | ||
text: text.to_string(), | ||
}); | ||
continue; | ||
} | ||
if let Some(names) = line.strip_prefix("diff --git ") { | ||
let (a, b) = names.split_once(" b/").unwrap_or((names, names)); | ||
files.push(File { | ||
old: path(a), | ||
new: b.to_string(), | ||
..File::default() | ||
}); | ||
continue; | ||
} | ||
let Some(file) = files.last_mut() else { | ||
continue; | ||
}; | ||
if line.starts_with("new file mode") { | ||
file.status = Status::Added; | ||
} else if line.starts_with("deleted file mode") { | ||
file.status = Status::Deleted; | ||
} else if let Some(p) = line.strip_prefix("rename from ") { | ||
(file.old, file.status) = (p.to_string(), Status::Renamed); | ||
} else if let Some(p) = line.strip_prefix("rename to ") { | ||
file.new = p.to_string(); | ||
} else if line.starts_with("Binary files ") { | ||
file.binary = true; | ||
} else if let Some(p) = line.strip_prefix("--- ").filter(|p| *p != "/dev/null") { | ||
file.old = path(p); | ||
} else if let Some(p) = line.strip_prefix("+++ ").filter(|p| *p != "/dev/null") { | ||
file.new = path(p); | ||
} else if line.starts_with("@@ ") { | ||
let mut sides = line.split(' ').skip(1); | ||
let (Some((o, ol)), Some((n, nl))) = | ||
(sides.next().and_then(range), sides.next().and_then(range)) | ||
else { | ||
continue; | ||
}; | ||
(old_at, old_left, new_at, new_left) = (o, ol, n, nl); | ||
file.rows.push(Row { | ||
kind: Kind::Hunk, | ||
old: None, | ||
new: None, | ||
text: line.to_string(), | ||
}); | ||
} else if line.starts_with('\\') { | ||
file.rows.push(Row { | ||
kind: Kind::Note, | ||
old: None, | ||
new: None, | ||
text: line.to_string(), | ||
}); | ||
} | ||
} | ||
files | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
fn shown(f: &File) -> Vec<String> { | ||
f.rows | ||
.iter() | ||
.map(|r| { | ||
let n = |x: Option<u32>| x.map(|x| x.to_string()).unwrap_or_default(); | ||
format!("{:?} {} {} {}", r.kind, n(r.old), n(r.new), r.text) | ||
}) | ||
.collect() | ||
} | ||
| ||
#[test] | ||
fn reads_a_patch() { | ||
let patch = "\ | ||
diff --git a/src/a.rs b/src/a.rs | ||
index 1..2 100644 | ||
--- a/src/a.rs | ||
+++ b/src/a.rs | ||
@@ -1,3 +1,3 @@ fn a() | ||
keep | ||
--- not a header | ||
+++ nor this | ||
end | ||
@@ -9 +9,2 @@ | ||
-x | ||
\\ No newline at end of file | ||
+x | ||
+y | ||
diff --git a/new file.txt b/new file.txt | ||
new file mode 100644 | ||
--- /dev/null | ||
+++ b/new file.txt | ||
@@ -0,0 +1 @@ | ||
+hi | ||
diff --git a/gone b/gone | ||
deleted file mode 100644 | ||
--- a/gone | ||
+++ /dev/null | ||
@@ -1 +0,0 @@ | ||
-bye | ||
diff --git a/old.png b/new.png | ||
similarity index 90% | ||
rename from old.png | ||
rename to new.png | ||
Binary files a/old.png and b/new.png differ | ||
diff --git a/run b/run | ||
old mode 100644 | ||
new mode 100755 | ||
@@ broken @@ | ||
\\ stray | ||
"; | ||
let files = parse(patch); | ||
assert_eq!(files.len(), 5); | ||
let a = &files[0]; | ||
assert_eq!( | ||
(a.old.as_str(), a.new.as_str(), a.status), | ||
("src/a.rs", "src/a.rs", Status::Modified) | ||
); | ||
assert_eq!((a.added, a.removed), (3, 2)); | ||
assert_eq!( | ||
shown(a), | ||
[ | ||
"Hunk @@ -1,3 +1,3 @@ fn a()", | ||
"Context 1 1 keep", | ||
"Del 2 -- not a header", | ||
"Add 2 ++ nor this", | ||
"Context 3 3 end", | ||
"Hunk @@ -9 +9,2 @@", | ||
"Del 9 x", | ||
"Note \\ No newline at end of file", | ||
"Add 9 x", | ||
"Add 10 y", | ||
] | ||
); | ||
assert_eq!( | ||
(files[1].new.as_str(), files[1].status, files[1].added), | ||
("new file.txt", Status::Added, 1) | ||
); | ||
assert_eq!( | ||
(files[2].old.as_str(), files[2].status, files[2].removed), | ||
("gone", Status::Deleted, 1) | ||
); | ||
let moved = &files[3]; | ||
assert_eq!( | ||
( | ||
moved.old.as_str(), | ||
moved.new.as_str(), | ||
moved.status, | ||
moved.binary | ||
), | ||
("old.png", "new.png", Status::Renamed, true) | ||
); | ||
assert_eq!(shown(&files[4]), ["Note \\ stray"]); | ||
assert!(Kind::Add.is_line() && !Kind::Hunk.is_line()); | ||
assert!(parse("index 1..2\n@@ -1 +1 @@\n x\n").is_empty()); | ||
assert_eq!(range("-3"), Some((3, 1))); | ||
assert_eq!(range("+a,1"), None); | ||
} | ||
} | ||
crates/knotless-serve/src/firehose.rsnew+212−0
| @@ -0,0 +1,212 @@ | ||
//! Keeps mirrors warm from `com.atproto.sync.subscribeRepos`: a commit that | ||
//! touches a knotless collection marks its account's mirrors stale and | ||
//! refreshes them, so the next clone needs no round trip to the PDS. | ||
| ||
use crate::App; | ||
use futures::StreamExt; | ||
use knotless::records::nsid; | ||
use std::sync::Arc; | ||
use std::sync::atomic::Ordering; | ||
use std::time::Duration; | ||
use tokio_tungstenite::tungstenite::Message; | ||
| ||
/// What a firehose frame says happened. | ||
#[derive(Debug, Default, PartialEq)] | ||
pub struct Event { | ||
pub seq: Option<i64>, | ||
/// The account, when it wrote to a knotless collection or its identity | ||
/// or repo changed under it. | ||
pub did: Option<String>, | ||
/// Writes to the knotless collections. | ||
pub ops: Vec<Op>, | ||
} | ||
| ||
#[derive(Debug, PartialEq)] | ||
pub struct Op { | ||
/// `create`, `update` or `delete`. | ||
pub action: String, | ||
pub collection: String, | ||
pub rkey: String, | ||
/// The record as written, when the source carries it decoded (Jetstream | ||
/// does; the firehose's is CBOR in a CAR, so it is fetched instead). | ||
pub record: Option<serde_json::Value>, | ||
} | ||
| ||
pub(crate) fn knotless(collection: &str) -> bool { | ||
nsid::ALL.contains(&collection) || nsid::FORGE.contains(&collection) | ||
} | ||
| ||
/// Reads a frame. `None` when it is not one. | ||
pub fn parse(frame: &[u8]) -> Option<Event> { | ||
let mut reader = frame; | ||
let header: ciborium::Value = ciborium::from_reader(&mut reader).ok()?; | ||
let body: ciborium::Value = ciborium::from_reader(&mut reader).ok()?; | ||
let field = |v: &ciborium::Value, name: &str| -> Option<ciborium::Value> { | ||
v.as_map()? | ||
.iter() | ||
.find(|(k, _)| k.as_text() == Some(name)) | ||
.map(|(_, v)| v.clone()) | ||
}; | ||
let text = |v: &ciborium::Value, name: &str| field(v, name)?.as_text().map(str::to_string); | ||
let kind = text(&header, "t")?; | ||
let mut event = Event { | ||
seq: field(&body, "seq") | ||
.and_then(|s| s.as_integer()) | ||
.and_then(|i| i64::try_from(i).ok()), | ||
..Event::default() | ||
}; | ||
match kind.as_str() { | ||
"#commit" => { | ||
for op in field(&body, "ops")?.as_array()? { | ||
let (Some(action), Some(path)) = (text(op, "action"), text(op, "path")) else { | ||
continue; | ||
}; | ||
let Some((collection, rkey)) = path.split_once('/') else { | ||
continue; | ||
}; | ||
if knotless(collection) { | ||
event.ops.push(Op { | ||
action, | ||
collection: collection.into(), | ||
rkey: rkey.into(), | ||
record: None, | ||
}); | ||
} | ||
} | ||
if !event.ops.is_empty() { | ||
event.did = Some(text(&body, "repo")?); | ||
} | ||
} | ||
"#sync" | "#account" | "#identity" => event.did = Some(text(&body, "did")?), | ||
_ => {} | ||
} | ||
Some(event) | ||
} | ||
| ||
/// The subscribeRepos endpoint for a relay or PDS base URL. | ||
pub fn endpoint(base: &str, cursor: Option<i64>) -> String { | ||
let mut url = base.trim_end_matches('/').to_string(); | ||
if !url.contains("/xrpc/") { | ||
url.push_str("/xrpc/com.atproto.sync.subscribeRepos"); | ||
} | ||
if let Some(c) = cursor { | ||
url.push_str(&format!("?cursor={c}")); | ||
} | ||
url | ||
} | ||
| ||
/// Follows the firehose until the process ends, reconnecting from the last | ||
/// sequence number. While disconnected, mirrors fall back to the TTL. | ||
pub async fn watch(app: Arc<App>, base: String) { | ||
let mut cursor = None; | ||
loop { | ||
if let Ok((mut ws, _)) = tokio_tungstenite::connect_async(endpoint(&base, cursor)).await { | ||
app.watching.store(true, Ordering::Relaxed); | ||
while let Some(Ok(msg)) = ws.next().await { | ||
if let Message::Binary(frame) = msg | ||
&& let Some(event) = parse(&frame) | ||
{ | ||
cursor = event.seq.or(cursor); | ||
app.observe(event); | ||
} | ||
} | ||
} | ||
app.watching.store(false, Ordering::Relaxed); | ||
tokio::time::sleep(Duration::from_millis(500)).await; | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use ciborium::Value as V; | ||
| ||
fn frame(header: V, body: V) -> Vec<u8> { | ||
let mut out = Vec::new(); | ||
ciborium::into_writer(&header, &mut out).unwrap(); | ||
ciborium::into_writer(&body, &mut out).unwrap(); | ||
out | ||
} | ||
| ||
fn map(pairs: Vec<(&str, V)>) -> V { | ||
V::Map( | ||
pairs | ||
.into_iter() | ||
.map(|(k, v)| (V::Text(k.into()), v)) | ||
.collect(), | ||
) | ||
} | ||
| ||
#[test] | ||
fn frames() { | ||
let commit = |ops: Vec<V>| { | ||
frame( | ||
map(vec![ | ||
("op", V::Integer(1.into())), | ||
("t", V::Text("#commit".into())), | ||
]), | ||
map(vec![ | ||
("seq", V::Integer(7.into())), | ||
("repo", V::Text("did:plc:a".into())), | ||
("ops", V::Array(ops)), | ||
]), | ||
) | ||
}; | ||
let op = |action: &str, path: &str| { | ||
map(vec![ | ||
("action", V::Text(action.into())), | ||
("path", V::Text(path.into())), | ||
]) | ||
}; | ||
let event = parse(&commit(vec![ | ||
op("create", "dev.knotless.ref/k.x"), | ||
op("delete", "dev.knotless.issue/3"), | ||
op("create", "app.bsky.feed.post/x"), | ||
op("create", "no-slash"), | ||
map(vec![("path", V::Text("dev.knotless.ref/y".into()))]), | ||
])) | ||
.unwrap(); | ||
assert_eq!(event.seq, Some(7)); | ||
assert_eq!(event.did.as_deref(), Some("did:plc:a")); | ||
assert_eq!( | ||
event.ops, | ||
[ | ||
Op { | ||
action: "create".into(), | ||
collection: "dev.knotless.ref".into(), | ||
rkey: "k.x".into(), | ||
record: None | ||
}, | ||
Op { | ||
action: "delete".into(), | ||
collection: "dev.knotless.issue".into(), | ||
rkey: "3".into(), | ||
record: None | ||
}, | ||
] | ||
); | ||
let elsewhere = parse(&commit(vec![op("create", "app.bsky.feed.post/x")])).unwrap(); | ||
assert_eq!((elsewhere.did, elsewhere.ops.len()), (None, 0)); | ||
let identity = frame( | ||
map(vec![("t", V::Text("#identity".into()))]), | ||
map(vec![("did", V::Text("did:plc:b".into()))]), | ||
); | ||
assert_eq!(parse(&identity).unwrap().did.as_deref(), Some("did:plc:b")); | ||
let info = frame(map(vec![("t", V::Text("#info".into()))]), map(vec![])); | ||
assert_eq!(parse(&info), Some(Event::default())); | ||
assert_eq!(parse(b"\xff"), None); | ||
assert_eq!(parse(&frame(map(vec![]), map(vec![]))), None); | ||
} | ||
| ||
#[test] | ||
fn endpoints() { | ||
assert_eq!( | ||
endpoint("wss://relay/", None), | ||
"wss://relay/xrpc/com.atproto.sync.subscribeRepos" | ||
); | ||
assert_eq!( | ||
endpoint("ws://h/xrpc/com.atproto.sync.subscribeRepos", Some(3)), | ||
"ws://h/xrpc/com.atproto.sync.subscribeRepos?cursor=3" | ||
); | ||
} | ||
} | ||
crates/knotless-serve/src/gateway.rsnew+401−0
| @@ -0,0 +1,401 @@ | ||
//! The gateway: writes on owners' behalf, for collaborators and for batches. | ||
//! | ||
//! An owner hands over an app password (`dev.knotless.putDelegation`). After | ||
//! that, anyone on a repository's collaborator list can push | ||
//! (`dev.knotless.pushRefs`) or commit (`dev.knotless.createCommit`) with a service | ||
//! token from their own PDS, and the gateway writes as the owner. | ||
//! | ||
//! Commits to one branch queue up while a write to it is in flight. The | ||
//! next write takes the whole queue: each commit is built on the one before | ||
//! it, and the chain goes out as one ref update. Sixteen agents committing | ||
//! to one branch cost about two writes, not sixteen rounds of losing a race. | ||
| ||
use crate::{App, AppError}; | ||
use axum::Json; | ||
use axum::body::Bytes; | ||
use axum::extract::State; | ||
use axum::http::{HeaderMap, StatusCode, header}; | ||
use knotless::commit::{self, Committed, Spec}; | ||
use knotless::gateway::{ | ||
self, COMMIT, CommitRequest, CommitResponse, DELEGATE, PUSH, PushResults, RefResult, | ||
}; | ||
use knotless::mirror::Mirror; | ||
use knotless::push::{self, Update}; | ||
use knotless::remote::{Remote, Snapshot}; | ||
use knotless::session::{self, Session}; | ||
use knotless::url::RepoUrl; | ||
use serde::Deserialize; | ||
use serde_json::{Value, json}; | ||
use std::collections::HashMap; | ||
use std::sync::atomic::{AtomicU64, Ordering}; | ||
use std::sync::{Arc, Mutex}; | ||
use tokio::sync::oneshot; | ||
| ||
/// How many times a batch is rebuilt on a branch that moved under it. | ||
const RETRIES: u32 = 8; | ||
| ||
pub struct Gateway { | ||
pub did: String, | ||
pub endpoint: String, | ||
/// Batches written, each one ref update however many commits it holds. | ||
pub writes: AtomicU64, | ||
/// Batches rebuilt because the branch moved under them. | ||
pub rebuilt: AtomicU64, | ||
repos: Mutex<HashMap<(String, String), Lock>>, | ||
branches: Mutex<HashMap<(String, String, String), Arc<Branch>>>, | ||
} | ||
| ||
/// Held while a repository is written. | ||
type Lock = Arc<tokio::sync::Mutex<()>>; | ||
| ||
#[derive(Default)] | ||
struct Branch { | ||
queue: Mutex<Vec<Pending>>, | ||
turn: tokio::sync::Mutex<()>, | ||
} | ||
| ||
struct Pending { | ||
spec: Spec, | ||
reply: oneshot::Sender<Result<CommitResponse, AppError>>, | ||
} | ||
| ||
impl Gateway { | ||
/// A gateway reached at `endpoint`, an `https://` or `http://` URL, as | ||
/// the `did:web` of its host. | ||
pub fn new(endpoint: &str) -> Result<Gateway, String> { | ||
let endpoint = endpoint.trim_end_matches('/'); | ||
let host = endpoint | ||
.strip_prefix("https://") | ||
.or_else(|| endpoint.strip_prefix("http://")) | ||
.filter(|h| !h.is_empty() && !h.contains('/')) | ||
.ok_or(format!( | ||
"{endpoint}: a gateway URL is http(s)://host[:port]" | ||
))?; | ||
Ok(Gateway { | ||
did: format!("did:web:{}", host.replace(':', "%3A")), | ||
endpoint: endpoint.to_string(), | ||
writes: Default::default(), | ||
rebuilt: Default::default(), | ||
repos: Mutex::default(), | ||
branches: Mutex::default(), | ||
}) | ||
} | ||
} | ||
| ||
fn forbidden(msg: impl Into<String>) -> AppError { | ||
AppError(StatusCode::FORBIDDEN, msg.into()) | ||
} | ||
| ||
fn bad(msg: impl Into<String>) -> AppError { | ||
AppError(StatusCode::BAD_REQUEST, msg.into()) | ||
} | ||
| ||
pub(crate) fn gateway(app: &App) -> Result<&Gateway, AppError> { | ||
app.gateway | ||
.as_ref() | ||
.ok_or_else(|| AppError(StatusCode::NOT_FOUND, "this server is not a gateway".into())) | ||
} | ||
| ||
/// The DID that signed the request's service token for `lxm`. | ||
pub(crate) async fn caller(app: &App, headers: &HeaderMap, lxm: &str) -> Result<String, AppError> { | ||
let gw = gateway(app)?; | ||
let token = headers | ||
.get(header::AUTHORIZATION) | ||
.and_then(|v| v.to_str().ok()) | ||
.and_then(|v| v.strip_prefix("Bearer ")) | ||
.ok_or_else(|| { | ||
AppError( | ||
StatusCode::UNAUTHORIZED, | ||
"a service token is required".into(), | ||
) | ||
})?; | ||
knotless::auth::verify(&app.ctx, token, &gw.did, lxm) | ||
.await | ||
.map_err(|e| AppError(StatusCode::UNAUTHORIZED, format!("{e:#}"))) | ||
} | ||
| ||
pub async fn did_document(State(app): State<Arc<App>>) -> Result<Json<Value>, AppError> { | ||
let gw = gateway(&app)?; | ||
Ok(Json(json!({ | ||
"@context": ["https://www.w3.org/ns/did/v1"], | ||
"id": gw.did, | ||
"service": [{ | ||
"id": gateway::SERVICE, | ||
"type": "KnotlessGateway", | ||
"serviceEndpoint": gw.endpoint, | ||
}, { | ||
"id": knotless::moderation::LABELER, | ||
"type": "AtprotoLabeler", | ||
"serviceEndpoint": gw.endpoint, | ||
}], | ||
}))) | ||
} | ||
| ||
#[derive(Deserialize)] | ||
pub struct Delegation { | ||
password: String, | ||
} | ||
| ||
/// Takes an owner's app password, and keeps the session it opens. | ||
pub async fn delegate( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
Json(body): Json<Delegation>, | ||
) -> Result<Json<Value>, AppError> { | ||
let did = caller(&app, &headers, DELEGATE).await?; | ||
session::login(&app.ctx, &did, &body.password) | ||
.await | ||
.map_err(|e| forbidden(format!("that password does not log in as {did}: {e:#}")))?; | ||
Ok(Json(json!({}))) | ||
} | ||
| ||
/// The repository, as its owner, for a caller who may write to it. | ||
async fn writable( | ||
app: &App, | ||
caller: &str, | ||
owner: &str, | ||
name: &str, | ||
) -> Result<(Remote, Session, Snapshot), AppError> { | ||
let url = | ||
RepoUrl::parse(&format!("knotless://{owner}/{name}")).map_err(|e| bad(e.to_string()))?; | ||
let remote = Remote::open(&app.ctx, &url) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
let session = session::ensure(&app.ctx, &remote.identity) | ||
.await | ||
.map_err(|_| forbidden(format!("{owner} has not delegated writes to this gateway")))?; | ||
let snap = remote | ||
.snapshot(&app.ctx) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
let Some(repo) = &snap.repo else { | ||
return Err(AppError( | ||
StatusCode::NOT_FOUND, | ||
format!("{url}: no such repository"), | ||
)); | ||
}; | ||
if caller != remote.did() && !repo.value.is_collaborator(caller) { | ||
return Err(forbidden(format!( | ||
"{caller} is not a collaborator on {url}" | ||
))); | ||
} | ||
Ok((remote, session, snap)) | ||
} | ||
| ||
async fn mirror(app: &App, remote: &Remote, snap: &Snapshot) -> Result<Mirror, AppError> { | ||
let key = snap.key().expect("a writable repository exists"); | ||
let mirror = Mirror::open(&app.ctx, remote.did(), key) | ||
.await | ||
.map_err(AppError::internal)?; | ||
mirror | ||
.sync(remote, snap) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
Ok(mirror) | ||
} | ||
| ||
pub async fn push( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
body: Bytes, | ||
) -> Result<Json<PushResults>, AppError> { | ||
let caller = caller(&app, &headers, PUSH).await?; | ||
let (header, pack) = gateway::split_push(&body).map_err(|e| bad(format!("{e:#}")))?; | ||
let mut updates = Vec::new(); | ||
for u in &header.updates { | ||
if u.src.as_deref().is_some_and(|s| !gateway::is_sha(s)) || !u.dst.starts_with("refs/") { | ||
return Err(bad(format!("bad update of {}", u.dst))); | ||
} | ||
updates.push(Update { | ||
src: u.src.clone(), | ||
dst: u.dst.clone(), | ||
force: u.force, | ||
}); | ||
} | ||
let lock = gateway(&app)? | ||
.repos | ||
.lock() | ||
.unwrap() | ||
.entry((header.repo.clone(), header.name.clone())) | ||
.or_default() | ||
.clone(); | ||
let _turn = lock.lock().await; | ||
let (remote, session, snap) = writable(&app, &caller, &header.repo, &header.name).await?; | ||
let mirror = mirror(&app, &remote, &snap).await?; | ||
if !pack.is_empty() { | ||
mirror | ||
.git | ||
.index_pack(pack) | ||
.await | ||
.map_err(|e| bad(format!("{e:#}")))?; | ||
} | ||
let tips: Vec<String> = updates.iter().filter_map(|u| u.src.clone()).collect(); | ||
if let Some(sha) = mirror | ||
.git | ||
.missing(&tips) | ||
.await | ||
.map_err(AppError::internal)? | ||
.into_iter() | ||
.next() | ||
{ | ||
return Err(bad(format!("the pack does not hold {sha}"))); | ||
} | ||
let report = push::push( | ||
&app.ctx, | ||
&remote, | ||
&snap, | ||
&mirror.git, | ||
&session.access_jwt, | ||
&updates, | ||
) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
Ok(Json(PushResults { | ||
results: report | ||
.into_iter() | ||
.map(|(name, res)| RefResult { | ||
name, | ||
error: res.err(), | ||
}) | ||
.collect(), | ||
})) | ||
} | ||
| ||
pub async fn commit( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
Json(req): Json<CommitRequest>, | ||
) -> Result<Json<CommitResponse>, AppError> { | ||
let caller = caller(&app, &headers, COMMIT).await?; | ||
if req.branch.is_empty() || req.message.is_empty() { | ||
return Err(bad("a commit needs a branch and a message")); | ||
} | ||
// Checked before queueing, so a batch holds only commits that may land. | ||
writable(&app, &caller, &req.repo, &req.name).await?; | ||
let branch = gateway(&app)? | ||
.branches | ||
.lock() | ||
.unwrap() | ||
.entry((req.repo.clone(), req.name.clone(), req.branch.clone())) | ||
.or_default() | ||
.clone(); | ||
let (reply, answer) = oneshot::channel(); | ||
branch.queue.lock().unwrap().push(Pending { | ||
spec: req.spec(), | ||
reply, | ||
}); | ||
{ | ||
let _turn = branch.turn.lock().await; | ||
let batch = std::mem::take(&mut *branch.queue.lock().unwrap()); | ||
// Empty when the write before this one took this commit along. | ||
if !batch.is_empty() { | ||
land(&app, &caller, &req, batch).await; | ||
} | ||
} | ||
answer | ||
.await | ||
.expect("every queued commit is answered") | ||
.map(Json) | ||
} | ||
| ||
/// Builds the batch on the branch and writes it, rebuilding on a branch | ||
/// that moved, and answers every commit in it. | ||
async fn land(app: &App, caller: &str, req: &CommitRequest, batch: Vec<Pending>) { | ||
match attempt(app, caller, req, &batch).await { | ||
Ok(results) => { | ||
for (pending, result) in batch.into_iter().zip(results) { | ||
let _ = pending.reply.send(result); | ||
} | ||
} | ||
Err(err) => { | ||
for pending in batch { | ||
let _ = pending.reply.send(Err(AppError(err.0, err.1.clone()))); | ||
} | ||
} | ||
} | ||
} | ||
| ||
type Answers = Vec<Result<CommitResponse, AppError>>; | ||
| ||
async fn attempt( | ||
app: &App, | ||
caller: &str, | ||
req: &CommitRequest, | ||
batch: &[Pending], | ||
) -> Result<Answers, AppError> { | ||
let refname = format!("refs/heads/{}", req.branch); | ||
let mut round = 0; | ||
loop { | ||
let (remote, session, snap) = writable(app, caller, &req.repo, &req.name).await?; | ||
let mirror = mirror(app, &remote, &snap).await?; | ||
let start = snap.find_ref(&refname).and_then(|r| r.value.target.clone()); | ||
let mut tip = start.clone(); | ||
let mut built: Vec<Result<Committed, String>> = Vec::new(); | ||
for pending in batch { | ||
if let Err(err) = commit::conflicts(&mirror.git, &pending.spec, tip.as_deref()).await { | ||
built.push(Err(format!("{err:#}"))); | ||
continue; | ||
} | ||
let sha = commit::build( | ||
&app.ctx, | ||
&mirror.git, | ||
tip.as_deref(), | ||
&pending.spec, | ||
remote.did(), | ||
) | ||
.await | ||
.map_err(AppError::input)?; | ||
let parent = tip.clone(); | ||
if let Some(sha) = &sha { | ||
tip = Some(sha.clone()); | ||
} | ||
built.push(Ok(Committed { | ||
sha: tip.clone().unwrap_or_default(), | ||
parent, | ||
attempts: 1, | ||
changed: sha.is_some(), | ||
})); | ||
} | ||
let landed = built.iter().flatten().filter(|c| c.changed).count(); | ||
if tip != start { | ||
let update = Update { | ||
src: tip.clone(), | ||
dst: refname.clone(), | ||
force: true, | ||
}; | ||
let report = push::push( | ||
&app.ctx, | ||
&remote, | ||
&snap, | ||
&mirror.git, | ||
&session.access_jwt, | ||
&[update], | ||
) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
gateway(app)?.writes.fetch_add(1, Ordering::Relaxed); | ||
// Forced against the generation read above, so the one refusal | ||
// is that the branch moved: rebuild on where it went. | ||
if let Err(reason) = &report[0].1 { | ||
let gave_up = AppError(StatusCode::CONFLICT, format!("{refname}: {reason}")); | ||
(round < RETRIES).then_some(()).ok_or(gave_up)?; | ||
gateway(app)?.rebuilt.fetch_add(1, Ordering::Relaxed); | ||
round += 1; | ||
continue; | ||
} | ||
} | ||
let answers = built | ||
.into_iter() | ||
.map(|c| match c { | ||
Ok(c) => Ok(CommitResponse { | ||
sha: c.sha, | ||
parent: c.parent, | ||
changed: c.changed, | ||
batched: landed, | ||
}), | ||
Err(msg) => Err(AppError(StatusCode::CONFLICT, msg)), | ||
}) | ||
.collect(); | ||
return Ok(answers); | ||
} | ||
} | ||
crates/knotless-serve/src/highlight.rsnew+577−0
| @@ -0,0 +1,577 @@ | ||
//! Syntax highlighting as ranges, not markup: the page carries the text | ||
//! once, and a few lines of script hand these ranges to the CSS Custom | ||
//! Highlight API. Offsets are in UTF-16 code units, as the DOM counts. | ||
//! | ||
//! Code is parsed with tree-sitter, whose grammars take tens of megabytes a | ||
//! second; a language without one here is shown plain. | ||
| ||
use crate::diff::{Kind, Row}; | ||
use std::cell::RefCell; | ||
use std::fmt::Write; | ||
use std::sync::LazyLock; | ||
use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter}; | ||
| ||
/// Larger files are shown plain: highlighting them costs more than it gives. | ||
pub const LIMIT: usize = 1024 * 1024; | ||
| ||
/// The highlight names for code, each styled by `::highlight(name)`. | ||
const CODE: &[&str] = &[ | ||
"comment", "string", "keyword", "number", "function", "type", "tag", "attr", | ||
]; | ||
| ||
/// The captures the grammars' queries use, and the highlight each gives. | ||
/// A capture takes its longest listed prefix: `function.method` is a | ||
/// `function`. | ||
const CAPTURES: [(&str, usize); 23] = [ | ||
("comment", 0), | ||
("string", 1), | ||
("string.special.key", 7), | ||
("escape", 1), | ||
("character", 1), | ||
("keyword", 2), | ||
("variable.builtin", 2), | ||
("number", 3), | ||
("float", 3), | ||
("boolean", 3), | ||
("constant", 3), | ||
("constant.builtin", 3), | ||
("function", 4), | ||
("function.builtin", 4), | ||
("function.method", 4), | ||
("function.macro", 4), | ||
("type", 5), | ||
("type.builtin", 5), | ||
("constructor", 5), | ||
("tag", 6), | ||
("attribute", 7), | ||
("property.definition", 7), | ||
("label", 2), | ||
]; | ||
| ||
/// The names for a diff's rows: the code names, then the changed part of | ||
/// an added line, and of a removed one. | ||
const ROWS: &[&str] = &[ | ||
"comment", "string", "keyword", "number", "function", "type", "tag", "attr", "xins", "xdel", | ||
]; | ||
| ||
struct Language { | ||
name: &'static str, | ||
extensions: &'static [&'static str], | ||
config: HighlightConfiguration, | ||
} | ||
| ||
fn language( | ||
name: &'static str, | ||
extensions: &'static [&'static str], | ||
grammar: tree_sitter_language::LanguageFn, | ||
highlights: &[&str], | ||
injections: &str, | ||
) -> Language { | ||
let mut config = | ||
HighlightConfiguration::new(grammar.into(), name, &highlights.join("\n"), injections, "") | ||
.expect("a bundled highlight query compiles"); | ||
let names: Vec<&str> = CAPTURES.iter().map(|(n, _)| *n).collect(); | ||
config.configure(&names); | ||
Language { | ||
name, | ||
extensions, | ||
config, | ||
} | ||
} | ||
| ||
static LANGUAGES: LazyLock<Vec<Language>> = LazyLock::new(|| { | ||
use tree_sitter_javascript as js; | ||
use tree_sitter_typescript as ts; | ||
vec![ | ||
language( | ||
"rust", | ||
&["rs"], | ||
tree_sitter_rust::LANGUAGE, | ||
&[tree_sitter_rust::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"javascript", | ||
&["js", "mjs", "cjs", "jsx"], | ||
js::LANGUAGE, | ||
&[js::HIGHLIGHT_QUERY, js::JSX_HIGHLIGHT_QUERY], | ||
"", | ||
), | ||
language( | ||
"typescript", | ||
&["ts", "mts", "cts"], | ||
ts::LANGUAGE_TYPESCRIPT, | ||
&[js::HIGHLIGHT_QUERY, ts::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"tsx", | ||
&["tsx"], | ||
ts::LANGUAGE_TSX, | ||
&[ | ||
js::HIGHLIGHT_QUERY, | ||
js::JSX_HIGHLIGHT_QUERY, | ||
ts::HIGHLIGHTS_QUERY, | ||
], | ||
"", | ||
), | ||
language( | ||
"python", | ||
&["py", "pyi"], | ||
tree_sitter_python::LANGUAGE, | ||
&[tree_sitter_python::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"go", | ||
&["go"], | ||
tree_sitter_go::LANGUAGE, | ||
&[tree_sitter_go::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"c", | ||
&["c", "h"], | ||
tree_sitter_c::LANGUAGE, | ||
&[tree_sitter_c::HIGHLIGHT_QUERY], | ||
"", | ||
), | ||
language( | ||
"cpp", | ||
&["cc", "cpp", "cxx", "hpp", "hh", "hxx"], | ||
tree_sitter_cpp::LANGUAGE, | ||
&[ | ||
tree_sitter_c::HIGHLIGHT_QUERY, | ||
tree_sitter_cpp::HIGHLIGHT_QUERY, | ||
], | ||
"", | ||
), | ||
language( | ||
"java", | ||
&["java"], | ||
tree_sitter_java::LANGUAGE, | ||
&[tree_sitter_java::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"ruby", | ||
&["rb", "rake", "gemspec", "Gemfile", "Rakefile"], | ||
tree_sitter_ruby::LANGUAGE, | ||
&[tree_sitter_ruby::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"bash", | ||
&["sh", "bash", "zsh"], | ||
tree_sitter_bash::LANGUAGE, | ||
&[tree_sitter_bash::HIGHLIGHT_QUERY], | ||
"", | ||
), | ||
language( | ||
"json", | ||
&["json"], | ||
tree_sitter_json::LANGUAGE, | ||
// Where patterns overlap the last wins: keys over strings. | ||
&[ | ||
tree_sitter_json::HIGHLIGHTS_QUERY, | ||
"(pair key: (_) @string.special.key)", | ||
], | ||
"", | ||
), | ||
language( | ||
"toml", | ||
&["toml", "lock"], | ||
tree_sitter_toml_ng::LANGUAGE, | ||
&[tree_sitter_toml_ng::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"yaml", | ||
&["yml", "yaml"], | ||
tree_sitter_yaml::LANGUAGE, | ||
&[tree_sitter_yaml::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"html", | ||
&["html", "htm"], | ||
tree_sitter_html::LANGUAGE, | ||
&[tree_sitter_html::HIGHLIGHTS_QUERY], | ||
tree_sitter_html::INJECTIONS_QUERY, | ||
), | ||
language( | ||
"css", | ||
&["css"], | ||
tree_sitter_css::LANGUAGE, | ||
&[tree_sitter_css::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"lua", | ||
&["lua"], | ||
tree_sitter_lua::LANGUAGE, | ||
&[tree_sitter_lua::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
language( | ||
"zig", | ||
&["zig"], | ||
tree_sitter_zig::LANGUAGE, | ||
&[tree_sitter_zig::HIGHLIGHTS_QUERY], | ||
"", | ||
), | ||
] | ||
}); | ||
| ||
thread_local! { | ||
static HIGHLIGHTER: RefCell<Highlighter> = RefCell::new(Highlighter::new()); | ||
} | ||
| ||
fn by_name(name: &str) -> Option<&'static Language> { | ||
LANGUAGES.iter().find(|l| l.name == name) | ||
} | ||
| ||
/// The language of the file at `path`: by its extension, or its name, or | ||
/// the interpreter its first line names. | ||
fn language_for(path: &str, text: &str) -> Option<&'static Language> { | ||
let name = path.rsplit('/').next().unwrap_or(path); | ||
let ext = name.rsplit_once('.').map_or(name, |(_, ext)| ext); | ||
let by_ext = LANGUAGES.iter().find(|l| l.extensions.contains(&ext)); | ||
let shebang = text | ||
.lines() | ||
.next() | ||
.and_then(|l| l.strip_prefix("#!")) | ||
.unwrap_or_default(); | ||
let interpreter = shebang.rsplit(['/', ' ']).next().unwrap_or_default(); | ||
let by_shebang = match interpreter.trim_end_matches(|c: char| c.is_ascii_digit() || c == '.') { | ||
"sh" | "bash" | "zsh" => by_name("bash"), | ||
"python" => by_name("python"), | ||
"node" => by_name("javascript"), | ||
"ruby" => by_name("ruby"), | ||
_ => None, | ||
}; | ||
by_ext.or(by_shebang) | ||
} | ||
| ||
/// Runs of each named highlight, as `[start, end)` in UTF-16 units. | ||
#[derive(Debug, PartialEq)] | ||
pub struct Ranges { | ||
names: &'static [&'static str], | ||
runs: Vec<Vec<(u32, u32)>>, | ||
} | ||
| ||
impl Ranges { | ||
fn new(names: &'static [&'static str]) -> Ranges { | ||
Ranges { | ||
names, | ||
runs: vec![Vec::new(); names.len()], | ||
} | ||
} | ||
| ||
/// Adds a run, joining it to the one before when they touch. | ||
fn push(&mut self, class: usize, start: u32, end: u32) { | ||
let runs = &mut self.runs[class]; | ||
match runs.last_mut() { | ||
Some(last) if last.1 == start => last.1 = end, | ||
_ if start < end => runs.push((start, end)), | ||
_ => {} | ||
} | ||
} | ||
| ||
/// `name:gap,len,gap,len;…`, each gap from the end of the run before: | ||
/// short, and safe in an attribute as it is. | ||
pub fn encode(&self) -> String { | ||
let mut out = String::new(); | ||
let named = self | ||
.names | ||
.iter() | ||
.zip(&self.runs) | ||
.filter(|(_, r)| !r.is_empty()); | ||
for (name, runs) in named { | ||
let sep = if out.is_empty() { "" } else { ";" }; | ||
write!(out, "{sep}{name}:").unwrap(); | ||
let mut at = 0; | ||
for (i, (start, end)) in runs.iter().enumerate() { | ||
let sep = if i == 0 { "" } else { "," }; | ||
write!(out, "{sep}{},{}", start - at, end - start).unwrap(); | ||
at = *end; | ||
} | ||
} | ||
out | ||
} | ||
} | ||
| ||
fn utf16_len(s: &str) -> u32 { | ||
s.chars().map(|c| c.len_utf16() as u32).sum() | ||
} | ||
| ||
/// The highlighting of `text`, the file at `path`: `None` for a language | ||
/// without a grammar here, or a file too large. | ||
pub fn code(path: &str, text: &str) -> Option<Ranges> { | ||
if text.len() > LIMIT { | ||
return None; | ||
} | ||
let language = language_for(path, text)?; | ||
HIGHLIGHTER.with_borrow_mut(|highlighter| { | ||
let events = highlighter | ||
.highlight(&language.config, text.as_bytes(), None, None, |name| { | ||
by_name(name).map(|l| &l.config) | ||
}) | ||
.ok()?; | ||
let mut ranges = Ranges::new(CODE); | ||
let mut stack: Vec<usize> = Vec::new(); | ||
// Sources come in order, so offsets are counted once, as they go. | ||
let (mut byte, mut unit) = (0, 0u32); | ||
for event in events { | ||
match event.ok()? { | ||
HighlightEvent::HighlightStart(h) => stack.push(CAPTURES[h.0].1), | ||
HighlightEvent::HighlightEnd => { | ||
stack.pop(); | ||
} | ||
HighlightEvent::Source { start, end } => { | ||
unit += utf16_len(&text[byte..start]); | ||
let len = utf16_len(&text[start..end]); | ||
if let Some(&class) = stack.last() { | ||
ranges.push(class, unit, unit + len); | ||
} | ||
(byte, unit) = (end, unit + len); | ||
} | ||
} | ||
} | ||
Some(ranges) | ||
}) | ||
} | ||
| ||
/// The highlighting of a file's changed lines, shown a row each: offsets | ||
/// run over the lines' text as if joined by newlines. Removed lines are | ||
/// highlighted as the old side reads, in its context, and the rest as the | ||
/// new side reads; of a removed line and the added line paired with it, | ||
/// the part that differs is marked too. | ||
pub fn rows(path: &str, rows: &[Row]) -> Ranges { | ||
let lines: Vec<&Row> = rows.iter().filter(|r| r.kind.is_line()).collect(); | ||
let mut starts = Vec::with_capacity(lines.len()); | ||
let mut at = 0; | ||
for line in &lines { | ||
starts.push(at); | ||
at += utf16_len(&line.text) + 1; | ||
} | ||
let mut runs: Vec<Vec<(u32, u32)>> = vec![Vec::new(); ROWS.len()]; | ||
for side in [Kind::Del, Kind::Add] { | ||
let other = if side == Kind::Del { | ||
Kind::Add | ||
} else { | ||
Kind::Del | ||
}; | ||
let picked: Vec<usize> = (0..lines.len()) | ||
.filter(|&i| lines[i].kind != other) | ||
.collect(); | ||
let text: String = picked | ||
.iter() | ||
.map(|&i| format!("{}\n", lines[i].text)) | ||
.collect(); | ||
let Some(ranges) = code(path, &text) else { | ||
break; | ||
}; | ||
let mut side_starts = Vec::with_capacity(picked.len()); | ||
let mut at = 0; | ||
for &i in &picked { | ||
side_starts.push(at); | ||
at += utf16_len(&lines[i].text) + 1; | ||
} | ||
for (class, list) in ranges.runs.iter().enumerate() { | ||
for &(s, e) in list { | ||
let mut j = side_starts.partition_point(|&x| x <= s) - 1; | ||
while j < picked.len() && side_starts[j] < e { | ||
let i = picked[j]; | ||
let (from, to) = ( | ||
s.max(side_starts[j]), | ||
e.min(side_starts[j] + utf16_len(&lines[i].text)), | ||
); | ||
let shown = side == Kind::Add || lines[i].kind == Kind::Del; | ||
if shown && from < to { | ||
let base = starts[i] - side_starts[j]; | ||
runs[class].push((base + from, base + to)); | ||
} | ||
j += 1; | ||
} | ||
} | ||
} | ||
} | ||
// Each run of removed lines, and the added lines after it. | ||
let mut i = 0; | ||
while i < lines.len() { | ||
let dels = lines[i..] | ||
.iter() | ||
.take_while(|l| l.kind == Kind::Del) | ||
.count(); | ||
let adds = lines[i + dels..] | ||
.iter() | ||
.take_while(|l| l.kind == Kind::Add) | ||
.count(); | ||
for k in 0..dels.min(adds) { | ||
let (d, a) = (i + k, i + dels + k); | ||
if let Some((dr, ar)) = changed(&lines[d].text, &lines[a].text) { | ||
runs[ROWS.len() - 1].push((starts[d] + dr.0, starts[d] + dr.1)); | ||
runs[ROWS.len() - 2].push((starts[a] + ar.0, starts[a] + ar.1)); | ||
} | ||
} | ||
i += (dels + adds).max(1); | ||
} | ||
let mut out = Ranges::new(ROWS); | ||
for (class, mut list) in runs.into_iter().enumerate() { | ||
list.sort_unstable(); | ||
for (s, e) in list { | ||
out.push(class, s, e); | ||
} | ||
} | ||
out | ||
} | ||
| ||
/// Where `old` and `new` differ, in each, between what they start and end | ||
/// with alike: `None` when that is less than half the shorter, as then the | ||
/// line was rewritten rather than edited. | ||
fn changed(old: &str, new: &str) -> Option<((u32, u32), (u32, u32))> { | ||
let (o, n): (Vec<char>, Vec<char>) = (old.chars().collect(), new.chars().collect()); | ||
let prefix = o.iter().zip(&n).take_while(|(a, b)| a == b).count(); | ||
let room = o.len().min(n.len()) - prefix; | ||
let suffix = o | ||
.iter() | ||
.rev() | ||
.zip(n.iter().rev()) | ||
.take(room) | ||
.take_while(|(a, b)| a == b) | ||
.count(); | ||
if (prefix + suffix) * 2 < o.len().min(n.len()) || prefix + suffix == 0 { | ||
return None; | ||
} | ||
let units = |c: &[char]| c.iter().map(|c| c.len_utf16() as u32).sum::<u32>(); | ||
let span = |c: &[char]| (units(&c[..prefix]), units(&c[..c.len() - suffix])); | ||
Some((span(&o), span(&n))) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
/// The text each run of `name` covers. | ||
fn spans(r: &Ranges, text: &str, name: &str) -> Vec<String> { | ||
let units: Vec<u16> = text.encode_utf16().collect(); | ||
let class = r.names.iter().position(|n| *n == name).unwrap(); | ||
r.runs[class] | ||
.iter() | ||
.map(|(s, e)| String::from_utf16(&units[*s as usize..*e as usize]).unwrap()) | ||
.collect() | ||
} | ||
| ||
#[test] | ||
fn rust() { | ||
let text = "// é😀 hi\nfn main() { let s: u8 = \"x\"; 42 }\n"; | ||
let r = code("src/main.rs", text).unwrap(); | ||
assert_eq!(spans(&r, text, "comment"), ["// é😀 hi"]); | ||
assert_eq!(spans(&r, text, "string"), ["\"x\""]); | ||
assert_eq!(spans(&r, text, "number"), ["42"]); | ||
assert_eq!(spans(&r, text, "function"), ["main"]); | ||
assert_eq!(spans(&r, text, "type"), ["u8"]); | ||
assert!(spans(&r, text, "keyword").contains(&"fn".to_string())); | ||
let encoded = r.encode(); | ||
assert!(encoded.starts_with("comment:0,9;string:"), "{encoded}"); | ||
} | ||
| ||
#[test] | ||
fn every_language_parses() { | ||
let samples = [ | ||
("a.js", "const f = () => <b x=\"1\">hi</b>; // c\n"), | ||
("a.ts", "let x: number = 1;\n"), | ||
("a.tsx", "const e = <div class=\"x\" />;\n"), | ||
("a.py", "def f():\n return 'x'\n"), | ||
("a.go", "package main\nfunc main() {}\n"), | ||
("a.c", "int main(void) { return 0; }\n"), | ||
("a.cpp", "class A {};\n"), | ||
("A.java", "class A { void f() {} }\n"), | ||
("Gemfile", "gem 'x'\n"), | ||
("a.json", "{\"key\": 1}\n"), | ||
("Cargo.lock", "[[package]]\nname = \"x\"\n"), | ||
("a.yml", "a: 'b'\n"), | ||
("a.css", "a { color: red; }\n"), | ||
("a.lua", "local x = 'y'\n"), | ||
("a.zig", "const x = 1;\n"), | ||
("run", "#!/usr/bin/env bash\necho 'hi'\n"), | ||
("tool", "#!/usr/bin/python3.12\nx = 'y'\n"), | ||
("cli", "#!/usr/bin/env node\nlet a = 'b'\n"), | ||
("task", "#!/usr/bin/ruby\nputs 'x'\n"), | ||
]; | ||
for (path, text) in samples { | ||
let r = code(path, text).expect(path); | ||
assert!(!r.encode().is_empty(), "{path}"); | ||
} | ||
let json = "{\"key\": 1}\n"; | ||
assert_eq!( | ||
spans(&code("a.json", json).unwrap(), json, "attr"), | ||
["\"key\""] | ||
); | ||
} | ||
| ||
#[test] | ||
fn html_carries_its_scripts_and_styles() { | ||
let html = | ||
"<a href=\"x\">y</a><script>let a = 'q';</script><style>b { color: red }</style>\n"; | ||
let r = code("index.html", html).unwrap(); | ||
assert_eq!(spans(&r, html, "attr"), ["href"]); | ||
assert!( | ||
spans(&r, html, "string").contains(&"'q'".to_string()), | ||
"{:?}", | ||
r | ||
); | ||
assert!(spans(&r, html, "tag").contains(&"script".to_string())); | ||
} | ||
| ||
#[test] | ||
fn what_it_leaves_plain() { | ||
assert_eq!(code("notes.unknown-kind", "plain\n"), None); | ||
assert_eq!(code("script", "#!/bin/perl\n"), None); | ||
assert_eq!(code("big.rs", &"x".repeat(LIMIT + 1)), None); | ||
let mut empty = Ranges::new(CODE); | ||
empty.push(0, 4, 4); | ||
assert_eq!(empty.encode(), ""); | ||
} | ||
| ||
#[test] | ||
fn diff_rows() { | ||
let row = |kind, text: &str| Row { | ||
kind, | ||
old: None, | ||
new: None, | ||
text: text.into(), | ||
}; | ||
let rows = [ | ||
row(Kind::Hunk, "@@ -1,3 +1,3 @@"), | ||
row(Kind::Context, "/* a"), | ||
row(Kind::Del, "b */ let x = 1;"), | ||
row(Kind::Add, "b */ let x = 2;"), | ||
row(Kind::Add, "let é = \"q\";"), | ||
row(Kind::Note, "\\ No newline at end of file"), | ||
row(Kind::Del, "gone"), | ||
row(Kind::Add, "here"), | ||
]; | ||
let r = rows_named(&rows); | ||
let shown = "/* a\nb */ let x = 1;\nb */ let x = 2;\nlet é = \"q\";\ngone\nhere\n"; | ||
// The comment spans two rows, and is cut at the row between. | ||
assert_eq!(spans(&r, shown, "comment"), ["/* a", "b */", "b */"]); | ||
assert_eq!(spans(&r, shown, "number"), ["1", "2"]); | ||
assert_eq!(spans(&r, shown, "string"), ["\"q\""]); | ||
assert_eq!(spans(&r, shown, "xdel"), ["1"]); | ||
assert_eq!(spans(&r, shown, "xins"), ["2"]); | ||
assert_eq!( | ||
super::rows("notes.unknown-kind", &rows).encode(), | ||
"xins:34,1;xdel:18,1" | ||
); | ||
assert_eq!(changed("abc", "xyz"), None); | ||
assert_eq!(changed("gone", "here"), None); | ||
assert_eq!(changed("aé", "aéb"), Some(((2, 2), (2, 3)))); | ||
assert_eq!(changed("aa", "a"), Some(((1, 2), (1, 1)))); | ||
} | ||
| ||
fn rows_named(rows: &[Row]) -> Ranges { | ||
super::rows("a.rs", rows) | ||
} | ||
} | ||
crates/knotless-serve/src/index.rsnew+727−0
| @@ -0,0 +1,727 @@ | ||
//! The appview's index: every public repository and every pull, issue, | ||
//! comment and state change anyone has written about one, from the firehose | ||
//! and a backfill. Private repositories never reach it; their records are | ||
//! not on the firehose. | ||
//! | ||
//! It keeps what moderation needs too: whom owners block from all their | ||
//! repositories, labels from the labelers the server follows, and, kept | ||
//! when the rest is rebuilt, its own takedowns, the labels it gives out for | ||
//! them, and the reports it was sent. | ||
| ||
use knotless::forge::Entry; | ||
use knotless::records::{RepoRef, nsid}; | ||
use rusqlite::{Connection, OptionalExtension, params}; | ||
use serde::Serialize; | ||
use serde_json::Value; | ||
use std::path::Path; | ||
use std::sync::Mutex; | ||
| ||
pub struct Index { | ||
db: Mutex<Connection>, | ||
} | ||
| ||
/// A repository as the index knows it. | ||
#[derive(Serialize, Debug, Clone, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct Listed { | ||
pub did: String, | ||
pub name: String, | ||
#[serde(skip_serializing_if = "Option::is_none")] | ||
pub description: Option<String>, | ||
#[serde(skip_serializing_if = "Option::is_none")] | ||
pub fork_of: Option<String>, | ||
pub created_at: String, | ||
} | ||
| ||
/// The shape of the tables. An index of another shape is dropped and | ||
/// built again: it is a cache, and `--backfill` refills it. | ||
const VERSION: i64 = 5; | ||
| ||
/// Drops the tables that are a cache. Takedowns, the labels given out for | ||
/// them and reports are not, and stay. | ||
const CACHE: &str = "DROP TABLE IF EXISTS repo; DROP TABLE IF EXISTS forge; | ||
DROP TABLE IF EXISTS moved; DROP TABLE IF EXISTS block; DROP TABLE IF EXISTS label; | ||
DROP TABLE IF EXISTS labeler;"; | ||
| ||
impl Index { | ||
pub fn open(path: &Path) -> rusqlite::Result<Index> { | ||
let db = Connection::open(path)?; | ||
let version: i64 = db.query_row("PRAGMA user_version", [], |row| row.get(0))?; | ||
if version != VERSION { | ||
db.execute_batch(CACHE)?; | ||
} | ||
// A repository is known by its owner and key; its name, and the | ||
// URIs others wrote naming it, can change. | ||
db.execute_batch(&format!( | ||
"PRAGMA journal_mode = WAL; | ||
PRAGMA synchronous = NORMAL; | ||
PRAGMA user_version = {VERSION}; | ||
CREATE TABLE IF NOT EXISTS repo ( | ||
uri TEXT PRIMARY KEY, did TEXT NOT NULL, name TEXT NOT NULL, key TEXT NOT NULL, | ||
description TEXT, fork_of TEXT, fork_did TEXT, fork_key TEXT, | ||
from_did TEXT, from_key TEXT, created_at TEXT NOT NULL, value TEXT NOT NULL); | ||
CREATE INDEX IF NOT EXISTS repo_key ON repo (did, key); | ||
CREATE INDEX IF NOT EXISTS repo_fork ON repo (fork_did, fork_key); | ||
CREATE INDEX IF NOT EXISTS repo_created ON repo (created_at); | ||
CREATE TABLE IF NOT EXISTS forge ( | ||
uri TEXT PRIMARY KEY, author TEXT NOT NULL, repo_did TEXT NOT NULL, | ||
repo_key TEXT NOT NULL, value TEXT NOT NULL); | ||
CREATE INDEX IF NOT EXISTS forge_repo ON forge (repo_did, repo_key); | ||
CREATE TABLE IF NOT EXISTS moved ( | ||
uri TEXT PRIMARY KEY, from_did TEXT NOT NULL, from_key TEXT NOT NULL, | ||
to_did TEXT NOT NULL, to_key TEXT NOT NULL); | ||
CREATE TABLE IF NOT EXISTS block ( | ||
uri TEXT PRIMARY KEY, did TEXT NOT NULL, subject TEXT NOT NULL); | ||
CREATE INDEX IF NOT EXISTS block_did ON block (did); | ||
CREATE TABLE IF NOT EXISTS label ( | ||
src TEXT NOT NULL, uri TEXT NOT NULL, val TEXT NOT NULL, exp TEXT, | ||
PRIMARY KEY (src, uri, val)); | ||
CREATE TABLE IF NOT EXISTS labeler (src TEXT PRIMARY KEY, seq INTEGER NOT NULL); | ||
CREATE TABLE IF NOT EXISTS takedown ( | ||
subject TEXT PRIMARY KEY, value TEXT NOT NULL, at TEXT NOT NULL); | ||
CREATE TABLE IF NOT EXISTS emitted (seq INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT NOT NULL); | ||
CREATE TABLE IF NOT EXISTS report ( | ||
id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL); | ||
CREATE TEMP VIEW IF NOT EXISTS down AS | ||
SELECT subject FROM takedown | ||
UNION SELECT uri FROM label | ||
WHERE exp IS NULL OR exp > strftime('%Y-%m-%dT%H:%M:%fZ', 'now');" | ||
))?; | ||
Ok(Index { db: Mutex::new(db) }) | ||
} | ||
| ||
/// Records the record at `uri`, by `did`, replacing what was there. | ||
pub fn put( | ||
&self, | ||
did: &str, | ||
collection: &str, | ||
rkey: &str, | ||
value: &Value, | ||
) -> rusqlite::Result<()> { | ||
let uri = format!("at://{did}/{collection}/{rkey}"); | ||
let db = self.db.lock().unwrap(); | ||
let text = value.to_string(); | ||
let named = |field: &str| serde_json::from_value::<RepoRef>(value[field].clone()).ok(); | ||
let key = value["key"].as_str().unwrap_or_default(); | ||
// An owner's word that the repository moved to another account: | ||
// its offer, or the redirect left when the old copy went. | ||
let away = match collection { | ||
nsid::REPO => named("transfer").map(|to| (key.to_string(), to)), | ||
nsid::REDIRECT => named("to").map(|to| (to.key.clone(), to)), | ||
_ => None, | ||
}; | ||
db.execute("DELETE FROM moved WHERE uri = ?1", [&uri])?; | ||
if let Some((from_key, to)) = away.filter(|(_, to)| to.did() != did) { | ||
let moved = db.execute( | ||
"INSERT INTO moved VALUES (?1, ?2, ?3, ?4, ?5)", | ||
params![uri, did, from_key, to.did(), to.key], | ||
); | ||
moved?; | ||
} | ||
// Bound, then checked: a `?` alone on a line counts as a line of its | ||
// own for coverage. | ||
let wrote = if collection == nsid::REPO { | ||
let fork = named("forkOf"); | ||
let from = named("transferredFrom"); | ||
db.execute( | ||
"INSERT OR REPLACE INTO repo VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", | ||
params![ | ||
uri, | ||
did, | ||
rkey, | ||
key, | ||
value["description"].as_str(), | ||
fork.as_ref().map(|f| &f.uri), | ||
fork.as_ref().map(RepoRef::did), | ||
fork.as_ref().map(|f| &f.key), | ||
from.as_ref().map(RepoRef::did), | ||
from.as_ref().map(|f| &f.key), | ||
value["createdAt"].as_str().unwrap_or_default(), | ||
text | ||
], | ||
) | ||
} else if let Some(repo) = named("repo") { | ||
db.execute( | ||
"INSERT OR REPLACE INTO forge VALUES (?1, ?2, ?3, ?4, ?5)", | ||
params![uri, did, repo.did(), repo.key, text], | ||
) | ||
} else if collection == nsid::BLOCK { | ||
// A block naming no repository keeps its subject out of all the | ||
// blocker's. | ||
let subject = value["subject"].as_str().unwrap_or_default(); | ||
db.execute( | ||
"INSERT OR REPLACE INTO block VALUES (?1, ?2, ?3)", | ||
params![uri, did, subject], | ||
) | ||
} else { | ||
Ok(0) | ||
}; | ||
wrote?; | ||
Ok(()) | ||
} | ||
| ||
pub fn delete(&self, did: &str, collection: &str, rkey: &str) -> rusqlite::Result<()> { | ||
let uri = format!("at://{did}/{collection}/{rkey}"); | ||
let db = self.db.lock().unwrap(); | ||
db.execute("DELETE FROM repo WHERE uri = ?1", [&uri])?; | ||
db.execute("DELETE FROM forge WHERE uri = ?1", [&uri])?; | ||
db.execute("DELETE FROM moved WHERE uri = ?1", [&uri])?; | ||
db.execute("DELETE FROM block WHERE uri = ?1", [&uri])?; | ||
Ok(()) | ||
} | ||
| ||
/// Every forge record about `did`'s repository `key` that stands: not | ||
/// taken down, not by an account taken down, and not kept out by the | ||
/// repository's owner. | ||
pub fn forge(&self, did: &str, key: &str) -> rusqlite::Result<Vec<Entry>> { | ||
let db = self.db.lock().unwrap(); | ||
// With those about the repository it moved from, when both owners | ||
// say so: it names that one, and that one's owner said it moved here. | ||
let mut stmt = db.prepare_cached( | ||
"SELECT * FROM ( | ||
SELECT uri, author, value FROM forge WHERE repo_did = ?1 AND repo_key = ?2 | ||
UNION ALL | ||
SELECT f.uri, f.author, f.value FROM repo r | ||
JOIN moved m ON m.from_did = r.from_did AND m.from_key = r.from_key | ||
AND m.to_did = r.did AND m.to_key = r.key | ||
JOIN forge f ON f.repo_did = r.from_did AND f.repo_key = r.from_key | ||
WHERE r.did = ?1 AND r.key = ?2 | ||
GROUP BY f.uri) | ||
WHERE uri NOT IN down AND author NOT IN down | ||
AND coalesce(json_extract(value, '$.subject'), '') NOT IN down", | ||
)?; | ||
let entries = stmt | ||
.query_map([did, key], entry)? | ||
.collect::<rusqlite::Result<_>>()?; | ||
let mut stmt = db.prepare_cached("SELECT subject FROM block WHERE did = ?1")?; | ||
let blocked = stmt | ||
.query_map([did], |row| row.get(0))? | ||
.collect::<rusqlite::Result<Vec<String>>>()?; | ||
Ok(knotless::forge::moderate(did, key, &blocked, entries)) | ||
} | ||
| ||
/// Whether any of `subjects`, accounts or records, is taken down here | ||
/// or by a labeler this server follows. | ||
pub fn down(&self, subjects: &[&str]) -> rusqlite::Result<bool> { | ||
let db = self.db.lock().unwrap(); | ||
let mut stmt = db.prepare_cached("SELECT 1 FROM down WHERE subject = ?1")?; | ||
for s in subjects { | ||
if stmt.exists([s])? { | ||
return Ok(true); | ||
} | ||
} | ||
Ok(false) | ||
} | ||
| ||
/// Takes `subject` down, or puts it back: `value` is how the takedown | ||
/// names it. Returns the label that says so, and its place among those | ||
/// this server has given out. | ||
pub fn take_down( | ||
&self, | ||
src: &str, | ||
subject: &str, | ||
value: &Value, | ||
applied: bool, | ||
) -> rusqlite::Result<(i64, Value)> { | ||
let db = self.db.lock().unwrap(); | ||
let at = knotless::records::now(); | ||
if applied { | ||
db.execute( | ||
"INSERT OR REPLACE INTO takedown VALUES (?1, ?2, ?3)", | ||
params![subject, value.to_string(), at], | ||
)?; | ||
} else { | ||
db.execute("DELETE FROM takedown WHERE subject = ?1", [subject])?; | ||
} | ||
let mut label = serde_json::json!({ | ||
"ver": 1, "src": src, "uri": subject, "val": knotless::moderation::TAKEDOWN, "cts": at, | ||
}); | ||
if !applied { | ||
label["neg"] = true.into(); | ||
} | ||
db.execute( | ||
"INSERT INTO emitted (label) VALUES (?1)", | ||
[label.to_string()], | ||
)?; | ||
Ok((db.last_insert_rowid(), label)) | ||
} | ||
| ||
/// How a takedown of `subject` named it, if it is taken down. | ||
pub fn taken_down(&self, subject: &str) -> rusqlite::Result<Option<Value>> { | ||
let db = self.db.lock().unwrap(); | ||
db.query_row( | ||
"SELECT value FROM takedown WHERE subject = ?1", | ||
[subject], | ||
json, | ||
) | ||
.optional() | ||
} | ||
| ||
/// The labels this server gave out after `seq`, oldest first. | ||
pub fn emitted(&self, seq: i64) -> rusqlite::Result<Vec<(i64, Value)>> { | ||
let db = self.db.lock().unwrap(); | ||
let mut stmt = | ||
db.prepare_cached("SELECT seq, label FROM emitted WHERE seq > ?1 ORDER BY seq")?; | ||
stmt.query_map([seq], numbered)?.collect() | ||
} | ||
| ||
/// Takedowns standing whose subject starts with `prefix`, as labels | ||
/// from `src`. | ||
pub fn labels(&self, src: &str, prefix: &str) -> rusqlite::Result<Vec<Value>> { | ||
let db = self.db.lock().unwrap(); | ||
let mut stmt = db.prepare_cached( | ||
"SELECT subject, at FROM takedown WHERE substr(subject, 1, length(?1)) = ?1 | ||
ORDER BY subject LIMIT 250", | ||
)?; | ||
let rows = stmt | ||
.query_map([prefix], pair)? | ||
.collect::<rusqlite::Result<Vec<(String, String)>>>()?; | ||
let label = |(uri, at)| { | ||
serde_json::json!({ | ||
"ver": 1, "src": src, "uri": uri, "val": knotless::moderation::TAKEDOWN, "cts": at, | ||
}) | ||
}; | ||
Ok(rows.into_iter().map(label).collect()) | ||
} | ||
| ||
/// Takes in a label from the labeler `src`, seen at `seq` of its stream. | ||
/// Only those that take things out of view are kept. | ||
pub fn label(&self, src: &str, seq: i64, label: &Value) -> rusqlite::Result<()> { | ||
let db = self.db.lock().unwrap(); | ||
let val = label["val"].as_str().unwrap_or_default(); | ||
let uri = label["uri"].as_str().unwrap_or_default(); | ||
let known = [knotless::moderation::TAKEDOWN, knotless::moderation::HIDE]; | ||
if label["src"] == src && known.contains(&val) { | ||
if label["neg"] == true { | ||
db.execute( | ||
"DELETE FROM label WHERE src = ?1 AND uri = ?2 AND val = ?3", | ||
[src, uri, val], | ||
)?; | ||
} else { | ||
db.execute( | ||
"INSERT OR REPLACE INTO label VALUES (?1, ?2, ?3, ?4)", | ||
params![src, uri, val, label["exp"].as_str()], | ||
)?; | ||
} | ||
} | ||
db.execute( | ||
"INSERT OR REPLACE INTO labeler VALUES (?1, ?2)", | ||
params![src, seq], | ||
)?; | ||
Ok(()) | ||
} | ||
| ||
/// Where the labeler `src`'s stream was last read to. | ||
pub fn labeler_seq(&self, src: &str) -> rusqlite::Result<Option<i64>> { | ||
let db = self.db.lock().unwrap(); | ||
db.query_row("SELECT seq FROM labeler WHERE src = ?1", [src], |row| { | ||
row.get(0) | ||
}) | ||
.optional() | ||
} | ||
| ||
/// Keeps a report, as `com.atproto.moderation.createReport` answers it | ||
/// without its `id`, and returns it with the `id` it was given. | ||
pub fn report(&self, mut report: Value) -> rusqlite::Result<Value> { | ||
let db = self.db.lock().unwrap(); | ||
db.execute( | ||
"INSERT INTO report (value) VALUES (?1)", | ||
[report.to_string()], | ||
)?; | ||
report["id"] = db.last_insert_rowid().into(); | ||
Ok(report) | ||
} | ||
| ||
/// Reports, newest first, from before `before` when it is given. | ||
pub fn reports(&self, limit: usize, before: Option<i64>) -> rusqlite::Result<Vec<Value>> { | ||
let db = self.db.lock().unwrap(); | ||
let mut stmt = db.prepare_cached( | ||
"SELECT id, value FROM report WHERE id < ?1 ORDER BY id DESC LIMIT ?2", | ||
)?; | ||
let before = before.unwrap_or(i64::MAX); | ||
stmt.query_map(params![before, limit as i64], numbered)? | ||
.map(|r| { | ||
r.map(|(id, mut report): (i64, Value)| { | ||
report["id"] = id.into(); | ||
report | ||
}) | ||
}) | ||
.collect() | ||
} | ||
| ||
/// The name `did`'s repository `key` has now. | ||
pub fn name(&self, did: &str, key: &str) -> rusqlite::Result<Option<String>> { | ||
let db = self.db.lock().unwrap(); | ||
db.query_row( | ||
"SELECT name FROM repo WHERE did = ?1 AND key = ?2", | ||
[did, key], | ||
|row| row.get(0), | ||
) | ||
.optional() | ||
} | ||
| ||
/// The repository record at `uri`. | ||
pub fn repo(&self, uri: &str) -> rusqlite::Result<Option<Value>> { | ||
let db = self.db.lock().unwrap(); | ||
db.query_row("SELECT value FROM repo WHERE uri = ?1", [uri], |row| { | ||
row.get::<_, String>(0) | ||
}) | ||
.optional() | ||
.map(|v| v.and_then(|v| serde_json::from_str(&v).ok())) | ||
} | ||
| ||
/// Repositories, newest first, whose name or description holds `query`. | ||
pub fn repos(&self, query: &str, limit: usize) -> rusqlite::Result<Vec<Listed>> { | ||
self.repos_page(query, limit, None).map(|(repos, _)| repos) | ||
} | ||
| ||
/// A page of [`Index::repos`] after `cursor`, and the cursor of the | ||
/// next page when there may be one. | ||
pub fn repos_page( | ||
&self, | ||
query: &str, | ||
limit: usize, | ||
cursor: Option<&str>, | ||
) -> rusqlite::Result<(Vec<Listed>, Option<String>)> { | ||
let db = self.db.lock().unwrap(); | ||
let pattern = format!("%{}%", query.replace(['%', '_'], "")); | ||
// Newest first, ties broken by URI, so a page ends at a point the | ||
// next can start strictly after. | ||
let (at, uri) = cursor | ||
.and_then(|c| c.split_once(' ')) | ||
.unwrap_or(("\u{10ffff}", "")); | ||
let mut stmt = db.prepare_cached( | ||
"SELECT did, name, description, fork_of, created_at, uri FROM repo | ||
WHERE (name LIKE ?1 OR description LIKE ?1) | ||
AND (created_at < ?2 OR (created_at = ?2 AND uri < ?3)) | ||
AND did NOT IN down AND uri NOT IN down | ||
ORDER BY created_at DESC, uri DESC LIMIT ?4", | ||
)?; | ||
let rows: Vec<(Listed, String)> = stmt | ||
.query_map(params![pattern, at, uri, limit as i64], paged)? | ||
.collect::<rusqlite::Result<_>>()?; | ||
let next = (rows.len() == limit) | ||
.then(|| { | ||
rows.last() | ||
.map(|(r, uri)| format!("{} {uri}", r.created_at)) | ||
}) | ||
.flatten(); | ||
Ok((rows.into_iter().map(|(r, _)| r).collect(), next)) | ||
} | ||
| ||
/// Forks of `did`'s repository `key`. | ||
pub fn forks(&self, did: &str, key: &str) -> rusqlite::Result<Vec<Listed>> { | ||
let db = self.db.lock().unwrap(); | ||
let mut stmt = db.prepare_cached( | ||
"SELECT did, name, description, fork_of, created_at FROM repo | ||
WHERE fork_did = ?1 AND fork_key = ?2 AND did NOT IN down AND uri NOT IN down | ||
ORDER BY created_at", | ||
)?; | ||
stmt.query_map([did, key], listed)?.collect() | ||
} | ||
} | ||
| ||
/// A number and the JSON after it. | ||
fn numbered(row: &rusqlite::Row) -> rusqlite::Result<(i64, Value)> { | ||
Ok((row.get(0)?, json(row)?)) | ||
} | ||
| ||
fn pair(row: &rusqlite::Row) -> rusqlite::Result<(String, String)> { | ||
Ok((row.get(0)?, row.get(1)?)) | ||
} | ||
| ||
/// The JSON in a row's last column. | ||
fn json(row: &rusqlite::Row) -> rusqlite::Result<Value> { | ||
let text: String = row.get(row.as_ref().column_count() - 1)?; | ||
Ok(serde_json::from_str(&text).unwrap_or_default()) | ||
} | ||
| ||
fn entry(row: &rusqlite::Row) -> rusqlite::Result<Entry> { | ||
let value: String = row.get(2)?; | ||
Ok(Entry { | ||
uri: row.get(0)?, | ||
author: row.get(1)?, | ||
value: serde_json::from_str(&value).unwrap_or_default(), | ||
}) | ||
} | ||
| ||
/// A listed repository, and its URI to page from. | ||
fn paged(row: &rusqlite::Row) -> rusqlite::Result<(Listed, String)> { | ||
Ok((listed(row)?, row.get(5)?)) | ||
} | ||
| ||
fn listed(row: &rusqlite::Row) -> rusqlite::Result<Listed> { | ||
Ok(Listed { | ||
did: row.get(0)?, | ||
name: row.get(1)?, | ||
description: row.get(2)?, | ||
fork_of: row.get(3)?, | ||
created_at: row.get(4)?, | ||
}) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use serde_json::json; | ||
| ||
#[test] | ||
fn not_a_database() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let path = dir.path().join("i.db"); | ||
std::fs::write(&path, vec![7u8; 4096]).unwrap(); | ||
assert!(Index::open(&path).is_err()); | ||
} | ||
| ||
#[test] | ||
fn a_database_of_another_shape() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let path = dir.path().join("i.db"); | ||
// Claiming to be this version, so it is kept: every use fails. | ||
Connection::open(&path) | ||
.unwrap() | ||
.execute_batch(&format!( | ||
"PRAGMA user_version = {VERSION}; | ||
CREATE TABLE repo (did, key, fork_did, fork_key, created_at); | ||
CREATE TABLE forge (repo_did, repo_key);" | ||
)) | ||
.unwrap(); | ||
let index = Index::open(&path).unwrap(); | ||
assert!(index.put("did:a", nsid::REPO, "r", &json!({})).is_err()); | ||
let issue = json!({"repo": {"uri": "at://did:a/dev.knotless.repo/r", "key": "k"}}); | ||
assert!(index.put("did:a", nsid::ISSUE, "1", &issue).is_err()); | ||
assert!(index.repos("", 1).is_err()); | ||
assert!(index.forks("did:a", "k").is_err()); | ||
assert!(index.forge("did:a", "k").is_err()); | ||
assert!(index.name("did:a", "k").is_err()); | ||
} | ||
| ||
#[test] | ||
fn moderation() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let index = Index::open(&dir.path().join("i.db")).unwrap(); | ||
let named = json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"}); | ||
let issue = |did: &str, rkey: &str| { | ||
index | ||
.put( | ||
did, | ||
nsid::ISSUE, | ||
rkey, | ||
&json!({"repo": named, "title": "t"}), | ||
) | ||
.unwrap(); | ||
format!("at://{did}/dev.knotless.issue/{rkey}") | ||
}; | ||
let forge = || -> Vec<String> { | ||
index | ||
.forge("did:a", "k") | ||
.unwrap() | ||
.into_iter() | ||
.map(|e| e.uri) | ||
.collect() | ||
}; | ||
// Blocks from all the owner's repositories, and from this one. | ||
let (first, second) = (issue("did:b", "1"), issue("did:c", "2")); | ||
index | ||
.put("did:a", nsid::BLOCK, "b", &json!({"subject": "did:c"})) | ||
.unwrap(); | ||
assert_eq!(forge(), std::slice::from_ref(&first)); | ||
index.delete("did:a", nsid::BLOCK, "b").unwrap(); | ||
assert_eq!(forge().len(), 2); | ||
index | ||
.put( | ||
"did:a", | ||
nsid::BLOCK, | ||
"c", | ||
&json!({"subject": "did:c", "repo": named, "createdAt": "t"}), | ||
) | ||
.unwrap(); | ||
assert_eq!( | ||
forge(), | ||
[first.clone(), "at://did:a/dev.knotless.block/c".into()] | ||
); | ||
index.delete("did:a", nsid::BLOCK, "c").unwrap(); | ||
assert_eq!(forge().len(), 2); | ||
| ||
// Takedowns, and the labels that say so. | ||
let subject = json!({"$type": "com.atproto.repo.strongRef", "uri": first}); | ||
let (seq, label) = index | ||
.take_down("did:web:me", &first, &subject, true) | ||
.unwrap(); | ||
assert_eq!((seq, label["val"].as_str()), (1, Some("!takedown"))); | ||
assert_eq!(forge(), std::slice::from_ref(&second)); | ||
assert!(index.down(&["did:x", &first]).unwrap()); | ||
assert_eq!(index.taken_down(&first).unwrap(), Some(subject.clone())); | ||
assert_eq!(index.labels("did:web:me", "at://did:b/").unwrap().len(), 1); | ||
assert!(index.labels("did:web:me", "did:").unwrap().is_empty()); | ||
let (_, undone) = index | ||
.take_down("did:web:me", &first, &subject, false) | ||
.unwrap(); | ||
assert_eq!(undone["neg"], true); | ||
assert_eq!(index.taken_down(&first).unwrap(), None); | ||
assert_eq!(index.emitted(0).unwrap().len(), 2); | ||
assert_eq!(index.emitted(1).unwrap()[0].1, undone); | ||
| ||
// Labels from a labeler: only its own, only those that hide, and | ||
// not past their expiry. | ||
let label = |src: &str, uri: &str, val: &str, extra: Value| { | ||
let mut l = json!({"src": src, "uri": uri, "val": val}); | ||
l.as_object_mut() | ||
.unwrap() | ||
.extend(extra.as_object().unwrap().clone()); | ||
l | ||
}; | ||
assert_eq!(index.labeler_seq("did:web:l").unwrap(), None); | ||
let none = json!({}); | ||
index | ||
.label( | ||
"did:web:l", | ||
3, | ||
&label("did:web:other", "did:c", "!takedown", none.clone()), | ||
) | ||
.unwrap(); | ||
index | ||
.label( | ||
"did:web:l", | ||
4, | ||
&label("did:web:l", "did:c", "porn", none.clone()), | ||
) | ||
.unwrap(); | ||
index | ||
.label( | ||
"did:web:l", | ||
5, | ||
&label( | ||
"did:web:l", | ||
"did:b", | ||
"!hide", | ||
json!({"exp": "2000-01-01T00:00:00Z"}), | ||
), | ||
) | ||
.unwrap(); | ||
assert_eq!(forge().len(), 2); | ||
index | ||
.label( | ||
"did:web:l", | ||
6, | ||
&label("did:web:l", "did:c", "!hide", none.clone()), | ||
) | ||
.unwrap(); | ||
assert_eq!(forge(), std::slice::from_ref(&first)); | ||
index | ||
.label( | ||
"did:web:l", | ||
7, | ||
&label("did:web:l", "did:c", "!hide", json!({"neg": true})), | ||
) | ||
.unwrap(); | ||
assert_eq!(forge().len(), 2); | ||
assert_eq!(index.labeler_seq("did:web:l").unwrap(), Some(7)); | ||
| ||
// Reports, newest first, a page at a time. | ||
for n in 0..3 { | ||
let report = index.report(json!({"reason": n})).unwrap(); | ||
assert_eq!(report["id"], n + 1); | ||
} | ||
let page = index.reports(2, None).unwrap(); | ||
assert_eq!( | ||
(page[0]["id"].as_i64(), page[1]["reason"].as_i64()), | ||
(Some(3), Some(1)) | ||
); | ||
assert_eq!(index.reports(2, Some(2)).unwrap()[0]["id"], 1); | ||
| ||
// Takedowns and reports outlive a rebuild; the cache does not. | ||
drop(index); | ||
let db = Connection::open(dir.path().join("i.db")).unwrap(); | ||
db.execute_batch("PRAGMA user_version = 1").unwrap(); | ||
drop(db); | ||
let index = Index::open(&dir.path().join("i.db")).unwrap(); | ||
assert_eq!(index.reports(10, None).unwrap().len(), 3); | ||
assert_eq!(index.emitted(0).unwrap().len(), 2); | ||
assert_eq!(index.labeler_seq("did:web:l").unwrap(), None); | ||
} | ||
| ||
#[test] | ||
fn an_index_of_another_version_is_rebuilt() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let path = dir.path().join("i.db"); | ||
Connection::open(&path) | ||
.unwrap() | ||
.execute_batch("CREATE TABLE repo (did, created_at); CREATE TABLE forge (repo);") | ||
.unwrap(); | ||
let index = Index::open(&path).unwrap(); | ||
index | ||
.put("did:a", nsid::REPO, "r", &json!({"key": "k"})) | ||
.unwrap(); | ||
assert_eq!(index.name("did:a", "k").unwrap().as_deref(), Some("r")); | ||
} | ||
| ||
#[test] | ||
fn records_in_and_out() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let index = Index::open(&dir.path().join("i.db")).unwrap(); | ||
let repo = | ||
json!({"name": "r", "key": "k", "description": "a thing", "createdAt": "2026-01-01"}); | ||
index.put("did:a", nsid::REPO, "r", &repo).unwrap(); | ||
let uri = "at://did:a/dev.knotless.repo/r"; | ||
let named = json!({"uri": uri, "key": "k"}); | ||
let fork = json!({"name": "r", "key": "k2", "forkOf": named, "createdAt": "2026-01-02"}); | ||
index.put("did:b", nsid::REPO, "r", &fork).unwrap(); | ||
index | ||
.put( | ||
"did:b", | ||
nsid::ISSUE, | ||
"1", | ||
&json!({"repo": named, "title": "t"}), | ||
) | ||
.unwrap(); | ||
// About a repository since deleted, whose name was taken again. | ||
let gone = json!({"uri": uri, "key": "k0"}); | ||
index | ||
.put( | ||
"did:b", | ||
nsid::ISSUE, | ||
"3", | ||
&json!({"repo": gone, "title": "old"}), | ||
) | ||
.unwrap(); | ||
index | ||
.put("did:b", nsid::ISSUE, "2", &json!({"title": "no repo"})) | ||
.unwrap(); | ||
assert_eq!(index.repo(uri).unwrap(), Some(repo)); | ||
assert_eq!(index.repo("at://did:x/dev.knotless.repo/r").unwrap(), None); | ||
assert_eq!(index.forge("did:a", "k").unwrap().len(), 1); | ||
assert_eq!(index.name("did:a", "k").unwrap().as_deref(), Some("r")); | ||
assert_eq!(index.name("did:a", "k0").unwrap(), None); | ||
let all = index.repos("", 10).unwrap(); | ||
assert_eq!( | ||
all.iter().map(|r| r.did.as_str()).collect::<Vec<_>>(), | ||
["did:b", "did:a"] | ||
); | ||
assert_eq!(index.repos("THING", 10).unwrap().len(), 1); | ||
let (page, cursor) = index.repos_page("", 1, None).unwrap(); | ||
assert_eq!( | ||
(page[0].did.as_str(), cursor.as_deref()), | ||
("did:b", Some("2026-01-02 at://did:b/dev.knotless.repo/r")) | ||
); | ||
let (page, cursor) = index.repos_page("", 1, cursor.as_deref()).unwrap(); | ||
assert_eq!(page[0].did, "did:a"); | ||
assert_eq!( | ||
index.repos_page("", 1, cursor.as_deref()).unwrap(), | ||
(vec![], None) | ||
); | ||
let listed = serde_json::to_value(&page[0]).unwrap(); | ||
assert!(listed.get("forkOf").is_none() && listed["description"] == "a thing"); | ||
assert_eq!(index.forks("did:a", "k").unwrap()[0].did, "did:b"); | ||
let all = index.repos("", 10).unwrap(); | ||
assert_eq!(all[0].fork_of.as_deref(), Some(uri)); | ||
index.delete("did:b", nsid::ISSUE, "1").unwrap(); | ||
index.delete("did:a", nsid::REPO, "r").unwrap(); | ||
assert!(index.forge("did:a", "k").unwrap().is_empty()); | ||
assert_eq!(index.repos("", 10).unwrap().len(), 1); | ||
// A second open finds the tables already there. | ||
drop(index); | ||
assert!(Index::open(&dir.path().join("i.db")).is_ok()); | ||
assert!(Index::open(dir.path()).is_err()); | ||
} | ||
} | ||
crates/knotless-serve/src/jetstream.rsnew+209−0
| @@ -0,0 +1,209 @@ | ||
//! Keeps mirrors warm and the index fed from Jetstream v2 | ||
//! (`network.bsky.jetstream.subscribeEvents`), the firehose re-encoded as | ||
//! JSON and filtered server side. Asking only for `dev.knotless.*` commits, | ||
//! a server receives its own collections' writes and every account's | ||
//! identity, account and sync events, with each record decoded inline, so | ||
//! indexing it needs no `getRecord` round trip. | ||
//! | ||
//! Jetstream frames are not signed: the record is what the Jetstream | ||
//! operator says the commit held. That is the same trust a server places in | ||
//! the relay it follows, and the index is a cache either way. | ||
| ||
use crate::App; | ||
use crate::firehose::{Event, Op, knotless}; | ||
use futures::StreamExt; | ||
use serde_json::Value; | ||
use std::sync::Arc; | ||
use std::sync::atomic::Ordering; | ||
use std::time::Duration; | ||
use tokio_tungstenite::tungstenite::Message; | ||
use tokio_tungstenite::tungstenite::client::IntoClientRequest; | ||
use tokio_tungstenite::tungstenite::http::HeaderValue; | ||
| ||
/// The collections asked for: every knotless and forge NSID. | ||
pub const COLLECTIONS: &str = "dev.knotless.*"; | ||
| ||
/// The one framing the endpoint speaks. | ||
const SUBPROTOCOL: &str = "xrpc.v1.json"; | ||
| ||
/// The payload `$type`s, after this prefix, name the event kind. | ||
const KIND: &str = "network.bsky.jetstream.subscribeEvents#"; | ||
| ||
/// The subscribeEvents endpoint for a Jetstream host, from `cursor`, which | ||
/// it replays inclusively. | ||
pub fn endpoint(base: &str, cursor: Option<i64>) -> String { | ||
let mut url = base.trim_end_matches('/').to_string(); | ||
if !url.contains("/xrpc/") { | ||
url.push_str("/xrpc/network.bsky.jetstream.subscribeEvents"); | ||
} | ||
url.push_str(&format!( | ||
"?collections={COLLECTIONS}&kinds=commit&kinds=identity&kinds=account&kinds=sync" | ||
)); | ||
if let Some(c) = cursor { | ||
url.push_str(&format!("&cursor={c}")); | ||
} | ||
url | ||
} | ||
| ||
/// Reads a text frame. `None` when it is not a message: an error frame, or | ||
/// not Jetstream at all. | ||
pub fn parse(frame: &str) -> Option<Event> { | ||
let mut frame: Value = serde_json::from_str(frame).ok()?; | ||
if frame["$type"] != "message" { | ||
return None; | ||
} | ||
let mut payload = frame["payload"].take(); | ||
let kind = payload["$type"].as_str()?.strip_prefix(KIND)?.to_string(); | ||
let mut event = Event { | ||
seq: payload["seq"].as_i64(), | ||
..Event::default() | ||
}; | ||
match kind.as_str() { | ||
"commit" => { | ||
let collection = text(&payload, "collection")?; | ||
if knotless(&collection) { | ||
let record = payload.get_mut("record").map(Value::take); | ||
event.ops.push(Op { | ||
action: text(&payload, "operation")?, | ||
collection, | ||
rkey: text(&payload, "rkey")?, | ||
record, | ||
}); | ||
event.did = Some(text(&payload, "did")?); | ||
} | ||
} | ||
"identity" | "account" | "sync" => event.did = Some(text(&payload, "did")?), | ||
_ => {} | ||
} | ||
Some(event) | ||
} | ||
| ||
fn text(v: &Value, name: &str) -> Option<String> { | ||
v[name].as_str().map(str::to_string) | ||
} | ||
| ||
/// Follows Jetstream until the process ends, reconnecting after the last | ||
/// sequence number seen. While disconnected, mirrors fall back to the TTL. | ||
pub async fn watch(app: Arc<App>, base: String) { | ||
let mut cursor = None; | ||
loop { | ||
if let Ok(mut request) = endpoint(&base, cursor).into_client_request() { | ||
request.headers_mut().insert( | ||
"Sec-WebSocket-Protocol", | ||
HeaderValue::from_static(SUBPROTOCOL), | ||
); | ||
if let Ok((mut ws, _)) = tokio_tungstenite::connect_async(request).await { | ||
app.watching.store(true, Ordering::Relaxed); | ||
while let Some(Ok(msg)) = ws.next().await { | ||
if let Message::Text(frame) = msg | ||
&& let Some(event) = parse(&frame) | ||
{ | ||
// The cursor is inclusive: resume after the last one. | ||
cursor = event.seq.map(|s| s + 1).or(cursor); | ||
app.observe(event); | ||
} | ||
} | ||
} | ||
} | ||
app.watching.store(false, Ordering::Relaxed); | ||
tokio::time::sleep(Duration::from_millis(500)).await; | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use knotless::records::nsid; | ||
| ||
const TIME: &str = "1970-01-01T00:00:00.000001Z"; | ||
| ||
fn commit(seq: i64, op: &str, collection: &str, record: &str) -> String { | ||
format!( | ||
r#"{{"$type":"message","payload":{{"$type":"{KIND}commit","seq":{seq},"did":"did:plc:a","time":"{TIME}","rev":"r","operation":"{op}","collection":"{collection}","rkey":"k"{record}}}}}"# | ||
) | ||
} | ||
| ||
#[test] | ||
fn the_filter_covers_every_collection() { | ||
let prefix = COLLECTIONS.trim_end_matches('*'); | ||
for collection in nsid::ALL.iter().chain(nsid::FORGE.iter()) { | ||
assert!(collection.starts_with(prefix), "{collection}"); | ||
} | ||
} | ||
| ||
#[test] | ||
fn frames() { | ||
let created = parse(&commit( | ||
42, | ||
"create", | ||
"dev.knotless.issue", | ||
r#","cid":"bafy","record":{"$type":"dev.knotless.issue","title":"hi"}"#, | ||
)) | ||
.unwrap(); | ||
assert_eq!(created.seq, Some(42)); | ||
assert_eq!(created.did.as_deref(), Some("did:plc:a")); | ||
assert_eq!( | ||
created.ops, | ||
[Op { | ||
action: "create".into(), | ||
collection: "dev.knotless.issue".into(), | ||
rkey: "k".into(), | ||
record: Some(serde_json::json!({"$type": "dev.knotless.issue", "title": "hi"})), | ||
}] | ||
); | ||
let deleted = parse(&commit(43, "delete", "dev.knotless.ref", "")).unwrap(); | ||
assert_eq!(deleted.ops[0].record, None); | ||
let elsewhere = parse(&commit(44, "create", "app.bsky.feed.post", "")).unwrap(); | ||
assert_eq!( | ||
(elsewhere.seq, elsewhere.did, elsewhere.ops.len()), | ||
(Some(44), None, 0) | ||
); | ||
| ||
let identity = format!( | ||
r#"{{"$type":"message","payload":{{"$type":"{KIND}identity","seq":5,"did":"did:plc:b","time":"{TIME}","identity":{{"did":"did:plc:b","handle":"b.test","seq":9,"time":"{TIME}"}}}}}}"# | ||
); | ||
let identity = parse(&identity).unwrap(); | ||
assert_eq!( | ||
(identity.seq, identity.did.as_deref()), | ||
(Some(5), Some("did:plc:b")) | ||
); | ||
let info = format!( | ||
r#"{{"$type":"message","payload":{{"$type":"{KIND}info","name":"OutdatedCursor","message":"starting at seq 200"}}}}"# | ||
); | ||
assert_eq!(parse(&info), Some(Event::default())); | ||
| ||
assert_eq!( | ||
parse(r#"{"$type":"error","error":"ConsumerTooSlow"}"#), | ||
None | ||
); | ||
assert_eq!( | ||
parse(r#"{"$type":"message","payload":{"$type":"app.bsky.x"}}"#), | ||
None | ||
); | ||
assert_eq!(parse(r#"{"$type":"message","payload":{}}"#), None); | ||
assert_eq!(parse("not json"), None); | ||
let no_rkey = format!( | ||
r#"{{"$type":"message","payload":{{"$type":"{KIND}commit","seq":1,"did":"did:plc:a","operation":"create","collection":"dev.knotless.repo"}}}}"# | ||
); | ||
assert_eq!(parse(&no_rkey), None); | ||
} | ||
| ||
#[test] | ||
fn endpoints() { | ||
let filter = | ||
"collections=dev.knotless.*&kinds=commit&kinds=identity&kinds=account&kinds=sync"; | ||
assert_eq!( | ||
endpoint("wss://jetstream.us-east.bsky.network/", None), | ||
format!( | ||
"wss://jetstream.us-east.bsky.network/xrpc/network.bsky.jetstream.subscribeEvents?{filter}" | ||
) | ||
); | ||
assert_eq!( | ||
endpoint( | ||
"ws://h/xrpc/network.bsky.jetstream.subscribeEvents", | ||
Some(3) | ||
), | ||
format!("ws://h/xrpc/network.bsky.jetstream.subscribeEvents?{filter}&cursor=3") | ||
); | ||
} | ||
} | ||
crates/knotless-serve/src/lib.rsnew+539−0
| @@ -0,0 +1,539 @@ | ||
//! A read accelerator for knotless repositories. | ||
//! | ||
//! A PDS can hand out records and blobs and nothing else: no partial clone, | ||
//! no shallow fetch, no "what is in this file at this commit". This server | ||
//! keeps a bare mirror per repository and answers those from it: | ||
//! | ||
//! - git smart HTTP (protocol v0 and v2, `--filter`, `--depth`) at | ||
//! `/<handle or did>/<name>`, read only; | ||
//! - a JSON and raw API at `/api/<handle or did>/<name>/...` for agents that | ||
//! want a file, a tree, a log or a diff without a clone. | ||
//! | ||
//! It holds no state of its own: every mirror is a cache of the owner's | ||
//! records, revalidated against them (one round trip) when older than the | ||
//! TTL, or kept warm by the firehose or Jetstream. Every object it serves is named by its | ||
//! hash, so a client can check it. | ||
| ||
mod api; | ||
mod appview; | ||
pub mod diff; | ||
pub mod firehose; | ||
pub mod gateway; | ||
pub mod highlight; | ||
pub mod index; | ||
pub mod jetstream; | ||
pub mod moderation; | ||
pub mod objects; | ||
mod smart; | ||
pub mod web; | ||
| ||
use anyhow::Result; | ||
use axum::Router; | ||
use axum::http::StatusCode; | ||
use axum::response::{IntoResponse, Response}; | ||
use axum::routing::{get, post}; | ||
use knotless::env::Ctx; | ||
use knotless::git::Git; | ||
use knotless::mirror::Mirror; | ||
use knotless::remote::Remote; | ||
use knotless::url::RepoUrl; | ||
use std::collections::HashMap; | ||
use std::sync::atomic::{AtomicBool, Ordering}; | ||
use std::sync::{Arc, Mutex}; | ||
use std::time::{Duration, Instant}; | ||
| ||
pub struct App { | ||
pub ctx: Ctx, | ||
/// How long a mirror is trusted without asking the PDS, when the | ||
/// firehose is not watching. | ||
pub ttl: Duration, | ||
repos: Mutex<HashMap<(String, String), Shared<Entry>>>, | ||
/// Firehose events seen per DID. | ||
events: Mutex<HashMap<String, u64>>, | ||
pub watching: AtomicBool, | ||
/// Set when this server also takes writes for owners who delegate them. | ||
pub gateway: Option<gateway::Gateway>, | ||
/// Set when this server is an appview, indexing what the firehose says. | ||
pub index: Option<Arc<index::Index>>, | ||
/// The labels this server gives out, to those following them. | ||
pub labels: tokio::sync::broadcast::Sender<(i64, serde_json::Value)>, | ||
} | ||
| ||
type Shared<T> = Arc<tokio::sync::Mutex<T>>; | ||
| ||
#[derive(Default)] | ||
struct Entry { | ||
git: Option<Git>, | ||
/// The repository's owner, key and name now, which differ from what | ||
/// was asked for after a move or a rename. | ||
found: Option<(String, String, String)>, | ||
synced: Option<Instant>, | ||
events: u64, | ||
} | ||
| ||
/// A repository found by name, and its mirror. | ||
pub struct Located { | ||
pub git: Git, | ||
pub did: String, | ||
pub key: String, | ||
/// Its name now: after a rename, not the name it was asked for by. | ||
pub name: String, | ||
/// Whether it is in another account than the one asked for: it moved. | ||
pub moved: bool, | ||
} | ||
| ||
impl App { | ||
pub fn new(ctx: Ctx, ttl: Duration) -> Arc<App> { | ||
App::build(ctx, ttl, None, None) | ||
} | ||
| ||
pub fn build( | ||
ctx: Ctx, | ||
ttl: Duration, | ||
gateway: Option<gateway::Gateway>, | ||
index: Option<index::Index>, | ||
) -> Arc<App> { | ||
Arc::new(App { | ||
ctx, | ||
ttl, | ||
repos: Mutex::default(), | ||
events: Mutex::default(), | ||
watching: AtomicBool::new(false), | ||
gateway, | ||
index: index.map(Arc::new), | ||
labels: tokio::sync::broadcast::channel(256).0, | ||
}) | ||
} | ||
| ||
/// Takes in a firehose or Jetstream event: refreshes the mirrors of an account whose | ||
/// repositories moved, and indexes what it wrote. | ||
pub fn observe(self: &Arc<App>, event: firehose::Event) { | ||
let Some(did) = event.did else { | ||
return; | ||
}; | ||
let repository = event | ||
.ops | ||
.iter() | ||
.any(|op| knotless::records::nsid::ALL.contains(&op.collection.as_str())); | ||
if event.ops.is_empty() || repository { | ||
self.touch(&did); | ||
} | ||
if let Some(index) = self.index.clone() | ||
&& !event.ops.is_empty() | ||
{ | ||
// What the event carries in full is indexed now, in order, so a | ||
// later write cannot land before an earlier one. | ||
if event | ||
.ops | ||
.iter() | ||
.all(|op| op.action == "delete" || op.record.is_some()) | ||
{ | ||
for op in &event.ops { | ||
let _ = index_inline(&index, &did, op); | ||
} | ||
return; | ||
} | ||
let ctx = self.ctx.clone(); | ||
tokio::spawn(async move { | ||
for op in event.ops { | ||
let _ = index_op(&ctx, &index, &did, &op).await; | ||
} | ||
}); | ||
} | ||
} | ||
| ||
/// Firehose events seen for `did`. | ||
pub fn events_for(&self, did: &str) -> u64 { | ||
self.events.lock().unwrap().get(did).copied().unwrap_or(0) | ||
} | ||
| ||
/// Records that `did` wrote to a repository collection, and refreshes its | ||
/// mirrors in the background. | ||
pub fn touch(self: &Arc<App>, did: &str) { | ||
*self | ||
.events | ||
.lock() | ||
.unwrap() | ||
.entry(did.to_string()) | ||
.or_default() += 1; | ||
let names: Vec<String> = self | ||
.repos | ||
.lock() | ||
.unwrap() | ||
.keys() | ||
.filter(|(d, _)| d == did) | ||
.map(|(_, n)| n.clone()) | ||
.collect(); | ||
for name in names { | ||
let app = self.clone(); | ||
let did = did.to_string(); | ||
tokio::spawn(async move { | ||
let _ = app.mirror(&did, &name).await; | ||
}); | ||
} | ||
} | ||
| ||
/// The mirror of `authority/name`, synced if it may be stale. | ||
pub async fn mirror(&self, authority: &str, name: &str) -> Result<Git, AppError> { | ||
Ok(self.located(authority, name).await?.git) | ||
} | ||
| ||
/// The mirror of `authority/name`, with the repository's key and the | ||
/// name it has now. | ||
pub async fn located(&self, authority: &str, name: &str) -> Result<Located, AppError> { | ||
let url = RepoUrl::parse(&format!("knotless://{authority}/{name}")) | ||
.map_err(|e| AppError(StatusCode::BAD_REQUEST, e.to_string()))?; | ||
let remote = Remote::connect(&self.ctx, &url) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
let did = remote.did().to_string(); | ||
let entry = self | ||
.repos | ||
.lock() | ||
.unwrap() | ||
.entry((did.clone(), url.name.clone())) | ||
.or_default() | ||
.clone(); | ||
let mut entry = entry.lock().await; | ||
let events = self.events_for(&did); | ||
let fresh = match (&entry.git, entry.synced) { | ||
(Some(_), Some(_)) if self.watching.load(Ordering::Relaxed) => entry.events == events, | ||
(Some(_), Some(at)) => at.elapsed() < self.ttl, | ||
_ => false, | ||
}; | ||
if !fresh { | ||
let snap = remote | ||
.snapshot(&self.ctx) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
let (remote, snap) = remote | ||
.follow(&self.ctx, snap) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
let Some(repo) = &snap.repo else { | ||
return Err(AppError( | ||
StatusCode::NOT_FOUND, | ||
format!("{url}: no such repository"), | ||
)); | ||
}; | ||
let key = repo.value.key.as_str(); | ||
let mirror = Mirror::open(&self.ctx, remote.did(), key) | ||
.await | ||
.map_err(AppError::internal)?; | ||
mirror | ||
.sync(&remote, &snap) | ||
.await | ||
.map_err(AppError::upstream)?; | ||
entry.found = Some((remote.did().to_string(), key.to_string(), repo.rkey.clone())); | ||
entry.git = Some(mirror.git); | ||
entry.synced = Some(Instant::now()); | ||
entry.events = events; | ||
} | ||
let (now, key, name) = entry.found.clone().expect("a fresh entry has a mirror"); | ||
let uri = knotless::forge::repo_uri(&now, &name); | ||
if let Some(index) = &self.index | ||
&& index | ||
.down(&[&now, &uri]) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)? | ||
{ | ||
return Err(AppError( | ||
StatusCode::NOT_FOUND, | ||
format!("{url}: taken down"), | ||
)); | ||
} | ||
Ok(Located { | ||
git: entry.git.clone().expect("a fresh entry has a mirror"), | ||
moved: now != did, | ||
did: now, | ||
key, | ||
name, | ||
}) | ||
} | ||
} | ||
| ||
/// Indexes a write the event carried in full: a deletion, or a record | ||
/// delivered inline. `false` when the record has to be fetched. | ||
fn index_inline(index: &index::Index, did: &str, op: &firehose::Op) -> rusqlite::Result<bool> { | ||
match (&op.record, op.action.as_str()) { | ||
(_, "delete") => index.delete(did, &op.collection, &op.rkey)?, | ||
(Some(record), _) => index.put(did, &op.collection, &op.rkey, record)?, | ||
(None, _) => return Ok(false), | ||
} | ||
Ok(true) | ||
} | ||
| ||
/// Indexes one write: the record the event carried, else the record as its | ||
/// PDS now has it, or its absence. | ||
pub async fn index_op(ctx: &Ctx, index: &index::Index, did: &str, op: &firehose::Op) -> Result<()> { | ||
if index_inline(index, did, op)? { | ||
return Ok(()); | ||
} | ||
let id = knotless::identity::resolve(ctx, did).await?; | ||
let out: serde_json::Value = knotless::xrpc::Xrpc::new(ctx.http.clone(), &id.pds) | ||
.query( | ||
"com.atproto.repo.getRecord", | ||
&[ | ||
("repo", did), | ||
("collection", &op.collection), | ||
("rkey", &op.rkey), | ||
], | ||
None, | ||
) | ||
.await?; | ||
index.put(did, &op.collection, &op.rkey, &out["value"])?; | ||
Ok(()) | ||
} | ||
| ||
/// Backfills from `service`, saying how it went. | ||
pub async fn backfilled(ctx: &Ctx, index: &index::Index, service: &str) -> String { | ||
match backfill(ctx, index, service).await { | ||
Ok(n) => format!("backfilled {n} records from {service}"), | ||
Err(err) => format!("backfilling from {service}: {err:#}"), | ||
} | ||
} | ||
| ||
/// Indexes every account on `service`, a PDS or relay: each account's | ||
/// repositories and forge records. | ||
pub async fn backfill(ctx: &Ctx, index: &index::Index, service: &str) -> Result<usize> { | ||
#[derive(serde::Deserialize)] | ||
struct Repos { | ||
repos: Vec<Account>, | ||
cursor: Option<String>, | ||
} | ||
#[derive(serde::Deserialize)] | ||
struct Account { | ||
did: String, | ||
} | ||
let xrpc = knotless::xrpc::Xrpc::new(ctx.http.clone(), service); | ||
let mut dids = Vec::new(); | ||
let mut cursor: Option<String> = None; | ||
loop { | ||
let mut params = vec![("limit", "1000")]; | ||
if let Some(c) = &cursor { | ||
params.push(("cursor", c)); | ||
} | ||
let page: Repos = xrpc | ||
.query("com.atproto.sync.listRepos", ¶ms, None) | ||
.await?; | ||
let done = page.repos.is_empty(); | ||
dids.extend(page.repos.into_iter().map(|r| r.did)); | ||
match page.cursor { | ||
Some(next) if !done => cursor = Some(next), | ||
_ => break, | ||
} | ||
} | ||
let mut indexed = 0; | ||
for did in dids { | ||
let url = RepoUrl::parse(&format!("knotless://{did}/_"))?; | ||
let remote = Remote::connect(ctx, &url).await?; | ||
let collections = | ||
std::iter::once(knotless::records::nsid::REPO).chain(knotless::records::nsid::FORGE); | ||
for collection in collections { | ||
for r in remote.list_all::<serde_json::Value>(collection).await? { | ||
index.put(&did, collection, &r.rkey, &r.value)?; | ||
indexed += 1; | ||
} | ||
} | ||
} | ||
Ok(indexed) | ||
} | ||
| ||
/// A status and a message, as the response. | ||
#[derive(Debug)] | ||
pub struct AppError(pub StatusCode, pub String); | ||
| ||
impl AppError { | ||
pub(crate) fn upstream(err: anyhow::Error) -> AppError { | ||
AppError(StatusCode::BAD_GATEWAY, format!("{err:#}")) | ||
} | ||
/// 400 for input the request got wrong, else 500. | ||
pub(crate) fn input(err: anyhow::Error) -> AppError { | ||
let status = match err.downcast_ref::<knotless::read::BadInput>() { | ||
Some(_) => StatusCode::BAD_REQUEST, | ||
None => StatusCode::INTERNAL_SERVER_ERROR, | ||
}; | ||
AppError(status, format!("{err:#}")) | ||
} | ||
| ||
pub(crate) fn internal(err: anyhow::Error) -> AppError { | ||
AppError(StatusCode::INTERNAL_SERVER_ERROR, format!("{err:#}")) | ||
} | ||
} | ||
| ||
impl IntoResponse for AppError { | ||
fn into_response(self) -> Response { | ||
(self.0, self.1).into_response() | ||
} | ||
} | ||
| ||
pub fn router(app: Arc<App>) -> Router { | ||
Router::new() | ||
.route("/healthz", get(|| async { "ok" })) | ||
.route("/{authority}/{repo}/info/refs", get(smart::info_refs)) | ||
.route( | ||
"/{authority}/{repo}/git-upload-pack", | ||
post(smart::upload_pack), | ||
) | ||
.route("/api/{authority}/{repo}/refs", get(api::refs)) | ||
.route("/api/{authority}/{repo}/tree", get(api::tree)) | ||
.route("/api/{authority}/{repo}/blob", get(api::blob)) | ||
.route("/api/{authority}/{repo}/log", get(api::log)) | ||
.route("/api/{authority}/{repo}/diff", get(api::diff)) | ||
.route("/api/{authority}/{repo}/object/{sha}", get(api::object)) | ||
.route("/.well-known/did.json", get(gateway::did_document)) | ||
.route("/xrpc/dev.knotless.putDelegation", post(gateway::delegate)) | ||
.route( | ||
"/xrpc/dev.knotless.pushRefs", | ||
post(gateway::push).layer(axum::extract::DefaultBodyLimit::max(PUSH_LIMIT)), | ||
) | ||
.route("/xrpc/dev.knotless.createCommit", post(gateway::commit)) | ||
.route("/xrpc/dev.knotless.getForge", get(appview::get_forge)) | ||
.route("/xrpc/dev.knotless.listRepos", get(appview::list_repos)) | ||
.route( | ||
"/xrpc/com.atproto.moderation.createReport", | ||
post(moderation::create_report), | ||
) | ||
.route( | ||
"/xrpc/com.atproto.admin.updateSubjectStatus", | ||
post(moderation::update_subject_status), | ||
) | ||
.route( | ||
"/xrpc/com.atproto.admin.getSubjectStatus", | ||
get(moderation::get_subject_status), | ||
) | ||
.route( | ||
"/xrpc/dev.knotless.listReports", | ||
get(moderation::list_reports), | ||
) | ||
.route( | ||
"/xrpc/com.atproto.label.queryLabels", | ||
get(moderation::query_labels), | ||
) | ||
.route( | ||
"/xrpc/com.atproto.label.subscribeLabels", | ||
get(moderation::subscribe_labels), | ||
) | ||
.merge(web::routes()) | ||
.with_state(app) | ||
} | ||
| ||
/// The largest push a gateway takes. | ||
const PUSH_LIMIT: usize = 512 << 20; | ||
| ||
/// `name` with a trailing `.git` dropped, as clone URLs often carry one. | ||
fn repo_name(name: &str) -> &str { | ||
name.strip_suffix(".git").unwrap_or(name) | ||
} | ||
| ||
/// The arguments `knotless-serve` takes. | ||
#[derive(Debug, PartialEq)] | ||
pub struct Options { | ||
pub listen: String, | ||
pub ttl: Duration, | ||
pub firehose: Option<String>, | ||
/// A Jetstream v2 host, followed like the firehose. | ||
pub jetstream: Option<String>, | ||
/// The URL this server is reached at, which makes it a gateway. | ||
pub public_url: Option<String>, | ||
/// Where to keep the appview index, which makes it an appview. | ||
pub index: Option<std::path::PathBuf>, | ||
/// A PDS or relay to index every account of at startup. | ||
pub backfill: Option<String>, | ||
/// Labelers, by DID, whose takedowns hold here too. | ||
pub labelers: Vec<String>, | ||
} | ||
| ||
pub fn parse_args(args: &[String]) -> Result<Options, String> { | ||
let mut opts = Options { | ||
listen: "127.0.0.1:2590".into(), | ||
ttl: Duration::from_secs(5), | ||
firehose: None, | ||
jetstream: None, | ||
public_url: None, | ||
index: None, | ||
backfill: None, | ||
labelers: Vec::new(), | ||
}; | ||
let mut it = args.iter().skip(1); | ||
while let Some(flag) = it.next() { | ||
let mut value = || it.next().cloned().ok_or(format!("{flag} takes a value")); | ||
match flag.as_str() { | ||
"--listen" => opts.listen = value()?, | ||
"--ttl" => { | ||
opts.ttl = Duration::from_secs( | ||
value()? | ||
.parse() | ||
.map_err(|_| "--ttl takes seconds".to_string())?, | ||
) | ||
} | ||
"--firehose" => opts.firehose = Some(value()?), | ||
"--jetstream" => opts.jetstream = Some(value()?), | ||
"--public-url" => opts.public_url = Some(value()?), | ||
"--index" => opts.index = Some(value()?.into()), | ||
"--backfill" => opts.backfill = Some(value()?), | ||
"--labeler" => opts.labelers.push(value()?), | ||
other => { | ||
return Err(format!( | ||
"unexpected {other}\nusage: knotless-serve [--listen <addr>] [--ttl <secs>] [--firehose <wss://relay or pds>] [--jetstream <wss://host>] [--public-url <https://host>] [--index <file>] [--backfill <https://pds or relay>] [--labeler <did>]..." | ||
)); | ||
} | ||
} | ||
} | ||
Ok(opts) | ||
} | ||
| ||
/// Serves until SIGTERM or SIGINT. Prints the bound address first. | ||
pub async fn run(opts: Options, ctx: Ctx) -> Result<()> { | ||
let gateway = match &opts.public_url { | ||
Some(url) => Some(gateway::Gateway::new(url).map_err(anyhow::Error::msg)?), | ||
None => None, | ||
}; | ||
let index = match &opts.index { | ||
Some(path) => Some(index::Index::open(path)?), | ||
None => None, | ||
}; | ||
let app = App::build(ctx, opts.ttl, gateway, index); | ||
if let (Some(service), Some(index)) = (opts.backfill, app.index.clone()) { | ||
let ctx = app.ctx.clone(); | ||
tokio::spawn(async move { eprintln!("{}", backfilled(&ctx, &index, &service).await) }); | ||
} | ||
for did in opts.labelers { | ||
tokio::spawn(moderation::follow(app.clone(), did)); | ||
} | ||
if let Some(url) = opts.firehose { | ||
app.watching.store(true, Ordering::Relaxed); | ||
tokio::spawn(firehose::watch(app.clone(), url)); | ||
} | ||
if let Some(url) = opts.jetstream { | ||
app.watching.store(true, Ordering::Relaxed); | ||
tokio::spawn(jetstream::watch(app.clone(), url)); | ||
} | ||
let listener = tokio::net::TcpListener::bind(&opts.listen).await?; | ||
println!("listening on http://{}", listener.local_addr()?); | ||
axum::serve(listener, router(app)) | ||
.with_graceful_shutdown(async { | ||
let mut term = | ||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) | ||
.expect("installing a SIGTERM handler"); | ||
tokio::select! { | ||
_ = term.recv() => {} | ||
_ = tokio::signal::ctrl_c() => {} | ||
} | ||
}) | ||
.await?; | ||
Ok(()) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn input_errors() { | ||
let bad = AppError::input(knotless::read::BadInput("no".into()).into()); | ||
assert_eq!(bad.0, StatusCode::BAD_REQUEST); | ||
let broken = AppError::input(anyhow::anyhow!("disk on fire")); | ||
assert_eq!(broken.0, StatusCode::INTERNAL_SERVER_ERROR); | ||
} | ||
} | ||
crates/knotless-serve/src/main.rsnew+16−0
| @@ -0,0 +1,16 @@ | ||
fn main() { | ||
let args: Vec<String> = std::env::args().collect(); | ||
let opts = match knotless_serve::parse_args(&args) { | ||
Ok(opts) => opts, | ||
Err(usage) => { | ||
eprintln!("{usage}"); | ||
std::process::exit(2); | ||
} | ||
}; | ||
let ctx = knotless::env::Ctx::new(knotless::env::Env::from_process()); | ||
let runtime = tokio::runtime::Runtime::new().expect("a runtime builds"); | ||
if let Err(err) = runtime.block_on(knotless_serve::run(opts, ctx)) { | ||
eprintln!("knotless-serve: {err:#}"); | ||
std::process::exit(1); | ||
} | ||
} | ||
crates/knotless-serve/src/moderation.rsnew+369−0
| @@ -0,0 +1,369 @@ | ||
//! The moderation service: reports in, takedowns out, as atproto has them. | ||
//! | ||
//! Anyone sends a report with a service token from their PDS | ||
//! (`com.atproto.moderation.createReport`). The operator, by the admin | ||
//! password (`KNOTLESS_ADMIN_PASSWORD`), reads them (`dev.knotless.listReports`) | ||
//! and takes accounts and records down (`com.atproto.admin.updateSubjectStatus`). | ||
//! A takedown leaves the index, the pages and the clones here at once, and | ||
//! goes out as a `!takedown` label (`com.atproto.label.subscribeLabels`, | ||
//! `queryLabels`) for other appviews that follow this one; this one follows | ||
//! the labelers it is given the same way (`--labeler`). Labels go out | ||
//! unsigned: a follower reads them from the endpoint the labeler's DID | ||
//! document names. | ||
| ||
use crate::gateway::{caller, gateway}; | ||
use crate::index::Index; | ||
use crate::{App, AppError}; | ||
use anyhow::Context; | ||
use axum::Json; | ||
use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; | ||
use axum::extract::{Query, State}; | ||
use axum::http::{HeaderMap, StatusCode, header}; | ||
use axum::response::Response; | ||
use base64::Engine; | ||
use futures::{SinkExt, StreamExt}; | ||
use knotless::moderation::{GET_STATUS, LABELER, REPORT}; | ||
use serde::Deserialize; | ||
use serde_json::{Value, json}; | ||
use std::collections::VecDeque; | ||
use std::sync::Arc; | ||
use std::time::Duration; | ||
| ||
fn index(app: &App) -> Result<&Index, AppError> { | ||
app.index | ||
.as_deref() | ||
.ok_or_else(|| AppError(StatusCode::NOT_FOUND, "this server keeps no index".into())) | ||
} | ||
| ||
fn bad(msg: impl Into<String>) -> AppError { | ||
AppError(StatusCode::BAD_REQUEST, msg.into()) | ||
} | ||
| ||
/// Lets the request through if it carries the admin password, as a PDS | ||
/// takes it: `Basic` for the user `admin`. | ||
fn admin(app: &App, headers: &HeaderMap) -> Result<(), AppError> { | ||
let denied = || { | ||
AppError( | ||
StatusCode::UNAUTHORIZED, | ||
"the admin password is required".into(), | ||
) | ||
}; | ||
let password = app.ctx.env.admin_password.as_deref().ok_or_else(denied)?; | ||
let expected = base64::engine::general_purpose::STANDARD.encode(format!("admin:{password}")); | ||
let given = headers | ||
.get(header::AUTHORIZATION) | ||
.and_then(|v| v.to_str().ok()) | ||
.and_then(|v| v.strip_prefix("Basic ")); | ||
match given { | ||
Some(given) if given == expected => Ok(()), | ||
_ => Err(denied()), | ||
} | ||
} | ||
| ||
/// The account or record a subject names: a DID or an `at://` URI. | ||
fn subject_key(subject: &Value) -> Result<String, AppError> { | ||
let key = match subject["$type"].as_str() { | ||
Some("com.atproto.admin.defs#repoRef") => subject["did"].as_str(), | ||
Some("com.atproto.repo.strongRef") => subject["uri"].as_str(), | ||
_ => None, | ||
}; | ||
key.map(str::to_string).ok_or_else(|| { | ||
bad("a subject is a com.atproto.admin.defs#repoRef or a com.atproto.repo.strongRef") | ||
}) | ||
} | ||
| ||
pub async fn create_report( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
Json(body): Json<Value>, | ||
) -> Result<Json<Value>, AppError> { | ||
let index = index(&app)?; | ||
let by = caller(&app, &headers, REPORT).await?; | ||
subject_key(&body["subject"])?; | ||
let reason_type = body["reasonType"] | ||
.as_str() | ||
.ok_or_else(|| bad("a report has a reasonType"))?; | ||
let mut report = json!({ | ||
"reasonType": reason_type, | ||
"subject": body["subject"], | ||
"reportedBy": by, | ||
"createdAt": knotless::records::now(), | ||
}); | ||
if let Some(reason) = body["reason"].as_str() { | ||
report["reason"] = reason.into(); | ||
} | ||
Ok(Json( | ||
index | ||
.report(report) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?, | ||
)) | ||
} | ||
| ||
#[derive(Deserialize)] | ||
pub struct ReportsParams { | ||
limit: Option<usize>, | ||
cursor: Option<i64>, | ||
} | ||
| ||
pub async fn list_reports( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
Query(p): Query<ReportsParams>, | ||
) -> Result<Json<Value>, AppError> { | ||
admin(&app, &headers)?; | ||
let limit = p.limit.unwrap_or(50).clamp(1, 500); | ||
let reports = index(&app)? | ||
.reports(limit, p.cursor) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
let mut out = json!({ "reports": reports }); | ||
if reports.len() == limit { | ||
out["cursor"] = reports.last().map(|r| r["id"].to_string()).into(); | ||
} | ||
Ok(Json(out)) | ||
} | ||
| ||
pub async fn update_subject_status( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
Json(body): Json<Value>, | ||
) -> Result<Json<Value>, AppError> { | ||
admin(&app, &headers)?; | ||
let index = index(&app)?; | ||
let src = &gateway(&app)?.did; | ||
let subject = &body["subject"]; | ||
let key = subject_key(subject)?; | ||
let mut out = json!({ "subject": subject }); | ||
if let Some(applied) = body["takedown"]["applied"].as_bool() { | ||
let (seq, label) = index | ||
.take_down(src, &key, subject, applied) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
let _ = app.labels.send((seq, label)); | ||
out["takedown"] = json!({ "applied": applied }); | ||
} | ||
Ok(Json(out)) | ||
} | ||
| ||
#[derive(Deserialize)] | ||
pub struct StatusParams { | ||
did: Option<String>, | ||
uri: Option<String>, | ||
} | ||
| ||
pub async fn get_subject_status( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
Query(p): Query<StatusParams>, | ||
) -> Result<Json<Value>, AppError> { | ||
admin(&app, &headers)?; | ||
let (key, named) = match (p.did, p.uri) { | ||
(Some(did), _) => ( | ||
did.clone(), | ||
json!({"$type": "com.atproto.admin.defs#repoRef", "did": did}), | ||
), | ||
(None, Some(uri)) => ( | ||
uri.clone(), | ||
json!({"$type": "com.atproto.repo.strongRef", "uri": uri}), | ||
), | ||
(None, None) => return Err(bad(format!("{GET_STATUS} takes a did or a uri"))), | ||
}; | ||
let found = index(&app)? | ||
.taken_down(&key) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
let applied = found.is_some(); | ||
Ok(Json(json!({ | ||
"subject": found.unwrap_or(named), | ||
"takedown": { "applied": applied }, | ||
}))) | ||
} | ||
| ||
/// Takedowns standing here, as labels, for each of the `uriPatterns`: a | ||
/// URI, or a prefix ending in `*`. | ||
pub async fn query_labels( | ||
State(app): State<Arc<App>>, | ||
Query(params): Query<Vec<(String, String)>>, | ||
) -> Result<Json<Value>, AppError> { | ||
let index = index(&app)?; | ||
let src = &gateway(&app)?.did; | ||
let mut labels = Vec::new(); | ||
for (_, pattern) in params.iter().filter(|(k, _)| k == "uriPatterns") { | ||
let (prefix, exact) = match pattern.strip_suffix('*') { | ||
Some(prefix) => (prefix, false), | ||
None => (pattern.as_str(), true), | ||
}; | ||
let found = index | ||
.labels(src, prefix) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
labels.extend(found.into_iter().filter(|l| !exact || l["uri"] == *pattern)); | ||
} | ||
Ok(Json(json!({ "labels": labels }))) | ||
} | ||
| ||
#[derive(Deserialize)] | ||
pub struct SubscribeParams { | ||
cursor: Option<i64>, | ||
} | ||
| ||
/// The labels this server gives out, as they are given: those after | ||
/// `cursor` first, when there is one. | ||
pub async fn subscribe_labels( | ||
State(app): State<Arc<App>>, | ||
Query(p): Query<SubscribeParams>, | ||
ws: WebSocketUpgrade, | ||
) -> Result<Response, AppError> { | ||
index(&app)?; | ||
gateway(&app)?; | ||
Ok(ws.on_upgrade(move |socket| stream(app, socket, p.cursor))) | ||
} | ||
| ||
async fn stream(app: Arc<App>, socket: WebSocket, cursor: Option<i64>) { | ||
let (mut tx, mut rx) = socket.split(); | ||
let mut live = app.labels.subscribe(); | ||
let mut past: VecDeque<(i64, Value)> = match (cursor, &app.index) { | ||
(Some(cursor), Some(index)) => index.emitted(cursor).unwrap_or_default().into(), | ||
_ => VecDeque::new(), | ||
}; | ||
// What was given out while the past was read arrives live as well. | ||
let mut sent = cursor.unwrap_or(i64::MIN); | ||
let mut open = true; | ||
while open { | ||
let next = match past.pop_front() { | ||
Some(next) => Some(next), | ||
None => tokio::select! { | ||
got = live.recv() => got.ok(), | ||
_ = closed(&mut rx) => None, | ||
}, | ||
}; | ||
let Some((seq, label)) = next else { return }; | ||
if seq > sent { | ||
sent = seq; | ||
open = tx | ||
.send(Message::Binary(frame(seq, &label).into())) | ||
.await | ||
.is_ok(); | ||
} | ||
} | ||
} | ||
| ||
/// Resolves when the subscriber hangs up: it has nothing to say. | ||
async fn closed(rx: &mut futures::stream::SplitStream<WebSocket>) { | ||
while let Some(Ok(_)) = rx.next().await {} | ||
} | ||
| ||
/// A `#labels` frame, as `subscribeLabels` sends one: a CBOR header and body. | ||
pub fn frame(seq: i64, label: &Value) -> Vec<u8> { | ||
let mut out = Vec::new(); | ||
let header = json!({"op": 1, "t": "#labels"}); | ||
let body = json!({"seq": seq, "labels": [label]}); | ||
ciborium::into_writer(&header, &mut out).expect("JSON encodes as CBOR"); | ||
ciborium::into_writer(&body, &mut out).expect("JSON encodes as CBOR"); | ||
out | ||
} | ||
| ||
/// Reads a `#labels` frame: its sequence number and labels, each with the | ||
/// fields that matter here. `None` for any other frame. | ||
pub fn parse(frame: &[u8]) -> Option<(i64, Vec<Value>)> { | ||
use ciborium::Value as V; | ||
let mut reader = frame; | ||
let header: V = ciborium::from_reader(&mut reader).ok()?; | ||
let body: V = ciborium::from_reader(&mut reader).ok()?; | ||
let field = |v: &V, name: &str| -> Option<V> { | ||
v.as_map()? | ||
.iter() | ||
.find(|(k, _)| k.as_text() == Some(name)) | ||
.map(|(_, v)| v.clone()) | ||
}; | ||
if field(&header, "t")?.as_text() != Some("#labels") { | ||
return None; | ||
} | ||
let seq = i64::try_from(field(&body, "seq")?.as_integer()?).ok()?; | ||
let labels = field(&body, "labels")? | ||
.as_array()? | ||
.iter() | ||
.map(|l| { | ||
let mut out = json!({}); | ||
for name in ["src", "uri", "val", "exp"] { | ||
if let Some(text) = field(l, name).and_then(|v| v.as_text().map(str::to_string)) { | ||
out[name] = text.into(); | ||
} | ||
} | ||
out["neg"] = field(l, "neg") | ||
.and_then(|v| v.as_bool()) | ||
.unwrap_or(false) | ||
.into(); | ||
out | ||
}) | ||
.collect(); | ||
Some((seq, labels)) | ||
} | ||
| ||
/// Follows the labeler `did` until the process ends: its `!takedown` and | ||
/// `!hide` labels take things out of view here, as this server's own | ||
/// takedowns do. Reconnects from where it read to. | ||
pub async fn follow(app: Arc<App>, did: String) { | ||
loop { | ||
if let Err(err) = follow_once(&app, &did).await { | ||
eprintln!("following the labeler {did}: {err:#}"); | ||
} | ||
tokio::time::sleep(Duration::from_millis(500)).await; | ||
} | ||
} | ||
| ||
async fn follow_once(app: &App, did: &str) -> anyhow::Result<()> { | ||
let index = app | ||
.index | ||
.as_ref() | ||
.ok_or_else(|| anyhow::anyhow!("following a labeler takes an index"))?; | ||
let doc = knotless::identity::document(&app.ctx, did).await?; | ||
let endpoint = knotless::identity::service(&doc, LABELER) | ||
.ok_or_else(|| anyhow::anyhow!("{did} names no {LABELER} service"))?; | ||
let cursor = index.labeler_seq(did)?.unwrap_or(0); | ||
let url = format!( | ||
"{}/xrpc/com.atproto.label.subscribeLabels?cursor={cursor}", | ||
endpoint.replacen("http", "ws", 1) | ||
); | ||
let (mut ws, _) = tokio_tungstenite::connect_async(url).await?; | ||
loop { | ||
let msg = ws.next().await.context("the labeler ended the stream")??; | ||
let (seq, labels) = parse(&msg.into_data()).unwrap_or_default(); | ||
for label in &labels { | ||
index.label(did, seq, label)?; | ||
} | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn frames() { | ||
let label = json!({"src": "did:web:l", "uri": "did:plc:a", "val": "!takedown", "cts": "t"}); | ||
let (seq, labels) = parse(&frame(7, &label)).unwrap(); | ||
assert_eq!(seq, 7); | ||
assert_eq!( | ||
labels, | ||
[json!({"src": "did:web:l", "uri": "did:plc:a", "val": "!takedown", "neg": false})] | ||
); | ||
let mut other = Vec::new(); | ||
ciborium::into_writer(&json!({"op": 1, "t": "#info"}), &mut other).unwrap(); | ||
ciborium::into_writer(&json!({"name": "OutdatedCursor"}), &mut other).unwrap(); | ||
assert_eq!(parse(&other), None); | ||
assert_eq!(parse(b"\xff"), None); | ||
} | ||
| ||
#[test] | ||
fn subjects() { | ||
assert_eq!( | ||
subject_key(&json!({"$type": "com.atproto.admin.defs#repoRef", "did": "did:a"})) | ||
.unwrap(), | ||
"did:a" | ||
); | ||
assert!(subject_key(&json!({"$type": "com.atproto.repo.strongRef"})).is_err()); | ||
assert!(subject_key(&json!({"did": "did:a"})).is_err()); | ||
} | ||
} | ||
crates/knotless-serve/src/objects.rsnew+292−0
| @@ -0,0 +1,292 @@ | ||
//! A mirror's objects read in process, for pages that must be quick: | ||
//! starting a git process costs more than these reads take. Each mirror is | ||
//! opened once; new packs a sync brings are found when first needed. | ||
| ||
use anyhow::{Context, Result}; | ||
use gix::revision::walk::Sorting; | ||
use gix::traverse::commit::simple::CommitTimeOrder; | ||
use knotless::read::{Author, Commit, Entry}; | ||
use std::collections::HashMap; | ||
use std::path::{Path, PathBuf}; | ||
use std::sync::{LazyLock, Mutex}; | ||
| ||
static OPEN: LazyLock<Mutex<HashMap<PathBuf, gix::ThreadSafeRepository>>> = | ||
LazyLock::new(Mutex::default); | ||
| ||
pub struct Objects(gix::Repository); | ||
| ||
/// The repository at `dir`, a bare mirror. | ||
pub fn open(dir: &Path) -> Result<Objects> { | ||
let mut open = OPEN.lock().unwrap(); | ||
let safe = match open.get(dir) { | ||
Some(safe) => safe.clone(), | ||
None => { | ||
let safe = gix::ThreadSafeRepository::open(dir)?; | ||
open.insert(dir.to_path_buf(), safe.clone()); | ||
safe | ||
} | ||
}; | ||
let mut repo = safe.to_thread_local(); | ||
repo.object_cache_size_if_unset(4 << 20); | ||
Ok(Objects(repo)) | ||
} | ||
| ||
/// The id of whatever `path` names in `tree`, if anything. | ||
fn id_at(tree: &gix::Tree<'_>, path: &str) -> Result<Option<gix::ObjectId>> { | ||
Ok(tree.lookup_entry_by_path(path)?.map(|e| e.oid().to_owned())) | ||
} | ||
| ||
impl Objects { | ||
/// The branch HEAD names, if it names one. | ||
pub fn head(&self) -> Option<String> { | ||
let name = self.0.head_name().ok()??; | ||
Some(name.shorten().to_string()) | ||
} | ||
| ||
/// Every branch and tag, by its short name. | ||
pub fn names(&self) -> Result<Vec<String>> { | ||
let platform = self.0.references()?; | ||
let mut names = Vec::new(); | ||
for r in platform.all()? { | ||
let r = r?; | ||
let full = r.name().as_bstr().to_string(); | ||
if let Some(short) = full | ||
.strip_prefix("refs/heads/") | ||
.or(full.strip_prefix("refs/tags/")) | ||
{ | ||
names.push(short.to_string()); | ||
} | ||
} | ||
Ok(names) | ||
} | ||
| ||
fn commit(&self, rev: &str) -> Result<gix::Commit<'_>> { | ||
let id = self | ||
.0 | ||
.rev_parse_single(rev) | ||
.with_context(|| format!("no revision {rev}"))?; | ||
Ok(id.object()?.peel_to_commit()?) | ||
} | ||
| ||
/// The commit `rev` names, by its full id. | ||
pub fn resolve(&self, rev: &str) -> Result<String> { | ||
Ok(self.commit(rev)?.id.to_string()) | ||
} | ||
| ||
pub fn tree(&self, rev: &str, path: &str) -> Result<Vec<Entry>> { | ||
let mut tree = self.commit(rev)?.tree()?; | ||
if !path.is_empty() { | ||
let entry = tree | ||
.lookup_entry_by_path(path)? | ||
.with_context(|| format!("no {path} at {rev}"))?; | ||
tree = entry | ||
.object()? | ||
.try_into_tree() | ||
.with_context(|| format!("{path} is not a directory"))?; | ||
} | ||
let mut out = Vec::new(); | ||
for e in tree.iter() { | ||
let e = e?; | ||
let mode = e.mode(); | ||
let kind = if mode.is_tree() { | ||
"tree" | ||
} else if mode.is_commit() { | ||
"commit" | ||
} else { | ||
"blob" | ||
}; | ||
let size = match kind { | ||
"blob" => Some(self.0.find_header(e.oid())?.size()), | ||
_ => None, | ||
}; | ||
out.push(Entry { | ||
name: e.filename().to_string(), | ||
mode: mode.kind().as_octal_str().to_string(), | ||
kind: kind.into(), | ||
sha: e.oid().to_string(), | ||
size, | ||
}); | ||
} | ||
Ok(out) | ||
} | ||
| ||
pub fn blob(&self, rev: &str, path: &str) -> Result<Vec<u8>> { | ||
let tree = self.commit(rev)?.tree()?; | ||
let entry = tree | ||
.lookup_entry_by_path(path)? | ||
.with_context(|| format!("no {path} at {rev}"))?; | ||
let object = entry.object()?; | ||
anyhow::ensure!( | ||
object.kind == gix::object::Kind::Blob, | ||
"{path} is not a file" | ||
); | ||
Ok(object.detach().data) | ||
} | ||
| ||
/// Whether `commit` changed `path` from each of its parents: a commit | ||
/// that left it as one parent had it is not part of its history. | ||
fn touches(&self, commit: &gix::Commit<'_>, path: &str) -> Result<bool> { | ||
let here = id_at(&commit.tree()?, path)?; | ||
let mut parents = commit.parent_ids().peekable(); | ||
if parents.peek().is_none() { | ||
return Ok(here.is_some()); | ||
} | ||
for parent in parents { | ||
if id_at(&parent.object()?.peel_to_commit()?.tree()?, path)? == here { | ||
return Ok(false); | ||
} | ||
} | ||
Ok(true) | ||
} | ||
| ||
/// History from `rev`, newest first, touching `path` if it is not | ||
/// empty, after its first `skip` commits. | ||
pub fn log(&self, rev: &str, path: &str, skip: usize, limit: usize) -> Result<Vec<Commit>> { | ||
let tip = self.commit(rev)?.id; | ||
let walk = self | ||
.0 | ||
.rev_walk([tip]) | ||
.sorting(Sorting::ByCommitTime(CommitTimeOrder::NewestFirst)) | ||
.all()?; | ||
let mut out = Vec::new(); | ||
let mut skipped = 0; | ||
for info in walk { | ||
let commit = info?.object()?; | ||
if !path.is_empty() && !self.touches(&commit, path)? { | ||
continue; | ||
} | ||
if skipped < skip { | ||
skipped += 1; | ||
continue; | ||
} | ||
let author = commit.author()?; | ||
let date = author | ||
.time()? | ||
.format(gix::date::time::format::ISO8601_STRICT)?; | ||
out.push(Commit { | ||
sha: commit.id.to_string(), | ||
parents: commit.parent_ids().map(|p| p.to_string()).collect(), | ||
author: Author { | ||
name: author.name.to_string(), | ||
email: author.email.to_string(), | ||
date, | ||
}, | ||
message: commit.message_raw()?.to_string().trim_end().to_string(), | ||
}); | ||
if out.len() == limit { | ||
break; | ||
} | ||
} | ||
Ok(out) | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use knotless::git::Git; | ||
| ||
async fn git(dir: &Path, args: &[&str]) -> String { | ||
Git::new(dir).text(args, None).await.unwrap() | ||
} | ||
| ||
#[tokio::test] | ||
async fn reads_as_git_does() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let d = dir.path(); | ||
let run = |args: Vec<&str>| { | ||
let out = std::process::Command::new("git") | ||
.args(args) | ||
.current_dir(d) | ||
.env("GIT_CONFIG_GLOBAL", "/dev/null") | ||
.env("GIT_AUTHOR_NAME", "A") | ||
.env("GIT_AUTHOR_EMAIL", "a@x") | ||
.env("GIT_COMMITTER_NAME", "A") | ||
.env("GIT_COMMITTER_EMAIL", "a@x") | ||
.output() | ||
.unwrap(); | ||
assert!(out.status.success(), "{out:?}"); | ||
}; | ||
run(vec!["init", "-q", "-b", "main"]); | ||
std::fs::create_dir(d.join("src")).unwrap(); | ||
std::fs::write(d.join("src/a.rs"), "fn a() {}\n").unwrap(); | ||
std::fs::write(d.join("b"), "b\n").unwrap(); | ||
run(vec!["add", "-A"]); | ||
run(vec!["commit", "-q", "-m", "one\n\nwith a body"]); | ||
std::fs::write(d.join("b"), "bb\n").unwrap(); | ||
run(vec!["commit", "-qam", "two"]); | ||
run(vec!["checkout", "-q", "-b", "side", "HEAD~1"]); | ||
std::fs::write(d.join("src/a.rs"), "fn a() { 1 }\n").unwrap(); | ||
run(vec!["commit", "-qam", "three"]); | ||
run(vec!["checkout", "-q", "main"]); | ||
run(vec!["merge", "-q", "--no-edit", "side"]); | ||
let one = git(&d.join(".git"), &["rev-parse", "HEAD~1"]).await; | ||
run(vec![ | ||
"update-index", | ||
"--add", | ||
"--cacheinfo", | ||
&format!("160000,{one},vendored"), | ||
]); | ||
run(vec!["commit", "-qm", "vendor"]); | ||
run(vec!["tag", "v1"]); | ||
run(vec!["update-ref", "refs/notes/x", "HEAD"]); | ||
| ||
let o = open(&d.join(".git")).unwrap(); | ||
let again = open(&d.join(".git")).unwrap(); | ||
assert_eq!(o.head().as_deref(), Some("main")); | ||
assert_eq!( | ||
again.resolve("v1").unwrap(), | ||
git(&d.join(".git"), &["rev-parse", "HEAD"]).await | ||
); | ||
let mut names = o.names().unwrap(); | ||
names.sort(); | ||
assert_eq!(names, ["main", "side", "v1"]); | ||
| ||
let tree = o.tree("main", "").unwrap(); | ||
let shown: Vec<(&str, &str, Option<u64>)> = tree | ||
.iter() | ||
.map(|e| (e.name.as_str(), e.kind.as_str(), e.size)) | ||
.collect(); | ||
assert_eq!( | ||
shown, | ||
[ | ||
("b", "blob", Some(3)), | ||
("src", "tree", None), | ||
("vendored", "commit", None) | ||
] | ||
); | ||
assert_eq!(tree[1].mode, "40000"); | ||
assert_eq!(o.tree("main", "src").unwrap()[0].name, "a.rs"); | ||
assert!(o.tree("main", "b").is_err()); | ||
assert!(o.tree("main", "nope").is_err()); | ||
assert!(o.tree("nope", "").is_err()); | ||
assert_eq!(o.blob("main", "src/a.rs").unwrap(), b"fn a() { 1 }\n"); | ||
assert!(o.blob("main", "src").is_err()); | ||
assert!(o.blob("main", "nope").is_err()); | ||
| ||
let subjects = |log: Vec<Commit>| { | ||
log.iter() | ||
.map(|c| c.message.lines().next().unwrap().to_string()) | ||
.collect::<Vec<_>>() | ||
}; | ||
let all = o.log("main", "", 0, 10).unwrap(); | ||
assert_eq!(all.len(), 5); | ||
assert_eq!(all[1].parents.len(), 2); | ||
assert_eq!(all[4].message, "one\n\nwith a body"); | ||
assert_eq!(all[4].author.name, "A"); | ||
assert!(all[4].author.date.contains('T'), "{}", all[4].author.date); | ||
assert_eq!(o.log("main", "", 1, 2).unwrap().len(), 2); | ||
assert_eq!(subjects(o.log("main", "b", 0, 10).unwrap()), ["two", "one"]); | ||
assert_eq!( | ||
subjects(o.log("main", "src", 0, 10).unwrap()), | ||
["three", "one"] | ||
); | ||
assert_eq!(subjects(o.log("main", "src", 1, 10).unwrap()), ["one"]); | ||
assert!(o.log("main", "nowhere", 0, 10).unwrap().is_empty()); | ||
assert!(o.log("nope", "", 0, 10).is_err()); | ||
assert!(open(&d.join("missing")).is_err()); | ||
| ||
run(vec!["checkout", "-q", "--detach"]); | ||
assert_eq!(open(&d.join(".git")).unwrap().head(), None); | ||
} | ||
} | ||
crates/knotless-serve/src/smart.rsnew+123−0
| @@ -0,0 +1,123 @@ | ||
//! git smart HTTP, read only, by `git upload-pack --stateless-rpc`. | ||
| ||
use crate::{App, AppError, repo_name}; | ||
use anyhow::Context; | ||
use axum::body::{Body, Bytes}; | ||
use axum::extract::{Path, Query, State}; | ||
use axum::http::{HeaderMap, StatusCode, header}; | ||
use axum::response::Response; | ||
use knotless::git::Git; | ||
use serde::Deserialize; | ||
use std::io::Read; | ||
use std::process::Stdio; | ||
use std::sync::Arc; | ||
use tokio::io::AsyncWriteExt; | ||
use tokio::process::Command; | ||
use tokio_util::io::ReaderStream; | ||
| ||
#[derive(Deserialize)] | ||
pub struct Service { | ||
service: Option<String>, | ||
} | ||
| ||
fn protocol(headers: &HeaderMap) -> String { | ||
headers | ||
.get("git-protocol") | ||
.and_then(|v| v.to_str().ok()) | ||
.unwrap_or_default() | ||
.to_string() | ||
} | ||
| ||
fn command(git: &Git, protocol: &str, advertise: bool) -> Command { | ||
let mut cmd = Command::new("git"); | ||
cmd.arg("upload-pack").arg("--stateless-rpc"); | ||
if advertise { | ||
cmd.arg("--http-backend-info-refs"); | ||
} | ||
cmd.arg(&git.dir) | ||
.env("GIT_PROTOCOL", protocol) | ||
.stdin(Stdio::piped()) | ||
.stdout(Stdio::piped()) | ||
.stderr(Stdio::null()) | ||
.kill_on_drop(true); | ||
cmd | ||
} | ||
| ||
fn pkt(line: &str) -> String { | ||
format!("{:04x}{line}", line.len() + 4) | ||
} | ||
| ||
pub async fn info_refs( | ||
State(app): State<Arc<App>>, | ||
Path((authority, repo)): Path<(String, String)>, | ||
Query(q): Query<Service>, | ||
headers: HeaderMap, | ||
) -> Result<Response, AppError> { | ||
if q.service.as_deref() != Some("git-upload-pack") { | ||
return Err(AppError( | ||
StatusCode::FORBIDDEN, | ||
"read only: push with knotless:// instead".into(), | ||
)); | ||
} | ||
let git = app.mirror(&authority, repo_name(&repo)).await?; | ||
let protocol = protocol(&headers); | ||
let out = command(&git, &protocol, true) | ||
.output() | ||
.await | ||
.context("running git upload-pack") | ||
.map_err(AppError::internal)?; | ||
let mut body = Vec::new(); | ||
if !protocol.contains("version=2") { | ||
body.extend(pkt("# service=git-upload-pack\n").as_bytes()); | ||
body.extend(b"0000"); | ||
} | ||
body.extend(out.stdout); | ||
Ok(Response::builder() | ||
.header( | ||
header::CONTENT_TYPE, | ||
"application/x-git-upload-pack-advertisement", | ||
) | ||
.header(header::CACHE_CONTROL, "no-cache") | ||
.body(Body::from(body)) | ||
.expect("static headers are valid")) | ||
} | ||
| ||
pub async fn upload_pack( | ||
State(app): State<Arc<App>>, | ||
Path((authority, repo)): Path<(String, String)>, | ||
headers: HeaderMap, | ||
body: Bytes, | ||
) -> Result<Response, AppError> { | ||
let input = if headers | ||
.get(header::CONTENT_ENCODING) | ||
.is_some_and(|v| v == "gzip") | ||
{ | ||
let mut out = Vec::new(); | ||
flate2::read::GzDecoder::new(&body[..]) | ||
.read_to_end(&mut out) | ||
.map_err(|e| AppError(StatusCode::BAD_REQUEST, format!("bad gzip body: {e}")))?; | ||
out | ||
} else { | ||
body.to_vec() | ||
}; | ||
let git = app.mirror(&authority, repo_name(&repo)).await?; | ||
let mut child = command(&git, &protocol(&headers), false) | ||
.spawn() | ||
.context("running git upload-pack") | ||
.map_err(AppError::internal)?; | ||
let mut stdin = child.stdin.take().expect("stdin is piped"); | ||
tokio::spawn(async move { | ||
let _ = stdin.write_all(&input).await; | ||
}); | ||
let stdout = child.stdout.take().expect("stdout is piped"); | ||
// The child is reaped when the stream ends; kill_on_drop covers a | ||
// client that hangs up first. | ||
tokio::spawn(async move { | ||
let _ = child.wait().await; | ||
}); | ||
Ok(Response::builder() | ||
.header(header::CONTENT_TYPE, "application/x-git-upload-pack-result") | ||
.header(header::CACHE_CONTROL, "no-cache") | ||
.body(Body::from_stream(ReaderStream::new(stdout))) | ||
.expect("static headers are valid")) | ||
} | ||
crates/knotless-serve/src/web.rsnew+1821−0
| @@ -0,0 +1,1821 @@ | ||
//! The web interface: read-only pages over the mirrors and the index. | ||
//! Writing is for git, the CLI and agents over MCP; these pages only show. | ||
//! | ||
//! Each page is one HTML document with its styles inline and two small | ||
//! scripts: speculation rules, which prerender a link as the pointer | ||
//! settles on it, and the page script, which hands precomputed ranges to | ||
//! the CSS Custom Highlight API so code arrives as plain text, and copies | ||
//! commands to the clipboard. | ||
| ||
use crate::diff::{self, Kind, Status}; | ||
use crate::objects::{self, Objects}; | ||
use crate::{App, AppError, highlight, repo_name}; | ||
use axum::Router; | ||
use axum::extract::{Path, Query, Request, State}; | ||
use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; | ||
use axum::middleware::{self, Next}; | ||
use axum::response::{IntoResponse, Response}; | ||
use axum::routing::get; | ||
use base64::Engine; | ||
use knotless::forge::{self, Item, Thread}; | ||
use knotless::git::Git; | ||
use knotless::read; | ||
use knotless::records::{RepoRecord, RepoRef}; | ||
use knotless::remote::Remote; | ||
use knotless::url::RepoUrl; | ||
use serde::Deserialize; | ||
use sha2::{Digest, Sha256}; | ||
use std::collections::HashMap; | ||
use std::fmt::Write; | ||
use std::sync::{Arc, LazyLock, Mutex}; | ||
| ||
pub fn routes() -> Router<Arc<App>> { | ||
Router::new() | ||
.route("/", get(home)) | ||
.route("/{authority}", get(profile)) | ||
.route("/{authority}/{repo}", get(code)) | ||
.route("/{authority}/{repo}/tree/{*rest}", get(tree)) | ||
.route("/{authority}/{repo}/blob/{*rest}", get(blob)) | ||
.route("/{authority}/{repo}/raw/{*rest}", get(raw)) | ||
.route("/{authority}/{repo}/commits/{*rest}", get(commits)) | ||
.route("/{authority}/{repo}/commit/{rev}", get(commit)) | ||
.route("/{authority}/{repo}/pulls", get(pulls)) | ||
.route("/{authority}/{repo}/issues", get(issues)) | ||
.route("/{authority}/{repo}/{kind}/{did}/{rkey}", get(item)) | ||
.layer(middleware::from_fn(renamed)) | ||
.layer(tower_http::compression::CompressionLayer::new()) | ||
} | ||
| ||
/// Where a repository is now, on the answer to a request that used a name | ||
/// it had before: its name, and its owner if it moved to another account. | ||
#[derive(Clone)] | ||
struct Renamed(String); | ||
| ||
/// Points a request made with a repository's old name at the same page | ||
/// under its new one. The redirect is temporary: the old name can be given | ||
/// to another repository. | ||
async fn renamed(req: Request, next: Next) -> Response { | ||
let path = req.uri().path().to_string(); | ||
let query = req | ||
.uri() | ||
.query() | ||
.map(|q| format!("?{q}")) | ||
.unwrap_or_default(); | ||
let mut res = next.run(req).await; | ||
if let Some(Renamed(now)) = res.extensions_mut().remove::<Renamed>() { | ||
// "", the authority, the name, and the rest. | ||
let mut parts: Vec<&str> = path.splitn(4, '/').collect(); | ||
match now.split_once('/') { | ||
Some((owner, name)) => (parts[1], parts[2]) = (owner, name), | ||
None => parts[2] = &now, | ||
} | ||
let to = format!("{}{query}", parts.join("/")); | ||
res.headers_mut() | ||
.insert(header::LOCATION, HeaderValue::from_str(&to).unwrap()); | ||
} | ||
res | ||
} | ||
| ||
/// Text inlined above, past this, is a link to the raw file instead. | ||
const INLINE: usize = 1024 * 1024; | ||
const COMMITS_PER_PAGE: usize = 50; | ||
/// A file's diff with more rows than this is folded away. | ||
const DIFF_ROWS: usize = 2000; | ||
| ||
const CSS: &str = r#" | ||
:root{color-scheme:light;--bg:#fff;--fg:#1c1d20;--mute:#6a6d75;--faint:#9a9ca3;--line:#e6e6e9;--soft:#f6f6f7;--panel:#fbfbfc;--accent:#2f6af5;--accent-soft:#eef3ff;--open:#1d8348;--merged:#7a48d6;--closed:#c2410c;--ins:#e9f8ee;--ins-n:#d3f2dd;--ins-x:#b2e8c4;--del:#fdeeee;--del-n:#fbdcdc;--del-x:#f6bcbc;--hunk:#f0f5ff;--target:#fff4c2;--comment:#8a8f98;--string:#0b7a4b;--keyword:#c0285e;--number:#a85d00;--function:#2b59c3;--type:#7a4fc2;--tag:#c0285e;--attr:#a85d00;--mono:ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;--sans:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",sans-serif;--r:10px} | ||
@media (prefers-color-scheme:dark){:root{color-scheme:dark;--bg:#0d0e10;--fg:#e8e8ea;--mute:#999ba3;--faint:#6b6e76;--line:#26282d;--soft:#17181b;--panel:#121316;--accent:#7aa2ff;--accent-soft:#18213a;--open:#4cc38a;--merged:#b197fc;--closed:#f0884f;--ins:#0f2418;--ins-n:#163421;--ins-x:#1f5232;--del:#2a1214;--del-n:#3b191c;--del-x:#6a2a2f;--hunk:#121b2e;--target:#3b3313;--comment:#6c717a;--string:#7dd3a8;--keyword:#ff8fb3;--number:#efb36a;--function:#8db3ff;--type:#c4a5ff;--tag:#ff8fb3;--attr:#efb36a}} | ||
@view-transition{navigation:auto} | ||
*{box-sizing:border-box} | ||
html{-webkit-text-size-adjust:100%} | ||
body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.55 var(--sans);-webkit-font-smoothing:antialiased;min-height:100vh;display:flex;flex-direction:column} | ||
a{color:inherit;text-decoration:none} | ||
a:hover{color:var(--accent)} | ||
.wrap{width:100%;max-width:76rem;margin:0 auto;padding:0 1.5rem} | ||
.top{border-bottom:1px solid var(--line);view-transition-name:top} | ||
.top .wrap{display:flex;align-items:center;gap:1.5rem;height:3.5rem} | ||
.brand{display:flex;align-items:center;gap:.55rem;font-weight:650;font-size:15px;letter-spacing:-.01em} | ||
.brand svg{width:22px;height:22px} | ||
.top nav{margin-left:auto;display:flex;gap:1.25rem;color:var(--mute)} | ||
main.wrap{flex:1;padding-top:1.75rem;padding-bottom:4rem} | ||
.foot{color:var(--faint);font-size:12.5px;padding-top:1.25rem;padding-bottom:2rem;border-top:1px solid var(--line)} | ||
h1,h2,h3{letter-spacing:-.015em;margin:0} | ||
h1{font-size:20px;font-weight:600} | ||
h2{font-size:16px;font-weight:600} | ||
h3{font-size:14px;font-weight:600} | ||
p{margin:0} | ||
code,pre,.mono{font-family:var(--mono);font-size:12.5px} | ||
.mute{color:var(--mute)} | ||
.end{margin-left:auto} | ||
svg.i{width:16px;height:16px;flex:none;fill:currentColor;vertical-align:-3px} | ||
.av{display:inline-grid;place-items:center;flex:none;width:20px;height:20px;border-radius:50%;font:600 10px var(--sans);color:#fff;background:hsl(var(--h) 45% 52%);text-transform:uppercase} | ||
.av.lg{width:32px;height:32px;font-size:13px} | ||
.av.xl{width:88px;height:88px;font-size:34px;font-weight:600} | ||
.profile{display:flex;align-items:center;gap:1.5rem;margin:.75rem 0 2.25rem;min-width:0} | ||
.profile>div{min-width:0} | ||
.profile h1{font-size:24px;letter-spacing:-.02em;overflow-wrap:anywhere} | ||
.profile .who{color:var(--mute);margin-top:.15rem;overflow-wrap:anywhere} | ||
.profile .did{font:12px var(--mono);color:var(--faint);margin-top:.35rem;overflow-wrap:anywhere} | ||
.profile .bio{margin-top:.6rem;max-width:40rem;white-space:pre-line} | ||
.btn{display:inline-flex;align-items:center;gap:.4rem;height:28px;padding:0 .7rem;border:1px solid var(--line);border-radius:7px;background:var(--bg);font:500 12.5px var(--sans);color:var(--fg);cursor:pointer;white-space:nowrap} | ||
.btn:hover{background:var(--soft);color:var(--fg)} | ||
.box{border:1px solid var(--line);border-radius:var(--r);overflow:hidden;background:var(--bg)} | ||
.box+.box{margin-top:1rem} | ||
.boxhead{display:flex;flex-wrap:wrap;align-items:center;gap:.5rem .75rem;padding:.6rem 1rem;background:var(--panel);border-bottom:1px solid var(--line);min-height:2.75rem} | ||
.boxhead:last-child{border-bottom:0} | ||
.list{list-style:none;margin:0;padding:0} | ||
.list li{display:flex;gap:.75rem;align-items:center;padding:.6rem 1rem;border-top:1px solid var(--line)} | ||
.list li:first-child{border-top:0} | ||
.list li:hover{background:var(--panel)} | ||
.list li>.mute{display:inline-flex;align-items:center;gap:.3rem} | ||
.list li>svg.i{color:var(--faint);margin-top:2px} | ||
.grow{flex:1;min-width:0} | ||
.clip{overflow:hidden;text-overflow:ellipsis;white-space:nowrap} | ||
.title{font-weight:600} | ||
.meta{color:var(--mute);font-size:12.5px;margin-top:.1rem;display:flex;flex-wrap:wrap;align-items:center;gap:.35rem} | ||
.sha{font:12px var(--mono);color:var(--mute);padding:.15rem .45rem;border:1px solid var(--line);border-radius:6px} | ||
.sha:hover{color:var(--accent)} | ||
.empty{padding:2.5rem 1rem;text-align:center;color:var(--mute)} | ||
.hero{padding:2.5rem 0 2.25rem;max-width:44rem} | ||
.hero h1{font-size:clamp(28px,5vw,40px);line-height:1.12;letter-spacing:-.03em;font-weight:650} | ||
.hero p{font-size:16px;color:var(--mute);margin-top:.9rem;max-width:36rem} | ||
.steps{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:1rem;margin-bottom:3rem} | ||
.step{border:1px solid var(--line);border-radius:var(--r);padding:1.1rem 1.1rem 1rem;background:var(--panel);min-width:0} | ||
.step h3{display:flex;align-items:center;gap:.55rem} | ||
.step h3 b{display:inline-grid;place-items:center;width:20px;height:20px;border-radius:50%;background:var(--fg);color:var(--bg);font-size:11px} | ||
.step p{color:var(--mute);margin:.4rem 0 .8rem} | ||
.step details{margin-top:.6rem;color:var(--mute);font-size:12.5px} | ||
.step summary{cursor:pointer} | ||
.step details pre{margin-top:.5rem} | ||
.cmd{display:flex;align-items:flex-start;gap:.5rem;border:1px solid var(--line);border-radius:8px;background:var(--bg);padding:.5rem .5rem .5rem .75rem} | ||
.cmd+.cmd{margin-top:.5rem} | ||
.cmd pre{flex:1;min-width:0;margin:0;padding:.2rem 0;white-space:pre-wrap;overflow-wrap:anywhere} | ||
.cmd .btn{height:24px;padding:0 .5rem;font-size:11.5px} | ||
.section{display:flex;flex-wrap:wrap;align-items:center;gap:.75rem;margin:0 0 .9rem} | ||
.search{flex:1;min-width:12rem;max-width:22rem;margin-left:auto;font:inherit;color:inherit;background:var(--bg);border:1px solid var(--line);border-radius:8px;padding:.4rem .75rem;height:32px} | ||
.search:focus{outline:2px solid var(--accent);outline-offset:-1px} | ||
.repohead{display:flex;flex-wrap:wrap;align-items:flex-start;gap:1rem 2rem} | ||
.repohead h1{font-weight:500;display:flex;flex-wrap:wrap;align-items:center;gap:.35rem} | ||
.repohead h1 b{font-weight:650} | ||
.repohead .about{flex:1;min-width:16rem} | ||
.repohead .about p{color:var(--mute);margin-top:.35rem} | ||
.clone{display:flex;align-items:center;gap:.5rem;border:1px solid var(--line);border-radius:8px;padding:.3rem .3rem .3rem .75rem;max-width:100%;background:var(--panel)} | ||
.clone code{overflow-x:auto;white-space:nowrap;color:var(--mute)} | ||
.tabs{display:flex;gap:.25rem;border-bottom:1px solid var(--line);margin:1.25rem 0 1.25rem;view-transition-name:tabs;overflow-x:auto;scrollbar-width:none} | ||
.tabs a{display:flex;align-items:center;gap:.45rem;padding:.55rem .75rem;color:var(--mute);border-bottom:2px solid transparent;margin-bottom:-1px;white-space:nowrap} | ||
.tabs a:hover{color:var(--fg)} | ||
.tabs a[aria-current]{color:var(--fg);border-color:var(--accent);font-weight:550} | ||
.count{font-size:11.5px;padding:0 .4rem;border-radius:9px;background:var(--soft);color:var(--mute);line-height:18px} | ||
.toolbar{display:flex;flex-wrap:wrap;align-items:center;gap:.6rem .9rem;margin:0 0 .9rem} | ||
.branch{display:inline-flex;align-items:center;gap:.35rem;height:28px;padding:0 .65rem;border:1px solid var(--line);border-radius:7px;background:var(--panel);font:500 12.5px var(--sans);max-width:14rem} | ||
.branch span{overflow:hidden;text-overflow:ellipsis;white-space:nowrap} | ||
.crumbs{font-size:15px;min-width:0;overflow-wrap:anywhere} | ||
.crumbs a{color:var(--accent)}.crumbs b{font-weight:600} | ||
.crumbs .sep{color:var(--faint);margin:0 .3rem} | ||
.files li{padding:.45rem 1rem} | ||
.files svg.i{color:var(--faint)} | ||
.files .dir svg.i{color:var(--accent)} | ||
.files .size{color:var(--faint);font-size:12px} | ||
.code{display:grid;grid-template-columns:auto minmax(0,1fr)} | ||
.code pre{margin:0;padding:.75rem 0;line-height:20px;tab-size:4;font-size:12.5px} | ||
.code .ln{text-align:right;color:var(--faint);user-select:none;padding:.75rem .75rem .75rem 1rem;min-width:3.5rem} | ||
.code .ln a{color:inherit} | ||
.code .ln a:target{color:var(--fg)} | ||
.code .src{overflow-x:auto;padding:.75rem 1rem} | ||
.prose{padding:1.25rem 2rem 1.75rem;overflow-wrap:anywhere;font-size:15px;line-height:1.65} | ||
.prose>:first-child{margin-top:0}.prose>:last-child{margin-bottom:0} | ||
.prose h1,.prose h2{padding-bottom:.35rem;border-bottom:1px solid var(--line);margin:1.5rem 0 .75rem} | ||
.prose h1{font-size:24px}.prose h2{font-size:19px}.prose h3{font-size:16px;margin:1.25rem 0 .5rem} | ||
.prose p,.prose ul,.prose ol,.prose table,.prose pre,.prose blockquote{margin:0 0 1rem} | ||
.prose a{color:var(--accent)}.prose a:hover{text-decoration:underline} | ||
.prose img{max-width:100%} | ||
.prose pre{background:var(--soft);border-radius:8px;padding:.85rem 1rem;overflow:auto;line-height:1.5} | ||
.prose :not(pre)>code{background:var(--soft);padding:.12rem .35rem;border-radius:5px;font-size:.85em} | ||
.prose table{border-collapse:collapse;display:block;overflow-x:auto}.prose td,.prose th{border:1px solid var(--line);padding:.35rem .7rem} | ||
.prose blockquote{color:var(--mute);border-left:3px solid var(--line);padding-left:1rem;margin-left:0} | ||
.day{display:flex;align-items:center;gap:.5rem;color:var(--mute);font-weight:500;font-size:13px;margin:1.5rem 0 .6rem} | ||
.day:first-of-type{margin-top:0} | ||
.more{display:flex;justify-content:center;margin-top:1.25rem} | ||
.commit-msg{padding:1rem} | ||
.commit-msg pre{white-space:pre-wrap;color:var(--mute);margin:.6rem 0 0;font-size:12.5px} | ||
.summary{margin:1.5rem 0 .9rem;color:var(--mute)} | ||
.summary b{color:var(--fg)} | ||
.add{color:var(--open)}.rem{color:#cf222e} | ||
@media (prefers-color-scheme:dark){.rem{color:#f47067}} | ||
.bars{display:inline-flex;gap:1px;vertical-align:-1px}.bars i{width:7px;height:7px;border-radius:1px;background:var(--line)}.bars i.a{background:var(--open)}.bars i.d{background:#cf222e} | ||
.file{border:1px solid var(--line);border-radius:var(--r);margin:0 0 1rem;overflow:hidden} | ||
.file>summary{display:flex;flex-wrap:wrap;align-items:center;gap:.4rem .75rem;padding:.55rem 1rem;background:var(--panel);cursor:pointer;list-style:none;position:sticky;top:0} | ||
.file>summary::-webkit-details-marker{display:none} | ||
.file>summary::before{content:"";width:6px;height:6px;border:solid var(--mute);border-width:0 1.5px 1.5px 0;transform:rotate(-45deg);transition:transform .15s;margin-right:.1rem} | ||
.file[open]>summary::before{transform:rotate(45deg)} | ||
.file[open]>summary{border-bottom:1px solid var(--line)} | ||
.file .path{font:500 12.5px var(--mono);overflow-wrap:anywhere;min-width:0} | ||
.file .stat{font:12px var(--mono);display:flex;align-items:center;gap:.4rem;margin-left:auto} | ||
.tag{font:500 11px var(--sans);padding:.05rem .45rem;border-radius:9px;border:1px solid var(--line);color:var(--mute)} | ||
.note{padding:1rem;color:var(--mute);text-align:center} | ||
.scroll{overflow-x:auto} | ||
.diff{border-collapse:collapse;width:100%;font:12px/20px var(--mono);tab-size:4} | ||
.diff td{padding:0;vertical-align:top} | ||
.diff .n{width:1%;min-width:3rem;padding:0 .6rem;text-align:right;color:var(--faint);user-select:none} | ||
.diff .n::before{content:attr(data-n)} | ||
.diff .c{padding:0 1rem 0 1.4rem;white-space:pre;position:relative} | ||
.diff .c::before{position:absolute;left:.5rem;user-select:none} | ||
.diff code{font:inherit} | ||
.diff .a .c{background:var(--ins)}.diff .a .n{background:var(--ins-n)}.diff .a .c::before{content:"+"} | ||
.diff .d .c{background:var(--del)}.diff .d .n{background:var(--del-n)}.diff .d .c::before{content:"-"} | ||
.diff .h td{background:var(--hunk);color:var(--mute);padding-top:.2rem;padding-bottom:.2rem} | ||
.diff .h .c{white-space:pre-wrap} | ||
.diff .x .c{color:var(--mute);font-style:italic} | ||
.filter{display:flex;gap:1rem} | ||
.filter a{display:flex;align-items:center;gap:.35rem;color:var(--mute)} | ||
.filter a[aria-current]{color:var(--fg);font-weight:600} | ||
.st.open{color:var(--open)}.st.merged{color:var(--merged)}.st.closed{color:var(--closed)} | ||
.pill{display:inline-flex;align-items:center;gap:.35rem;height:26px;padding:0 .7rem;border-radius:13px;color:#fff;font-weight:550;font-size:13px} | ||
.pill.open{background:var(--open)}.pill.merged{background:var(--merged)}.pill.closed{background:var(--closed)} | ||
@media (prefers-color-scheme:dark){.pill{color:#0d0e10}} | ||
.item-title{font-size:24px;font-weight:500;letter-spacing:-.02em;overflow-wrap:anywhere} | ||
.item-meta{display:flex;flex-wrap:wrap;align-items:center;gap:.5rem .75rem;margin:.6rem 0 1.5rem;padding-bottom:1.25rem;border-bottom:1px solid var(--line);color:var(--mute)} | ||
.item-meta b{color:var(--fg);font-weight:600} | ||
.item-meta .end{display:inline-flex;align-items:center;gap:.35rem} | ||
.ref{font:12px var(--mono);padding:.1rem .45rem;border-radius:6px;background:var(--accent-soft);color:var(--accent)} | ||
.timeline{display:flex;flex-direction:column;gap:1rem;max-width:56rem} | ||
.comment .boxhead{gap:.5rem} | ||
.comment .prose{padding:1rem 1.25rem;font-size:14px} | ||
.verdict{display:inline-flex;align-items:center;gap:.3rem;font-weight:550;font-size:12.5px} | ||
.verdict.approve{color:var(--open)}.verdict.changes{color:var(--closed)} | ||
.event{display:flex;align-items:center;gap:.6rem;color:var(--mute);padding:0 1rem} | ||
.event .dot{display:inline-grid;place-items:center;width:28px;height:28px;border-radius:50%;background:var(--merged);color:#fff} | ||
.error{padding:4rem 0;text-align:center} | ||
.error h1{font-size:44px;letter-spacing:-.04em} | ||
.error p{color:var(--mute);margin-top:.5rem} | ||
@media (max-width:860px){.steps{grid-template-columns:minmax(0,1fr)}} | ||
@media (max-width:640px){ | ||
.av.xl{width:64px;height:64px;font-size:26px} | ||
.profile{gap:1rem;align-items:flex-start} | ||
.wrap{padding:0 1rem} | ||
main.wrap{padding-top:1.25rem} | ||
.hero{padding:1.25rem 0 1.75rem} | ||
.hero p{font-size:15px} | ||
.top nav{gap:1rem} | ||
.tabs a{padding:.55rem .5rem;gap:.35rem} | ||
.clone{width:100%} | ||
.search{max-width:none;min-width:100%} | ||
.prose{padding:1rem 1.1rem 1.25rem} | ||
.list li{align-items:flex-start} | ||
.list .sha,.list .hide-sm{display:none} | ||
.file .stat{margin-left:0} | ||
.code .ln{min-width:2.75rem;padding-left:.5rem} | ||
.diff .n{min-width:2.25rem;padding:0 .35rem} | ||
.item-title{font-size:20px} | ||
} | ||
::highlight(comment){color:var(--comment)}::highlight(string){color:var(--string)}::highlight(keyword){color:var(--keyword)}::highlight(number){color:var(--number)}::highlight(function){color:var(--function)}::highlight(type){color:var(--type)}::highlight(tag){color:var(--tag)}::highlight(attr){color:var(--attr)} | ||
::highlight(xins){background-color:var(--ins-x)}::highlight(xdel){background-color:var(--del-x)}::highlight(target){background-color:var(--target)} | ||
"#; | ||
| ||
/// Reads each `data-hl` element's ranges into named highlights, over the | ||
/// text of its `code` elements as if joined by newlines; marks the line a | ||
/// `#L12` fragment names; and copies a `data-copy` button's text. | ||
const SCRIPT: &str = r#"(()=>{for(const b of document.querySelectorAll("[data-copy]"))b.onclick=()=>navigator.clipboard.writeText(b.dataset.copy).then(()=>{b.textContent="Copied";setTimeout(()=>b.textContent="Copy",1500)});if(!window.CSS||!CSS.highlights)return;const H={};const h=n=>H[n]||(CSS.highlights.set(n,H[n]=new Highlight()),H[n]);for(const p of document.querySelectorAll("[data-hl]")){const N=[];let s=0;for(const c of p.querySelectorAll("code")){c.normalize();const t=c.firstChild;if(t)N.push([s,t]);s+=(t?t.length:0)+1}for(const g of p.dataset.hl.split(";")){if(!g)continue;const[n,v]=g.split(":"),x=v.split(",");let o=0,j=0;for(let i=0;i<x.length;i+=2){o+=+x[i];const e=o+ +x[i+1];while(j<N.length&&N[j][0]+N[j][1].length<=o)j++;for(let k=j;k<N.length&&N[k][0]<e;k++){const[a,t]=N[k],r=new Range();r.setStart(t,Math.max(o,a)-a);r.setEnd(t,Math.min(e,a+t.length)-a);h(n).add(r)}o=e}}}const L=()=>{CSS.highlights.delete("target");const m=/^#L(\d+)$/.exec(location.hash),c=document.querySelector(".src code");if(!m||!c||!c.firstChild)return;const t=c.firstChild,x=t.data;let a=0;for(let i=1;i<+m[1];i++){a=x.indexOf("\n",a)+1;if(!a)return}const e=x.indexOf("\n",a),r=new Range();r.setStart(t,a);r.setEnd(t,e<0?x.length:e+1);CSS.highlights.set("target",new Highlight(r))};L();addEventListener("hashchange",L)})()"#; | ||
| ||
/// Prerenders a same-site page as the pointer settles on its link. | ||
const SPECULATION: &str = r#"{"prerender":[{"where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":"/*/*/raw/*"}}]},"eagerness":"moderate"}]}"#; | ||
| ||
static POLICY: LazyLock<String> = LazyLock::new(|| { | ||
let hash = |s: &str| base64::engine::general_purpose::STANDARD.encode(Sha256::digest(s)); | ||
format!( | ||
"default-src 'none'; img-src 'self' https: data:; style-src 'unsafe-inline'; script-src 'sha256-{}' 'sha256-{}'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'", | ||
hash(SCRIPT), | ||
hash(SPECULATION) | ||
) | ||
}); | ||
| ||
/// The mark: two commits on one straight thread. | ||
const LOGO: &str = r#"<svg viewBox="0 0 24 24" aria-hidden="true"><rect width="24" height="24" rx="6.5" fill="currentColor"/><path d="M7 12h10" stroke="var(--bg)" stroke-width="2" stroke-linecap="round"/><circle cx="7" cy="12" r="2.6" fill="var(--bg)"/><circle cx="17" cy="12" r="2.6" fill="var(--bg)"/></svg>"#; | ||
| ||
/// Icons, each a 16-unit path. | ||
mod icon { | ||
pub const CODE: &str = "M4.7 3.3a.75.75 0 0 1 0 1.06L1.06 8l3.64 3.64a.75.75 0 1 1-1.06 1.06L-.53 8.53a.75.75 0 0 1 0-1.06L3.64 3.3a.75.75 0 0 1 1.06 0Zm6.6 0a.75.75 0 0 1 1.06 0l4.17 4.17a.75.75 0 0 1 0 1.06l-4.17 4.17a.75.75 0 1 1-1.06-1.06L14.94 8 11.3 4.36a.75.75 0 0 1 0-1.06Z"; | ||
pub const COMMIT: &str = "M11.93 8.5a4 4 0 0 1-7.86 0H.75a.75.75 0 0 1 0-1.5h3.32a4 4 0 0 1 7.86 0h3.32a.75.75 0 0 1 0 1.5Zm-1.43-.75a2.5 2.5 0 1 0-5 0 2.5 2.5 0 0 0 5 0Z"; | ||
pub const PULL: &str = "M1.5 3.25a2.25 2.25 0 1 1 3 2.12v5.26a2.25 2.25 0 1 1-1.5 0V5.37a2.25 2.25 0 0 1-1.5-2.12Zm5.72-.53L9.47.47a.75.75 0 0 1 1.06 1.06l-.97.97h1.69A2.75 2.75 0 0 1 14 5.25v5.38a2.25 2.25 0 1 1-1.5 0V5.25c0-.69-.56-1.25-1.25-1.25H9.56l.97.97a.75.75 0 0 1-1.06 1.06L7.22 3.78a.75.75 0 0 1 0-1.06ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm9.5 0a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"; | ||
pub const ISSUE: &str = "M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"; | ||
pub const DONE: &str = "M10.28 6.28a.75.75 0 0 0-1.06-1.06L7 7.44 6.03 6.47a.75.75 0 0 0-1.06 1.06l1.5 1.5a.75.75 0 0 0 1.06 0ZM8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"; | ||
pub const MERGE: &str = "M5.45 5.15A3.75 3.75 0 0 0 8.75 7h1.62a2.25 2.25 0 1 1 0 1.5H8.75a5.23 5.23 0 0 1-3.5-1.34v3.47a2.25 2.25 0 1 1-1.5 0V5.37a2.25 2.25 0 1 1 1.7-.22ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.5-4.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"; | ||
pub const DIR: &str = "M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.22.78 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"; | ||
pub const FILE: &str = "M2 1.75C2 .78 2.78 0 3.75 0h6.59c.46 0 .9.18 1.23.51l2.92 2.92c.33.33.51.77.51 1.23v9.59A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .14.11.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.06V4.25c0 .14.11.25.25.25h2.69L10.5 1.56Z"; | ||
pub const BRANCH: &str = "M9.5 3.25a2.25 2.25 0 1 1 3 2.12V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.13a2.25 2.25 0 1 1-1.5 0V5.37a2.25 2.25 0 1 1 1.5 0v1.84A2.49 2.49 0 0 1 6 7h4a1 1 0 0 0 1-1v-.63a2.25 2.25 0 0 1-1.5-2.12ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM3.5 3.25a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"; | ||
pub const CHAT: &str = "M1 2.75C1 1.78 1.78 1 2.75 1h10.5c.97 0 1.75.78 1.75 1.75v7.5A1.75 1.75 0 0 1 13.25 12H9.06l-2.57 2.57A1.46 1.46 0 0 1 4 13.54V12H2.75A1.75 1.75 0 0 1 1 10.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .14.11.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.75.75 0 0 1 .53-.22h4.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"; | ||
pub const REPO: &str = "M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.71 1.7.75.75 0 1 1-1.08 1.05A2.5 2.5 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.71A2.49 2.49 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.09a.25.25 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"; | ||
pub const FORK: &str = "M5 5.37v.88c0 .41.34.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.88a2.25 2.25 0 1 1 1.5 0v.88A2.25 2.25 0 0 1 10.25 8.5h-1.5v2.13a2.25 2.25 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.88a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"; | ||
} | ||
| ||
fn svg(path: &str) -> String { | ||
format!(r#"<svg class="i" viewBox="0 0 16 16" aria-hidden="true"><path d="{path}"/></svg>"#) | ||
} | ||
| ||
fn esc(s: &str) -> String { | ||
let mut out = String::with_capacity(s.len()); | ||
for c in s.chars() { | ||
match c { | ||
'&' => out.push_str("&"), | ||
'<' => out.push_str("<"), | ||
'>' => out.push_str(">"), | ||
'"' => out.push_str("""), | ||
'\'' => out.push_str("'"), | ||
c => out.push(c), | ||
} | ||
} | ||
out | ||
} | ||
| ||
/// A path for a URL: each segment percent-encoded, the slashes kept. | ||
fn url_path(path: &str) -> String { | ||
let mut out = String::new(); | ||
for b in path.bytes() { | ||
match b { | ||
b'a'..=b'z' | ||
| b'A'..=b'Z' | ||
| b'0'..=b'9' | ||
| b'-' | ||
| b'.' | ||
| b'_' | ||
| b'~' | ||
| b'/' | ||
| b':' | ||
| b'@' => out.push(b as char), | ||
b => write!(out, "%{b:02X}").unwrap(), | ||
} | ||
} | ||
out | ||
} | ||
| ||
/// `2026-10-20T…` as `Oct 20, 2026`; anything else as it is. | ||
fn date(iso: &str) -> String { | ||
const MONTHS: [&str; 12] = [ | ||
"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", | ||
]; | ||
let parts = (iso.get(..4), iso.get(5..7), iso.get(8..10)); | ||
let (Some(y), Some(m), Some(d)) = parts else { | ||
return iso.to_string(); | ||
}; | ||
let (Some(month), Ok(day)) = ( | ||
m.parse::<usize>() | ||
.ok() | ||
.and_then(|m| MONTHS.get(m.wrapping_sub(1))), | ||
d.parse::<u8>(), | ||
) else { | ||
return iso.to_string(); | ||
}; | ||
format!("{month} {day}, {y}") | ||
} | ||
| ||
fn subject(message: &str) -> &str { | ||
message.lines().next().unwrap_or_default() | ||
} | ||
| ||
/// A round mark with a name's first letter, in a color its name picks. | ||
fn avatar(name: &str, class: &str) -> String { | ||
let hue = name | ||
.bytes() | ||
.fold(7u32, |h, b| h.wrapping_mul(31).wrapping_add(b as u32)) | ||
% 360; | ||
let letter = name.chars().find(|c| c.is_alphanumeric()).unwrap_or('?'); | ||
format!( | ||
r#"<span class="av{class}" style="--h:{hue}" aria-hidden="true">{}</span>"#, | ||
esc(&letter.to_string()) | ||
) | ||
} | ||
| ||
fn copy(text: &str) -> String { | ||
format!( | ||
r#"<button class="btn" type="button" data-copy="{}">Copy</button>"#, | ||
esc(text) | ||
) | ||
} | ||
| ||
/// A handle, linked to its profile. | ||
fn person(handle: &str) -> String { | ||
format!( | ||
r#"<a href="/{}"><b>{}</b></a>"#, | ||
url_path(handle), | ||
esc(handle) | ||
) | ||
} | ||
| ||
/// A command, shown with a button that copies it. | ||
fn command(text: &str) -> String { | ||
format!( | ||
r#"<div class="cmd"><pre>{}</pre>{}</div>"#, | ||
esc(text), | ||
copy(text) | ||
) | ||
} | ||
| ||
fn layout(title: &str, body: &str) -> String { | ||
format!( | ||
r#"<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="color-scheme" content="light dark"><title>{}</title><style>{CSS}</style><script type="speculationrules">{SPECULATION}</script></head><body><header class="top"><div class="wrap"><a class="brand" href="/">{LOGO}<span>knotless</span></a><nav><a href="/#repositories">Repositories</a><a href="/#connect">Connect</a></nav></div></header><main class="wrap">{body}</main><footer class="wrap foot">Git repositories stored in atproto accounts. This server only reads them.</footer><script>{SCRIPT}</script></body></html>"#, | ||
esc(title) | ||
) | ||
} | ||
| ||
/// Caching for a page about `rev`: forever when it names a commit by hash, | ||
/// else checked again each time, as a branch can move. | ||
fn cache_for(rev: &str) -> &'static str { | ||
if read::is_pinned(rev) { | ||
"public, max-age=31536000, immutable" | ||
} else { | ||
"no-cache" | ||
} | ||
} | ||
| ||
fn page(status: StatusCode, cache: &'static str, title: &str, body: &str) -> Response { | ||
let mut res = (status, layout(title, body)).into_response(); | ||
let h = res.headers_mut(); | ||
h.insert( | ||
header::CONTENT_TYPE, | ||
HeaderValue::from_static("text/html; charset=utf-8"), | ||
); | ||
h.insert(header::CACHE_CONTROL, HeaderValue::from_static(cache)); | ||
h.insert( | ||
header::CONTENT_SECURITY_POLICY, | ||
HeaderValue::from_str(&POLICY).unwrap(), | ||
); | ||
h.insert( | ||
header::X_CONTENT_TYPE_OPTIONS, | ||
HeaderValue::from_static("nosniff"), | ||
); | ||
h.insert( | ||
header::REFERRER_POLICY, | ||
HeaderValue::from_static("same-origin"), | ||
); | ||
res | ||
} | ||
| ||
/// An error, as a page. | ||
pub struct Shown(StatusCode, String); | ||
| ||
impl From<AppError> for Shown { | ||
fn from(e: AppError) -> Shown { | ||
Shown(e.0, e.1) | ||
} | ||
} | ||
| ||
impl IntoResponse for Shown { | ||
fn into_response(self) -> Response { | ||
if self.0 == StatusCode::TEMPORARY_REDIRECT { | ||
let mut res = self.0.into_response(); | ||
res.extensions_mut().insert(Renamed(self.1)); | ||
return res; | ||
} | ||
let reason = self.0.canonical_reason().unwrap_or("Error"); | ||
let body = format!( | ||
r#"<div class="error"><h1>{}</h1><p><b>{reason}</b></p><p>{}</p></div>"#, | ||
self.0.as_u16(), | ||
esc(&self.1) | ||
); | ||
page(self.0, "no-store", reason, &body) | ||
} | ||
} | ||
| ||
fn not_found(e: anyhow::Error) -> Shown { | ||
Shown(StatusCode::NOT_FOUND, format!("{e:#}")) | ||
} | ||
| ||
type Result<T> = std::result::Result<T, Shown>; | ||
| ||
static HANDLES: LazyLock<Mutex<HashMap<String, String>>> = LazyLock::new(Mutex::default); | ||
| ||
/// The handle `did` claims, if it resolves back to `did`; else the DID. | ||
async fn handle(app: &App, did: &str) -> String { | ||
if let Some(known) = HANDLES.lock().unwrap().get(did) { | ||
return known.clone(); | ||
} | ||
let doc = knotless::identity::document(&app.ctx, did).await.ok(); | ||
let claimed = doc | ||
.as_ref() | ||
.and_then(|d| d["alsoKnownAs"][0].as_str()) | ||
.and_then(|a| a.strip_prefix("at://")) | ||
.unwrap_or(did); | ||
let verified = knotless::identity::resolve_handle(&app.ctx, claimed) | ||
.await | ||
.ok(); | ||
let resolves = |_: &&str| verified.as_deref() == Some(did); | ||
let shown = Some(claimed).filter(resolves).unwrap_or(did); | ||
HANDLES | ||
.lock() | ||
.unwrap() | ||
.insert(did.to_string(), shown.to_string()); | ||
shown.to_string() | ||
} | ||
| ||
/// A repository as `handle/name`, under the name it has now if the index | ||
/// knows it, and its page. | ||
async fn repo_link(app: &App, repo: &RepoRef) -> Option<(String, String)> { | ||
let (did, _, named) = forge::parse_uri(&repo.uri).ok()?; | ||
let now = app | ||
.index | ||
.as_ref() | ||
.and_then(|i| i.name(did, &repo.key).ok().flatten()); | ||
let name = now.as_deref().unwrap_or(named); | ||
let owner = handle(app, did).await; | ||
Some(( | ||
format!("{owner}/{name}"), | ||
format!("/{}/{}", url_path(&owner), url_path(name)), | ||
)) | ||
} | ||
| ||
#[derive(Clone, Copy, PartialEq)] | ||
enum Tab { | ||
Code, | ||
Commits, | ||
Pulls, | ||
Issues, | ||
} | ||
| ||
/// A repository being shown. | ||
struct Repo { | ||
authority: String, | ||
name: String, | ||
did: String, | ||
key: String, | ||
git: Git, | ||
/// The default branch. | ||
head: String, | ||
} | ||
| ||
impl Repo { | ||
async fn open(app: &App, authority: &str, name: &str) -> Result<Repo> { | ||
let name = repo_name(name); | ||
let found = app.located(authority, name).await?; | ||
if found.moved { | ||
let now = format!("{}/{}", found.did, found.name); | ||
return Err(Shown(StatusCode::TEMPORARY_REDIRECT, now)); | ||
} | ||
if found.name != name { | ||
return Err(Shown(StatusCode::TEMPORARY_REDIRECT, found.name)); | ||
} | ||
let git = found.git; | ||
let head = objects::open(&git.dir) | ||
.map_err(AppError::internal)? | ||
.head() | ||
.unwrap_or_default(); | ||
Ok(Repo { | ||
authority: authority.to_string(), | ||
name: name.to_string(), | ||
did: found.did, | ||
key: found.key, | ||
git, | ||
head, | ||
}) | ||
} | ||
| ||
/// Its objects, read in process. Not held across an await: a reader | ||
/// belongs to one thread. | ||
fn objects(&self) -> Result<Objects> { | ||
Ok(objects::open(&self.git.dir).map_err(AppError::internal)?) | ||
} | ||
| ||
fn href(&self, tail: &str) -> String { | ||
format!( | ||
"/{}/{}{tail}", | ||
url_path(&self.authority), | ||
url_path(&self.name) | ||
) | ||
} | ||
| ||
fn uri(&self) -> String { | ||
forge::repo_uri(&self.did, &self.name) | ||
} | ||
| ||
/// The record, as the index has it. | ||
fn record(&self, app: &App) -> Option<RepoRecord> { | ||
let value = app.index.as_ref()?.repo(&self.uri()).ok()??; | ||
serde_json::from_value(value).ok() | ||
} | ||
| ||
/// Its open pulls and issues, as the index knows them. | ||
fn open_items(&self, app: &App) -> Option<(usize, usize)> { | ||
let entries = app.index.as_ref()?.forge(&self.did, &self.key).ok()?; | ||
let record = self.record(app).unwrap_or_default(); | ||
let items = forge::assemble(&record, &self.did, &entries); | ||
let open = |kind: &str| { | ||
items | ||
.iter() | ||
.filter(|i| i.kind == kind && i.state == "open") | ||
.count() | ||
}; | ||
Some((open("pull"), open("issue"))) | ||
} | ||
| ||
async fn header(&self, app: &App, tab: Tab) -> String { | ||
let record = self.record(app); | ||
let mut out = format!( | ||
r#"<div class="repohead"><div class="about"><h1>{}<a href="/{}">{}</a><span class="mute">/</span><a href="{}"><b>{}</b></a></h1>"#, | ||
avatar(&self.authority, ""), | ||
url_path(&self.authority), | ||
esc(&self.authority), | ||
self.href(""), | ||
esc(&self.name) | ||
); | ||
if let Some(description) = record.as_ref().and_then(|r| r.description.as_deref()) { | ||
write!(out, "<p>{}</p>", esc(description)).unwrap(); | ||
} | ||
if let Some(from) = record.as_ref().and_then(|r| r.fork_of.as_ref()) | ||
&& let Some((shown, href)) = repo_link(app, from).await | ||
{ | ||
write!( | ||
out, | ||
r#"<p>{} Forked from <a href="{href}">{}</a></p>"#, | ||
svg(icon::FORK), | ||
esc(&shown) | ||
) | ||
.unwrap(); | ||
} | ||
let clone = format!("git clone knotless://{}/{}", self.authority, self.name); | ||
write!( | ||
out, | ||
r#"</div><div class="clone"><code>{}</code>{}</div></div><nav class="tabs">"#, | ||
esc(&clone), | ||
copy(&clone) | ||
) | ||
.unwrap(); | ||
let (pulls, issues) = self.open_items(app).unwrap_or_default(); | ||
let count = |n: usize| { | ||
if n > 0 { | ||
format!(r#"<span class="count">{n}</span>"#) | ||
} else { | ||
String::new() | ||
} | ||
}; | ||
let tabs = [ | ||
(Tab::Code, icon::CODE, "Code", self.href(""), String::new()), | ||
( | ||
Tab::Commits, | ||
icon::COMMIT, | ||
"Commits", | ||
self.href(&format!("/commits/{}", url_path(&self.head))), | ||
String::new(), | ||
), | ||
( | ||
Tab::Pulls, | ||
icon::PULL, | ||
"Pulls", | ||
self.href("/pulls"), | ||
count(pulls), | ||
), | ||
( | ||
Tab::Issues, | ||
icon::ISSUE, | ||
"Issues", | ||
self.href("/issues"), | ||
count(issues), | ||
), | ||
]; | ||
for (t, path, label, href, count) in tabs { | ||
let current = if t == tab { | ||
r#" aria-current="page""# | ||
} else { | ||
"" | ||
}; | ||
write!( | ||
out, | ||
r#"<a href="{href}"{current}>{}{label}{count}</a>"#, | ||
svg(path) | ||
) | ||
.unwrap(); | ||
} | ||
out.push_str("</nav>"); | ||
out | ||
} | ||
| ||
/// Splits `rest` into a revision and a path: the longest branch or tag | ||
/// that starts it, else its first segment. | ||
fn locate<'a>(&self, rest: &'a str) -> Result<(String, &'a str)> { | ||
let names = self.objects()?.names().map_err(AppError::internal)?; | ||
let longest = names | ||
.iter() | ||
.filter(|n| rest == n.as_str() || rest.starts_with(&format!("{n}/"))) | ||
.map(String::len) | ||
.max(); | ||
let cut = longest.unwrap_or(rest.find('/').unwrap_or(rest.len())); | ||
let rev = read::rev(Some(&rest[..cut])).map_err(not_found)?; | ||
Ok((rev, rest[cut..].trim_start_matches('/'))) | ||
} | ||
| ||
/// The revision, and links up the path, each segment to its tree; the | ||
/// last is the page itself. | ||
fn toolbar(&self, rev: &str, path: &str) -> String { | ||
let shown = if read::is_pinned(rev) { &rev[..7] } else { rev }; | ||
let mut out = format!( | ||
r#"<div class="toolbar"><span class="branch" title="{}">{}<span>{}</span></span><nav class="crumbs">"#, | ||
esc(rev), | ||
svg(icon::BRANCH), | ||
esc(shown) | ||
); | ||
let segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); | ||
let root = self.href(&format!("/tree/{}", url_path(rev))); | ||
if segments.is_empty() { | ||
write!(out, "<b>{}</b>", esc(&self.name)).unwrap(); | ||
} else { | ||
write!(out, r#"<a href="{root}">{}</a>"#, esc(&self.name)).unwrap(); | ||
} | ||
let mut at = String::new(); | ||
for (i, segment) in segments.iter().enumerate() { | ||
at = if at.is_empty() { | ||
segment.to_string() | ||
} else { | ||
format!("{at}/{segment}") | ||
}; | ||
out.push_str(r#"<span class="sep">/</span>"#); | ||
if i + 1 == segments.len() { | ||
write!(out, "<b>{}</b>", esc(segment)).unwrap(); | ||
} else { | ||
write!( | ||
out, | ||
r#"<a href="{root}/{}">{}</a>"#, | ||
url_path(&at), | ||
esc(segment) | ||
) | ||
.unwrap(); | ||
} | ||
} | ||
out.push_str("</nav></div>"); | ||
out | ||
} | ||
} | ||
| ||
#[derive(Deserialize, Default)] | ||
pub struct HomeQuery { | ||
q: Option<String>, | ||
} | ||
| ||
/// What knotless is, how to start, and the repositories the index knows. | ||
async fn home( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
Query(query): Query<HomeQuery>, | ||
) -> Result<Response> { | ||
let host = headers | ||
.get(header::HOST) | ||
.and_then(|h| h.to_str().ok()) | ||
.unwrap_or("localhost"); | ||
let mcp = r#"{ "mcpServers": { "knotless": { "command": "knotless-mcp" } } }"#; | ||
let mut body = format!( | ||
r#"<section class="hero"><h1>Git that lives in your atproto account.</h1><p>Each repository is a few records on your own PDS: no git server to run, nothing to install on the PDS. Clone and push with plain git, and let agents read, commit, review and merge over MCP.</p></section> | ||
<section class="steps" id="connect"> | ||
<div class="step"><h3><b>1</b>Install and log in</h3><p>From a checkout of knotless. Use an app password.</p>{}{}</div> | ||
<div class="step"><h3><b>2</b>Push</h3><p>Any repository, to any account on any PDS.</p>{}{}</div> | ||
<div class="step"><h3><b>3</b>Connect your agent</h3><p>The MCP server acts as the account you logged in as.</p>{}<details><summary>Other MCP clients</summary><p>Add this to the client's MCP settings, such as Claude Desktop's or Cursor's:</p>{}</details></div> | ||
</section>"#, | ||
command("cargo install --path crates/knotless\ncargo install --path crates/knotless-mcp"), | ||
command("knotless login you.bsky.social"), | ||
command("git push knotless://you.bsky.social/project main"), | ||
command(&format!("git clone http://{host}/you.bsky.social/project")), | ||
command("claude mcp add knotless -- knotless-mcp"), | ||
command(mcp), | ||
); | ||
let Some(index) = &app.index else { | ||
return Ok(page(StatusCode::OK, "no-cache", "knotless", &body)); | ||
}; | ||
let q = query.q.unwrap_or_default(); | ||
write!( | ||
body, | ||
r#"<section id="repositories"><div class="section"><h2>Repositories</h2><form action="/" role="search" class="search-form"><input class="search" type="search" name="q" value="{}" placeholder="Find a repository" aria-label="Find a repository"></form></div>"#, | ||
esc(&q) | ||
) | ||
.unwrap(); | ||
let repos = index | ||
.repos(&q, 50) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
let owners = futures::future::join_all(repos.iter().map(|r| handle(&app, &r.did))).await; | ||
body.push_str(r#"<div class="box"><ul class="list">"#); | ||
for (r, owner) in repos.iter().zip(owners) { | ||
write!( | ||
body, | ||
r#"<li>{}<div class="grow"><a class="clip" href="/{}/{}"><span class="mute">{}/</span><b>{}</b></a><div class="meta clip">{}</div></div><span class="mute hide-sm">{}</span></li>"#, | ||
avatar(&owner, " lg"), | ||
url_path(&owner), | ||
url_path(&r.name), | ||
esc(&owner), | ||
esc(&r.name), | ||
esc(r.description.as_deref().unwrap_or("No description")), | ||
date(&r.created_at) | ||
) | ||
.unwrap(); | ||
} | ||
if repos.is_empty() { | ||
body.push_str(r#"<li class="empty">No repositories found.</li>"#); | ||
} | ||
body.push_str("</ul></div></section>"); | ||
Ok(page(StatusCode::OK, "no-cache", "knotless", &body)) | ||
} | ||
| ||
/// An account: who it is, as its Bluesky profile says if it has one, and | ||
/// its public repositories, read from its PDS. | ||
async fn profile(State(app): State<Arc<App>>, Path(authority): Path<String>) -> Result<Response> { | ||
let at = RepoUrl { | ||
authority: authority.clone(), | ||
name: "_".into(), | ||
}; | ||
let remote = Remote::connect(&app.ctx, &at).await.map_err(not_found)?; | ||
let did = remote.did().to_string(); | ||
let shown = handle(&app, &did).await; | ||
let about = remote | ||
.get_record::<serde_json::Value>("app.bsky.actor.profile", "self") | ||
.await | ||
.ok() | ||
.flatten() | ||
.map(|r| r.value) | ||
.unwrap_or_default(); | ||
let name = about["displayName"] | ||
.as_str() | ||
.filter(|n| !n.trim().is_empty()); | ||
let mut repos = remote.list_repos().await.map_err(AppError::upstream)?; | ||
repos.sort_by(|a, b| b.value.created_at.cmp(&a.value.created_at)); | ||
let mut body = format!( | ||
r#"<section class="profile">{}<div><h1>{}</h1>"#, | ||
avatar(&shown, " xl"), | ||
esc(name.unwrap_or(&shown)) | ||
); | ||
if name.is_some() { | ||
write!(body, r#"<p class="who">{}</p>"#, esc(&shown)).unwrap(); | ||
} | ||
write!(body, r#"<p class="did">{}</p>"#, esc(&did)).unwrap(); | ||
if let Some(bio) = about["description"].as_str() { | ||
write!(body, r#"<p class="bio">{}</p>"#, esc(bio)).unwrap(); | ||
} | ||
write!( | ||
body, | ||
r#"</div></section><div class="section"><h2>Repositories</h2><span class="count">{}</span></div><div class="box"><ul class="list">"#, | ||
repos.len() | ||
) | ||
.unwrap(); | ||
let forks = futures::future::join_all(repos.iter().map(|r| async { | ||
match &r.value.fork_of { | ||
Some(from) => repo_link(&app, from).await, | ||
None => None, | ||
} | ||
})) | ||
.await; | ||
for (r, fork) in repos.iter().zip(forks) { | ||
let href = format!("/{}/{}", url_path(&shown), url_path(&r.value.name)); | ||
let from = fork.map(|(from, at)| { | ||
format!( | ||
r#"<div class="meta">{}Forked from <a href="{at}">{}</a></div>"#, | ||
svg(icon::FORK), | ||
esc(&from) | ||
) | ||
}); | ||
write!( | ||
body, | ||
r#"<li>{}<div class="grow"><a class="title" href="{href}">{}</a><div class="meta">{}</div>{}</div><span class="mute hide-sm">{}</span></li>"#, | ||
svg(icon::REPO), | ||
esc(&r.value.name), | ||
esc(r.value.description.as_deref().unwrap_or("No description")), | ||
from.unwrap_or_default(), | ||
date(&r.value.created_at) | ||
) | ||
.unwrap(); | ||
} | ||
if repos.is_empty() { | ||
write!( | ||
body, | ||
r#"<li class="empty"><p>No public repositories yet.</p><div style="max-width:32rem;margin:1rem auto 0;text-align:left">{}</div></li>"#, | ||
command(&format!("git push knotless://{shown}/project main")) | ||
) | ||
.unwrap(); | ||
} | ||
body.push_str("</ul></div>"); | ||
Ok(page( | ||
StatusCode::OK, | ||
"no-cache", | ||
name.unwrap_or(&shown), | ||
&body, | ||
)) | ||
} | ||
| ||
type RepoPath = Path<(String, String)>; | ||
type RestPath = Path<(String, String, String)>; | ||
| ||
async fn code(State(app): State<Arc<App>>, Path((authority, name)): RepoPath) -> Result<Response> { | ||
let repo = Repo::open(&app, &authority, &name).await?; | ||
let mut body = repo.header(&app, Tab::Code).await; | ||
if repo.objects()?.tree("HEAD", "").is_err() { | ||
write!( | ||
body, | ||
r#"<div class="box"><div class="empty"><h2>Nothing pushed yet</h2><p style="margin:.5rem 0 1rem">Push a branch to start this repository.</p><div style="max-width:32rem;margin:0 auto;text-align:left">{}</div></div></div>"#, | ||
command(&format!("git push knotless://{}/{} main", repo.authority, repo.name)) | ||
) | ||
.unwrap(); | ||
return Ok(page(StatusCode::OK, "no-cache", &repo.name, &body)); | ||
} | ||
let head = repo.head.clone(); | ||
listing(&repo, &head, "", body) | ||
} | ||
| ||
async fn tree( | ||
State(app): State<Arc<App>>, | ||
Path((authority, name, rest)): RestPath, | ||
) -> Result<Response> { | ||
let repo = Repo::open(&app, &authority, &name).await?; | ||
let (rev, path) = repo.locate(&rest)?; | ||
let body = repo.header(&app, Tab::Code).await; | ||
listing(&repo, &rev, path, body) | ||
} | ||
| ||
/// A directory, its latest commit, and its README. | ||
fn listing(repo: &Repo, rev: &str, path: &str, mut body: String) -> Result<Response> { | ||
let o = repo.objects()?; | ||
let mut entries = o.tree(rev, path).map_err(not_found)?; | ||
entries.sort_by_key(|e| (e.kind != "tree", e.name.to_lowercase())); | ||
let last = o.log(rev, path, 0, 1).map_err(not_found)?; | ||
body.push_str(&repo.toolbar(rev, path)); | ||
body.push_str(r#"<div class="box">"#); | ||
if let Some(c) = last.first() { | ||
write!( | ||
body, | ||
r#"<div class="boxhead">{}<b>{}</b><a class="grow clip mute" href="{}">{}</a><a class="sha" href="{2}">{}</a><span class="mute hide-sm">{}</span><a class="btn" href="{}">{}History</a></div>"#, | ||
avatar(&c.author.name, ""), | ||
esc(&c.author.name), | ||
repo.href(&format!("/commit/{}", c.sha)), | ||
esc(subject(&c.message)), | ||
&c.sha[..7], | ||
date(&c.author.date), | ||
repo.href(&format!("/commits/{}/{}", url_path(rev), url_path(path))).trim_end_matches('/'), | ||
svg(icon::COMMIT) | ||
) | ||
.unwrap(); | ||
} | ||
body.push_str(r#"<ul class="list files">"#); | ||
let prefix = if path.is_empty() { | ||
String::new() | ||
} else { | ||
format!("{path}/") | ||
}; | ||
for e in &entries { | ||
let (view, class, glyph) = if e.kind == "tree" { | ||
("tree", "dir", icon::DIR) | ||
} else { | ||
("blob", "", icon::FILE) | ||
}; | ||
let href = repo.href(&format!( | ||
"/{view}/{}/{}", | ||
url_path(rev), | ||
url_path(&format!("{prefix}{}", e.name)) | ||
)); | ||
let size = e.size.map(size).unwrap_or_default(); | ||
write!( | ||
body, | ||
r#"<li class="{class}">{}<a class="grow clip" href="{href}">{}</a><span class="size">{size}</span></li>"#, | ||
svg(glyph), | ||
esc(&e.name) | ||
) | ||
.unwrap(); | ||
} | ||
body.push_str("</ul></div>"); | ||
let readme = entries | ||
.iter() | ||
.find(|e| e.kind == "blob" && e.name.to_lowercase().starts_with("readme")); | ||
if let Some(readme) = readme { | ||
let at = format!("{prefix}{}", readme.name); | ||
let bytes = o.blob(rev, &at).map_err(not_found)?; | ||
let text = String::from_utf8_lossy(&bytes); | ||
let dir = |view: &str| { | ||
repo.href(&format!("/{view}/{}/{}", url_path(rev), url_path(path))) | ||
.trim_end_matches('/') | ||
.to_string() | ||
}; | ||
write!( | ||
body, | ||
r#"<article class="box"><div class="boxhead">{}<b>{}</b></div>"#, | ||
svg(icon::FILE), | ||
esc(&readme.name) | ||
) | ||
.unwrap(); | ||
if readme.name.to_lowercase().ends_with(".md") { | ||
write!( | ||
body, | ||
r#"<div class="prose">{}</div></article>"#, | ||
markdown(&text, &dir("blob"), &dir("raw")) | ||
) | ||
.unwrap(); | ||
} else { | ||
write!( | ||
body, | ||
r#"<div class="prose"><pre>{}</pre></div></article>"#, | ||
esc(&text) | ||
) | ||
.unwrap(); | ||
} | ||
} | ||
let title = if path.is_empty() { | ||
repo.name.clone() | ||
} else { | ||
format!("{path} · {}", repo.name) | ||
}; | ||
Ok(page(StatusCode::OK, cache_for(rev), &title, &body)) | ||
} | ||
| ||
fn size(bytes: u64) -> String { | ||
match bytes { | ||
b if b < 1024 => format!("{b} B"), | ||
b if b < 1024 * 1024 => format!("{:.1} KB", b as f64 / 1024.0), | ||
b => format!("{:.1} MB", b as f64 / (1024.0 * 1024.0)), | ||
} | ||
} | ||
| ||
/// Markdown, with raw HTML shown as text and links made relative to the | ||
/// directory it was found in. | ||
fn markdown(text: &str, blobs: &str, raws: &str) -> String { | ||
use pulldown_cmark::{Event, Options, Parser, Tag}; | ||
let options = | ||
Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH | Options::ENABLE_TASKLISTS; | ||
let events = Parser::new_ext(text, options).map(|event| match event { | ||
Event::Html(html) | Event::InlineHtml(html) => Event::Text(html), | ||
Event::Start(Tag::Link { | ||
link_type, | ||
dest_url, | ||
title, | ||
id, | ||
}) => Event::Start(Tag::Link { | ||
link_type, | ||
dest_url: link(&dest_url, blobs).into(), | ||
title, | ||
id, | ||
}), | ||
Event::Start(Tag::Image { | ||
link_type, | ||
dest_url, | ||
title, | ||
id, | ||
}) => Event::Start(Tag::Image { | ||
link_type, | ||
dest_url: link(&dest_url, raws).into(), | ||
title, | ||
id, | ||
}), | ||
other => other, | ||
}); | ||
let mut out = String::new(); | ||
pulldown_cmark::html::push_html(&mut out, events); | ||
out | ||
} | ||
| ||
/// A link from a README: web and mail links as they are, other schemes | ||
/// nowhere, and paths into the repository. | ||
fn link(dest: &str, base: &str) -> String { | ||
let scheme = dest | ||
.split_once(':') | ||
.map(|(s, _)| s) | ||
.filter(|s| !s.contains(['/', '#', '?'])); | ||
match scheme.map(str::to_ascii_lowercase).as_deref() { | ||
Some("http" | "https" | "mailto") => dest.to_string(), | ||
Some(_) => "#".to_string(), | ||
None if dest.starts_with(['#', '/']) => dest.to_string(), | ||
None => format!("{base}/{}", dest.trim_start_matches("./")), | ||
} | ||
} | ||
| ||
fn is_image(path: &str) -> Option<&'static str> { | ||
let ext = path.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase())?; | ||
Some(match ext.as_str() { | ||
"png" => "image/png", | ||
"jpg" | "jpeg" => "image/jpeg", | ||
"gif" => "image/gif", | ||
"webp" => "image/webp", | ||
"avif" => "image/avif", | ||
"svg" => "image/svg+xml", | ||
"ico" => "image/x-icon", | ||
_ => return None, | ||
}) | ||
} | ||
| ||
fn is_binary(bytes: &[u8]) -> bool { | ||
bytes[..bytes.len().min(8000)].contains(&0) || std::str::from_utf8(bytes).is_err() | ||
} | ||
| ||
async fn blob( | ||
State(app): State<Arc<App>>, | ||
Path((authority, name, rest)): RestPath, | ||
) -> Result<Response> { | ||
let repo = Repo::open(&app, &authority, &name).await?; | ||
let (rev, path) = repo.locate(&rest)?; | ||
let bytes = repo.objects()?.blob(&rev, path).map_err(not_found)?; | ||
let raw = repo.href(&format!("/raw/{}/{}", url_path(&rev), url_path(path))); | ||
let mut body = repo.header(&app, Tab::Code).await; | ||
body.push_str(&repo.toolbar(&rev, path)); | ||
let text = String::from_utf8_lossy(&bytes); | ||
let lines = text.lines().count().max(1); | ||
let shape = if is_image(path).is_some() || is_binary(&bytes) { | ||
size(bytes.len() as u64) | ||
} else { | ||
format!("{lines} lines · {}", size(bytes.len() as u64)) | ||
}; | ||
write!( | ||
body, | ||
r#"<div class="box"><div class="boxhead"><span class="mute">{shape}</span><span class="end"></span><a class="btn" href="{raw}">Raw</a><a class="btn" href="{}">{}History</a></div>"#, | ||
repo.href(&format!("/commits/{}/{}", url_path(&rev), url_path(path))), | ||
svg(icon::COMMIT) | ||
) | ||
.unwrap(); | ||
if is_image(path).is_some() { | ||
write!( | ||
body, | ||
r#"<div class="prose" style="text-align:center"><img src="{raw}" alt="{}"></div>"#, | ||
esc(path) | ||
) | ||
.unwrap(); | ||
} else if is_binary(&bytes) { | ||
body.push_str(r#"<div class="note">A binary file, not shown.</div>"#); | ||
} else if bytes.len() > INLINE { | ||
body.push_str(r#"<div class="note">Too large to show here: view it raw.</div>"#); | ||
} else { | ||
let mut gutter = String::with_capacity(lines * 24); | ||
for n in 1..=lines { | ||
writeln!(gutter, "<a id=\"L{n}\" href=\"#L{n}\">{n}</a>").unwrap(); | ||
} | ||
let ranges = highlight::code(path, &text) | ||
.map(|r| r.encode()) | ||
.unwrap_or_default(); | ||
write!( | ||
body, | ||
r#"<div class="code"><pre class="ln" aria-hidden="true">{gutter}</pre><pre class="src" data-hl="{ranges}"><code>{}</code></pre></div>"#, | ||
esc(&text) | ||
) | ||
.unwrap(); | ||
} | ||
body.push_str("</div>"); | ||
Ok(page( | ||
StatusCode::OK, | ||
cache_for(&rev), | ||
&format!("{path} · {}", repo.name), | ||
&body, | ||
)) | ||
} | ||
| ||
/// The file itself. Never run as a page: sandboxed, and typed only as text | ||
/// or an image. | ||
async fn raw( | ||
State(app): State<Arc<App>>, | ||
Path((authority, name, rest)): RestPath, | ||
) -> Result<Response> { | ||
let repo = Repo::open(&app, &authority, &name).await?; | ||
let (rev, path) = repo.locate(&rest)?; | ||
let bytes = repo.objects()?.blob(&rev, path).map_err(not_found)?; | ||
let kind = match is_image(path) { | ||
Some(image) => image, | ||
None if is_binary(&bytes) => "application/octet-stream", | ||
None => "text/plain; charset=utf-8", | ||
}; | ||
Ok(( | ||
[ | ||
(header::CONTENT_TYPE, kind), | ||
(header::CACHE_CONTROL, cache_for(&rev)), | ||
( | ||
header::CONTENT_SECURITY_POLICY, | ||
"sandbox; default-src 'none'; style-src 'unsafe-inline'", | ||
), | ||
(header::X_CONTENT_TYPE_OPTIONS, "nosniff"), | ||
], | ||
bytes, | ||
) | ||
.into_response()) | ||
} | ||
| ||
#[derive(Deserialize, Default)] | ||
pub struct PageQuery { | ||
page: Option<usize>, | ||
} | ||
| ||
/// History, a day at a time. | ||
async fn commits( | ||
State(app): State<Arc<App>>, | ||
Path((authority, name, rest)): RestPath, | ||
Query(q): Query<PageQuery>, | ||
) -> Result<Response> { | ||
let repo = Repo::open(&app, &authority, &name).await?; | ||
let (rev, path) = repo.locate(&rest)?; | ||
let n = q.page.unwrap_or(0); | ||
let log = repo | ||
.objects()? | ||
.log(&rev, path, n * COMMITS_PER_PAGE, COMMITS_PER_PAGE) | ||
.map_err(not_found)?; | ||
let mut body = repo.header(&app, Tab::Commits).await; | ||
body.push_str(&repo.toolbar(&rev, path)); | ||
let mut days: Vec<(String, Vec<&read::Commit>)> = Vec::new(); | ||
for c in &log { | ||
let on = date(&c.author.date); | ||
match days.last_mut() { | ||
Some((day, commits)) if *day == on => commits.push(c), | ||
_ => days.push((on, vec![c])), | ||
} | ||
} | ||
for (day, commits) in &days { | ||
write!( | ||
body, | ||
r#"<h3 class="day">{}Commits on {day}</h3><div class="box"><ul class="list">"#, | ||
svg(icon::COMMIT) | ||
) | ||
.unwrap(); | ||
for c in commits { | ||
let href = repo.href(&format!("/commit/{}", c.sha)); | ||
write!( | ||
body, | ||
r#"<li><div class="grow"><a class="title clip" href="{href}">{}</a><div class="meta">{}<b>{}</b> committed</div></div><a class="sha" href="{href}">{}</a></li>"#, | ||
esc(subject(&c.message)), | ||
avatar(&c.author.name, ""), | ||
esc(&c.author.name), | ||
&c.sha[..7] | ||
) | ||
.unwrap(); | ||
} | ||
body.push_str("</ul></div>"); | ||
} | ||
if log.is_empty() { | ||
body.push_str(r#"<div class="box"><div class="empty">No commits here.</div></div>"#); | ||
} | ||
if log.len() == COMMITS_PER_PAGE { | ||
write!( | ||
body, | ||
r#"<div class="more"><a class="btn" href="?page={}">Older commits</a></div>"#, | ||
n + 1 | ||
) | ||
.unwrap(); | ||
} | ||
Ok(page( | ||
StatusCode::OK, | ||
cache_for(&rev), | ||
&format!("Commits · {}", repo.name), | ||
&body, | ||
)) | ||
} | ||
| ||
/// Five squares, filled in proportion to what a file gained and lost. | ||
fn bars(added: usize, removed: usize) -> String { | ||
let total = added + removed; | ||
let share = |n: usize| (n * 5 + total / 2).checked_div(total).unwrap_or(0); | ||
let (a, d) = (share(added), share(removed).min(5 - share(added))); | ||
let mut out = String::from(r#"<span class="bars" aria-hidden="true">"#); | ||
for i in 0..5 { | ||
out.push_str(match i { | ||
i if i < a => r#"<i class="a"></i>"#, | ||
i if i < a + d => r#"<i class="d"></i>"#, | ||
_ => "<i></i>", | ||
}); | ||
} | ||
out.push_str("</span>"); | ||
out | ||
} | ||
| ||
/// One file of a diff: its header, then a row per line with the line's old | ||
/// and new numbers. | ||
fn file_diff(f: &diff::File) -> String { | ||
let path = match f.status { | ||
Status::Renamed => format!("{} → {}", esc(&f.old), esc(&f.new)), | ||
Status::Deleted => esc(&f.old), | ||
_ => esc(&f.new), | ||
}; | ||
let tag = match f.status { | ||
Status::Added => r#"<span class="tag">new</span>"#, | ||
Status::Deleted => r#"<span class="tag">deleted</span>"#, | ||
Status::Renamed => r#"<span class="tag">renamed</span>"#, | ||
Status::Modified => "", | ||
}; | ||
let mut out = format!( | ||
r#"<details class="file" open><summary><span class="path">{path}</span>{tag}<span class="stat"><span class="add">+{}</span><span class="rem">−{}</span>{}</span></summary>"#, | ||
f.added, | ||
f.removed, | ||
bars(f.added, f.removed) | ||
); | ||
let note = if f.binary { | ||
Some("A binary file, not shown.") | ||
} else if f.rows.is_empty() { | ||
Some(if f.status == Status::Renamed { | ||
"Moved without changes." | ||
} else { | ||
"No changes to the content." | ||
}) | ||
} else if f.rows.len() > DIFF_ROWS { | ||
Some("A large diff, not shown.") | ||
} else { | ||
None | ||
}; | ||
if let Some(note) = note { | ||
write!(out, r#"<div class="note">{note}</div></details>"#).unwrap(); | ||
return out; | ||
} | ||
let path = if f.status == Status::Deleted { | ||
&f.old | ||
} else { | ||
&f.new | ||
}; | ||
write!( | ||
out, | ||
r#"<div class="scroll"><table class="diff" data-hl="{}">"#, | ||
highlight::rows(path, &f.rows).encode() | ||
) | ||
.unwrap(); | ||
let n = |x: Option<u32>| x.map(|x| x.to_string()).unwrap_or_default(); | ||
for row in &f.rows { | ||
let class = match row.kind { | ||
Kind::Hunk => "h", | ||
Kind::Note => "x", | ||
Kind::Add => "a", | ||
Kind::Del => "d", | ||
Kind::Context => "", | ||
}; | ||
if row.kind.is_line() { | ||
write!( | ||
out, | ||
r#"<tr class="{class}"><td class="n" data-n="{}"></td><td class="n" data-n="{}"></td><td class="c"><code>{}</code></td></tr>"#, | ||
n(row.old), | ||
n(row.new), | ||
esc(&row.text) | ||
) | ||
.unwrap(); | ||
} else { | ||
write!( | ||
out, | ||
r#"<tr class="{class}"><td class="n" colspan="2"></td><td class="c">{}</td></tr>"#, | ||
esc(&row.text) | ||
) | ||
.unwrap(); | ||
} | ||
} | ||
out.push_str("</table></div></details>"); | ||
out | ||
} | ||
| ||
async fn commit( | ||
State(app): State<Arc<App>>, | ||
Path((authority, name, rev)): RestPath, | ||
) -> Result<Response> { | ||
let repo = Repo::open(&app, &authority, &name).await?; | ||
let rev = read::rev(Some(&rev)).map_err(not_found)?; | ||
let c = repo | ||
.objects()? | ||
.log(&rev, "", 0, 1) | ||
.map_err(not_found)? | ||
.remove(0); | ||
let patch = repo | ||
.git | ||
.text( | ||
&[ | ||
"-c", | ||
"core.quotePath=false", | ||
"show", | ||
"--format=", | ||
"--patch", | ||
"-M", | ||
"--no-color", | ||
"--no-ext-diff", | ||
&c.sha, | ||
], | ||
None, | ||
) | ||
.await | ||
.map_err(not_found)?; | ||
let files = diff::parse(&patch); | ||
let mut body = repo.header(&app, Tab::Commits).await; | ||
let (title, rest) = c.message.split_once('\n').unwrap_or((&c.message, "")); | ||
write!( | ||
body, | ||
r#"<div class="box"><div class="commit-msg"><h2>{}</h2>"#, | ||
esc(title) | ||
) | ||
.unwrap(); | ||
if !rest.trim().is_empty() { | ||
write!(body, "<pre>{}</pre>", esc(rest.trim())).unwrap(); | ||
} | ||
write!( | ||
body, | ||
r#"</div><div class="boxhead" style="border-top:1px solid var(--line);border-bottom:0">{}<b>{}</b><span class="mute">committed on {}</span><span class="end mute">"#, | ||
avatar(&c.author.name, ""), | ||
esc(&c.author.name), | ||
date(&c.author.date) | ||
) | ||
.unwrap(); | ||
for p in &c.parents { | ||
write!( | ||
body, | ||
r#"parent <a class="sha" href="{}">{}</a> "#, | ||
repo.href(&format!("/commit/{p}")), | ||
&p[..7] | ||
) | ||
.unwrap(); | ||
} | ||
write!( | ||
body, | ||
r#"commit <span class="sha">{}</span></span></div></div>"#, | ||
&c.sha[..7] | ||
) | ||
.unwrap(); | ||
let (added, removed) = files | ||
.iter() | ||
.fold((0, 0), |(a, r), f| (a + f.added, r + f.removed)); | ||
let plural = | ||
|n: usize, one: &str, many: &str| format!("{n} {}", if n == 1 { one } else { many }); | ||
write!( | ||
body, | ||
r#"<div class="summary">Showing <b>{}</b> with <b class="add">{}</b> and <b class="rem">{}</b></div>"#, | ||
plural(files.len(), "changed file", "changed files"), | ||
plural(added, "addition", "additions"), | ||
plural(removed, "deletion", "deletions") | ||
) | ||
.unwrap(); | ||
for f in &files { | ||
body.push_str(&file_diff(f)); | ||
} | ||
Ok(page( | ||
StatusCode::OK, | ||
cache_for(&rev), | ||
&format!("{} · {}", subject(&c.message), repo.name), | ||
&body, | ||
)) | ||
} | ||
| ||
#[derive(Deserialize, Default)] | ||
pub struct StateQuery { | ||
state: Option<String>, | ||
} | ||
| ||
async fn pulls( | ||
State(app): State<Arc<App>>, | ||
Path((authority, name)): RepoPath, | ||
Query(q): Query<StateQuery>, | ||
) -> Result<Response> { | ||
items(app, authority, name, "pull", q).await | ||
} | ||
| ||
async fn issues( | ||
State(app): State<Arc<App>>, | ||
Path((authority, name)): RepoPath, | ||
Query(q): Query<StateQuery>, | ||
) -> Result<Response> { | ||
items(app, authority, name, "issue", q).await | ||
} | ||
| ||
/// Everything the index knows about the repository's pulls and issues. | ||
fn assembled(app: &App, repo: &Repo) -> Result<Vec<forge::Entry>> { | ||
let index = app.index.as_ref().ok_or(Shown( | ||
StatusCode::NOT_FOUND, | ||
"This server keeps no index, so it knows no pulls or issues.".into(), | ||
))?; | ||
let entries = index | ||
.forge(&repo.did, &repo.key) | ||
.map_err(anyhow::Error::from) | ||
.map_err(AppError::internal)?; | ||
Ok(entries) | ||
} | ||
| ||
/// The icon for an item in a state. | ||
fn state_icon(kind: &str, state: &str) -> &'static str { | ||
match (kind, state) { | ||
("pull", "merged") => icon::MERGE, | ||
("pull", _) => icon::PULL, | ||
(_, "open") => icon::ISSUE, | ||
_ => icon::DONE, | ||
} | ||
} | ||
| ||
fn capitalized(s: &str) -> String { | ||
let mut c = s.chars(); | ||
c.next() | ||
.map(|f| f.to_uppercase().chain(c).collect()) | ||
.unwrap_or_default() | ||
} | ||
| ||
async fn items( | ||
app: Arc<App>, | ||
authority: String, | ||
name: String, | ||
kind: &str, | ||
q: StateQuery, | ||
) -> Result<Response> { | ||
let repo = Repo::open(&app, &authority, &name).await?; | ||
let entries = assembled(&app, &repo)?; | ||
let record = repo.record(&app).unwrap_or_default(); | ||
let all: Vec<Item> = forge::assemble(&record, &repo.did, &entries) | ||
.into_iter() | ||
.filter(|i| i.kind == kind) | ||
.collect(); | ||
let showing_open = q.state.as_deref() != Some("closed"); | ||
let (open, closed): (Vec<&Item>, Vec<&Item>) = all.iter().partition(|i| i.state == "open"); | ||
let (tab, label) = if kind == "pull" { | ||
(Tab::Pulls, "Pulls") | ||
} else { | ||
(Tab::Issues, "Issues") | ||
}; | ||
let mut body = repo.header(&app, tab).await; | ||
let current = |yes: bool| if yes { r#" aria-current="page""# } else { "" }; | ||
write!( | ||
body, | ||
r#"<div class="box"><div class="boxhead"><nav class="filter"><a href="?state=open"{}>{}{} Open</a><a href="?state=closed"{}>{}{} Closed</a></nav></div><ul class="list">"#, | ||
current(showing_open), | ||
svg(state_icon(kind, "open")), | ||
open.len(), | ||
current(!showing_open), | ||
svg(icon::DONE), | ||
closed.len() | ||
) | ||
.unwrap(); | ||
let shown = if showing_open { &open } else { &closed }; | ||
let authors = futures::future::join_all(shown.iter().map(|i| handle(&app, &i.author))).await; | ||
for (i, author) in shown.iter().zip(authors) { | ||
let (did, _, rkey) = forge::parse_uri(&i.uri).map_err(not_found)?; | ||
let comments = if i.comments > 0 { | ||
format!( | ||
r#"<span class="mute">{}{}</span>"#, | ||
svg(icon::CHAT), | ||
i.comments | ||
) | ||
} else { | ||
String::new() | ||
}; | ||
write!( | ||
body, | ||
r#"<li><span class="st {}" title="{}">{}</span><div class="grow"><a class="title" href="{}">{}</a><div class="meta">opened on {} by {}</div></div>{comments}</li>"#, | ||
i.state, | ||
i.state, | ||
svg(state_icon(kind, &i.state)), | ||
repo.href(&format!("/{kind}/{did}/{rkey}")), | ||
esc(&i.title), | ||
date(&i.created_at), | ||
person(&author) | ||
) | ||
.unwrap(); | ||
} | ||
if shown.is_empty() { | ||
let which = if showing_open { "open" } else { "closed" }; | ||
write!( | ||
body, | ||
r#"<li class="empty">No {which} {}.</li>"#, | ||
label.to_lowercase() | ||
) | ||
.unwrap(); | ||
} | ||
body.push_str("</ul></div>"); | ||
Ok(page( | ||
StatusCode::OK, | ||
"no-cache", | ||
&format!("{label} · {}", repo.name), | ||
&body, | ||
)) | ||
} | ||
| ||
async fn item( | ||
State(app): State<Arc<App>>, | ||
Path((authority, name, kind, did, rkey)): Path<(String, String, String, String, String)>, | ||
) -> Result<Response> { | ||
let repo = Repo::open(&app, &authority, &name).await?; | ||
let collection = match kind.as_str() { | ||
"pull" => knotless::records::nsid::PULL, | ||
"issue" => knotless::records::nsid::ISSUE, | ||
_ => { | ||
return Err(Shown( | ||
StatusCode::NOT_FOUND, | ||
format!("no such page: {kind}"), | ||
)); | ||
} | ||
}; | ||
let entries = assembled(&app, &repo)?; | ||
let record = repo.record(&app).unwrap_or_default(); | ||
let uri = format!("at://{did}/{collection}/{rkey}"); | ||
let Thread { item, comments } = | ||
forge::thread(&record, &repo.did, &entries, &uri).map_err(not_found)?; | ||
let tab = if kind == "pull" { | ||
Tab::Pulls | ||
} else { | ||
Tab::Issues | ||
}; | ||
let mut body = repo.header(&app, tab).await; | ||
let author = handle(&app, &item.author).await; | ||
write!( | ||
body, | ||
r#"<h2 class="item-title">{}</h2><div class="item-meta"><span class="pill {}">{}{}</span>"#, | ||
esc(&item.title), | ||
item.state, | ||
svg(state_icon(&kind, &item.state)), | ||
capitalized(&item.state) | ||
) | ||
.unwrap(); | ||
if let (Some(base), Some(head)) = (&item.base, &item.head) { | ||
let from = repo_link(&app, &head.repo) | ||
.await | ||
.map(|(shown, _)| shown) | ||
.unwrap_or_default(); | ||
write!( | ||
body, | ||
r#"<span>{} wants to merge into <span class="ref">{}</span> from <span class="ref">{}:{}</span></span>"#, | ||
person(&author), | ||
esc(base), | ||
esc(&from), | ||
esc(&head.branch) | ||
) | ||
.unwrap(); | ||
} else { | ||
write!( | ||
body, | ||
r#"<span>{} opened this issue on {}</span>"#, | ||
person(&author), | ||
date(&item.created_at) | ||
) | ||
.unwrap(); | ||
} | ||
let said = comments.len() + item.body.is_some() as usize; | ||
let plural = if comments.len() == 1 { "" } else { "s" }; | ||
write!( | ||
body, | ||
r#"<span class="end">{}{} comment{plural}</span></div><div class="timeline">"#, | ||
svg(icon::CHAT), | ||
comments.len() | ||
) | ||
.unwrap(); | ||
if let Some(text) = &item.body { | ||
write!( | ||
body, | ||
r#"<div class="box comment"><div class="boxhead">{}{}<span class="mute">opened on {}</span></div><div class="prose">{}</div></div>"#, | ||
avatar(&author, ""), | ||
person(&author), | ||
date(&item.created_at), | ||
markdown(text, &repo.href(""), &repo.href("")) | ||
) | ||
.unwrap(); | ||
} | ||
let names = futures::future::join_all(comments.iter().map(|c| handle(&app, &c.author))).await; | ||
for (c, who) in comments.iter().zip(names) { | ||
let verdict = match c.value["verdict"].as_str() { | ||
Some("approve") => format!( | ||
r#"<span class="verdict approve">{}Approved</span>"#, | ||
svg(icon::DONE) | ||
), | ||
Some("changes") => format!( | ||
r#"<span class="verdict changes">{}Asked for changes</span>"#, | ||
svg(icon::ISSUE) | ||
), | ||
_ => String::new(), | ||
}; | ||
write!( | ||
body, | ||
r#"<div class="box comment"><div class="boxhead">{}{}<span class="mute">commented on {}</span><span class="end">{verdict}</span></div><div class="prose">{}</div></div>"#, | ||
avatar(&who, ""), | ||
person(&who), | ||
date(c.value["createdAt"].as_str().unwrap_or_default()), | ||
markdown(c.value["body"].as_str().unwrap_or_default(), &repo.href(""), &repo.href("")) | ||
) | ||
.unwrap(); | ||
} | ||
if let Some(sha) = &item.merged { | ||
write!( | ||
body, | ||
r#"<div class="event"><span class="dot">{}</span><span>Merged as <a class="sha" href="{}">{}</a></span></div>"#, | ||
svg(icon::MERGE), | ||
repo.href(&format!("/commit/{sha}")), | ||
&sha[..7] | ||
) | ||
.unwrap(); | ||
} | ||
if said == 0 { | ||
body.push_str(r#"<div class="box"><div class="empty">Nothing said yet.</div></div>"#); | ||
} | ||
body.push_str("</div>"); | ||
Ok(page( | ||
StatusCode::OK, | ||
"no-cache", | ||
&format!("{} · {}", item.title, repo.name), | ||
&body, | ||
)) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn readme_links() { | ||
let base = "/a/r/blob/main"; | ||
assert_eq!(link("https://x.dev", base), "https://x.dev"); | ||
assert_eq!(link("MAILTO:a@b.c", base), "MAILTO:a@b.c"); | ||
assert_eq!(link("javascript:alert(1)", base), "#"); | ||
assert_eq!(link("#usage", base), "#usage"); | ||
assert_eq!(link("/elsewhere", base), "/elsewhere"); | ||
assert_eq!(link("./docs/a.md", base), "/a/r/blob/main/docs/a.md"); | ||
assert_eq!( | ||
link("docs/a.md?x=y:z", base), | ||
"/a/r/blob/main/docs/a.md?x=y:z" | ||
); | ||
let html = markdown( | ||
"<script>x</script>\n\n[a](b.md)  <b>t</b>", | ||
"/B", | ||
"/R", | ||
); | ||
assert!( | ||
html.contains("<script>") && html.contains("<b>"), | ||
"{html}" | ||
); | ||
assert!( | ||
html.contains(r#"href="/B/b.md""#) && html.contains(r#"src="/R/i.png""#), | ||
"{html}" | ||
); | ||
} | ||
| ||
#[test] | ||
fn small_things() { | ||
assert_eq!( | ||
esc(r#"<a href="x">'&'</a>"#), | ||
"<a href="x">'&'</a>" | ||
); | ||
assert_eq!(url_path("a b/c#d?é"), "a%20b/c%23d%3F%C3%A9"); | ||
assert_eq!( | ||
(size(10), size(2048), size(3 << 20)), | ||
("10 B".into(), "2.0 KB".into(), "3.0 MB".into()) | ||
); | ||
assert_eq!(date("2026-10-20T10:00:00Z"), "Oct 20, 2026"); | ||
assert_eq!(date("2026-01-05"), "Jan 5, 2026"); | ||
assert_eq!(date("soon"), "soon"); | ||
assert_eq!(date("2026-13-01"), "2026-13-01"); | ||
assert_eq!(date("2026-00-01"), "2026-00-01"); | ||
assert_eq!(is_image("a/B.PNG"), Some("image/png")); | ||
assert_eq!(is_image("Makefile"), None); | ||
assert!(is_binary(&[0, 1]) && is_binary(&[0xff, 0xfe]) && !is_binary(b"text")); | ||
assert!(POLICY.contains("script-src 'sha256-")); | ||
assert!(avatar("bob.test", "").contains(">b<") && avatar("", "").contains(">?<")); | ||
assert_eq!(capitalized("open"), "Open"); | ||
assert_eq!(capitalized(""), ""); | ||
} | ||
| ||
#[test] | ||
fn bars_split_five_squares() { | ||
let shown = |a, d| { | ||
bars(a, d).matches("class=\"a\"").count() * 10 | ||
+ bars(a, d).matches("class=\"d\"").count() | ||
}; | ||
assert_eq!(shown(0, 0), 0); | ||
assert_eq!(shown(3, 0), 50); | ||
assert_eq!(shown(0, 3), 5); | ||
assert_eq!(shown(1, 1), 32); | ||
assert_eq!(shown(100, 1), 50); | ||
assert_eq!(shown(9, 2), 41); | ||
} | ||
| ||
#[test] | ||
fn files_of_a_diff() { | ||
let file = |status, binary, rows: usize| diff::File { | ||
old: "a.rs".into(), | ||
new: "b.rs".into(), | ||
status, | ||
binary, | ||
rows: (0..rows) | ||
.map(|i| diff::Row { | ||
kind: Kind::Add, | ||
old: None, | ||
new: Some(i as u32), | ||
text: "x".into(), | ||
}) | ||
.collect(), | ||
added: rows, | ||
removed: 0, | ||
}; | ||
assert!( | ||
file_diff(&file(Status::Renamed, false, 0)) | ||
.contains("a.rs → b.rs</span><span class=\"tag\">renamed</span>") | ||
); | ||
assert!(file_diff(&file(Status::Renamed, false, 0)).contains("Moved without changes.")); | ||
assert!( | ||
file_diff(&file(Status::Modified, false, 0)).contains("No changes to the content.") | ||
); | ||
assert!(file_diff(&file(Status::Added, true, 0)).contains("A binary file")); | ||
assert!(file_diff(&file(Status::Deleted, false, DIFF_ROWS + 1)).contains("A large diff")); | ||
let deleted = file_diff(&file(Status::Deleted, false, 1)); | ||
assert!( | ||
deleted.contains(r#"<span class="path">a.rs</span><span class="tag">deleted</span>"#), | ||
"{deleted}" | ||
); | ||
let mut mixed = file(Status::Modified, false, 0); | ||
let row = |kind, old, new, text: &str| diff::Row { | ||
kind, | ||
old, | ||
new, | ||
text: text.into(), | ||
}; | ||
mixed.rows = vec![ | ||
row(Kind::Hunk, None, None, "@@ -1,2 +1,2 @@"), | ||
row(Kind::Context, Some(1), Some(1), "a"), | ||
row(Kind::Del, Some(2), None, "b"), | ||
row(Kind::Note, None, None, "\\ No newline at end of file"), | ||
row(Kind::Add, None, Some(2), "c"), | ||
]; | ||
let shown = file_diff(&mixed); | ||
for want in [ | ||
r#"<tr class="h"><td class="n" colspan="2"></td><td class="c">@@ -1,2 +1,2 @@</td>"#, | ||
r#"<tr class=""><td class="n" data-n="1"></td><td class="n" data-n="1"></td>"#, | ||
r#"<tr class="d"><td class="n" data-n="2"></td><td class="n" data-n=""></td><td class="c"><code>b</code>"#, | ||
r#"<tr class="x">"#, | ||
] { | ||
assert!(shown.contains(want), "{want} not in {shown}"); | ||
} | ||
assert!(file_diff(&file(Status::Added, false, 1)).contains( | ||
r#"<tr class="a"><td class="n" data-n=""></td><td class="n" data-n="0"></td>"# | ||
)); | ||
} | ||
} | ||
crates/knotless-serve/tests/forge.rsnew+897−0
| @@ -0,0 +1,897 @@ | ||
//! Forks, pulls, issues and reviews, with an appview that follows the | ||
//! reference PDS's firehose. | ||
| ||
#[path = "../../knotless/tests/common/mod.rs"] | ||
mod common; | ||
use common::*; | ||
| ||
use futures::{SinkExt, StreamExt}; | ||
use knotless_serve::gateway::Gateway; | ||
use knotless_serve::index::Index; | ||
use knotless_serve::{App, router}; | ||
use serde_json::Value; | ||
use std::sync::Arc; | ||
use std::time::Duration; | ||
| ||
struct Appview { | ||
base: String, | ||
did: String, | ||
app: Arc<App>, | ||
rt: tokio::runtime::Runtime, | ||
_sandbox: Sandbox, | ||
} | ||
| ||
/// An appview that is a gateway too, following the PDS. | ||
fn appview() -> Appview { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
s.set("KNOTLESS_DID_WEB_SCHEME", "http"); | ||
s.set("KNOTLESS_ADMIN_PASSWORD", "hunter2"); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let base = format!("http://localhost:{}", listener.local_addr().unwrap().port()); | ||
let gw = Gateway::new(&base).unwrap(); | ||
let did = gw.did.clone(); | ||
let index = Index::open(&s.path("index.db")).unwrap(); | ||
let app = App::build(ctx(&s), Duration::ZERO, Some(gw), Some(index)); | ||
let router = router(app.clone()); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
let ws = net().pds.replace("http://", "ws://"); | ||
rt.spawn(knotless_serve::firehose::watch(app.clone(), ws)); | ||
Appview { | ||
base, | ||
did, | ||
app, | ||
rt, | ||
_sandbox: s, | ||
} | ||
} | ||
| ||
fn person(account: Account, appview: &Appview) -> Sandbox { | ||
let mut s = Sandbox::with(account); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
s.set("KNOTLESS_DID_WEB_SCHEME", "http"); | ||
s.set("KNOTLESS_APPVIEW", &appview.base); | ||
let password = s.account.password.clone(); | ||
let handle = s.account.handle.clone(); | ||
assert_eq!( | ||
s.cli(&["login", &handle, "--password-stdin"], password.as_bytes()) | ||
.0, | ||
0 | ||
); | ||
s | ||
} | ||
| ||
fn ok(s: &Sandbox, args: &[&str]) -> String { | ||
let (code, out) = s.cli(args, b""); | ||
assert_eq!(code, 0, "{args:?}: {out}"); | ||
out | ||
} | ||
| ||
fn fails(s: &Sandbox, args: &[&str]) { | ||
let (code, out) = s.cli(args, b""); | ||
assert_eq!(code, 1, "{args:?} should fail: {out}"); | ||
} | ||
| ||
/// Waits until `list` shows `line`. | ||
fn listed(s: &Sandbox, url: &str, line: &str) { | ||
wait_until(|| s.cli(&["list", url], b"").1.contains(line)); | ||
} | ||
| ||
#[test] | ||
fn forks_pulls_reviews_and_merges() { | ||
let av = appview(); | ||
let owner = person(account(), &av); | ||
let contrib = person(account(), &av); | ||
let url = owner.url("project"); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
| ||
// A fork in the contributor's account, with a branch to offer. | ||
let fork = ok(&contrib, &["fork", &url]); | ||
assert_eq!(fork, format!("knotless://{}/project", contrib.account.did)); | ||
let clone = contrib.path("clone"); | ||
contrib.git(contrib.dir.path(), &["clone", "-q", &fork, "clone"]); | ||
contrib.git(&clone, &["checkout", "-q", "-b", "feature"]); | ||
let feature = contrib.commit_file(&clone, "b", "from the fork\n"); | ||
contrib.git(&clone, &["push", "-q", "origin", "feature"]); | ||
| ||
fails( | ||
&contrib, | ||
&["pull", &url, "main", &fork, "no-such-branch", "nope"], | ||
); | ||
let pull = ok( | ||
&contrib, | ||
&[ | ||
"pull", &url, "main", &fork, "feature", "Add b", "-m", "please", | ||
], | ||
); | ||
assert!(pull.starts_with(&format!("at://{}/dev.knotless.pull/", contrib.account.did))); | ||
listed(&owner, &url, &format!("pull\topen\t{pull}\tAdd b")); | ||
| ||
ok( | ||
&contrib, | ||
&["comment", &url, &pull, "looks right to me", "--approve"], | ||
); | ||
ok(&owner, &["comment", &url, &pull, "thanks"]); | ||
fails(&owner, &["comment", &url, &pull, "?", "--bogus"]); | ||
wait_until(|| owner.cli(&["show", &url, &pull], b"").1.contains("thanks")); | ||
let shown = ok(&owner, &["show", &url, &pull]); | ||
assert!(shown.starts_with("open pull by"), "{shown}"); | ||
assert!( | ||
shown.contains("please") && shown.contains("[approve]: looks right"), | ||
"{shown}" | ||
); | ||
| ||
// The base has not moved: a fast-forward. | ||
let merged = ok(&owner, &["merge", &url, &pull]); | ||
assert_eq!(merged, feature); | ||
let (_, refs) = owner.cli(&["refs", &url], b""); | ||
assert!( | ||
refs.contains(&format!("{feature}\trefs/heads/main")), | ||
"{refs}" | ||
); | ||
listed(&owner, &url, &format!("pull\tmerged\t{pull}")); | ||
fails(&owner, &["merge", &url, &pull]); | ||
| ||
// The base moved: a merge commit with both parents. | ||
owner.git(&work, &["pull", "-q", &url, "main"]); | ||
owner.commit_file(&work, "c", "owner's\n"); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
contrib.git(&clone, &["checkout", "-q", "-b", "two", &feature]); | ||
let two = contrib.commit_file(&clone, "d", "more\n"); | ||
contrib.git(&clone, &["push", "-q", "origin", "two"]); | ||
let second = ok(&contrib, &["pull", &url, "main", &fork, "two", "Add d"]); | ||
listed(&owner, &url, &second); | ||
let merge = ok(&owner, &["merge", &url, &second]); | ||
let check = owner.path("check"); | ||
owner.git(owner.dir.path(), &["clone", "-q", &url, "check"]); | ||
assert_eq!(owner.git(&check, &["rev-parse", "HEAD"]), merge); | ||
assert_eq!(owner.git(&check, &["rev-parse", "HEAD^2"]), two); | ||
assert!(check.join("c").exists() && check.join("d").exists()); | ||
| ||
// Both sides changed one file: a conflict, and nothing moves. | ||
owner.git(&work, &["pull", "-q", &url, "main"]); | ||
owner.commit_file(&work, "f", "owner's f\n"); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
contrib.git(&clone, &["checkout", "-q", "-b", "clash", &two]); | ||
contrib.commit_file(&clone, "f", "contributor's f\n"); | ||
contrib.git(&clone, &["push", "-q", "origin", "clash"]); | ||
let clash = ok( | ||
&contrib, | ||
&["pull", &url, "main", &fork, "clash", "Change f"], | ||
); | ||
listed(&owner, &url, &clash); | ||
let (merged, _, err) = owner.knotless(&["merge", &url, &clash], b""); | ||
assert!(!merged && err.contains("conflicts in f"), "{err}"); | ||
| ||
// Branches can go between opening a pull and merging it. | ||
contrib.git(&clone, &["push", "-q", "origin", ":clash"]); | ||
let (merged, _, err) = owner.knotless(&["merge", &url, &clash], b""); | ||
assert!(!merged && err.ends_with("has no branch clash"), "{err}"); | ||
fails( | ||
&contrib, | ||
&["pull", &url, "release", &fork, "two", "Into release"], | ||
); | ||
owner.git(&work, &["push", "-q", &url, "main:release"]); | ||
let release = ok( | ||
&contrib, | ||
&["pull", &url, "release", &fork, "two", "Into release"], | ||
); | ||
owner.git(&work, &["push", "-q", &url, ":release"]); | ||
listed(&owner, &url, &release); | ||
let (merged, _, err) = owner.knotless(&["merge", &url, &release], b""); | ||
assert!(!merged && err.ends_with("has no branch release"), "{err}"); | ||
| ||
// A pull outlives its branches by pointing at others: the target by | ||
// anyone who may close it, the head by its author alone. | ||
owner.git(&work, &["push", "-q", &url, "main:stable"]); | ||
for bad in [ | ||
vec!["retarget", &url, &release], | ||
vec!["retarget", &url, &release, "--sideways", "x"], | ||
vec!["retarget", &url, &release, "--base", "nope"], | ||
vec!["retarget", &url, &release, "--head", "two"], | ||
vec!["retarget", &url, &pull, "--base", "stable"], | ||
] { | ||
fails(&owner, &bad); | ||
} | ||
ok(&owner, &["retarget", &url, &release, "--base", "stable"]); | ||
fails(&contrib, &["retarget", &url, &clash, "--head", "nope"]); | ||
ok(&contrib, &["retarget", &url, &clash, "--head", "two"]); | ||
let issue = ok(&contrib, &["issue", &url, "Not a pull"]); | ||
fails(&owner, &["retarget", &url, &issue, "--base", "main"]); | ||
// Once the index has the new target, the pull merges into it. | ||
wait_until(|| owner.knotless(&["merge", &url, &release], b"").0); | ||
let check = owner.path("stable"); | ||
owner.git( | ||
owner.dir.path(), | ||
&["clone", "-q", "-b", "stable", &url, "stable"], | ||
); | ||
owner.git(&check, &["merge-base", "--is-ancestor", &two, "stable"]); | ||
| ||
// A collaborator merges, through the gateway. | ||
ok(&owner, &["delegate", &owner.account.handle, &av.did]); | ||
ok(&owner, &["share", &url, &contrib.account.handle, "--write"]); | ||
contrib.git(&clone, &["fetch", "-q", &url, "main"]); | ||
contrib.git(&clone, &["checkout", "-q", "-b", "three", &merge]); | ||
contrib.commit_file(&clone, "e", "collaborating\n"); | ||
contrib.git(&clone, &["push", "-q", "origin", "three"]); | ||
let third = ok(&contrib, &["pull", &url, "main", &fork, "three", "Add e"]); | ||
listed(&contrib, &url, &third); | ||
let sha = ok(&contrib, &["merge", &url, &third]); | ||
let (_, refs) = owner.cli(&["refs", &url], b""); | ||
assert!(refs.contains(&format!("{sha}\trefs/heads/main")), "{refs}"); | ||
| ||
// Someone else's commit, elsewhere on the firehose, changes nothing. | ||
av.app.observe(knotless_serve::firehose::Event::default()); | ||
assert_eq!(av.app.events_for(&account().did), 0); | ||
| ||
// A record gone before the index fetched it is an error to skip. | ||
let gone = knotless_serve::firehose::Op { | ||
action: "create".into(), | ||
collection: "dev.knotless.issue".into(), | ||
rkey: "3zzzzzzzzzzzz".into(), | ||
record: None, | ||
}; | ||
let index = av.app.index.as_ref().unwrap(); | ||
assert!( | ||
av.rt | ||
.block_on(knotless_serve::index_op( | ||
&av.app.ctx, | ||
index, | ||
&owner.account.did, | ||
&gone | ||
)) | ||
.is_err() | ||
); | ||
| ||
// The index knows the fork, and finds repositories by name. | ||
let found: Value = | ||
reqwest::blocking::get(format!("{}/xrpc/dev.knotless.listRepos?q=proj", av.base)) | ||
.unwrap() | ||
.json() | ||
.unwrap(); | ||
assert_eq!(found["repos"].as_array().unwrap().len(), 2); | ||
assert!(found.get("cursor").is_none()); | ||
// A page at a time, the same two. | ||
let list = |cursor: Option<&str>| -> Value { | ||
let mut query = vec![("q", "proj"), ("limit", "1")]; | ||
query.extend(cursor.map(|c| ("cursor", c))); | ||
reqwest::blocking::Client::new() | ||
.get(format!("{}/xrpc/dev.knotless.listRepos", av.base)) | ||
.query(&query) | ||
.send() | ||
.unwrap() | ||
.json() | ||
.unwrap() | ||
}; | ||
let first = list(None); | ||
let second = list(first["cursor"].as_str()); | ||
assert_eq!( | ||
[&first["repos"][0], &second["repos"][0]], | ||
[&found["repos"][0], &found["repos"][1]] | ||
); | ||
assert!( | ||
list(second["cursor"].as_str())["repos"] | ||
.as_array() | ||
.unwrap() | ||
.is_empty() | ||
); | ||
let uri = format!("at://{}/dev.knotless.repo/project", owner.account.did); | ||
let indexed = av.app.index.as_ref().unwrap(); | ||
let key = indexed.repo(&uri).unwrap().unwrap()["key"] | ||
.as_str() | ||
.unwrap() | ||
.to_string(); | ||
let forks = indexed.forks(&owner.account.did, &key); | ||
assert_eq!(forks.unwrap().len(), 1); | ||
| ||
// A second appview, backfilled, sees the same. | ||
let s = Sandbox::new(); | ||
let index = Index::open(&s.path("i.db")).unwrap(); | ||
let n = av | ||
.rt | ||
.block_on(knotless_serve::backfill(&ctx(&s), &index, &net().pds)) | ||
.unwrap(); | ||
assert!(n >= 10, "{n}"); | ||
let report = av.rt.block_on(knotless_serve::backfilled( | ||
&ctx(&s), | ||
&index, | ||
"http://127.0.0.1:9", | ||
)); | ||
assert!( | ||
report.starts_with("backfilling from http://127.0.0.1:9: "), | ||
"{report}" | ||
); | ||
let report = av | ||
.rt | ||
.block_on(knotless_serve::backfilled(&ctx(&s), &index, &net().pds)); | ||
assert!(report.starts_with("backfilled "), "{report}"); | ||
let did = owner.account.did.as_str(); | ||
let backfilled = index.forge(did, &key).unwrap().len(); | ||
wait_until(|| indexed.forge(did, &key).unwrap().len() == backfilled); | ||
| ||
// A deletion takes the record out of the index. | ||
let pull_rkey = third.rsplit('/').next().unwrap(); | ||
let deleted = knotless_serve::firehose::Op { | ||
action: "delete".into(), | ||
collection: "dev.knotless.pull".into(), | ||
rkey: pull_rkey.into(), | ||
record: None, | ||
}; | ||
av.rt | ||
.block_on(knotless_serve::index_op( | ||
&ctx(&s), | ||
&index, | ||
&contrib.account.did, | ||
&deleted, | ||
)) | ||
.unwrap(); | ||
assert_eq!(index.forge(did, &key).unwrap().len(), backfilled - 1); | ||
} | ||
| ||
#[test] | ||
fn issues_and_who_may_close_them() { | ||
let av = appview(); | ||
let owner = person(account(), &av); | ||
let author = person(account(), &av); | ||
let stranger = person(account(), &av); | ||
let url = owner.url("tracked"); | ||
ok(&owner, &["create", &url]); | ||
let issue = ok( | ||
&author, | ||
&["issue", &url, "It breaks", "-m", "when I do this"], | ||
); | ||
let quiet = ok(&author, &["issue", &url, "Also this"]); | ||
listed(&owner, &url, &issue); | ||
listed(&owner, &url, &quiet); | ||
assert_eq!(ok(&author, &["close", &url, &issue]), "closed"); | ||
listed(&owner, &url, &format!("issue\tclosed\t{issue}")); | ||
// A stranger's reopening counts for nothing; the owner's does. | ||
ok(&stranger, &["reopen", &url, &issue]); | ||
std::thread::sleep(Duration::from_millis(500)); | ||
let listing = ok(&owner, &["list", &url, "--issues"]); | ||
assert!( | ||
listing.contains(&format!("issue\tclosed\t{issue}")), | ||
"{listing}" | ||
); | ||
ok(&owner, &["reopen", &url, &issue]); | ||
listed(&owner, &url, &format!("issue\topen\t{issue}")); | ||
assert_eq!(ok(&owner, &["list", &url, "--pulls"]), ""); | ||
assert!(ok(&owner, &["show", &url, &quiet]).ends_with(": Also this")); | ||
fails(&owner, &["list", &url, "--bogus"]); | ||
fails(&owner, &["merge", &url, &issue]); | ||
fails( | ||
&owner, | ||
&["show", &url, "at://did:plc:nobody/dev.knotless.issue/x"], | ||
); | ||
// A comment is about a pull or an issue on this repository. | ||
fails( | ||
&owner, | ||
&[ | ||
"comment", | ||
&url, | ||
"at://did:plc:x/dev.knotless.state/3abc", | ||
"?", | ||
], | ||
); | ||
fails( | ||
&owner, | ||
&[ | ||
"comment", | ||
&url, | ||
&format!("{}x", issue.trim_end_matches(|c: char| c.is_alphanumeric())), | ||
"?", | ||
], | ||
); | ||
ok(&owner, &["create", &owner.url("elsewhere")]); | ||
fails(&owner, &["comment", &owner.url("elsewhere"), &issue, "?"]); | ||
fails(&owner, &["issue", &url, "t", "-x"]); | ||
fails(&owner, &["issue", &url, "t", "-m"]); | ||
fails(&owner, &["fork", &owner.url("nothing")]); | ||
| ||
// Without an appview there is nowhere to find them. | ||
let mut lost = person(owner.account.clone(), &av); | ||
lost.set("KNOTLESS_APPVIEW", ""); | ||
let (code, out) = lost.cli(&["list", &url], b""); | ||
assert_eq!(code, 1, "{out}"); | ||
| ||
// A server without an index says so. | ||
let s = Sandbox::new(); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let base = format!("http://{}", listener.local_addr().unwrap()); | ||
let bare = App::new(ctx(&s), Duration::ZERO); | ||
let router = router(bare.clone()); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
for path in [ | ||
"dev.knotless.getForge?repo=at://did:plc:a/dev.knotless.repo/r&key=k", | ||
"dev.knotless.listRepos", | ||
"com.atproto.label.queryLabels?uriPatterns=*", | ||
] { | ||
let res = reqwest::blocking::get(format!("{base}/xrpc/{path}")).unwrap(); | ||
assert_eq!(res.status().as_u16(), 404); | ||
} | ||
// Nor can it follow a labeler, and an account is not one: each try | ||
// fails, and it tries again. | ||
for (app, did) in [ | ||
(&bare, av.did.clone()), | ||
(&av.app, owner.account.did.clone()), | ||
] { | ||
let following = knotless_serve::moderation::follow(app.clone(), did); | ||
let _ = | ||
rt.block_on(async { tokio::time::timeout(Duration::from_secs(1), following).await }); | ||
} | ||
// A repository is named by its record's URI. | ||
let res = reqwest::blocking::get(format!( | ||
"{}/xrpc/dev.knotless.getForge?repo=x&key=k", | ||
av.base | ||
)) | ||
.unwrap(); | ||
assert_eq!(res.status().as_u16(), 400); | ||
} | ||
| ||
#[test] | ||
fn private_repositories_keep_their_issues_in_the_space() { | ||
if !net().spaces { | ||
eprintln!("skipped: the PDS is not on the spaces alpha"); | ||
return; | ||
} | ||
let av = appview(); | ||
let owner = person(account(), &av); | ||
let reader = person(account(), &av); | ||
let url = owner.url("inner"); | ||
ok(&owner, &["create", &url, "--private"]); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
ok(&owner, &["share", &url, &reader.account.handle]); | ||
| ||
let issue = ok(&reader, &["issue", &url, "Private trouble"]); | ||
owner.git(&work, &["checkout", "-q", "-b", "fix"]); | ||
let fix = owner.commit_file(&work, "fix", "fixed\n"); | ||
owner.git(&work, &["push", "-q", &url, "fix"]); | ||
let pull = ok(&owner, &["pull", &url, "main", &url, "fix", "Fix it"]); | ||
ok(&reader, &["comment", &url, &pull, "ship it", "--approve"]); | ||
// No appview involved: the members' repos in the space are read directly. | ||
let listing = ok(&owner, &["list", &url]); | ||
assert!( | ||
listing.contains(&format!("issue\topen\t{issue}")), | ||
"{listing}" | ||
); | ||
assert!( | ||
listing.contains(&format!("pull\topen\t{pull}")), | ||
"{listing}" | ||
); | ||
// Its pulls and issues are in the space, which cannot move. | ||
fails(&owner, &["transfer", &url, &reader.account.handle]); | ||
// The owner's gate and blocks hold in the space, with no appview. | ||
ok(&owner, &["hide", &url, &issue]); | ||
assert!(!ok(&reader, &["list", &url]).contains(&issue)); | ||
ok(&owner, &["unhide", &url, &issue]); | ||
ok(&owner, &["block", &reader.account.handle]); | ||
assert!(!ok(&owner, &["list", &url]).contains(&issue)); | ||
ok(&owner, &["unblock", &reader.account.handle]); | ||
assert!(ok(&owner, &["list", &url]).contains(&issue)); | ||
// A block from this repository alone is in the space with it. | ||
ok(&owner, &["block", &reader.account.handle, &url]); | ||
assert!(!ok(&owner, &["list", &url]).contains(&issue)); | ||
ok(&owner, &["unblock", &reader.account.handle, &url]); | ||
assert!(ok(&owner, &["list", &url]).contains(&issue)); | ||
assert_eq!(ok(&owner, &["merge", &url, &pull]), fix); | ||
let listing = ok(&reader, &["list", &url]); | ||
assert!( | ||
listing.contains(&format!("pull\tmerged\t{pull}")), | ||
"{listing}" | ||
); | ||
// The appview never saw any of it. | ||
let uri = format!("at://{}/dev.knotless.repo/inner", owner.account.did); | ||
assert_eq!(av.app.index.as_ref().unwrap().repo(&uri).unwrap(), None); | ||
| ||
// A private repository forks into a private one. | ||
let fork = ok(&reader, &["fork", &url, "inner-fork"]); | ||
let (_, refs) = reader.cli(&["refs", &fork], b""); | ||
assert!(refs.contains(&format!("{fix}\trefs/heads/main")), "{refs}"); | ||
let anonymous = Sandbox::new(); | ||
assert_eq!(anonymous.cli(&["refs", &fork], b"").0, 1); | ||
} | ||
| ||
#[test] | ||
fn moving_to_another_account() { | ||
let av = appview(); | ||
let owner = person(account(), &av); | ||
let heir = person(account(), &av); | ||
let stranger = person(account(), &av); | ||
let url = owner.url("moving"); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
let before = ok(&owner, &["refs", &url]); | ||
let closed = ok(&stranger, &["issue", &url, "Closed before the move"]); | ||
let open = ok(&stranger, &["issue", &url, "Still open"]); | ||
ok(&owner, &["close", &url, &closed]); | ||
listed(&owner, &url, &format!("issue\tclosed\t{closed}")); | ||
| ||
// Nothing moves until it is offered, and only to whom it is offered. | ||
fails(&heir, &["accept", &url]); | ||
fails(&owner, &["transfer", &url, &owner.account.handle]); | ||
ok(&owner, &["transfer", &url, &heir.account.handle]); | ||
fails(&stranger, &["accept", &url]); | ||
ok(&owner, &["transfer", &url, "--cancel"]); | ||
fails(&heir, &["accept", &url]); | ||
let elsewhere = format!("knotless://{}/moving", heir.account.handle); | ||
ok(&owner, &["transfer", &url, &elsewhere]); | ||
ok(&owner, &["transfer", &url, "--cancel"]); | ||
fails( | ||
&owner, | ||
&["transfer", &owner.url("nothing"), &heir.account.handle], | ||
); | ||
let offered = ok(&owner, &["transfer", &url, &heir.account.handle]); | ||
assert!(offered.contains("knotless accept"), "{offered}"); | ||
let moved = ok(&heir, &["accept", &url]); | ||
let new = format!("knotless://{}/moving", heir.account.did); | ||
assert_eq!(moved, format!("moved to {new}")); | ||
| ||
// The old URL leads to the new place, for reads and pushes alike. | ||
assert_eq!(ok(&heir, &["refs", &url]), before); | ||
heir.git(heir.dir.path(), &["clone", "-q", &url, "clone"]); | ||
let clone = heir.path("clone"); | ||
heir.commit_file(&clone, "g", "after the move\n"); | ||
let (pushed, _, err) = heir.git_raw(&clone, &["push", "-q", "origin", "main"]); | ||
assert!( | ||
pushed && err.contains(&format!("has moved to {new}")), | ||
"{err}" | ||
); | ||
assert_ne!(ok(&heir, &["refs", &new]), before); | ||
| ||
// Its issues came with it, and what the old owner did before the move | ||
// still counts; nothing they do after it does. | ||
listed(&heir, &new, &format!("issue\tclosed\t{closed}")); | ||
ok(&owner, &["close", &url, &open]); | ||
std::thread::sleep(Duration::from_secs(1)); | ||
listed(&heir, &new, &format!("issue\topen\t{open}")); | ||
ok(&heir, &["comment", &new, &open, "moved along"]); | ||
ok(&heir, &["close", &new, &open]); | ||
listed(&heir, &new, &format!("issue\tclosed\t{open}")); | ||
| ||
// The old copy can go, leaving a redirect in its place; its old names' | ||
// redirects lead to the new place too, and others' stay as they were. | ||
let key = owner.record("dev.knotless.repo", "moving")["key"].clone(); | ||
let at = format!("at://{}/dev.knotless.repo/moving", owner.account.did); | ||
owner.put_record( | ||
"dev.knotless.redirect", | ||
"once-moving", | ||
serde_json::json!({"$type": "dev.knotless.redirect", "to": {"uri": at, "key": key}, | ||
"createdAt": "2026-10-21T00:00:00.000Z"}), | ||
); | ||
ok(&owner, &["create", &owner.url("spare")]); | ||
ok(&owner, &["rename", &owner.url("spare"), "spare2"]); | ||
ok(&owner, &["delete", &url, "--yes"]); | ||
assert!( | ||
owner.record("dev.knotless.redirect", "once-moving")["to"]["uri"] | ||
.as_str() | ||
.unwrap() | ||
.starts_with(&format!("at://{}/", heir.account.did)) | ||
); | ||
assert_eq!( | ||
owner.record("dev.knotless.redirect", "spare")["to"]["uri"], | ||
format!("at://{}/dev.knotless.repo/spare2", owner.account.did) | ||
); | ||
assert!( | ||
owner.record("dev.knotless.redirect", "moving")["to"]["uri"] | ||
.as_str() | ||
.unwrap() | ||
.starts_with(&format!("at://{}/", heir.account.did)) | ||
); | ||
assert_eq!( | ||
ok(&stranger, &["refs", &url]), | ||
ok(&stranger, &["refs", &new]) | ||
); | ||
listed(&stranger, &url, &format!("issue\tclosed\t{closed}")); | ||
let direct = reqwest::blocking::Client::builder() | ||
.redirect(reqwest::redirect::Policy::none()) | ||
.build() | ||
.unwrap(); | ||
let old_page = || { | ||
direct | ||
.get(format!( | ||
"{}/{}/moving/issues", | ||
av.base, owner.account.handle | ||
)) | ||
.send() | ||
.unwrap() | ||
}; | ||
wait_until(|| old_page().status().as_u16() == 307); | ||
assert_eq!( | ||
old_page().headers()["location"], | ||
format!("/{}/moving/issues", heir.account.did) | ||
); | ||
| ||
// Saying a repository came from another proves nothing without its | ||
// owner's word: none of its issues come along. | ||
let other = owner.url("other"); | ||
owner.git(&work, &["push", "-q", &other, "main"]); | ||
let theirs = ok(&stranger, &["issue", &other, "Not yours"]); | ||
listed(&owner, &other, &theirs); | ||
let key = owner.record("dev.knotless.repo", "other")["key"].clone(); | ||
let from = serde_json::json!({"uri": format!("at://{}/dev.knotless.repo/other", owner.account.did), "key": key}); | ||
stranger.put_record( | ||
"dev.knotless.repo", | ||
"claimed", | ||
serde_json::json!({"$type": "dev.knotless.repo", "key": "3kabcdefghijk", "name": "claimed", | ||
"transferredFrom": from, "createdAt": "2026-10-21T00:00:00.000Z"}), | ||
); | ||
let claimed = stranger.url("claimed"); | ||
std::thread::sleep(Duration::from_secs(1)); | ||
assert!(!ok(&stranger, &["list", &claimed]).contains(&theirs)); | ||
} | ||
| ||
/// Waits until `list` no longer shows `line`. | ||
fn unlisted(s: &Sandbox, url: &str, line: &str) { | ||
wait_until(|| { | ||
let (code, out) = s.cli(&["list", url], b""); | ||
code == 0 && !out.contains(line) | ||
}); | ||
} | ||
| ||
#[test] | ||
fn moderating() { | ||
let av = appview(); | ||
let owner = person(account(), &av); | ||
let troll = person(account(), &av); | ||
let reporter = person(account(), &av); | ||
let url = owner.url("moderated"); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
let trolling = ok(&troll, &["issue", &url, "Trolling"]); | ||
let trolling = trolling.trim(); | ||
let fair = ok(&reporter, &["issue", &url, "A fair point"]); | ||
let fair = fair.trim(); | ||
let reply = ok(&troll, &["comment", &url, fair, "nonsense"]); | ||
let reply = reply.trim(); | ||
listed(&owner, &url, trolling); | ||
| ||
// The owner hides an issue, and what was said about it goes with it. | ||
assert_eq!(ok(&owner, &["hide", &url, fair]), "hidden"); | ||
unlisted(&reporter, &url, fair); | ||
assert_eq!(reporter.cli(&["show", &url, fair], b"").0, 1); | ||
ok(&owner, &["hide", &url, reply]); | ||
assert_eq!(ok(&owner, &["unhide", &url, fair]), "shown"); | ||
listed(&reporter, &url, fair); | ||
assert!(!ok(&reporter, &["show", &url, fair]).contains("nonsense")); | ||
ok(&owner, &["unhide", &url, reply]); | ||
wait_until(|| ok(&reporter, &["show", &url, fair]).contains("nonsense")); | ||
// Only the owner, and only what is said about a repository. | ||
fails(&troll, &["hide", &url, fair]); | ||
let repo_uri = format!("at://{}/dev.knotless.repo/moderated", owner.account.did); | ||
fails(&owner, &["hide", &url, &repo_uri]); | ||
| ||
// A block keeps out everything the account writes. | ||
let blocked = ok(&owner, &["block", &troll.account.handle]); | ||
assert_eq!(blocked, format!("blocked {}", troll.account.did)); | ||
ok(&owner, &["block", &troll.account.handle]); | ||
unlisted(&reporter, &url, trolling); | ||
wait_until(|| !ok(&reporter, &["show", &url, fair]).contains("nonsense")); | ||
fails(&owner, &["block", &owner.account.handle]); | ||
ok(&owner, &["unblock", &troll.account.handle]); | ||
listed(&reporter, &url, trolling); | ||
// Or keeps them out of one repository only, and only its owner can. | ||
ok(&owner, &["block", &troll.account.handle, &url]); | ||
ok(&owner, &["block", &troll.account.handle, &url]); | ||
unlisted(&reporter, &url, trolling); | ||
fails(&reporter, &["block", &troll.account.handle, &url]); | ||
ok(&owner, &["unblock", &troll.account.handle, &url]); | ||
listed(&reporter, &url, trolling); | ||
| ||
// Reports go to the appview's moderation service. | ||
let sent = ok( | ||
&reporter, | ||
&["report", trolling, "spam", "-m", "the same thing again"], | ||
); | ||
assert_eq!(sent, "report 1 sent"); | ||
ok(&reporter, &["report", &troll.account.handle, "rude"]); | ||
ok(&reporter, &["report", &url, "other"]); | ||
fails(&reporter, &["report", &url, "boring"]); | ||
fails(&reporter, &["report", &format!("{trolling}x"), "spam"]); | ||
let mut admin = person(account(), &av); | ||
fails(&admin, &["admin", "reports"]); | ||
admin.set("KNOTLESS_ADMIN_PASSWORD", "wrong"); | ||
fails(&admin, &["admin", "reports"]); | ||
admin.set("KNOTLESS_ADMIN_PASSWORD", "hunter2"); | ||
let reports = ok(&admin, &["admin", "reports"]); | ||
let lines: Vec<&str> = reports.lines().collect(); | ||
assert_eq!(lines.len(), 3, "{reports}"); | ||
assert!( | ||
lines[0].starts_with("3\t") && lines[0].contains(&repo_uri), | ||
"{reports}" | ||
); | ||
assert!( | ||
lines[1].contains(&format!("rude\t{}", troll.account.did)), | ||
"{reports}" | ||
); | ||
assert_eq!( | ||
lines[2], | ||
format!( | ||
"1\t{}\tspam\t{trolling}\tthe same thing again", | ||
reporter.account.did | ||
) | ||
); | ||
assert_eq!(ok(&admin, &["admin", "reports", "1"]).lines().count(), 1); | ||
fails(&admin, &["admin", "reports", "x"]); | ||
fails(&admin, &["admin", "nothing"]); | ||
// Without an appview, reports go where KNOTLESS_MODERATOR says. | ||
let mut direct = person(reporter.account.clone(), &av); | ||
direct.set("KNOTLESS_APPVIEW", ""); | ||
fails(&direct, &["report", trolling, "spam"]); | ||
direct.set("KNOTLESS_MODERATOR", &owner.account.did); | ||
fails(&direct, &["report", trolling, "spam"]); | ||
fails(&reporter, &["report", &owner.url("nothing"), "spam"]); | ||
direct.set("KNOTLESS_MODERATOR", &av.did); | ||
assert_eq!(ok(&direct, &["report", trolling, "spam"]), "report 4 sent"); | ||
| ||
// A second appview follows this one's labels. | ||
let other = appview(); | ||
other.rt.spawn(knotless_serve::moderation::follow( | ||
other.app.clone(), | ||
av.did.clone(), | ||
)); | ||
let down_there = |subject: &str| other.app.index.as_ref().unwrap().down(&[subject]).unwrap(); | ||
| ||
// The operator takes things down. | ||
assert_eq!(ok(&admin, &["admin", "status", trolling]), "up"); | ||
assert_eq!(ok(&admin, &["admin", "takedown", trolling]), "taken down"); | ||
assert_eq!(ok(&admin, &["admin", "status", trolling]), "taken down"); | ||
unlisted(&reporter, &url, trolling); | ||
wait_until(|| down_there(trolling)); | ||
fails(&admin, &["admin", "takedown", trolling, "--later"]); | ||
assert_eq!( | ||
ok(&admin, &["admin", "takedown", trolling, "--undo"]), | ||
"restored" | ||
); | ||
listed(&reporter, &url, trolling); | ||
wait_until(|| !down_there(trolling)); | ||
| ||
// An account taken down takes its repositories with it. | ||
let page = || { | ||
reqwest::blocking::get(format!("{}/{}/moderated", av.base, owner.account.handle)) | ||
.unwrap() | ||
.status() | ||
.as_u16() | ||
}; | ||
assert_eq!(page(), 200); | ||
ok(&admin, &["admin", "takedown", &owner.account.handle]); | ||
assert_eq!( | ||
ok(&admin, &["admin", "status", &owner.account.handle]), | ||
"taken down" | ||
); | ||
assert_eq!(page(), 404); | ||
let listing = reqwest::blocking::get(format!("{}/xrpc/dev.knotless.listRepos", av.base)) | ||
.unwrap() | ||
.text() | ||
.unwrap(); | ||
assert!(!listing.contains(&owner.account.did), "{listing}"); | ||
let labels: Value = reqwest::blocking::get(format!( | ||
"{}/xrpc/com.atproto.label.queryLabels?uriPatterns=did:*&uriPatterns={}", | ||
av.base, trolling | ||
)) | ||
.unwrap() | ||
.json() | ||
.unwrap(); | ||
assert_eq!(labels["labels"][0]["uri"], owner.account.did.as_str()); | ||
assert_eq!(labels["labels"][0]["val"], "!takedown"); | ||
assert_eq!(labels["labels"].as_array().unwrap().len(), 1); | ||
wait_until(|| down_there(&owner.account.did)); | ||
ok(&admin, &["admin", "takedown", &url]); | ||
assert_eq!(ok(&admin, &["admin", "status", &url]), "taken down"); | ||
ok( | ||
&admin, | ||
&["admin", "takedown", &owner.account.handle, "--undo"], | ||
); | ||
assert_eq!(page(), 404); | ||
ok(&admin, &["admin", "takedown", &url, "--undo"]); | ||
assert_eq!(page(), 200); | ||
| ||
// The calls themselves, by hand. | ||
let client = reqwest::blocking::Client::new(); | ||
let xrpc = |nsid: &str| format!("{}/xrpc/{nsid}", av.base); | ||
let post = |nsid: &str, auth: Option<&str>, body: Value| { | ||
let mut req = client.post(xrpc(nsid)).json(&body); | ||
if let Some(auth) = auth { | ||
req = req.header("authorization", auth); | ||
} | ||
req.send().unwrap().status().as_u16() | ||
}; | ||
let basic = "Basic YWRtaW46aHVudGVyMg=="; | ||
let status = "com.atproto.admin.updateSubjectStatus"; | ||
assert_eq!(post(status, None, serde_json::json!({})), 401); | ||
assert_eq!( | ||
post(status, Some(basic), serde_json::json!({"subject": {}})), | ||
400 | ||
); | ||
let account = | ||
serde_json::json!({"$type": "com.atproto.admin.defs#repoRef", "did": "did:plc:x"}); | ||
assert_eq!( | ||
post(status, Some(basic), serde_json::json!({"subject": account})), | ||
200 | ||
); | ||
let get_status = |q: &str| { | ||
client | ||
.get(xrpc(&format!("com.atproto.admin.getSubjectStatus{q}"))) | ||
.header("authorization", basic) | ||
.send() | ||
.unwrap() | ||
.status() | ||
.as_u16() | ||
}; | ||
assert_eq!(get_status(""), 400); | ||
assert_eq!(get_status("?did=did:plc:x"), 200); | ||
let report = "com.atproto.moderation.createReport"; | ||
assert_eq!(post(report, None, serde_json::json!({})), 401); | ||
let token = |ctx: &knotless::env::Ctx| { | ||
av.rt.block_on(async { | ||
let me = knotless::session::me(ctx).await.unwrap(); | ||
knotless::auth::service_token(ctx, &me, &av.did, report) | ||
.await | ||
.unwrap() | ||
}) | ||
}; | ||
let bearer = format!("Bearer {}", token(&ctx(&reporter))); | ||
assert_eq!( | ||
post( | ||
report, | ||
Some(&bearer), | ||
serde_json::json!({"subject": account}) | ||
), | ||
400 | ||
); | ||
let bearer = format!("Bearer {}", token(&ctx(&reporter))); | ||
assert_eq!( | ||
post(report, Some(&bearer), serde_json::json!({"subject": {}})), | ||
400 | ||
); | ||
let page2 = client | ||
.get(xrpc("dev.knotless.listReports?limit=2")) | ||
.header("authorization", basic) | ||
.send() | ||
.unwrap() | ||
.json::<Value>() | ||
.unwrap(); | ||
assert_eq!(page2["cursor"], "3"); | ||
| ||
// The label stream, from the start: everything given out so far, then | ||
// what comes; from past the end, only what comes after it. | ||
let stream = |cursor: Option<i64>| { | ||
let url = format!( | ||
"{}/xrpc/com.atproto.label.subscribeLabels{}", | ||
av.base.replace("http", "ws"), | ||
cursor.map(|c| format!("?cursor={c}")).unwrap_or_default() | ||
); | ||
av.rt | ||
.block_on(tokio_tungstenite::connect_async(url)) | ||
.unwrap() | ||
.0 | ||
}; | ||
let (mut from_start, mut from_end) = (stream(Some(0)), stream(Some(1000))); | ||
let next = |ws: &mut tokio_tungstenite::WebSocketStream<_>| { | ||
let msg = av.rt.block_on(ws.next()).unwrap().unwrap(); | ||
knotless_serve::moderation::parse(&msg.into_data()).unwrap() | ||
}; | ||
assert_eq!(next(&mut from_start).0, 1); | ||
for _ in 2..=6 { | ||
next(&mut from_start); | ||
} | ||
ok(&admin, &["admin", "takedown", trolling]); | ||
let (seq, labels) = next(&mut from_start); | ||
assert_eq!((seq, labels[0]["uri"].as_str()), (7, Some(trolling))); | ||
let ping = tokio_tungstenite::tungstenite::Message::Ping(vec![].into()); | ||
av.rt.block_on(from_start.send(ping)).unwrap(); | ||
av.rt.block_on(from_start.close(None)).unwrap(); | ||
av.rt.block_on(from_end.close(None)).unwrap(); | ||
av.rt.block_on(stream(None).close(None)).unwrap(); | ||
ok(&admin, &["admin", "takedown", trolling, "--undo"]); | ||
// Labels follow from where they were read to. | ||
let labels = other.app.index.as_ref().unwrap(); | ||
assert!(labels.labeler_seq(&av.did).unwrap().is_some()); | ||
} | ||
crates/knotless-serve/tests/gateway.rsnew+597−0
| @@ -0,0 +1,597 @@ | ||
//! The gateway against a reference PDS: owners delegate, collaborators push | ||
//! and commit through it, and commits to one branch are batched. | ||
| ||
#[path = "../../knotless/tests/common/mod.rs"] | ||
mod common; | ||
use common::*; | ||
| ||
use knotless_serve::gateway::Gateway; | ||
use knotless_serve::{App, router}; | ||
use serde_json::{Value, json}; | ||
use std::sync::atomic::Ordering; | ||
use std::sync::{Arc, Barrier}; | ||
use std::time::Duration; | ||
| ||
struct Server { | ||
base: String, | ||
did: String, | ||
app: Arc<App>, | ||
_rt: tokio::runtime::Runtime, | ||
_sandbox: Sandbox, | ||
} | ||
| ||
/// A gateway with config and cache of its own, where owners' delegated | ||
/// sessions live. | ||
fn gateway() -> Server { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
s.set("KNOTLESS_DID_WEB_SCHEME", "http"); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
// atproto takes a did:web with a port only for localhost. | ||
let base = format!("http://localhost:{}", listener.local_addr().unwrap().port()); | ||
let gw = Gateway::new(&base).unwrap(); | ||
let did = gw.did.clone(); | ||
let app = App::build(ctx(&s), Duration::ZERO, Some(gw), None); | ||
let router = router(app.clone()); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
Server { | ||
base, | ||
did, | ||
app, | ||
_rt: rt, | ||
_sandbox: s, | ||
} | ||
} | ||
| ||
fn web(mut s: Sandbox) -> Sandbox { | ||
s.set("KNOTLESS_DID_WEB_SCHEME", "http"); | ||
s | ||
} | ||
| ||
/// A sandbox logged in as `account`, with no password in its environment, | ||
/// so it acts only as itself. | ||
fn person(account: Account) -> Sandbox { | ||
let mut s = web(Sandbox::with(account)); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
let password = s.account.password.clone(); | ||
let handle = s.account.handle.clone(); | ||
assert_eq!( | ||
s.cli(&["login", &handle, "--password-stdin"], password.as_bytes()) | ||
.0, | ||
0 | ||
); | ||
s | ||
} | ||
| ||
/// A service token for `s` to call `lxm` on `aud`. | ||
fn token(s: &Sandbox, aud: &str, lxm: &str) -> String { | ||
let ctx = ctx(s); | ||
tokio::runtime::Runtime::new().unwrap().block_on(async { | ||
let id = knotless::identity::resolve(&ctx, &s.account.handle) | ||
.await | ||
.unwrap(); | ||
let me = knotless::session::ensure(&ctx, &id).await.unwrap(); | ||
knotless::auth::service_token(&ctx, &me, aud, lxm) | ||
.await | ||
.unwrap() | ||
}) | ||
} | ||
| ||
fn post(url: &str, token: Option<&str>, body: Vec<u8>, json: bool) -> (u16, String) { | ||
let client = reqwest::blocking::Client::new(); | ||
let mut req = client.post(url).body(body); | ||
if json { | ||
req = req.header("content-type", "application/json"); | ||
} | ||
if let Some(t) = token { | ||
req = req.bearer_auth(t); | ||
} | ||
let res = req.send().unwrap(); | ||
(res.status().as_u16(), res.text().unwrap()) | ||
} | ||
| ||
#[test] | ||
fn collaborators_push_and_commit_through_a_gateway() { | ||
let gw = gateway(); | ||
let owner = person(account()); | ||
let collab = person(account()); | ||
let outsider = person(account()); | ||
let url = owner.url("shared"); | ||
let work = owner.work("w", 2); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
| ||
// Sharing for writes needs a gateway to write through. | ||
let (code, _) = owner.cli(&["share", &url, &collab.account.handle, "--write"], b""); | ||
assert_eq!(code, 1); | ||
let (code, out) = owner.cli(&["delegate", &owner.account.handle, &gw.did], b""); | ||
assert_eq!(code, 0, "{out}"); | ||
assert_eq!( | ||
owner | ||
.cli(&["share", &url, &collab.account.handle, "--write"], b"") | ||
.0, | ||
0 | ||
); | ||
| ||
// A collaborator clones, commits, and pushes as git would. | ||
let clone = collab.path("clone"); | ||
collab.git(collab.dir.path(), &["clone", "-q", &url, "clone"]); | ||
let sha = collab.commit_file(&clone, "theirs", "from a collaborator\n"); | ||
collab.git( | ||
&clone, | ||
&["push", "-q", "origin", "main", "main:refs/heads/feature"], | ||
); | ||
let (_, refs) = owner.cli(&["refs", &url], b""); | ||
assert!(refs.contains(&format!("{sha}\trefs/heads/main")), "{refs}"); | ||
assert!( | ||
refs.contains(&format!("{sha}\trefs/heads/feature")), | ||
"{refs}" | ||
); | ||
collab.git(&clone, &["push", "-q", "origin", ":refs/heads/feature"]); | ||
let (_, refs) = owner.cli(&["refs", &url], b""); | ||
assert!(!refs.contains("feature"), "{refs}"); | ||
| ||
// git's rules hold at the gateway. | ||
owner.commit_file(&work, "f", "moved on\n"); | ||
owner.git(&work, &["pull", "-q", "--rebase", &url, "main"]); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
collab.commit_file(&clone, "theirs", "stale\n"); | ||
let (ok, _, err) = collab.git_raw(&clone, &["push", "origin", "main"]); | ||
assert!(!ok); | ||
assert!( | ||
err.contains("non-fast-forward") || err.contains("rejected"), | ||
"{err}" | ||
); | ||
| ||
// Commits without a checkout go through it too, and a base that the | ||
// branch has moved on from conflicts on the paths it touched. | ||
let note = collab.path("note"); | ||
std::fs::write(¬e, "n\n").unwrap(); | ||
let (code, out) = collab.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"note", | ||
"--put", | ||
"note", | ||
¬e.to_string_lossy(), | ||
"--json", | ||
], | ||
b"", | ||
); | ||
assert_eq!(code, 0, "{out}"); | ||
let committed: Value = serde_json::from_str(&out).unwrap(); | ||
let (code, out) = collab.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"late", | ||
"--put", | ||
"f", | ||
¬e.to_string_lossy(), | ||
"--base", | ||
&sha, | ||
], | ||
b"", | ||
); | ||
assert_eq!(code, 1, "f changed since: {out}"); | ||
let (code, out) = collab.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"clean", | ||
"--put", | ||
"elsewhere", | ||
¬e.to_string_lossy(), | ||
"--base", | ||
committed["sha"].as_str().unwrap(), | ||
], | ||
b"", | ||
); | ||
assert_eq!(code, 0, "{out}"); | ||
let (code, out) = collab.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"tidy", | ||
"--delete", | ||
"elsewhere", | ||
], | ||
b"", | ||
); | ||
assert_eq!(code, 0, "{out}"); | ||
// A path git refuses fails the batch it is in. | ||
let (code, _) = collab.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"bad", | ||
"--put", | ||
".git/x", | ||
¬e.to_string_lossy(), | ||
], | ||
b"", | ||
); | ||
assert_eq!(code, 1); | ||
// So does a base the owner alone writes against, directly. | ||
let (code, out) = owner.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"late", | ||
"--put", | ||
"note", | ||
¬e.to_string_lossy(), | ||
"--base", | ||
&sha, | ||
], | ||
b"", | ||
); | ||
assert_eq!(code, 1, "{out}"); | ||
assert!(out.is_empty()); | ||
| ||
// Not on the list: refused. | ||
let elsewhere = outsider.path("clone"); | ||
outsider.git(outsider.dir.path(), &["clone", "-q", &url, "clone"]); | ||
outsider.commit_file(&elsewhere, "x", "x\n"); | ||
let (ok, _, err) = outsider.git_raw(&elsewhere, &["push", "origin", "main"]); | ||
assert!(!ok); | ||
assert!(err.contains("not a collaborator"), "{err}"); | ||
| ||
// A repository with no gateway cannot take anyone else's push. | ||
let lonely = owner.url("lonely"); | ||
owner.git(&work, &["push", "-q", &lonely, "main"]); | ||
let (ok, _, err) = outsider.git_raw(&elsewhere, &["push", &lonely, "main"]); | ||
assert!(!ok); | ||
assert!(err.contains("names no gateway"), "{err}"); | ||
let (code, _) = outsider.cli( | ||
&["commit", &lonely, "-b", "main", "-m", "m", "--delete", "f"], | ||
b"", | ||
); | ||
assert_eq!(code, 1); | ||
} | ||
| ||
#[test] | ||
fn commits_to_one_branch_are_batched() { | ||
let gw = gateway(); | ||
let owner = person(account()); | ||
let url = owner.url("busy"); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
assert_eq!( | ||
owner | ||
.cli(&["delegate", &owner.account.handle, &gw.did], b"") | ||
.0, | ||
0 | ||
); | ||
let collab = person(account()); | ||
assert_eq!( | ||
owner | ||
.cli(&["share", &url, &collab.account.handle, "--write"], b"") | ||
.0, | ||
0 | ||
); | ||
| ||
let n = 16; | ||
let barrier = Arc::new(Barrier::new(n)); | ||
let shared = Arc::new((owner, collab)); | ||
let handles: Vec<_> = (0..n) | ||
.map(|i| { | ||
let barrier = barrier.clone(); | ||
let shared = shared.clone(); | ||
let url = url.clone(); | ||
std::thread::spawn(move || { | ||
let (owner, collab) = &*shared; | ||
let who = if i % 2 == 0 { owner } else { collab }; | ||
let file = who.path(&format!("in{i}")); | ||
std::fs::write(&file, format!("{i}\n")).unwrap(); | ||
barrier.wait(); | ||
let mut args = vec!["commit", &url, "-b", "main", "-m", "agent", "--json"]; | ||
let path = format!("agent{i}"); | ||
let file = file.to_string_lossy().to_string(); | ||
args.extend(["--put", &path, &file]); | ||
if who.account.did == owner.account.did { | ||
args.push("--batch"); | ||
} | ||
let (code, out) = who.cli(&args, b""); | ||
assert_eq!(code, 0, "{out}"); | ||
serde_json::from_str::<Value>(&out).unwrap() | ||
}) | ||
}) | ||
.collect(); | ||
let results: Vec<Value> = handles.into_iter().map(|h| h.join().unwrap()).collect(); | ||
let (owner, _) = &*shared; | ||
let clone = owner.path("check"); | ||
owner.git(owner.dir.path(), &["clone", "-q", &url, "check"]); | ||
let count = owner.git(&clone, &["rev-list", "--count", "HEAD"]); | ||
assert_eq!(count, (n + 1).to_string()); | ||
for i in 0..n { | ||
assert!(clone.join(format!("agent{i}")).exists()); | ||
} | ||
assert!(results.iter().all(|r| r["sha"].is_string())); | ||
// Fewer writes than commits: they arrived together. | ||
let writes = gw | ||
.app | ||
.gateway | ||
.as_ref() | ||
.unwrap() | ||
.writes | ||
.load(Ordering::Relaxed); | ||
assert!(writes < n as u64, "{writes} writes for {n} commits"); | ||
} | ||
| ||
#[test] | ||
fn racing_gateways_rebuild_on_the_moved_branch() { | ||
let (a, b) = (gateway(), gateway()); | ||
let owner = person(account()); | ||
let url = owner.url("contested"); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
for gw in [&a, &b] { | ||
assert_eq!( | ||
owner | ||
.cli(&["delegate", &owner.account.handle, &gw.did], b"") | ||
.0, | ||
0 | ||
); | ||
} | ||
let owner = Arc::new(owner); | ||
let mut round = 0; | ||
while a | ||
.app | ||
.gateway | ||
.as_ref() | ||
.unwrap() | ||
.rebuilt | ||
.load(Ordering::Relaxed) | ||
+ b.app | ||
.gateway | ||
.as_ref() | ||
.unwrap() | ||
.rebuilt | ||
.load(Ordering::Relaxed) | ||
== 0 | ||
{ | ||
round += 1; | ||
assert!(round < 50, "the gateways never raced"); | ||
let barrier = Arc::new(Barrier::new(2)); | ||
let handles: Vec<_> = [&a.did, &b.did] | ||
.into_iter() | ||
.enumerate() | ||
.map(|(i, did)| { | ||
let (owner, barrier, did) = (owner.clone(), barrier.clone(), did.clone()); | ||
std::thread::spawn(move || { | ||
let req = json!({ | ||
"repo": owner.account.did, "name": "contested", "branch": "main", | ||
"message": "race", "changes": [{"path": format!("r{round}-{i}"), "data": {"$bytes": "eA"}}], | ||
}); | ||
let t = token(&owner, &did, "dev.knotless.createCommit"); | ||
let base = did.strip_prefix("did:web:").unwrap().replace("%3A", ":"); | ||
barrier.wait(); | ||
post(&format!("http://{base}/xrpc/dev.knotless.createCommit"), Some(&t), req.to_string().into_bytes(), true) | ||
}) | ||
}) | ||
.collect(); | ||
for h in handles { | ||
let (status, body) = h.join().unwrap(); | ||
assert_eq!(status, 200, "{body}"); | ||
} | ||
} | ||
let (_, refs) = owner.cli(&["refs", &url], b""); | ||
assert!(refs.contains("refs/heads/main"), "{refs}"); | ||
} | ||
| ||
#[test] | ||
fn private_repositories_through_a_gateway() { | ||
if !net().spaces { | ||
eprintln!("skipped: the PDS is not on the spaces alpha"); | ||
return; | ||
} | ||
let gw = gateway(); | ||
let owner = person(account()); | ||
let collab = person(account()); | ||
let url = owner.url("hidden"); | ||
assert_eq!(owner.cli(&["create", &url, "--private"], b"").0, 0); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
assert_eq!( | ||
owner | ||
.cli(&["delegate", &owner.account.handle, &gw.did], b"") | ||
.0, | ||
0 | ||
); | ||
assert_eq!( | ||
owner | ||
.cli(&["share", &url, &collab.account.handle, "--write"], b"") | ||
.0, | ||
0 | ||
); | ||
| ||
let clone = collab.path("clone"); | ||
collab.git(collab.dir.path(), &["clone", "-q", &url, "clone"]); | ||
let sha = collab.commit_file(&clone, "theirs", "private work\n"); | ||
collab.git(&clone, &["push", "-q", "origin", "main"]); | ||
let (_, refs) = owner.cli(&["refs", &url], b""); | ||
assert_eq!(refs, format!("{sha}\trefs/heads/main")); | ||
let (_, members) = owner.cli(&["members", &url], b""); | ||
assert_eq!( | ||
members, | ||
format!( | ||
"{}\towner\n{}\twrite", | ||
owner.account.did, collab.account.did | ||
) | ||
); | ||
| ||
// The owner on another machine, which never delegated, keeps the | ||
// gateway the record already names. | ||
let elsewhere = person(owner.account.clone()); | ||
let third = account(); | ||
assert_eq!( | ||
elsewhere | ||
.cli(&["share", &url, &third.handle, "--write"], b"") | ||
.0, | ||
0 | ||
); | ||
} | ||
| ||
#[test] | ||
fn gateway_refusals() { | ||
let gw = gateway(); | ||
let owner = person(account()); | ||
let url = owner.url("guarded"); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
let push = format!("{}/xrpc/dev.knotless.pushRefs", gw.base); | ||
let commit = format!("{}/xrpc/dev.knotless.createCommit", gw.base); | ||
let delegate = format!("{}/xrpc/dev.knotless.putDelegation", gw.base); | ||
let header = |updates: Value| { | ||
let mut body = json!({"repo": owner.account.did, "name": "guarded", "updates": updates}) | ||
.to_string() | ||
.into_bytes(); | ||
body.push(b'\n'); | ||
body | ||
}; | ||
| ||
// No token, or one for another method or service. | ||
assert_eq!(post(&push, None, header(json!([])), false).0, 401); | ||
let wrong = token(&owner, &gw.did, "dev.knotless.createCommit"); | ||
assert_eq!(post(&push, Some(&wrong), header(json!([])), false).0, 401); | ||
let elsewhere = token(&owner, "did:web:elsewhere.test", "dev.knotless.pushRefs"); | ||
assert_eq!( | ||
post(&push, Some(&elsewhere), header(json!([])), false).0, | ||
401 | ||
); | ||
| ||
// A wrong password does not delegate; until one does, nothing writes. | ||
let t = token(&owner, &gw.did, "dev.knotless.putDelegation"); | ||
let (status, _) = post( | ||
&delegate, | ||
Some(&t), | ||
json!({"password": "nope"}).to_string().into_bytes(), | ||
true, | ||
); | ||
assert_eq!(status, 403); | ||
let t = token(&owner, &gw.did, "dev.knotless.pushRefs"); | ||
assert_eq!(post(&push, Some(&t), header(json!([])), false).0, 403); | ||
assert_eq!( | ||
owner | ||
.cli(&["delegate", &owner.account.handle, &gw.did], b"") | ||
.0, | ||
0 | ||
); | ||
// Nothing answers for this one, and that one is no gateway. | ||
let (code, _) = owner.cli( | ||
&["delegate", &owner.account.handle, "did:web:localhost%3A1"], | ||
b"", | ||
); | ||
assert_eq!(code, 1); | ||
let mut out = Vec::new(); | ||
let code = knotless::cli::main_with( | ||
vec![ | ||
"knotless".into(), | ||
"delegate".into(), | ||
owner.account.handle.clone(), | ||
owner.account.did.clone(), | ||
], | ||
owner.env(), | ||
&mut &b""[..], | ||
&mut out, | ||
); | ||
assert_eq!(code, 1); | ||
| ||
// Malformed pushes. | ||
let t = token(&owner, &gw.did, "dev.knotless.pushRefs"); | ||
assert_eq!(post(&push, Some(&t), b"no header".to_vec(), false).0, 400); | ||
let bad = header(json!([{"dst": "refs/heads/x", "src": "main"}])); | ||
assert_eq!(post(&push, Some(&t), bad, false).0, 400); | ||
let bad = header(json!([{"dst": "HEAD", "src": null}])); | ||
assert_eq!(post(&push, Some(&t), bad, false).0, 400); | ||
let mut garbage = header(json!([])); | ||
garbage.extend(b"PACK not really"); | ||
assert_eq!(post(&push, Some(&t), garbage, false).0, 400); | ||
let absent = header(json!([{"dst": "refs/heads/x", "src": "1".repeat(40)}])); | ||
let (status, body) = post(&push, Some(&t), absent, false); | ||
assert_eq!(status, 400); | ||
assert!(body.contains("does not hold"), "{body}"); | ||
let missing = json!({"repo": owner.account.did, "name": "nothing-here", "updates": []}); | ||
let mut body = missing.to_string().into_bytes(); | ||
body.push(b'\n'); | ||
assert_eq!(post(&push, Some(&t), body, false).0, 404); | ||
let mut body = json!({"repo": owner.account.did, "name": "bad name!", "updates": []}) | ||
.to_string() | ||
.into_bytes(); | ||
body.push(b'\n'); | ||
assert_eq!(post(&push, Some(&t), body, false).0, 400); | ||
| ||
// Malformed commits. | ||
let t = token(&owner, &gw.did, "dev.knotless.createCommit"); | ||
let empty = json!({"repo": owner.account.did, "name": "guarded", "branch": "", "message": "m", "changes": []}); | ||
assert_eq!( | ||
post(&commit, Some(&t), empty.to_string().into_bytes(), true).0, | ||
400 | ||
); | ||
// One that changes nothing lands nothing. | ||
let noop = json!({"repo": owner.account.did, "name": "guarded", "branch": "main", "message": "m", "changes": []}); | ||
let (status, body) = post(&commit, Some(&t), noop.to_string().into_bytes(), true); | ||
assert_eq!(status, 200, "{body}"); | ||
assert_eq!( | ||
serde_json::from_str::<Value>(&body).unwrap()["changed"], | ||
false | ||
); | ||
| ||
// The DID document says where to find it. | ||
let doc: Value = reqwest::blocking::get(format!("{}/.well-known/did.json", gw.base)) | ||
.unwrap() | ||
.json() | ||
.unwrap(); | ||
assert_eq!(doc["id"], gw.did); | ||
assert_eq!(doc["service"][0]["serviceEndpoint"], gw.base); | ||
assert!(Gateway::new("ftp://x").is_err()); | ||
assert!(Gateway::new("https://").is_err()); | ||
assert!(Gateway::new("https://a/b").is_err()); | ||
assert_eq!( | ||
Gateway::new("https://knot.example/").unwrap().did, | ||
"did:web:knot.example" | ||
); | ||
} | ||
| ||
#[test] | ||
fn a_server_that_is_not_a_gateway() { | ||
let s = Sandbox::new(); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let base = format!("http://{}", listener.local_addr().unwrap()); | ||
let router = router(App::new(ctx(&s), Duration::ZERO)); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
let res = reqwest::blocking::get(format!("{base}/.well-known/did.json")).unwrap(); | ||
assert_eq!(res.status().as_u16(), 404); | ||
let (status, _) = post( | ||
&format!("{base}/xrpc/dev.knotless.pushRefs"), | ||
Some("x"), | ||
vec![], | ||
false, | ||
); | ||
assert_eq!(status, 404); | ||
} | ||
crates/knotless-serve/tests/jetstream.rsnew+166−0
| @@ -0,0 +1,166 @@ | ||
//! An appview fed by Jetstream: a stand-in server speaks subscribeEvents | ||
//! frames, and the index fills from the records they carry without asking | ||
//! any PDS. | ||
| ||
use futures::SinkExt; | ||
use knotless_serve::App; | ||
use knotless_serve::index::Index; | ||
use std::sync::atomic::Ordering; | ||
use std::time::Duration; | ||
use tokio_tungstenite::tungstenite::Message; | ||
use tokio_tungstenite::tungstenite::handshake::server::{Request, Response}; | ||
| ||
const KIND: &str = "network.bsky.jetstream.subscribeEvents#"; | ||
const TIME: &str = "2026-10-20T00:00:00.000001Z"; | ||
const REPO: &str = "at://did:plc:owner/dev.knotless.repo/proj"; | ||
const KEY: &str = "3l3qo2vuowo2a"; | ||
| ||
fn message(payload: serde_json::Value) -> Message { | ||
Message::Text( | ||
serde_json::json!({"$type": "message", "payload": payload}) | ||
.to_string() | ||
.into(), | ||
) | ||
} | ||
| ||
fn commit( | ||
seq: i64, | ||
did: &str, | ||
op: &str, | ||
collection: &str, | ||
rkey: &str, | ||
record: Option<serde_json::Value>, | ||
) -> Message { | ||
let mut payload = serde_json::json!({ | ||
"$type": format!("{KIND}commit"), "seq": seq, "did": did, "time": TIME, | ||
"rev": "3l3qo2vutsw2b", "operation": op, "collection": collection, "rkey": rkey, | ||
}); | ||
if let Some(record) = record { | ||
payload["record"] = record; | ||
payload["cid"] = "bafyreia4w4bbvc5lqzkvexqlbwbagz5yfjfzzrqhfwqkhkk2xgekh4c6ai".into(); | ||
} | ||
message(payload) | ||
} | ||
| ||
fn wait_until(mut f: impl FnMut() -> bool) { | ||
for _ in 0..200 { | ||
if f() { | ||
return; | ||
} | ||
std::thread::sleep(Duration::from_millis(50)); | ||
} | ||
panic!("condition never held"); | ||
} | ||
| ||
// The handshake callback's error type is tungstenite's, not ours. | ||
#[allow(clippy::result_large_err)] | ||
#[test] | ||
fn the_index_fills_from_jetstream() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let addr = listener.local_addr().unwrap(); | ||
let (asked, mut asks) = tokio::sync::mpsc::unbounded_channel(); | ||
rt.spawn(async move { | ||
for connection in 0.. { | ||
let (tcp, _) = listener.accept().await.unwrap(); | ||
let asked = asked.clone(); | ||
let mut ws = tokio_tungstenite::accept_hdr_async(tcp, |req: &Request, mut res: Response| { | ||
let protocol = req.headers()["sec-websocket-protocol"].clone(); | ||
asked.send((req.uri().to_string(), protocol.to_str().unwrap().to_string())).unwrap(); | ||
res.headers_mut().insert("sec-websocket-protocol", protocol); | ||
Ok(res) | ||
}) | ||
.await | ||
.unwrap(); | ||
let frames = match connection { | ||
0 => vec![ | ||
message(serde_json::json!({"$type": format!("{KIND}info"), "name": "OutdatedCursor", "message": "starting at seq 10"})), | ||
Message::Text("not json".into()), | ||
Message::Binary(vec![1].into()), | ||
commit(10, "did:plc:owner", "create", "dev.knotless.repo", "proj", Some(serde_json::json!({ | ||
"$type": "dev.knotless.repo", "key": KEY, "description": "a project", "createdAt": TIME, | ||
}))), | ||
commit(11, "did:plc:other", "create", "dev.knotless.issue", "3l3qo2vuowo2b", Some(serde_json::json!({ | ||
"$type": "dev.knotless.issue", "repo": {"uri": REPO, "key": KEY}, "title": "It breaks", "createdAt": TIME, | ||
}))), | ||
commit(12, "did:plc:other", "create", "app.bsky.feed.post", "3l3qo2vuowo2c", Some(serde_json::json!({ | ||
"$type": "app.bsky.feed.post", "text": "elsewhere", "createdAt": TIME, | ||
}))), | ||
message(serde_json::json!({ | ||
"$type": format!("{KIND}identity"), "seq": 13, "did": "did:plc:moved", "time": TIME, | ||
"identity": {"did": "did:plc:moved", "handle": "moved.test", "seq": 99, "time": TIME}, | ||
})), | ||
Message::Text(r#"{"$type":"error","error":"ConsumerTooSlow"}"#.into()), | ||
], | ||
1 => vec![commit(14, "did:plc:other", "delete", "dev.knotless.issue", "3l3qo2vuowo2b", None)], | ||
_ => vec![], | ||
}; | ||
for frame in frames { | ||
ws.send(frame).await.unwrap(); | ||
} | ||
ws.close(None).await.unwrap(); | ||
} | ||
}); | ||
| ||
let env = knotless::env::Env::from_vars(|_| None); | ||
let index = Index::open(&dir.path().join("index.db")).unwrap(); | ||
let app = App::build( | ||
knotless::env::Ctx::new(env), | ||
Duration::ZERO, | ||
None, | ||
Some(index), | ||
); | ||
rt.spawn(knotless_serve::jetstream::watch( | ||
app.clone(), | ||
format!("ws://{addr}"), | ||
)); | ||
let index = app.index.clone().unwrap(); | ||
| ||
// It asks for knotless commits, blocks, and every account's identity, offering | ||
// the one framing Jetstream speaks. | ||
let (first, protocol) = rt.block_on(asks.recv()).unwrap(); | ||
assert_eq!(protocol, "xrpc.v1.json"); | ||
assert!( | ||
first.starts_with( | ||
"/xrpc/network.bsky.jetstream.subscribeEvents?collections=dev.knotless.*&kinds=commit" | ||
), | ||
"{first}" | ||
); | ||
assert!(!first.contains("cursor"), "{first}"); | ||
| ||
// The records it carried are indexed as they came. | ||
wait_until(|| index.forge("did:plc:owner", KEY).unwrap().len() == 1); | ||
assert_eq!( | ||
index.repo(REPO).unwrap().unwrap()["description"], | ||
"a project" | ||
); | ||
assert_eq!( | ||
index.forge("did:plc:owner", KEY).unwrap()[0].value["title"], | ||
"It breaks" | ||
); | ||
assert_eq!(app.events_for("did:plc:owner"), 1); | ||
assert_eq!(app.events_for("did:plc:other"), 0); | ||
assert_eq!(app.events_for("did:plc:moved"), 1); | ||
| ||
// After a drop it resumes past the last event it saw. | ||
let (second, _) = rt.block_on(asks.recv()).unwrap(); | ||
assert!(second.ends_with("&cursor=14"), "{second}"); | ||
wait_until(|| index.forge("did:plc:owner", KEY).unwrap().is_empty()); | ||
let (third, _) = rt.block_on(asks.recv()).unwrap(); | ||
assert!(third.ends_with("&cursor=15"), "{third}"); | ||
| ||
// A Jetstream it cannot reach, or cannot even name, leaves the TTL in | ||
// charge. | ||
for url in ["ws://127.0.0.1:9", "not a url"] { | ||
let lost = App::new( | ||
knotless::env::Ctx::new(knotless::env::Env::from_vars(|_| None)), | ||
Duration::ZERO, | ||
); | ||
lost.watching.store(true, Ordering::Relaxed); | ||
rt.spawn(knotless_serve::jetstream::watch(lost.clone(), url.into())); | ||
wait_until(|| !lost.watching.load(Ordering::Relaxed)); | ||
} | ||
} | ||
crates/knotless-serve/tests/serve.rsnew+426−0
| @@ -0,0 +1,426 @@ | ||
//! knotless-serve against a reference PDS: real git clients over smart | ||
//! HTTP, and the API. | ||
| ||
#[path = "../../knotless/tests/common/mod.rs"] | ||
mod common; | ||
use common::*; | ||
| ||
use knotless::commit::{Change, Spec}; | ||
use knotless_serve::{App, router}; | ||
use std::io::Write; | ||
use std::sync::Arc; | ||
use std::sync::atomic::Ordering; | ||
use std::time::Duration; | ||
| ||
struct Server { | ||
base: String, | ||
app: Arc<App>, | ||
rt: tokio::runtime::Runtime, | ||
} | ||
| ||
fn serve(s: &Sandbox, ttl: Duration) -> Server { | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let app = App::new(ctx(s), ttl); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let base = format!("http://{}", listener.local_addr().unwrap()); | ||
let router = router(app.clone()); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
Server { base, app, rt } | ||
} | ||
| ||
fn put(path: &str, data: &str) -> Change { | ||
Change::Put { | ||
path: path.into(), | ||
data: data.as_bytes().to_vec(), | ||
executable: false, | ||
} | ||
} | ||
| ||
fn commit(s: &Sandbox, name: &str, changes: Vec<Change>, message: &str) -> String { | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let url = knotless::url::RepoUrl::parse(&s.url(name)).unwrap(); | ||
let spec = Spec { | ||
branch: "main".into(), | ||
message: message.into(), | ||
changes, | ||
author: None, | ||
retries: 3, | ||
..Default::default() | ||
}; | ||
rt.block_on(knotless::commit::commit(&ctx(s), &url, &spec)) | ||
.unwrap() | ||
.sha | ||
} | ||
| ||
fn get(url: &str) -> (u16, String, reqwest::header::HeaderMap) { | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
rt.block_on(async { | ||
let res = reqwest::get(url).await.unwrap(); | ||
let status = res.status().as_u16(); | ||
let headers = res.headers().clone(); | ||
(status, res.text().await.unwrap(), headers) | ||
}) | ||
} | ||
| ||
#[test] | ||
fn git_clients_over_smart_http() { | ||
let s = Sandbox::new(); | ||
commit( | ||
&s, | ||
"web", | ||
vec![put("a.txt", "one\n"), put("dir/b.txt", "two\n")], | ||
"first", | ||
); | ||
let tip = commit(&s, "web", vec![put("a.txt", "three\n")], "second"); | ||
let server = serve(&s, Duration::ZERO); | ||
let url = format!("{}/{}/web.git", server.base, s.account.handle); | ||
| ||
let cases: [(&str, &[&str], &[&str]); 4] = [ | ||
("v2", &[], &[]), | ||
("v0", &["-c", "protocol.version=0"], &[]), | ||
("partial", &[], &["--filter=blob:none"]), | ||
("shallow", &[], &["--depth", "1"]), | ||
]; | ||
for (dir, config, flags) in cases { | ||
let path = s.path(dir); | ||
let mut args: Vec<&str> = config.to_vec(); | ||
args.extend(["clone", "-q"]); | ||
args.extend(flags); | ||
args.extend([url.as_str(), path.to_str().unwrap()]); | ||
s.git(s.dir.path(), &args); | ||
assert_eq!(s.git(&path, &["rev-parse", "HEAD"]), tip, "{dir}"); | ||
assert_eq!( | ||
std::fs::read_to_string(path.join("a.txt")).unwrap(), | ||
"three\n", | ||
"{dir}" | ||
); | ||
} | ||
assert_eq!( | ||
s.git(&s.path("shallow"), &["rev-list", "--count", "HEAD"]), | ||
"1" | ||
); | ||
| ||
// Pushing goes to the PDS, never here. | ||
let (status, _, _) = get(&format!("{url}/info/refs?service=git-receive-pack")); | ||
assert_eq!(status, 403); | ||
let (status, body, _) = get(&format!("{}/healthz", server.base)); | ||
assert_eq!((status, body.as_str()), (200, "ok")); | ||
drop(server.rt); | ||
} | ||
| ||
#[test] | ||
fn gzip_requests() { | ||
let s = Sandbox::new(); | ||
commit(&s, "gz", vec![put("a", "a")], "c"); | ||
let server = serve(&s, Duration::from_secs(60)); | ||
let url = format!("{}/{}/gz/git-upload-pack", server.base, s.account.handle); | ||
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); | ||
gz.write_all(b"0000").unwrap(); | ||
let body = gz.finish().unwrap(); | ||
server.rt.block_on(async { | ||
let client = reqwest::Client::new(); | ||
let ok = client | ||
.post(&url) | ||
.header("content-encoding", "gzip") | ||
.body(body) | ||
.send() | ||
.await | ||
.unwrap(); | ||
assert_eq!(ok.status(), 200); | ||
let bad = client | ||
.post(&url) | ||
.header("content-encoding", "gzip") | ||
.body("nope") | ||
.send() | ||
.await | ||
.unwrap(); | ||
assert_eq!(bad.status(), 400); | ||
}); | ||
} | ||
| ||
#[test] | ||
fn api_reads_without_a_clone() { | ||
let s = Sandbox::new(); | ||
let first = commit( | ||
&s, | ||
"api", | ||
vec![put("a.txt", "one\n"), put("dir/b.txt", "two\n")], | ||
"first", | ||
); | ||
let second = commit(&s, "api", vec![put("a.txt", "three\n")], "second\n\nbody"); | ||
let server = serve(&s, Duration::from_secs(60)); | ||
let api = format!("{}/api/{}/api", server.base, s.account.handle); | ||
| ||
let (status, body, _) = get(&format!("{api}/refs")); | ||
assert_eq!(status, 200); | ||
let v: serde_json::Value = serde_json::from_str(&body).unwrap(); | ||
assert_eq!(v["refs"][0]["target"], second); | ||
assert_eq!(v["head"], "refs/heads/main"); | ||
| ||
let (_, body, headers) = get(&format!("{api}/tree?rev={second}")); | ||
let v: serde_json::Value = serde_json::from_str(&body).unwrap(); | ||
let names: Vec<&str> = v["entries"] | ||
.as_array() | ||
.unwrap() | ||
.iter() | ||
.map(|e| e["name"].as_str().unwrap()) | ||
.collect(); | ||
assert_eq!(names, vec!["a.txt", "dir"]); | ||
assert!( | ||
headers["cache-control"] | ||
.to_str() | ||
.unwrap() | ||
.contains("immutable") | ||
); | ||
let (_, body, _) = get(&format!("{api}/tree?path=/dir/")); | ||
assert!(body.contains("b.txt")); | ||
| ||
let (status, body, headers) = get(&format!("{api}/blob?path=a.txt")); | ||
assert_eq!((status, body.as_str()), (200, "three\n")); | ||
assert!(headers.contains_key("etag")); | ||
let (_, body, _) = get(&format!("{api}/blob?path=a.txt&rev={first}")); | ||
assert_eq!(body, "one\n"); | ||
| ||
let (_, body, _) = get(&format!("{api}/log?limit=10")); | ||
let v: serde_json::Value = serde_json::from_str(&body).unwrap(); | ||
assert_eq!(v["commits"].as_array().unwrap().len(), 2); | ||
assert_eq!(v["commits"][0]["message"], "second\n\nbody"); | ||
assert_eq!(v["commits"][0]["parents"][0], first); | ||
let (_, body, _) = get(&format!("{api}/log?path=dir")); | ||
let v: serde_json::Value = serde_json::from_str(&body).unwrap(); | ||
assert_eq!(v["commits"].as_array().unwrap().len(), 1); | ||
| ||
let (_, body, _) = get(&format!("{api}/diff?base={first}&head={second}")); | ||
assert!(body.contains("-one") && body.contains("+three"), "{body}"); | ||
| ||
let (status, body, headers) = get(&format!("{api}/object/{second}")); | ||
assert_eq!(status, 200); | ||
assert!(body.starts_with("tree ")); | ||
assert_eq!(headers["x-git-type"], "commit"); | ||
| ||
for (path, want) in [ | ||
("/blob?rev=--output=x", 400), | ||
("/blob?path=nope", 404), | ||
("/object/xyz", 400), | ||
(&format!("/object/{}", "0".repeat(40)), 404), | ||
] { | ||
assert_eq!(get(&format!("{api}{path}")).0, want, "{path}"); | ||
} | ||
let (status, _, _) = get(&format!( | ||
"{}/api/{}/missing/refs", | ||
server.base, s.account.handle | ||
)); | ||
assert_eq!(status, 404); | ||
let (status, _, _) = get(&format!( | ||
"{}/api/{}/bad%20name/refs", | ||
server.base, s.account.handle | ||
)); | ||
assert_eq!(status, 400); | ||
let (status, _, _) = get(&format!("{}/api/nobody.invalid/x/refs", server.base)); | ||
assert_eq!(status, 502); | ||
} | ||
| ||
#[test] | ||
fn ttl_and_firehose_freshness() { | ||
let s = Sandbox::new(); | ||
commit(&s, "fresh", vec![put("a", "1")], "one"); | ||
let server = serve(&s, Duration::from_secs(3600)); | ||
let blob = format!("{}/api/{}/fresh/blob?path=a", server.base, s.account.handle); | ||
assert_eq!(get(&blob).1, "1"); | ||
// Within the TTL and unwatched, the mirror is trusted as is. | ||
commit(&s, "fresh", vec![put("a", "2")], "two"); | ||
assert_eq!(get(&blob).1, "1"); | ||
| ||
// The firehose marks it stale and refreshes it. | ||
let ws = format!( | ||
"{}/xrpc/com.atproto.sync.subscribeRepos", | ||
net().pds.replace("http", "ws") | ||
); | ||
server | ||
.rt | ||
.spawn(knotless_serve::firehose::watch(server.app.clone(), ws)); | ||
wait_until(|| server.app.watching.load(Ordering::Relaxed)); | ||
commit(&s, "fresh", vec![put("a", "3")], "three"); | ||
wait_until(|| get(&blob).1 == "3"); | ||
// Watched and quiet: served without asking the PDS. | ||
assert_eq!(get(&blob).1, "3"); | ||
| ||
// A firehose that cannot connect leaves the TTL in charge. | ||
let lost = App::new(ctx(&s), Duration::ZERO); | ||
lost.watching.store(true, Ordering::Relaxed); | ||
server.rt.spawn(knotless_serve::firehose::watch( | ||
lost.clone(), | ||
"ws://127.0.0.1:9".into(), | ||
)); | ||
wait_until(|| !lost.watching.load(Ordering::Relaxed)); | ||
} | ||
| ||
#[test] | ||
fn binary_lifecycle() { | ||
let s = Sandbox::new(); | ||
let exe = env!("CARGO_BIN_EXE_knotless-serve"); | ||
let out = s.command(exe, s.dir.path()).arg("--nope").output().unwrap(); | ||
assert_eq!(out.status.code(), Some(2)); | ||
let out = s | ||
.command(exe, s.dir.path()) | ||
.args(["--listen", "not-an-address"]) | ||
.output() | ||
.unwrap(); | ||
assert_eq!(out.status.code(), Some(1)); | ||
let out = s | ||
.command(exe, s.dir.path()) | ||
.args(["--public-url", "ftp://nope"]) | ||
.output() | ||
.unwrap(); | ||
assert_eq!(out.status.code(), Some(1)); | ||
| ||
let mut child = s | ||
.command(exe, s.dir.path()) | ||
.args([ | ||
"--listen", | ||
"127.0.0.1:0", | ||
"--ttl", | ||
"1", | ||
"--firehose", | ||
"ws://127.0.0.1:9", | ||
"--jetstream", | ||
"ws://127.0.0.1:9", | ||
"--public-url", | ||
"http://localhost:1", | ||
"--index", | ||
&s.path("index.db").to_string_lossy(), | ||
"--backfill", | ||
"http://127.0.0.1:9", | ||
"--labeler", | ||
"did:web:127.0.0.1%3A9", | ||
]) | ||
.stdout(std::process::Stdio::piped()) | ||
.spawn() | ||
.unwrap(); | ||
let mut line = String::new(); | ||
std::io::BufRead::read_line( | ||
&mut std::io::BufReader::new(child.stdout.take().unwrap()), | ||
&mut line, | ||
) | ||
.unwrap(); | ||
let base = line | ||
.trim() | ||
.strip_prefix("listening on ") | ||
.unwrap() | ||
.to_string(); | ||
assert_eq!(get(&format!("{base}/healthz")).1, "ok"); | ||
assert!( | ||
get(&format!("{base}/.well-known/did.json")) | ||
.1 | ||
.contains("did:web:localhost%3A1") | ||
); | ||
std::process::Command::new("kill") | ||
.args(["-TERM", &child.id().to_string()]) | ||
.status() | ||
.unwrap(); | ||
assert!(child.wait().unwrap().success()); | ||
} | ||
| ||
#[test] | ||
fn arguments() { | ||
use knotless_serve::parse_args; | ||
let args = |a: &[&str]| { | ||
parse_args( | ||
&std::iter::once("x") | ||
.chain(a.iter().copied()) | ||
.map(String::from) | ||
.collect::<Vec<_>>(), | ||
) | ||
}; | ||
let o = args(&[ | ||
"--listen", | ||
"0.0.0.0:1", | ||
"--ttl", | ||
"9", | ||
"--firehose", | ||
"wss://r", | ||
"--jetstream", | ||
"wss://jetstream.us-east.bsky.network", | ||
]) | ||
.unwrap(); | ||
assert_eq!(o.listen, "0.0.0.0:1"); | ||
assert_eq!(o.ttl, Duration::from_secs(9)); | ||
assert_eq!(o.firehose.as_deref(), Some("wss://r")); | ||
assert_eq!( | ||
o.jetstream.as_deref(), | ||
Some("wss://jetstream.us-east.bsky.network") | ||
); | ||
assert_eq!(o.public_url, None); | ||
let o = args(&["--public-url", "https://knot.example"]).unwrap(); | ||
assert_eq!(o.public_url.as_deref(), Some("https://knot.example")); | ||
let o = args(&["--labeler", "did:web:a", "--labeler", "did:web:b"]).unwrap(); | ||
assert_eq!(o.labelers, ["did:web:a", "did:web:b"]); | ||
assert!(args(&["--ttl", "x"]).is_err()); | ||
assert!(args(&["--ttl"]).is_err()); | ||
} | ||
| ||
#[test] | ||
fn firehose_frames_of_every_kind() { | ||
use futures::SinkExt; | ||
use tokio_tungstenite::tungstenite::Message; | ||
let s = Sandbox::new(); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let addr = listener.local_addr().unwrap(); | ||
rt.spawn(async move { | ||
let (tcp, _) = listener.accept().await.unwrap(); | ||
let mut ws = tokio_tungstenite::accept_async(tcp).await.unwrap(); | ||
let value = |pairs: Vec<(&str, ciborium::Value)>| { | ||
ciborium::Value::Map( | ||
pairs | ||
.into_iter() | ||
.map(|(k, v)| (ciborium::Value::Text(k.into()), v)) | ||
.collect(), | ||
) | ||
}; | ||
let mut frame = Vec::new(); | ||
ciborium::into_writer( | ||
&value(vec![("t", ciborium::Value::Text("#identity".into()))]), | ||
&mut frame, | ||
) | ||
.unwrap(); | ||
ciborium::into_writer( | ||
&value(vec![ | ||
("did", ciborium::Value::Text("did:plc:watched".into())), | ||
("seq", ciborium::Value::Integer(5.into())), | ||
]), | ||
&mut frame, | ||
) | ||
.unwrap(); | ||
ws.send(Message::Text("hello".into())).await.unwrap(); | ||
ws.send(Message::Binary(vec![0xff].into())).await.unwrap(); | ||
ws.send(Message::Binary(frame.into())).await.unwrap(); | ||
ws.close(None).await.unwrap(); | ||
}); | ||
let app = App::new(ctx(&s), Duration::ZERO); | ||
rt.spawn(knotless_serve::firehose::watch( | ||
app.clone(), | ||
format!("ws://{addr}"), | ||
)); | ||
wait_until(|| app.events_for("did:plc:watched") == 1); | ||
wait_until(|| !app.watching.load(Ordering::Relaxed)); | ||
} | ||
| ||
#[test] | ||
fn a_cache_it_cannot_write_is_a_server_error() { | ||
let mut s = Sandbox::new(); | ||
commit(&s, "nocache", vec![put("a", "a")], "c"); | ||
std::fs::write(s.path("blocked"), b"").unwrap(); | ||
let blocked = s.path("blocked").join("cache"); | ||
s.set("KNOTLESS_CACHE_DIR", blocked.to_str().unwrap()); | ||
let server = serve(&s, Duration::ZERO); | ||
let (status, _, _) = get(&format!( | ||
"{}/api/{}/nocache/refs", | ||
server.base, s.account.handle | ||
)); | ||
assert_eq!(status, 500); | ||
} | ||
crates/knotless-serve/tests/web.rsnew+496−0
| @@ -0,0 +1,496 @@ | ||
//! The web pages, over a reference PDS, with an index following its firehose. | ||
| ||
#[path = "../../knotless/tests/common/mod.rs"] | ||
mod common; | ||
use common::*; | ||
| ||
use knotless_serve::index::Index; | ||
use knotless_serve::{App, router}; | ||
use reqwest::blocking::{Client, Response}; | ||
use std::sync::Arc; | ||
use std::time::Duration; | ||
| ||
struct Site { | ||
base: String, | ||
_rt: tokio::runtime::Runtime, | ||
_sandbox: Sandbox, | ||
} | ||
| ||
fn serve(indexed: bool) -> Site { | ||
let s = Sandbox::new(); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let base = format!("http://{}", listener.local_addr().unwrap()); | ||
let index = indexed.then(|| Index::open(&s.path("index.db")).unwrap()); | ||
let app: Arc<App> = App::build(ctx(&s), Duration::ZERO, None, index); | ||
let router = router(app.clone()); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
if indexed { | ||
rt.spawn(knotless_serve::firehose::watch( | ||
app, | ||
net().pds.replace("http://", "ws://"), | ||
)); | ||
} | ||
Site { | ||
base, | ||
_rt: rt, | ||
_sandbox: s, | ||
} | ||
} | ||
| ||
impl Site { | ||
fn get(&self, path: &str) -> Response { | ||
Client::new() | ||
.get(format!("{}{path}", self.base)) | ||
.send() | ||
.unwrap() | ||
} | ||
| ||
/// The page at `path`, which must be there. | ||
fn page(&self, path: &str) -> String { | ||
let res = self.get(path); | ||
assert_eq!(res.status().as_u16(), 200, "{path}"); | ||
res.text().unwrap() | ||
} | ||
| ||
fn status(&self, path: &str) -> (u16, String) { | ||
let res = self.get(path); | ||
(res.status().as_u16(), res.text().unwrap()) | ||
} | ||
} | ||
| ||
fn person(account: Account, site: &Site) -> Sandbox { | ||
let mut s = Sandbox::with(account); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
s.set("KNOTLESS_APPVIEW", &site.base); | ||
let password = s.account.password.clone(); | ||
let handle = s.account.handle.clone(); | ||
assert_eq!( | ||
s.cli(&["login", &handle, "--password-stdin"], password.as_bytes()) | ||
.0, | ||
0 | ||
); | ||
s | ||
} | ||
| ||
fn ok(s: &Sandbox, args: &[&str]) -> String { | ||
let (code, out) = s.cli(args, b""); | ||
assert_eq!(code, 0, "{args:?}: {out}"); | ||
out | ||
} | ||
| ||
fn write(dir: &std::path::Path, path: &str, bytes: &[u8]) { | ||
let path = dir.join(path); | ||
std::fs::create_dir_all(path.parent().unwrap()).unwrap(); | ||
std::fs::write(path, bytes).unwrap(); | ||
} | ||
| ||
#[test] | ||
fn browsing_a_repository() { | ||
let site = serve(true); | ||
let s = person(account(), &site); | ||
let url = s.url("browsed"); | ||
let work = s.work("w", 51); | ||
write(&work, "README.md", b"# Browsed\n\n<script>alert(1)</script>\n\n[guide](docs/guide.md) [x](javascript:alert(1)) \n"); | ||
write(&work, "docs/README", b"plain <words>\n"); | ||
write(&work, "docs/guide.md", b"# Guide\n"); | ||
write( | ||
&work, | ||
"src/main.rs", | ||
"// héllo\nfn main() {\n println!(\"hi\");\n}\n".as_bytes(), | ||
); | ||
write(&work, "logo.png", b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR"); | ||
write(&work, "data.bin", &[0, 1, 2, 3]); | ||
write(&work, "big.txt", &vec![b'a'; 1100 * 1024]); | ||
write(&work, "notes.unknown-kind", b"no grammar\n"); | ||
s.git(&work, &["add", "-A"]); | ||
s.git( | ||
&work, | ||
&[ | ||
"commit", | ||
"-q", | ||
"-m", | ||
"Lay out the tree\n\nWith a body that says why.", | ||
], | ||
); | ||
let tip = s.git(&work, &["rev-parse", "HEAD"]); | ||
s.git(&work, &["push", "-q", &url, "main", "main:feature/x"]); | ||
let h = &s.account.handle; | ||
let repo = format!("/{h}/browsed"); | ||
| ||
// The front page finds it once the index has seen it. | ||
wait_until(|| site.page("/").contains("<b>browsed</b></a>")); | ||
assert!(site.page("/?q=BROWS").contains("<b>browsed</b></a>")); | ||
assert!(site.page("/?q=zzzz").contains("No repositories found.")); | ||
| ||
let res = site.get(&repo); | ||
let csp = res.headers()["content-security-policy"] | ||
.to_str() | ||
.unwrap() | ||
.to_string(); | ||
assert!( | ||
csp.starts_with("default-src 'none'") && csp.contains("script-src 'sha256-"), | ||
"{csp}" | ||
); | ||
assert_eq!(res.headers()["cache-control"], "no-cache"); | ||
let home = res.text().unwrap(); | ||
for want in [ | ||
">src</a>", | ||
">docs</a>", | ||
"Lay out the tree</a>", | ||
"git clone knotless://", | ||
"<h1>Browsed</h1>", | ||
"<script>alert(1)</script>", | ||
&format!(r#"href="{repo}/blob/main/docs/guide.md""#), | ||
r##"href="#""##, | ||
&format!(r#"src="{repo}/raw/main/logo.png""#), | ||
r#"aria-current="page"><svg"#, | ||
] { | ||
assert!(home.contains(want), "{want} not in\n{home}"); | ||
} | ||
assert!(home.find(">docs</a>").unwrap() < home.find(">README.md</a>").unwrap()); | ||
| ||
// Compressed for browsers that ask. | ||
let res = Client::builder() | ||
.no_gzip() | ||
.no_brotli() | ||
.build() | ||
.unwrap() | ||
.get(format!("{}{repo}", site.base)) | ||
.header("accept-encoding", "br") | ||
.send() | ||
.unwrap(); | ||
assert_eq!(res.headers()["content-encoding"], "br"); | ||
| ||
// A branch with a slash in its name, and a pinned commit, which never changes. | ||
let sub = site.page(&format!("{repo}/tree/feature/x/src")); | ||
assert!( | ||
sub.contains(&format!(r#"href="{repo}/blob/feature/x/src/main.rs""#)), | ||
"{sub}" | ||
); | ||
let res = site.get(&format!("{repo}/tree/{tip}/docs")); | ||
assert_eq!( | ||
res.headers()["cache-control"], | ||
"public, max-age=31536000, immutable" | ||
); | ||
assert!( | ||
res.text() | ||
.unwrap() | ||
.contains("<pre>plain <words>\n</pre>") | ||
); | ||
| ||
let file = site.page(&format!("{repo}/blob/main/src/main.rs")); | ||
assert!(file.contains(r##"<a id="L4" href="#L4">4</a>"##), "{file}"); | ||
assert!(file.contains("data-hl=\"comment:0,8;"), "{file}"); | ||
assert!(file.contains("// héllo\nfn main()")); | ||
assert!( | ||
site.page(&format!("{repo}/blob/main/notes.unknown-kind")) | ||
.contains(r#"data-hl="""#) | ||
); | ||
assert!( | ||
site.page(&format!("{repo}/blob/main/logo.png")) | ||
.contains(&format!(r#"<img src="{repo}/raw/main/logo.png""#)) | ||
); | ||
assert!( | ||
site.page(&format!("{repo}/blob/main/data.bin")) | ||
.contains("A binary file, not shown.") | ||
); | ||
assert!( | ||
site.page(&format!("{repo}/blob/main/big.txt")) | ||
.contains("Too large to show here") | ||
); | ||
| ||
for (path, kind) in [ | ||
("logo.png", "image/png"), | ||
("data.bin", "application/octet-stream"), | ||
("src/main.rs", "text/plain; charset=utf-8"), | ||
] { | ||
let res = site.get(&format!("{repo}/raw/main/{path}")); | ||
assert_eq!(res.headers()["content-type"], kind); | ||
assert!( | ||
res.headers()["content-security-policy"] | ||
.to_str() | ||
.unwrap() | ||
.starts_with("sandbox") | ||
); | ||
} | ||
| ||
let log = site.page(&format!("{repo}/commits/main")); | ||
assert_eq!(log.matches("<li>").count(), 50); | ||
assert!(log.contains("Commits on "), "{log}"); | ||
assert!(log.contains(r#"href="?page=1">Older"#)); | ||
let older = site.page(&format!("{repo}/commits/main?page=1")); | ||
assert_eq!(older.matches("<li>").count(), 2); | ||
assert!(!older.contains("Older")); | ||
assert_eq!( | ||
site.page(&format!("{repo}/commits/main/src")) | ||
.matches("<li>") | ||
.count(), | ||
1 | ||
); | ||
| ||
assert!( | ||
site.page(&format!("{repo}/commits/main/nowhere")) | ||
.contains("No commits here.") | ||
); | ||
let commit = site.page(&format!("{repo}/commit/{tip}")); | ||
for want in [ | ||
"<h2>Lay out the tree</h2>", | ||
"With a body that says why.", | ||
r#"<span class="path">src/main.rs</span><span class="tag">new</span>"#, | ||
r#"<table class="diff" data-hl="comment:0,8;"#, | ||
r#"<td class="n" data-n="4"></td><td class="c"><code>}</code>"#, | ||
"parent <a class=\"sha\"", | ||
"changed files</b>", | ||
] { | ||
assert!(commit.contains(want), "{want} not in\n{commit}"); | ||
} | ||
let first = s.git(&work, &["rev-list", "--max-parents=0", "HEAD"]); | ||
assert!( | ||
!site | ||
.page(&format!("{repo}/commit/{first}")) | ||
.contains("parent <a") | ||
); | ||
| ||
for missing in [ | ||
format!("{repo}/commit/0000000000000000000000000000000000000000"), | ||
format!("{repo}/tree/main/nope"), | ||
format!("{repo}/blob/main/nope"), | ||
format!("{repo}/raw/main/nope"), | ||
format!("{repo}/commits/nope"), | ||
format!("{repo}/tree/-x"), | ||
format!("{repo}/commit/-x"), | ||
format!("{repo}/wiki/did:plc:x/y"), | ||
format!("/{h}/missing"), | ||
] { | ||
let (status, body) = site.status(&missing); | ||
assert!(status == 404 || status == 502, "{missing}: {status}"); | ||
assert!(body.starts_with("<!doctype html>"), "{missing}"); | ||
} | ||
| ||
// A repository with nothing in it yet. | ||
ok(&s, &["create", &s.url("empty")]); | ||
assert!( | ||
site.page(&format!("/{h}/empty")) | ||
.contains("Nothing pushed yet") | ||
); | ||
} | ||
| ||
#[test] | ||
fn pulls_and_issues() { | ||
let site = serve(true); | ||
let owner = person(account(), &site); | ||
let other = person(account(), &site); | ||
let url = owner.url("tracked"); | ||
ok( | ||
&owner, | ||
&["create", &url, "--description", "Tracks <things>"], | ||
); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
let repo = format!("/{}/tracked", owner.account.handle); | ||
| ||
let fork = ok(&other, &["fork", &url]); | ||
let clone = other.path("clone"); | ||
other.git(other.dir.path(), &["clone", "-q", &fork, "clone"]); | ||
other.commit_file(&clone, "b", "b\n"); | ||
other.git(&clone, &["push", "-q", "origin", "main:topic"]); | ||
let pull = ok( | ||
&other, | ||
&[ | ||
"pull", | ||
&url, | ||
"main", | ||
&fork, | ||
"topic", | ||
"Add b", | ||
"-m", | ||
"Adds **b**.", | ||
], | ||
); | ||
let issue = ok(&other, &["issue", &url, "Broken <thing>"]); | ||
let quiet = ok(&other, &["issue", &url, "Quiet"]); | ||
ok(&owner, &["comment", &url, &pull, "Not yet", "--changes"]); | ||
ok(&other, &["comment", &url, &pull, "Fixed", "--approve"]); | ||
ok(&owner, &["comment", &url, &pull, "Thanks"]); | ||
ok(&owner, &["close", &url, &quiet]); | ||
let merged = ok(&owner, &["merge", &url, &pull]); | ||
| ||
let rkey = |uri: &str| uri.rsplit('/').next().unwrap().to_string(); | ||
let other_did = &other.account.did; | ||
let thread = format!("{repo}/pull/{other_did}/{}", rkey(&pull)); | ||
wait_until(|| site.page(&thread).contains("Merged as")); | ||
let page = site.page(&thread); | ||
for want in [ | ||
r#"<h2 class="item-title">Add b</h2>"#, | ||
"<strong>b</strong>", | ||
"Asked for changes", | ||
"Approved", | ||
"<p>Thanks</p>", | ||
"3 comments", | ||
&format!(">{}</a>", &merged[..7]), | ||
&format!( | ||
r#"<span class="ref">main</span> from <span class="ref">{}/tracked:topic</span>"#, | ||
other.account.handle | ||
), | ||
&format!( | ||
r#"<a href="/{0}"><b>{0}</b></a> wants to merge"#, | ||
other.account.handle | ||
), | ||
] { | ||
assert!(page.contains(want), "{want} not in\n{page}"); | ||
} | ||
| ||
wait_until(|| { | ||
site.page(&format!("{repo}/issues?state=closed")) | ||
.contains(">Quiet</a>") | ||
}); | ||
assert!(site.page(&repo).contains("<p>Tracks <things></p>")); | ||
let open = site.page(&format!("{repo}/issues")); | ||
assert!( | ||
open.contains("Broken <thing>") | ||
&& open.contains("1 Open") | ||
&& open.contains("1 Closed") | ||
&& open.contains(r#"Issues<span class="count">1</span>"#), | ||
"{open}" | ||
); | ||
assert!( | ||
site.page(&format!("{repo}/pulls")) | ||
.contains("No open pulls.") | ||
); | ||
let closed = site.page(&format!("{repo}/pulls?state=closed")); | ||
assert!( | ||
closed.contains(r#"<span class="st merged" title="merged">"#) | ||
&& closed.contains("</svg>3</span>"), | ||
"{closed}" | ||
); | ||
let page = site.page(&format!("{repo}/issue/{other_did}/{}", rkey(&issue))); | ||
assert!( | ||
page.contains(r#"<h2 class="item-title">Broken <thing></h2>"#), | ||
"{page}" | ||
); | ||
assert!( | ||
page.contains("Nothing said yet.") && page.contains("opened this issue on"), | ||
"{page}" | ||
); | ||
assert_eq!( | ||
site.status(&format!("{repo}/issue/{other_did}/nothing")).0, | ||
404 | ||
); | ||
| ||
// The fork says where it came from. | ||
let forked = site.page(&format!("/{}/tracked", other.account.handle)); | ||
assert!( | ||
forked.contains(&format!( | ||
"Forked from <a href=\"/{0}/tracked\">{0}/tracked</a>", | ||
owner.account.handle | ||
)), | ||
"{forked}" | ||
); | ||
| ||
// Each account has a page: its public repositories, newest first, and | ||
// who it is, as its Bluesky profile says. | ||
let theirs = site.page(&format!("/{}", other.account.handle)); | ||
assert!( | ||
theirs.contains(&format!( | ||
r#"Forked from <a href="/{0}/tracked">{0}/tracked</a>"#, | ||
owner.account.handle | ||
)) && theirs.contains(&other.account.did), | ||
"{theirs}" | ||
); | ||
let session: serde_json::Value = Client::new() | ||
.post(format!( | ||
"{}/xrpc/com.atproto.server.createSession", | ||
net().pds | ||
)) | ||
.json(&serde_json::json!({ | ||
"identifier": owner.account.handle, | ||
"password": owner.account.password, | ||
})) | ||
.send() | ||
.unwrap() | ||
.json() | ||
.unwrap(); | ||
let put = Client::new() | ||
.post(format!("{}/xrpc/com.atproto.repo.putRecord", net().pds)) | ||
.bearer_auth(session["accessJwt"].as_str().unwrap()) | ||
.json(&serde_json::json!({ | ||
"repo": owner.account.did, | ||
"collection": "app.bsky.actor.profile", | ||
"rkey": "self", | ||
"record": { | ||
"$type": "app.bsky.actor.profile", | ||
"displayName": "The <Owner>", | ||
"description": "Keeps things tracked.", | ||
}, | ||
})) | ||
.send() | ||
.unwrap(); | ||
assert!(put.status().is_success()); | ||
ok(&owner, &["create", &owner.url("later")]); | ||
let mine = site.page(&format!("/{}", owner.account.did)); | ||
assert!(mine.find(">later</a>").unwrap() < mine.find(">tracked</a>").unwrap()); | ||
for want in [ | ||
"<h1>The <Owner></h1>", | ||
&format!(r#"<p class="who">{}</p>"#, owner.account.handle), | ||
r#"<p class="bio">Keeps things tracked.</p>"#, | ||
&format!(r#"href="/{}/tracked">tracked</a>"#, owner.account.handle), | ||
"Tracks <things>", | ||
r#"<span class="count">2</span>"#, | ||
] { | ||
assert!(mine.contains(want), "{want} not in\n{mine}"); | ||
} | ||
let nobody = person(account(), &site); | ||
assert!( | ||
site.page(&format!("/{}", nobody.account.handle)) | ||
.contains("No public repositories yet.") | ||
); | ||
assert_eq!(site.status("/nobody.invalid").0, 404); | ||
| ||
// Renamed, it keeps its pulls and issues, even through the old name. | ||
// A page asked for by the old name redirects to the same page under the | ||
// new one, and the fork names its source as it is called now. | ||
ok(&owner, &["rename", &url, "tracked2"]); | ||
let moved = format!("/{}/tracked2", owner.account.handle); | ||
wait_until(|| { | ||
site.page(&format!("{moved}/issues")) | ||
.contains("Broken <thing>") | ||
}); | ||
assert!(ok(&owner, &["list", &url]).contains(&format!("pull\tmerged\t{pull}"))); | ||
let direct = Client::builder() | ||
.redirect(reqwest::redirect::Policy::none()) | ||
.build() | ||
.unwrap(); | ||
let old = || { | ||
direct | ||
.get(format!("{}{repo}/pulls?state=closed", site.base)) | ||
.send() | ||
.unwrap() | ||
}; | ||
// Once the firehose has told the server of the rename. | ||
wait_until(|| old().status().as_u16() == 307); | ||
assert_eq!( | ||
old().headers()["location"], | ||
format!("{moved}/pulls?state=closed") | ||
); | ||
assert!(site.page(&thread).contains("Merged as")); | ||
wait_until(|| { | ||
site.page(&format!("/{}/tracked", other.account.handle)) | ||
.contains(&format!( | ||
"Forked from <a href=\"{moved}\">{}/tracked2</a>", | ||
owner.account.handle | ||
)) | ||
}); | ||
| ||
// Without an index, there is no knowing. | ||
let bare = serve(false); | ||
let front = bare.page("/"); | ||
assert!(front.contains("Git that lives in your atproto account.")); | ||
assert!(front.contains("claude mcp add knotless -- knotless-mcp")); | ||
assert!(!front.contains("Find a repository")); | ||
let (status, body) = bare.status(&format!("{repo}/pulls")); | ||
assert_eq!(status, 404); | ||
assert!(body.contains("keeps no index")); | ||
} | ||
crates/knotless/Cargo.tomlnew+28−0
| @@ -0,0 +1,28 @@ | ||
[package] | ||
name = "knotless" | ||
description = "Git hosted natively in an atproto account, on any stock PDS" | ||
version.workspace = true | ||
edition.workspace = true | ||
license.workspace = true | ||
rust-version.workspace = true | ||
| ||
[dependencies] | ||
anyhow.workspace = true | ||
base64.workspace = true | ||
bytes = "1" | ||
futures.workspace = true | ||
humantime.workspace = true | ||
reqwest.workspace = true | ||
serde.workspace = true | ||
serde_json.workspace = true | ||
sha2.workspace = true | ||
k256.workspace = true | ||
p256.workspace = true | ||
rand_core.workspace = true | ||
bs58.workspace = true | ||
tokio.workspace = true | ||
| ||
[dev-dependencies] | ||
axum.workspace = true | ||
tempfile.workspace = true | ||
reqwest = { workspace = true, features = ["blocking"] } | ||
crates/knotless/examples/bench.rsnew+322−0
| @@ -0,0 +1,322 @@ | ||
//! Latency of the operations agents do, against the npm reference PDS, | ||
//! through a proxy that adds a chosen round-trip time. | ||
//! | ||
//! cargo build --release && cargo run --release --example bench -- --rtt 50 | ||
//! | ||
//! Builds nothing itself: it drives the release `git-remote-knotless` and | ||
//! `knotless` binaries through real `git`, the way an agent would. | ||
| ||
#[path = "../tests/common/mod.rs"] | ||
mod common; | ||
| ||
use common::*; | ||
use std::path::Path; | ||
use std::time::{Duration, Instant}; | ||
use tokio::io::{AsyncReadExt, AsyncWriteExt}; | ||
use tokio::net::{TcpListener, TcpStream}; | ||
| ||
/// Forwards to `upstream`, delivering every chunk `rtt / 2` after it was | ||
/// read, in each direction: latency without a bandwidth cap. | ||
async fn delay_proxy(upstream: String, rtt: Duration) -> String { | ||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); | ||
let addr = listener.local_addr().unwrap(); | ||
tokio::spawn(async move { | ||
loop { | ||
let (client, _) = listener.accept().await.unwrap(); | ||
let server = TcpStream::connect(&upstream).await.unwrap(); | ||
client.set_nodelay(true).unwrap(); | ||
server.set_nodelay(true).unwrap(); | ||
let (cr, cw) = client.into_split(); | ||
let (sr, sw) = server.into_split(); | ||
tokio::spawn(pipe(cr, sw, rtt / 2)); | ||
tokio::spawn(pipe(sr, cw, rtt / 2)); | ||
} | ||
}); | ||
format!("http://{addr}") | ||
} | ||
| ||
async fn pipe( | ||
mut from: tokio::net::tcp::OwnedReadHalf, | ||
mut to: tokio::net::tcp::OwnedWriteHalf, | ||
delay: Duration, | ||
) { | ||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<(Instant, Vec<u8>)>(); | ||
tokio::spawn(async move { | ||
while let Some((due, bytes)) = rx.recv().await { | ||
tokio::time::sleep_until(due.into()).await; | ||
if to.write_all(&bytes).await.is_err() { | ||
return; | ||
} | ||
} | ||
let _ = to.shutdown().await; | ||
}); | ||
let mut buf = vec![0u8; 64 * 1024]; | ||
loop { | ||
match from.read(&mut buf).await { | ||
Ok(0) | Err(_) => return, | ||
Ok(n) => { | ||
let _ = tx.send((Instant::now() + delay, buf[..n].to_vec())); | ||
} | ||
} | ||
} | ||
} | ||
| ||
fn time(f: impl FnOnce()) -> Duration { | ||
let start = Instant::now(); | ||
f(); | ||
start.elapsed() | ||
} | ||
| ||
struct Stats(Vec<Duration>); | ||
| ||
impl std::fmt::Display for Stats { | ||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | ||
let mut v = self.0.clone(); | ||
v.sort(); | ||
let ms = |d: Duration| d.as_secs_f64() * 1000.0; | ||
let p = |q: f64| ms(v[((v.len() - 1) as f64 * q).round() as usize]); | ||
write!( | ||
f, | ||
"p50 {:7.1} ms p95 {:7.1} ms n={}", | ||
p(0.5), | ||
p(0.95), | ||
v.len() | ||
) | ||
} | ||
} | ||
| ||
fn main() { | ||
let args: Vec<String> = std::env::args().collect(); | ||
let rtt = args | ||
.iter() | ||
.position(|a| a == "--rtt") | ||
.and_then(|i| args.get(i + 1)) | ||
.map_or(0, |v| v.parse().unwrap()); | ||
let n: usize = args | ||
.iter() | ||
.position(|a| a == "-n") | ||
.and_then(|i| args.get(i + 1)) | ||
.map_or(15, |v| v.parse().unwrap()); | ||
let release = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/release"); | ||
assert!( | ||
release.join("git-remote-knotless").exists(), | ||
"run `cargo build --release` first" | ||
); | ||
// SAFETY: set before any thread that reads the environment starts. | ||
unsafe { std::env::set_var("KNOTLESS_BIN_DIR", &release) }; | ||
| ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let upstream = net().pds.trim_start_matches("http://").to_string(); | ||
let proxy = rt.block_on(delay_proxy(upstream, Duration::from_millis(rtt))); | ||
println!( | ||
"knotless bench: reference PDS @ {} via {} ms RTT proxy\n", | ||
net().pds, | ||
rtt | ||
); | ||
| ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_SERVICE", &proxy); | ||
let handle = s.account.handle.clone(); | ||
let url = s.url("bench"); | ||
| ||
// Warm up: a login, the identity and the repository key are cached | ||
// after the first push, as they are for any agent past its first call. | ||
let work = s.work("work", 1); | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone.to_str().unwrap()], | ||
); | ||
| ||
let mut push = Vec::new(); | ||
let mut fetch = Vec::new(); | ||
for i in 0..n { | ||
s.commit_file(&work, "f", &format!("{i}\n")); | ||
push.push(time(|| { | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
})); | ||
fetch.push(time(|| { | ||
s.git(&clone, &["fetch", "-q", "origin"]); | ||
})); | ||
} | ||
println!("git push, one small commit {}", Stats(push)); | ||
println!("git fetch, one new commit {}", Stats(fetch)); | ||
| ||
let file = s.path("payload.txt"); | ||
let mut commit = Vec::new(); | ||
for i in 0..n { | ||
std::fs::write(&file, format!("{i}\n")).unwrap(); | ||
commit.push(time(|| { | ||
let (ok, _, err) = s.knotless( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"agent edit", | ||
"--put", | ||
"notes.txt", | ||
file.to_str().unwrap(), | ||
], | ||
b"", | ||
); | ||
assert!(ok, "{err}"); | ||
})); | ||
} | ||
println!("knotless commit, no checkout {}", Stats(commit)); | ||
| ||
let mut clones = Vec::new(); | ||
for i in 0..n.min(5) { | ||
let dir = s.path(&format!("c{i}")); | ||
clones.push(time(|| { | ||
s.git(s.dir.path(), &["clone", "-q", &url, dir.to_str().unwrap()]); | ||
})); | ||
} | ||
let packs = rt.block_on(async { | ||
let ctx = ctx(&s); | ||
let r = | ||
knotless::remote::Remote::connect(&ctx, &knotless::url::RepoUrl::parse(&url).unwrap()) | ||
.await | ||
.unwrap(); | ||
r.snapshot(&ctx).await.unwrap().packs.items.len() | ||
}); | ||
println!("git clone, {packs:3} packs {}", Stats(clones)); | ||
let (ok, _, err) = s.knotless(&["compact", &url], b""); | ||
assert!(ok, "{err}"); | ||
let mut clones = Vec::new(); | ||
for i in 0..n.min(5) { | ||
let dir = s.path(&format!("k{i}")); | ||
clones.push(time(|| { | ||
s.git(s.dir.path(), &["clone", "-q", &url, dir.to_str().unwrap()]); | ||
})); | ||
} | ||
println!("git clone, compacted {}", Stats(clones)); | ||
| ||
// A swarm: 16 agents, each pushing its own branch at the same moment. | ||
let agents = 16; | ||
let cmds: Vec<_> = (0..agents) | ||
.map(|i| { | ||
let dir = s.path(&format!("agent{i}")); | ||
s.git(s.dir.path(), &["clone", "-q", &url, dir.to_str().unwrap()]); | ||
s.git(&dir, &["checkout", "-q", "-b", &format!("agent/{i}")]); | ||
s.commit_file(&dir, "w", &format!("{i}\n")); | ||
let mut cmd = s.command("git", &dir); | ||
cmd.args(["push", "-q", "origin", &format!("agent/{i}")]); | ||
cmd | ||
}) | ||
.collect(); | ||
let wall = time(|| { | ||
let threads: Vec<_> = cmds | ||
.into_iter() | ||
.map(|mut c| std::thread::spawn(move || c.output().unwrap())) | ||
.collect(); | ||
for t in threads { | ||
assert!(t.join().unwrap().status.success()); | ||
} | ||
}); | ||
println!( | ||
"{agents} agents, own branches, at once wall {:7.1} ms", | ||
wall.as_secs_f64() * 1000.0 | ||
); | ||
| ||
// The same 16 agents all committing to main, replaying on conflict. | ||
let attempts = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); | ||
let wall = time(|| { | ||
let threads: Vec<_> = (0..agents) | ||
.map(|i| { | ||
let mut cmd = s.command(&release.join("knotless").to_string_lossy(), s.dir.path()); | ||
cmd.args([ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"swarm", | ||
"--put", | ||
&format!("swarm/{i}"), | ||
"-", | ||
"--retries", | ||
"100", | ||
"--json", | ||
]); | ||
cmd.stdin(std::process::Stdio::piped()) | ||
.stdout(std::process::Stdio::piped()) | ||
.stderr(std::process::Stdio::piped()); | ||
let attempts = attempts.clone(); | ||
std::thread::spawn(move || { | ||
let child = cmd.spawn().unwrap(); | ||
let out = child.wait_with_output().unwrap(); | ||
assert!( | ||
out.status.success(), | ||
"{}", | ||
String::from_utf8_lossy(&out.stderr) | ||
); | ||
let v: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); | ||
attempts.fetch_add( | ||
v["attempts"].as_u64().unwrap(), | ||
std::sync::atomic::Ordering::Relaxed, | ||
); | ||
}) | ||
}) | ||
.collect(); | ||
for t in threads { | ||
t.join().unwrap(); | ||
} | ||
}); | ||
println!( | ||
"{agents} agents, one branch, at once wall {:7.1} ms {} pushes for {agents} commits", | ||
wall.as_secs_f64() * 1000.0, | ||
attempts.load(std::sync::atomic::Ordering::Relaxed) | ||
); | ||
| ||
// Reads through knotless-serve: the mirror is local to the reader. | ||
let serve = release.join("knotless-serve"); | ||
if serve.exists() { | ||
let mut child = s | ||
.command(&serve.to_string_lossy(), s.dir.path()) | ||
.args(["--listen", "127.0.0.1:0", "--ttl", "5"]) | ||
.stdout(std::process::Stdio::piped()) | ||
.spawn() | ||
.unwrap(); | ||
let mut line = String::new(); | ||
std::io::BufRead::read_line( | ||
&mut std::io::BufReader::new(child.stdout.take().unwrap()), | ||
&mut line, | ||
) | ||
.unwrap(); | ||
let base = line.trim().trim_start_matches("listening on ").to_string(); | ||
let repo = format!("{base}/{handle}/bench"); | ||
let http = reqwest::blocking::Client::new(); | ||
http.get(format!("{base}/api/{handle}/bench/refs")) | ||
.send() | ||
.unwrap(); | ||
let mut shallow = Vec::new(); | ||
for i in 0..n.min(5) { | ||
let dir = s.path(&format!("s{i}")); | ||
shallow.push(time(|| { | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", "--depth", "1", &repo, dir.to_str().unwrap()], | ||
); | ||
})); | ||
} | ||
println!("clone --depth 1 via knotless-serve {}", Stats(shallow)); | ||
let mut reads = Vec::new(); | ||
for _ in 0..n { | ||
reads.push(time(|| { | ||
let r = http | ||
.get(format!("{base}/api/{handle}/bench/blob?path=notes.txt")) | ||
.send() | ||
.unwrap(); | ||
assert!(r.status().is_success()); | ||
})); | ||
} | ||
println!("API: read one file, warm {}", Stats(reads)); | ||
let _ = std::process::Command::new("kill") | ||
.args(["-TERM", &child.id().to_string()]) | ||
.status(); | ||
let _ = child.wait(); | ||
} | ||
} | ||
crates/knotless/src/auth.rsnew+53−0
| @@ -0,0 +1,53 @@ | ||
//! Service auth: short-lived JWTs an account's PDS signs with the account's | ||
//! key, naming the service they are for (`aud`) and the one method they | ||
//! allow (`lxm`). App passwords can get them, so they are how a push reaches | ||
//! a gateway on a collaborator's behalf. | ||
| ||
use crate::crypto::{self, PublicKey}; | ||
use crate::env::Ctx; | ||
use crate::identity; | ||
use crate::session::Session; | ||
use crate::xrpc::Xrpc; | ||
use anyhow::{Result, bail}; | ||
use serde::Deserialize; | ||
| ||
/// A token from `me`'s PDS for calling `lxm` on the service `aud`. | ||
pub async fn service_token(ctx: &Ctx, me: &Session, aud: &str, lxm: &str) -> Result<String> { | ||
#[derive(Deserialize)] | ||
struct Out { | ||
token: String, | ||
} | ||
let out: Out = Xrpc::new(ctx.http.clone(), &me.service) | ||
.query( | ||
"com.atproto.server.getServiceAuth", | ||
&[("aud", aud), ("lxm", lxm)], | ||
Some(&me.access_jwt), | ||
) | ||
.await?; | ||
Ok(out.token) | ||
} | ||
| ||
#[derive(Deserialize)] | ||
struct Claims { | ||
iss: String, | ||
aud: String, | ||
lxm: Option<String>, | ||
} | ||
| ||
/// Checks a service token for calling `lxm` on `aud`, and returns the DID | ||
/// that sent it. The signature is checked against the key the issuer's DID | ||
/// document names now. | ||
pub async fn verify(ctx: &Ctx, token: &str, aud: &str, lxm: &str) -> Result<String> { | ||
let (_, claims) = crypto::jwt_parts(token)?; | ||
let claims: Claims = serde_json::from_value(claims)?; | ||
let did = claims.iss.split('#').next().unwrap_or_default().to_string(); | ||
if claims.aud != aud { | ||
bail!("a token for {}, not {aud}", claims.aud); | ||
} | ||
if claims.lxm.as_deref() != Some(lxm) { | ||
bail!("a token for {:?}, not {lxm}", claims.lxm); | ||
} | ||
let key: PublicKey = identity::signing_key(&identity::document(ctx, &did).await?)?; | ||
crypto::verify_jwt::<serde_json::Value>(token, &key)?; | ||
Ok(did) | ||
} | ||
crates/knotless/src/base32.rsnew+81−0
| @@ -0,0 +1,81 @@ | ||
//! Unpadded base32 over an arbitrary 32-symbol alphabet. | ||
| ||
/// The sortable alphabet atproto uses for TIDs. | ||
pub const SORTABLE: &[u8; 32] = b"234567abcdefghijklmnopqrstuvwxyz"; | ||
/// RFC 4648 base32, lowercased. | ||
pub const LOWER: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; | ||
| ||
/// Encodes `bytes` most significant bit first, without padding. | ||
pub fn encode(alphabet: &[u8; 32], bytes: &[u8]) -> String { | ||
let mut out = String::with_capacity(bytes.len() * 8 / 5 + 1); | ||
let mut buffer: u32 = 0; | ||
let mut bits = 0; | ||
for &byte in bytes { | ||
buffer = (buffer << 8) | u32::from(byte); | ||
bits += 8; | ||
while bits >= 5 { | ||
bits -= 5; | ||
out.push(alphabet[((buffer >> bits) & 31) as usize] as char); | ||
} | ||
} | ||
if bits > 0 { | ||
out.push(alphabet[((buffer << (5 - bits)) & 31) as usize] as char); | ||
} | ||
out | ||
} | ||
| ||
/// Encodes a u64 as 13 symbols, so encodings sort the way the integers do. | ||
pub fn encode_u64(alphabet: &[u8; 32], value: u64) -> String { | ||
(0..13) | ||
.map(|i: u32| alphabet[((value >> (60 - 5 * i)) & 31) as usize] as char) | ||
.collect() | ||
} | ||
| ||
/// Inverts [`encode_u64`]. Returns `None` for anything that is not 13 symbols | ||
/// of `alphabet` or does not fit a u64. | ||
pub fn decode_u64(alphabet: &[u8; 32], text: &str) -> Option<u64> { | ||
if text.len() != 13 { | ||
return None; | ||
} | ||
let mut value: u64 = 0; | ||
for (i, c) in text.bytes().enumerate() { | ||
let symbol = alphabet.iter().position(|&a| a == c)? as u64; | ||
if i == 0 && symbol > 15 { | ||
return None; | ||
} | ||
value = (value << 5) | symbol; | ||
} | ||
Some(value) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn rfc4648_vectors() { | ||
assert_eq!(encode(LOWER, b""), ""); | ||
assert_eq!(encode(LOWER, b"f"), "my"); | ||
assert_eq!(encode(LOWER, b"fo"), "mzxq"); | ||
assert_eq!(encode(LOWER, b"foo"), "mzxw6"); | ||
assert_eq!(encode(LOWER, b"foobar"), "mzxw6ytboi"); | ||
} | ||
| ||
#[test] | ||
fn u64_round_trips_and_sorts() { | ||
for v in [0u64, 1, 31, 32, 1 << 40, u64::MAX >> 1, u64::MAX] { | ||
let s = encode_u64(SORTABLE, v); | ||
assert_eq!(s.len(), 13); | ||
assert_eq!(decode_u64(SORTABLE, &s), Some(v)); | ||
} | ||
assert!(encode_u64(SORTABLE, 5) < encode_u64(SORTABLE, 6)); | ||
assert!(encode_u64(SORTABLE, 1 << 50) > encode_u64(SORTABLE, (1 << 50) - 1)); | ||
} | ||
| ||
#[test] | ||
fn decode_rejects_garbage() { | ||
assert_eq!(decode_u64(SORTABLE, "short"), None); | ||
assert_eq!(decode_u64(SORTABLE, "1111111111111"), None); | ||
assert_eq!(decode_u64(SORTABLE, "z222222222222"), None); | ||
} | ||
} | ||
crates/knotless/src/bin/git-remote-knotless.rsnew+3−0
| @@ -0,0 +1,3 @@ | ||
fn main() { | ||
std::process::exit(knotless::cli::helper_main(std::env::args().collect())); | ||
} | ||
crates/knotless/src/bin/knotless.rsnew+3−0
| @@ -0,0 +1,3 @@ | ||
fn main() { | ||
std::process::exit(knotless::cli::main(std::env::args().collect())); | ||
} | ||
crates/knotless/src/cli.rsnew+549−0
| @@ -0,0 +1,549 @@ | ||
//! The two binaries: `git-remote-knotless` and `knotless`. | ||
| ||
use crate::commit::{self, Change, Spec}; | ||
use crate::compact; | ||
use crate::env::{Ctx, Env}; | ||
use crate::forge; | ||
use crate::git::Git; | ||
use crate::helper::{self, Knotless}; | ||
use crate::manage; | ||
use crate::moderation; | ||
use crate::remote::Remote; | ||
use crate::session; | ||
use crate::url::RepoUrl; | ||
use anyhow::{Context, Result, bail}; | ||
use serde_json::json; | ||
use std::io::Write; | ||
| ||
fn runtime() -> tokio::runtime::Runtime { | ||
tokio::runtime::Builder::new_current_thread() | ||
.enable_all() | ||
.build() | ||
.expect("a current-thread runtime always builds") | ||
} | ||
| ||
fn report(name: &str, res: Result<()>) -> i32 { | ||
match res { | ||
Ok(()) => 0, | ||
Err(err) => { | ||
eprintln!("{name}: {err:#}"); | ||
1 | ||
} | ||
} | ||
} | ||
| ||
/// `git-remote-knotless <remote> <url>`, run by git. | ||
pub fn helper_main(args: Vec<String>) -> i32 { | ||
let res = runtime().block_on(async { | ||
let Some(url) = args.get(2).or(args.get(1)) else { | ||
bail!("usage: git-remote-knotless <remote> <url>"); | ||
}; | ||
let url = RepoUrl::parse(url)?; | ||
let ctx = Ctx::new(Env::from_process()); | ||
let mut backend = Knotless::new(ctx, &url, Git::from_env()).await?; | ||
// git passes the URL twice for a URL given on the command line. | ||
if let [_, remote, given] = &args[..] | ||
&& remote != given | ||
{ | ||
let did = backend.remote.did().to_string(); | ||
helper::pin(&backend.git, remote, &url, &did).await?; | ||
} | ||
let stdin = tokio::io::BufReader::new(tokio::io::stdin()); | ||
helper::run(&mut backend, stdin, tokio::io::stdout()).await | ||
}); | ||
report("git-remote-knotless", res) | ||
} | ||
| ||
const USAGE: &str = "usage: knotless <command> | ||
| ||
login <handle> [--password-stdin] store a session (else KNOTLESS_PASSWORD) | ||
logout <handle> | ||
create <url> [--private] [--description <text>] [--default-branch <name>] | ||
delegate <handle> <gateway did> let a gateway write for you, so | ||
collaborators can push through it | ||
share <url> <handle> [--write] let an account read, or push too | ||
unshare <url> <handle> | ||
members <url> who may read and push | ||
ls <handle> list an account's repositories | ||
refs <url> print refs, one round trip | ||
commit <url> -b <branch> -m <message> [--put <path> <file>]... | ||
[--put-exec <path> <file>]... [--delete <path>]... | ||
[--author <name> <email>] [--base <commit>] [--batch] | ||
[--retries <n>] [--json] commit without a checkout ('-' reads stdin); | ||
--batch goes through the gateway as owner too | ||
fork <url> [<name>] copy a repository into your account | ||
pull <url> <base> <head url> <branch> <title> [-m <body>] | ||
ask to merge a branch into <base> | ||
issue <url> <title> [-m <body>] | ||
list <url> [--pulls | --issues] pulls and issues, newest first | ||
show <url> <at-uri> a pull or issue and its comments | ||
comment <url> <at-uri> <text> [--approve | --changes] | ||
close <url> <at-uri> | ||
reopen <url> <at-uri> | ||
retarget <url> <at-uri> [--base <branch>] [--head <branch>] | ||
point a pull at other branches; the head | ||
only by the pull's author | ||
merge <url> <at-uri> merge a pull: fast-forward or merge commit | ||
hide <url> <at-uri> as owner, hide a pull, issue or comment | ||
unhide <url> <at-uri> | ||
block <handle> [<url>] leave out what an account writes about | ||
unblock <handle> [<url>] your repositories, or about one | ||
report <handle, url or at-uri> <reason> [-m <text>] | ||
report to the moderation service; reasons: | ||
spam violation misleading sexual rude other | ||
admin takedown <handle, url or at-uri> [--undo] | ||
admin status <handle, url or at-uri> | ||
admin reports [<limit>] moderate the appview, with | ||
KNOTLESS_ADMIN_PASSWORD | ||
rename <url> <new-name> | ||
transfer <url> <handle or url> offer a repository to another account | ||
transfer <url> --cancel | ||
accept <url> take up a repository offered to you: | ||
it moves, pulls and issues included | ||
delete <url> --yes delete a repository and everything in it | ||
compact <url> fold packs into one | ||
mirror <url> sync the cache mirror, print its path | ||
| ||
URLs look like knotless://<handle or did>/<name>. Pulls and issues on public | ||
repositories come from the appview KNOTLESS_APPVIEW names."; | ||
| ||
/// `knotless ...`. | ||
pub fn main(args: Vec<String>) -> i32 { | ||
main_with( | ||
args, | ||
Env::from_process(), | ||
&mut std::io::stdin().lock(), | ||
&mut std::io::stdout(), | ||
) | ||
} | ||
| ||
pub fn main_with( | ||
args: Vec<String>, | ||
env: Env, | ||
stdin: &mut dyn std::io::Read, | ||
out: &mut dyn Write, | ||
) -> i32 { | ||
let ctx = Ctx::new(env); | ||
let res = runtime().block_on(command(&ctx, &args[1.min(args.len())..], stdin, out)); | ||
report("knotless", res) | ||
} | ||
| ||
struct Args<'a> { | ||
rest: std::slice::Iter<'a, String>, | ||
} | ||
| ||
impl<'a> Args<'a> { | ||
fn next(&mut self, what: &str) -> Result<&'a str> { | ||
self.rest | ||
.next() | ||
.map(String::as_str) | ||
.with_context(|| format!("missing {what}\n\n{USAGE}")) | ||
} | ||
| ||
fn url(&mut self) -> Result<RepoUrl> { | ||
RepoUrl::parse(self.next("<url>")?) | ||
} | ||
} | ||
| ||
/// An optional `-m <body>`. | ||
fn body<'a>(args: &mut Args<'a>) -> Result<Option<&'a str>> { | ||
match args.rest.next().map(String::as_str) { | ||
Some("-m") => Ok(Some(args.next("<body>")?)), | ||
Some(other) => bail!("unexpected {other}"), | ||
None => Ok(None), | ||
} | ||
} | ||
| ||
async fn command( | ||
ctx: &Ctx, | ||
argv: &[String], | ||
stdin: &mut dyn std::io::Read, | ||
out: &mut dyn Write, | ||
) -> Result<()> { | ||
let mut args = Args { rest: argv.iter() }; | ||
let Some(cmd) = args.rest.next() else { | ||
bail!("{USAGE}"); | ||
}; | ||
match cmd.as_str() { | ||
"login" => { | ||
let identifier = args.next("<handle>")?; | ||
let password = match args.rest.next().map(String::as_str) { | ||
Some("--password-stdin") => { | ||
let mut text = String::new(); | ||
stdin.read_to_string(&mut text)?; | ||
text.trim().to_string() | ||
} | ||
Some(other) => bail!("unexpected {other}"), | ||
None => ctx | ||
.env | ||
.password | ||
.clone() | ||
.context("pass --password-stdin or set KNOTLESS_PASSWORD")?, | ||
}; | ||
let s = session::login(ctx, identifier, &password).await?; | ||
session::remember(ctx, &s)?; | ||
writeln!(out, "logged in as {} ({})", s.handle, s.did)?; | ||
} | ||
"logout" => { | ||
let identifier = args.next("<handle>")?; | ||
let removed = session::logout(ctx, identifier).await?; | ||
writeln!(out, "{}", if removed { "logged out" } else { "no session" })?; | ||
} | ||
"create" => { | ||
let url = args.url()?; | ||
let mut description = None; | ||
let mut default_branch = None; | ||
let mut private = false; | ||
while let Some(flag) = args.rest.next() { | ||
match flag.as_str() { | ||
"--private" => private = true, | ||
"--description" => description = Some(args.next("<text>")?.to_string()), | ||
"--default-branch" => default_branch = Some(args.next("<name>")?.to_string()), | ||
other => bail!("unexpected {other}"), | ||
} | ||
} | ||
let new = manage::NewRepo { | ||
private, | ||
description, | ||
default_branch, | ||
..Default::default() | ||
}; | ||
manage::create(ctx, &url, &new).await?; | ||
writeln!(out, "created {url}")?; | ||
} | ||
"ls" => { | ||
let authority = args.next("<handle>")?; | ||
let remote = Remote::connect( | ||
ctx, | ||
&RepoUrl { | ||
authority: authority.into(), | ||
name: "_".into(), | ||
}, | ||
) | ||
.await?; | ||
for repo in remote.list_repos().await? { | ||
let v = repo.value; | ||
writeln!(out, "{}\t{}", v.name, v.description.unwrap_or_default())?; | ||
} | ||
} | ||
"delegate" => { | ||
let id = crate::identity::resolve(ctx, args.next("<handle>")?).await?; | ||
let gateway = args.next("<gateway did>")?; | ||
let me = session::ensure(ctx, &id).await?; | ||
crate::gateway::delegate(ctx, &me, gateway).await?; | ||
writeln!(out, "delegated to {gateway}")?; | ||
} | ||
"share" | "unshare" => { | ||
let url = args.url()?; | ||
let who = args.next("<handle>")?; | ||
let write = match args.rest.next().map(String::as_str) { | ||
Some("--write") if cmd == "share" => true, | ||
Some(other) => bail!("unexpected {other}"), | ||
None => false, | ||
}; | ||
match cmd.as_str() { | ||
"share" => manage::share(ctx, &url, who, write).await?, | ||
_ => manage::unshare(ctx, &url, who).await?, | ||
} | ||
let done = if cmd == "share" { "shared" } else { "unshared" }; | ||
writeln!(out, "{done}")?; | ||
} | ||
"members" => { | ||
let url = args.url()?; | ||
for m in manage::members(ctx, &url).await? { | ||
writeln!(out, "{}\t{}", m.did, m.access)?; | ||
} | ||
} | ||
"refs" => { | ||
let url = args.url()?; | ||
let (_, snap) = Remote::locate(ctx, &url).await?; | ||
if snap.repo.is_none() { | ||
bail!("{url}: no such repository"); | ||
} | ||
for (name, sha) in snap.live_refs() { | ||
writeln!(out, "{sha}\t{name}")?; | ||
} | ||
} | ||
"commit" => { | ||
let url = args.url()?; | ||
let mut spec = Spec { | ||
branch: String::new(), | ||
message: String::new(), | ||
changes: Vec::new(), | ||
author: None, | ||
base: None, | ||
retries: 5, | ||
via_gateway: false, | ||
}; | ||
let mut as_json = false; | ||
while let Some(flag) = args.rest.next() { | ||
match flag.as_str() { | ||
"-b" | "--branch" => spec.branch = args.next("<branch>")?.into(), | ||
"-m" | "--message" => spec.message = args.next("<message>")?.into(), | ||
"--put" | "--put-exec" => { | ||
let path = args.next("<path>")?.to_string(); | ||
let file = args.next("<file>")?; | ||
let data = if file == "-" { | ||
let mut data = Vec::new(); | ||
stdin.read_to_end(&mut data)?; | ||
data | ||
} else { | ||
std::fs::read(file).with_context(|| format!("reading {file}"))? | ||
}; | ||
spec.changes.push(Change::Put { | ||
path, | ||
data, | ||
executable: flag == "--put-exec", | ||
}); | ||
} | ||
"--delete" => spec.changes.push(Change::Delete { | ||
path: args.next("<path>")?.into(), | ||
}), | ||
"--author" => { | ||
let name = args.next("<name>")?.to_string(); | ||
spec.author = Some((name, args.next("<email>")?.to_string())); | ||
} | ||
"--retries" => { | ||
spec.retries = args | ||
.next("<n>")? | ||
.parse() | ||
.context("--retries takes a number")? | ||
} | ||
"--json" => as_json = true, | ||
"--base" => spec.base = Some(args.next("<commit>")?.into()), | ||
"--batch" => spec.via_gateway = true, | ||
other => bail!("unexpected {other}"), | ||
} | ||
} | ||
if spec.branch.is_empty() || spec.message.is_empty() { | ||
bail!("commit needs -b <branch> and -m <message>"); | ||
} | ||
let c = commit::commit(ctx, &url, &spec).await?; | ||
if as_json { | ||
let v = json!({"sha": c.sha, "parent": c.parent, "attempts": c.attempts, "changed": c.changed}); | ||
writeln!(out, "{v}")?; | ||
} else { | ||
writeln!(out, "{}", c.sha)?; | ||
} | ||
} | ||
"fork" => { | ||
let url = args.url()?; | ||
let name = args.rest.next().map(String::as_str); | ||
let fork = forge::fork(ctx, &url, name).await?; | ||
writeln!(out, "{fork}")?; | ||
} | ||
"pull" | "issue" => { | ||
let url = args.url()?; | ||
let uri = if cmd == "pull" { | ||
let base = args.next("<base>")?; | ||
let head = RepoUrl::parse(args.next("<head url>")?)?; | ||
let branch = args.next("<branch>")?; | ||
let title = args.next("<title>")?; | ||
let body = body(&mut args)?; | ||
forge::open_pull(ctx, &url, base, &head, branch, title, body).await? | ||
} else { | ||
let title = args.next("<title>")?; | ||
let body = body(&mut args)?; | ||
forge::open_issue(ctx, &url, title, body).await? | ||
}; | ||
writeln!(out, "{uri}")?; | ||
} | ||
"list" => { | ||
let url = args.url()?; | ||
let only = match args.rest.next().map(String::as_str) { | ||
Some("--pulls") => Some("pull"), | ||
Some("--issues") => Some("issue"), | ||
Some(other) => bail!("unexpected {other}"), | ||
None => None, | ||
}; | ||
for item in forge::list(ctx, &url).await? { | ||
if only.is_none_or(|k| k == item.kind) { | ||
let (kind, state) = (&item.kind, &item.state); | ||
writeln!(out, "{kind}\t{state}\t{}\t{}", item.uri, item.title)?; | ||
} | ||
} | ||
} | ||
"show" => { | ||
let url = args.url()?; | ||
let t = forge::show(ctx, &url, args.next("<at-uri>")?).await?; | ||
let i = &t.item; | ||
writeln!(out, "{} {} by {}: {}", i.state, i.kind, i.author, i.title)?; | ||
if let Some(body) = &t.item.body { | ||
writeln!(out, "\n{body}")?; | ||
} | ||
for c in &t.comments { | ||
let verdict = c.value["verdict"] | ||
.as_str() | ||
.map(|v| format!(" [{v}]")) | ||
.unwrap_or_default(); | ||
let body = c.value["body"].as_str().unwrap_or_default(); | ||
writeln!(out, "\n{}{verdict}: {body}", c.author)?; | ||
} | ||
} | ||
"comment" => { | ||
let url = args.url()?; | ||
let subject = args.next("<at-uri>")?; | ||
let text = args.next("<text>")?; | ||
let verdict = match args.rest.next().map(String::as_str) { | ||
Some("--approve") => Some("approve"), | ||
Some("--changes") => Some("changes"), | ||
Some(other) => bail!("unexpected {other}"), | ||
None => None, | ||
}; | ||
let uri = forge::comment(ctx, &url, subject, text, verdict).await?; | ||
writeln!(out, "{uri}")?; | ||
} | ||
"close" | "reopen" => { | ||
let url = args.url()?; | ||
let subject = args.next("<at-uri>")?; | ||
let state = if cmd == "close" { "closed" } else { "open" }; | ||
forge::set_state(ctx, &url, subject, state).await?; | ||
writeln!(out, "{state}")?; | ||
} | ||
"retarget" => { | ||
let url = args.url()?; | ||
let pull = args.next("<at-uri>")?.to_string(); | ||
let (mut base, mut head) = (None, None); | ||
while let Some(flag) = args.rest.next() { | ||
let slot = match flag.as_str() { | ||
"--base" => &mut base, | ||
"--head" => &mut head, | ||
other => bail!("unknown option {other}"), | ||
}; | ||
*slot = Some(args.next("<branch>")?.to_string()); | ||
} | ||
if base.is_none() && head.is_none() { | ||
bail!("retarget takes --base <branch>, --head <branch> or both"); | ||
} | ||
forge::retarget(ctx, &url, &pull, base.as_deref(), head.as_deref()).await?; | ||
writeln!(out, "retargeted")?; | ||
} | ||
"hide" | "unhide" => { | ||
let url = args.url()?; | ||
let subject = args.next("<at-uri>")?; | ||
let hide = cmd == "hide"; | ||
forge::hide(ctx, &url, subject, hide).await?; | ||
writeln!(out, "{}", if hide { "hidden" } else { "shown" })?; | ||
} | ||
"block" | "unblock" => { | ||
let who = args.next("<handle>")?; | ||
let repo = args.rest.next().map(|u| RepoUrl::parse(u)).transpose()?; | ||
let did = moderation::block(ctx, who, repo.as_ref(), cmd == "block").await?; | ||
writeln!(out, "{cmd}ed {did}")?; | ||
} | ||
"report" => { | ||
let target = args.next("<handle, url or at-uri>")?; | ||
let reason = args.next("<reason>")?; | ||
let text = body(&mut args)?; | ||
let made = moderation::report(ctx, target, reason, text).await?; | ||
writeln!(out, "report {} sent", made["id"])?; | ||
} | ||
"admin" => { | ||
let admin = moderation::Admin::new(ctx)?; | ||
match args.next("takedown, status or reports")? { | ||
"takedown" => { | ||
let target = args.next("<handle, url or at-uri>")?; | ||
let applied = match args.rest.next().map(String::as_str) { | ||
Some("--undo") => false, | ||
Some(other) => bail!("unexpected {other}"), | ||
None => true, | ||
}; | ||
admin.takedown(ctx, target, applied).await?; | ||
writeln!(out, "{}", if applied { "taken down" } else { "restored" })?; | ||
} | ||
"status" => { | ||
let status = admin | ||
.status(ctx, args.next("<handle, url or at-uri>")?) | ||
.await?; | ||
let down = status["takedown"]["applied"].as_bool() == Some(true); | ||
writeln!(out, "{}", if down { "taken down" } else { "up" })?; | ||
} | ||
"reports" => { | ||
let limit = match args.rest.next() { | ||
Some(n) => n.parse().context("a limit is a number")?, | ||
None => 50, | ||
}; | ||
for r in admin.reports(limit).await? { | ||
let subject = r["subject"]["uri"] | ||
.as_str() | ||
.or(r["subject"]["did"].as_str()); | ||
let reason = r["reasonType"].as_str().unwrap_or_default(); | ||
let reason = reason.rsplit("#reason").next().unwrap_or_default(); | ||
let line = format!( | ||
"{}\t{}\t{}\t{}\t{}", | ||
r["id"], | ||
r["reportedBy"].as_str().unwrap_or_default(), | ||
reason.to_lowercase(), | ||
subject.unwrap_or_default(), | ||
r["reason"].as_str().unwrap_or_default() | ||
); | ||
writeln!(out, "{line}")?; | ||
} | ||
} | ||
other => bail!("unknown admin command {other}\n\n{USAGE}"), | ||
} | ||
} | ||
"merge" => { | ||
let url = args.url()?; | ||
let sha = forge::merge(ctx, &url, args.next("<at-uri>")?).await?; | ||
writeln!(out, "{sha}")?; | ||
} | ||
"rename" => { | ||
let url = args.url()?; | ||
let to = manage::rename(ctx, &url, args.next("<new-name>")?).await?; | ||
writeln!(out, "renamed to {to}")?; | ||
} | ||
"transfer" => { | ||
let url = args.url()?; | ||
let to = args.next("<handle or url>")?; | ||
if to == "--cancel" { | ||
manage::transfer(ctx, &url, None).await?; | ||
writeln!(out, "offer withdrawn")?; | ||
} else { | ||
// A handle alone keeps the name. | ||
let to = match to.strip_prefix(crate::url::SCHEME) { | ||
Some(_) => RepoUrl::parse(to)?, | ||
None => RepoUrl::parse(&format!("{}{to}/{}", crate::url::SCHEME, url.name))?, | ||
}; | ||
manage::transfer(ctx, &url, Some(&to)).await?; | ||
let how = format!("it moves when they run `knotless accept {url}`"); | ||
writeln!(out, "offered to {to}: {how}")?; | ||
} | ||
} | ||
"accept" => { | ||
let url = args.url()?; | ||
let to = forge::accept(ctx, &url).await?; | ||
writeln!(out, "moved to {to}")?; | ||
} | ||
"delete" => { | ||
let url = args.url()?; | ||
if args.rest.next().map(String::as_str) != Some("--yes") { | ||
bail!("deleting {url} removes all of its history: pass --yes"); | ||
} | ||
manage::delete(ctx, &url).await?; | ||
writeln!(out, "deleted {url}")?; | ||
} | ||
"compact" => { | ||
let url = args.url()?; | ||
let remote = Remote::open(ctx, &url).await?; | ||
let auth = session::ensure(ctx, &remote.identity).await?; | ||
let c = compact::compact(ctx, &remote, &auth).await?; | ||
let (packs, folded, claims) = (c.before, c.removed, c.claims); | ||
let line = format!("{packs} packs, {folded} folded, {claims} old claims dropped"); | ||
writeln!(out, "{line}")?; | ||
} | ||
"mirror" => { | ||
let url = args.url()?; | ||
let (remote, snap) = Remote::locate(ctx, &url).await?; | ||
let Some(key) = snap.key() else { | ||
bail!("{url}: no such repository"); | ||
}; | ||
let mirror = crate::mirror::Mirror::open(ctx, remote.did(), key).await?; | ||
mirror.sync(&remote, &snap).await?; | ||
writeln!(out, "{}", mirror.git.dir.display())?; | ||
} | ||
"help" | "--help" | "-h" => writeln!(out, "{USAGE}")?, | ||
other => bail!("unknown command {other}\n\n{USAGE}"), | ||
} | ||
Ok(()) | ||
} | ||
crates/knotless/src/commit.rsnew+253−0
| @@ -0,0 +1,253 @@ | ||
//! Commits without a checkout. | ||
//! | ||
//! An agent that wants to change three files should not need a working copy | ||
//! of the repository. A commit here is built in the cache mirror from the | ||
//! branch tip and a list of changes, then pushed. When the branch moved in | ||
//! between, the same changes are applied to the new tip and pushed again: | ||
//! put and delete are defined by path, so replaying them is a rebase that | ||
//! cannot conflict. | ||
| ||
use crate::env::Ctx; | ||
use crate::gateway; | ||
use crate::git::Git; | ||
use crate::mirror::Mirror; | ||
use crate::push::{self, Update}; | ||
use crate::read::BadInput; | ||
use crate::remote::Remote; | ||
use crate::session; | ||
use crate::url::RepoUrl; | ||
use anyhow::{Result, bail}; | ||
use std::path::PathBuf; | ||
| ||
#[derive(Clone, Debug, PartialEq, Eq)] | ||
pub enum Change { | ||
Put { | ||
path: String, | ||
data: Vec<u8>, | ||
executable: bool, | ||
}, | ||
Delete { | ||
path: String, | ||
}, | ||
} | ||
| ||
#[derive(Clone, Debug, Default)] | ||
pub struct Spec { | ||
pub branch: String, | ||
pub message: String, | ||
pub changes: Vec<Change>, | ||
pub author: Option<(String, String)>, | ||
/// The commit the changes were made against. When the branch has moved | ||
/// on from it, a path it changed that these changes touch too is a | ||
/// conflict, where it would otherwise be overwritten. | ||
pub base: Option<String>, | ||
/// How many times to replay onto a branch that moved. | ||
pub retries: u32, | ||
/// Commit through the repository's gateway even as its owner, to be | ||
/// batched with other commits to the branch. | ||
pub via_gateway: bool, | ||
} | ||
| ||
#[derive(Clone, Debug, PartialEq, Eq)] | ||
pub struct Committed { | ||
pub sha: String, | ||
pub parent: Option<String>, | ||
/// Pushes it took, counting the one that landed. | ||
pub attempts: u32, | ||
/// False when the changes left the tree as it was and nothing was pushed. | ||
pub changed: bool, | ||
} | ||
| ||
pub async fn commit(ctx: &Ctx, url: &RepoUrl, spec: &Spec) -> Result<Committed> { | ||
let remote = Remote::locate(ctx, url).await?.0; | ||
if let Some(me) = session::reader(ctx, &remote.identity).await | ||
&& (me.did != remote.did() || spec.via_gateway) | ||
{ | ||
return gateway::commit(ctx, &remote, &me, spec).await; | ||
} | ||
let session = session::ensure(ctx, &remote.identity).await?; | ||
let refname = format!("refs/heads/{}", spec.branch); | ||
let mut attempts = 0; | ||
loop { | ||
attempts += 1; | ||
let snap = remote.snapshot(ctx).await?; | ||
let (git, _scratch) = match snap.key() { | ||
Some(key) => { | ||
let mirror = Mirror::open(ctx, remote.did(), key).await?; | ||
mirror.sync(&remote, &snap).await?; | ||
(mirror.git, None) | ||
} | ||
// A repository that does not exist yet starts from nothing, in a | ||
// scratch repository its first push mints the key for. | ||
None => { | ||
let dir = ctx.env.cache_dir.join("tmp").join(crate::keys::tid()); | ||
(Git::init_bare(&dir).await?, Some(Scratch(dir))) | ||
} | ||
}; | ||
let parent = snap.find_ref(&refname).and_then(|r| r.value.target.clone()); | ||
conflicts(&git, spec, parent.as_deref()).await?; | ||
let built = build(ctx, &git, parent.as_deref(), spec, remote.did()).await?; | ||
let Some(sha) = built else { | ||
return Ok(Committed { | ||
sha: parent.clone().unwrap_or_default(), | ||
parent, | ||
attempts, | ||
changed: false, | ||
}); | ||
}; | ||
// The claim pins the generation the parent was read at, so the | ||
// fast-forward check git would make is already guaranteed. | ||
let update = Update { | ||
src: Some(sha.clone()), | ||
dst: refname.clone(), | ||
force: true, | ||
}; | ||
let report = push::push(ctx, &remote, &snap, &git, &session.access_jwt, &[update]).await?; | ||
match &report[0].1 { | ||
Ok(()) => { | ||
return Ok(Committed { | ||
sha, | ||
parent, | ||
attempts, | ||
changed: true, | ||
}); | ||
} | ||
Err(reason) if reason == "fetch first" && attempts <= spec.retries => { | ||
backoff(attempts).await | ||
} | ||
Err(reason) => bail!("{refname}: {reason}"), | ||
} | ||
} | ||
} | ||
| ||
/// Fails when `spec` was made against a base the branch has moved on from, | ||
/// and touches paths changed since. | ||
pub async fn conflicts(git: &Git, spec: &Spec, tip: Option<&str>) -> Result<()> { | ||
let (Some(base), Some(tip)) = (&spec.base, tip) else { | ||
return Ok(()); | ||
}; | ||
let stale = gateway::stale_paths(git, base, tip, &spec.changes).await?; | ||
if !stale.is_empty() { | ||
bail!("conflict: {} changed since {base}", stale.join(", ")); | ||
} | ||
Ok(()) | ||
} | ||
| ||
/// A pause before replaying, random so that the writers that lost a round | ||
/// do not all arrive together for the next one, and longer the more rounds | ||
/// a writer has lost. | ||
async fn backoff(attempts: u32) { | ||
let ceiling = 20u64 << attempts.min(6); | ||
let jitter = crate::keys::now_micros() % 1000 * ceiling / 1000; | ||
tokio::time::sleep(std::time::Duration::from_millis(jitter)).await; | ||
} | ||
| ||
struct Scratch(PathBuf); | ||
| ||
impl Drop for Scratch { | ||
fn drop(&mut self) { | ||
let _ = std::fs::remove_dir_all(&self.0); | ||
} | ||
} | ||
| ||
/// Whether git would check `path` out: relative, with no empty, `.` or | ||
/// `..` segments, and nothing that names a `.git` directory, which git | ||
/// itself refuses to write into a working tree. | ||
pub fn valid_path(path: &str) -> bool { | ||
!path.is_empty() | ||
&& !path.contains(['\0', '\n']) | ||
&& path.split('/').all(|seg| { | ||
!seg.is_empty() && seg != "." && seg != ".." && !seg.eq_ignore_ascii_case(".git") | ||
}) | ||
} | ||
| ||
/// Writes the commit `spec` makes on `parent`. `None` when the tree comes | ||
/// out unchanged. | ||
pub async fn build( | ||
ctx: &Ctx, | ||
git: &Git, | ||
parent: Option<&str>, | ||
spec: &Spec, | ||
did: &str, | ||
) -> Result<Option<String>> { | ||
let index = ctx | ||
.env | ||
.cache_dir | ||
.join("tmp") | ||
.join(format!("index-{}", crate::keys::tid())); | ||
std::fs::create_dir_all(index.parent().expect("the index path has a parent"))?; | ||
let _cleanup = Scratch(index.clone()); | ||
let mut git = git.clone(); | ||
git.envs | ||
.push(("GIT_INDEX_FILE".into(), index.to_string_lossy().into())); | ||
match parent { | ||
Some(p) => git.run(&["read-tree", p], None).await?, | ||
None => git.run(&["read-tree", "--empty"], None).await?, | ||
}; | ||
let mut info = String::new(); | ||
for change in &spec.changes { | ||
let (Change::Put { path, .. } | Change::Delete { path }) = change; | ||
if !valid_path(path) { | ||
return Err(BadInput(format!("bad path {path:?}")).into()); | ||
} | ||
match change { | ||
Change::Put { | ||
path, | ||
data, | ||
executable, | ||
} => { | ||
let sha = git | ||
.text(&["hash-object", "-w", "--stdin"], Some(data)) | ||
.await?; | ||
let mode = if *executable { "100755" } else { "100644" }; | ||
info.push_str(&format!("{mode} {sha}\t{path}\n")); | ||
} | ||
Change::Delete { path } => { | ||
info.push_str(&format!("0 {}\t{path}\n", crate::git::ZERO)); | ||
} | ||
} | ||
} | ||
git.run(&["update-index", "--index-info"], Some(info.as_bytes())) | ||
.await?; | ||
let tree = git.text(&["write-tree"], None).await?; | ||
if let Some(p) = parent | ||
&& git | ||
.text(&["rev-parse", &format!("{p}^{{tree}}")], None) | ||
.await? | ||
== tree | ||
{ | ||
return Ok(None); | ||
} | ||
let (name, email) = spec | ||
.author | ||
.clone() | ||
.unwrap_or_else(|| ("knotless".into(), format!("{did}@knotless.invalid"))); | ||
git.envs.extend([ | ||
("GIT_AUTHOR_NAME".into(), name.clone()), | ||
("GIT_AUTHOR_EMAIL".into(), email.clone()), | ||
("GIT_COMMITTER_NAME".into(), name), | ||
("GIT_COMMITTER_EMAIL".into(), email), | ||
]); | ||
let mut args = vec!["commit-tree", tree.as_str(), "-m", spec.message.as_str()]; | ||
if let Some(p) = parent { | ||
args.extend(["-p", p]); | ||
} | ||
Ok(Some(git.text(&args, None).await?)) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn paths() { | ||
for ok in ["a", "a/b", "a.git", ".gitignore", "x/.github/y"] { | ||
assert!(valid_path(ok), "{ok}"); | ||
} | ||
for bad in [ | ||
"", "/a", "a/", "a//b", "./a", "a/..", ".git", "x/.GIT/y", "a\0b", "a\nb", | ||
] { | ||
assert!(!valid_path(bad), "{bad:?}"); | ||
} | ||
} | ||
} | ||
crates/knotless/src/compact.rsnew+115−0
| @@ -0,0 +1,115 @@ | ||
//! Compaction: fold a repository's packs into one. | ||
//! | ||
//! Every push adds a pack record, and a clone reads them all, so past a | ||
//! threshold the history is repacked into one pack holding everything the | ||
//! refs reach, and the old records are deleted with it. Packs younger than | ||
//! the grace period stay, so a reader partway through a walk still finds | ||
//! what it listed; a reader that loses the race restarts its walk once. | ||
//! Claims a day old go too: every push adds one per ref it moves. | ||
| ||
use crate::env::Ctx; | ||
use crate::git; | ||
use crate::keys; | ||
use crate::mirror::Mirror; | ||
use crate::push; | ||
use crate::records::{self, nsid}; | ||
use crate::remote::{self, Remote, Snapshot}; | ||
use crate::session::Session; | ||
use anyhow::Result; | ||
use std::time::SystemTime; | ||
| ||
#[derive(Debug, Default, PartialEq, Eq)] | ||
pub struct Compacted { | ||
pub before: usize, | ||
pub removed: usize, | ||
/// Claims dropped for being older than the claim grace period. | ||
pub claims: usize, | ||
} | ||
| ||
pub async fn compact(ctx: &Ctx, remote: &Remote, session: &Session) -> Result<Compacted> { | ||
let snap = remote.snapshot(ctx).await?; | ||
let Some(key) = snap.key().map(str::to_string) else { | ||
return Ok(Compacted::default()); | ||
}; | ||
let packs = remote | ||
.scan_all::<records::PackRecord>(nsid::PACK, &key) | ||
.await?; | ||
let cutoff = keys::now_micros().saturating_sub(ctx.env.compact_grace.as_micros() as u64); | ||
let old: Vec<String> = packs | ||
.iter() | ||
.filter(|p| { | ||
keys::suffix(&key, &p.rkey) | ||
.and_then(keys::tid_micros) | ||
.is_some_and(|t| t < cutoff) | ||
}) | ||
.map(|p| p.rkey.clone()) | ||
.collect(); | ||
let (mut writes, removed) = if old.len() >= 2 { | ||
( | ||
fold(ctx, remote, session, &snap, &key, &old).await?, | ||
old.len(), | ||
) | ||
} else { | ||
(Vec::new(), 0) | ||
}; | ||
// A claim only guards the generation it names while a writer may still | ||
// hold a read of it; a push refuses reads older than half the grace. | ||
let since = SystemTime::now() | ||
.checked_sub(ctx.env.claim_grace) | ||
.unwrap_or(SystemTime::UNIX_EPOCH); | ||
let claims: Vec<String> = remote | ||
.scan_all::<records::ClaimRecord>(nsid::CLAIM, &key) | ||
.await? | ||
.into_iter() | ||
.filter(|c| humantime::parse_rfc3339_weak(&c.value.created_at).is_ok_and(|t| t < since)) | ||
.map(|c| c.rkey) | ||
.collect(); | ||
writes.extend(claims.iter().map(|rkey| remote::delete(nsid::CLAIM, rkey))); | ||
for chunk in writes.chunks(200) { | ||
remote.apply(chunk.to_vec(), &session.access_jwt).await?; | ||
} | ||
Ok(Compacted { | ||
before: packs.len(), | ||
removed, | ||
claims: claims.len(), | ||
}) | ||
} | ||
| ||
/// The writes that replace the `old` packs with one holding everything the | ||
/// live refs reach. | ||
async fn fold( | ||
ctx: &Ctx, | ||
remote: &Remote, | ||
session: &Session, | ||
snap: &Snapshot, | ||
key: &str, | ||
old: &[String], | ||
) -> Result<Vec<serde_json::Value>> { | ||
let mirror = Mirror::open(ctx, remote.did(), key).await?; | ||
mirror.sync(remote, snap).await?; | ||
let tips: Vec<String> = snap.live_refs().into_iter().map(|(_, sha)| sha).collect(); | ||
let mut writes = Vec::new(); | ||
if !tips.is_empty() { | ||
let pack = mirror.git.pack_objects(&tips, &[]).await?; | ||
let objects = git::pack_objects_count(&pack); | ||
let mut record = push::place( | ||
ctx, | ||
remote, | ||
&session.access_jwt, | ||
key, | ||
pack, | ||
tips, | ||
Vec::new(), | ||
objects, | ||
) | ||
.await?; | ||
record.supersedes = old.to_vec(); | ||
writes.push(remote::create( | ||
nsid::PACK, | ||
&keys::pack_rkey(key), | ||
records::typed(nsid::PACK, &record), | ||
)); | ||
} | ||
writes.extend(old.iter().map(|rkey| remote::delete(nsid::PACK, rkey))); | ||
Ok(writes) | ||
} | ||
crates/knotless/src/crypto.rsnew+302−0
| @@ -0,0 +1,302 @@ | ||
//! The keys atproto uses: secp256k1 and P-256, as multikeys and `did:key`s, | ||
//! and the JWTs they sign. | ||
| ||
use anyhow::{Context, Result, bail}; | ||
use base64::Engine; | ||
use base64::engine::general_purpose::URL_SAFE_NO_PAD; | ||
use serde::Serialize; | ||
use serde::de::DeserializeOwned; | ||
use serde_json::Value; | ||
| ||
const K256_CODEC: [u8; 2] = [0xe7, 0x01]; | ||
const P256_CODEC: [u8; 2] = [0x80, 0x24]; | ||
| ||
/// A public key a DID document or `did:key` names. | ||
#[derive(Clone, Debug, PartialEq, Eq)] | ||
pub enum PublicKey { | ||
K256(k256::ecdsa::VerifyingKey), | ||
P256(p256::ecdsa::VerifyingKey), | ||
} | ||
| ||
impl PublicKey { | ||
/// Parses a multibase (`z`, base58btc) multicodec key. | ||
pub fn from_multibase(text: &str) -> Result<PublicKey> { | ||
let Some(b58) = text.strip_prefix('z') else { | ||
bail!("a multikey is base58btc, starting with z"); | ||
}; | ||
let bytes = bs58::decode(b58) | ||
.into_vec() | ||
.context("bad base58 in a multikey")?; | ||
let (codec, key) = bytes.split_at(2.min(bytes.len())); | ||
match <[u8; 2]>::try_from(codec) { | ||
Ok(K256_CODEC) => Ok(PublicKey::K256( | ||
k256::ecdsa::VerifyingKey::from_sec1_bytes(key).context("bad secp256k1 key")?, | ||
)), | ||
Ok(P256_CODEC) => Ok(PublicKey::P256( | ||
p256::ecdsa::VerifyingKey::from_sec1_bytes(key).context("bad P-256 key")?, | ||
)), | ||
_ => bail!("unsupported key type in a multikey"), | ||
} | ||
} | ||
| ||
/// Parses a DID document verification method: a `Multikey`, or one of | ||
/// the older types whose multibase holds the bare key with no codec. | ||
pub fn from_method(kind: &str, multibase: &str) -> Result<PublicKey> { | ||
let raw = || -> Result<Vec<u8>> { | ||
let Some(b58) = multibase.strip_prefix('z') else { | ||
bail!("a multikey is base58btc, starting with z"); | ||
}; | ||
bs58::decode(b58).into_vec().context("bad base58 in a key") | ||
}; | ||
match kind { | ||
"EcdsaSecp256k1VerificationKey2019" => Ok(PublicKey::K256( | ||
k256::ecdsa::VerifyingKey::from_sec1_bytes(&raw()?).context("bad secp256k1 key")?, | ||
)), | ||
"EcdsaSecp256r1VerificationKey2019" => Ok(PublicKey::P256( | ||
p256::ecdsa::VerifyingKey::from_sec1_bytes(&raw()?).context("bad P-256 key")?, | ||
)), | ||
_ => PublicKey::from_multibase(multibase), | ||
} | ||
} | ||
| ||
pub fn from_did_key(did: &str) -> Result<PublicKey> { | ||
match did.strip_prefix("did:key:") { | ||
Some(multibase) => PublicKey::from_multibase(multibase), | ||
None => bail!("{did} is not a did:key"), | ||
} | ||
} | ||
| ||
pub fn to_multibase(&self) -> String { | ||
let (codec, key) = match self { | ||
PublicKey::K256(k) => (K256_CODEC, k.to_encoded_point(true).as_bytes().to_vec()), | ||
PublicKey::P256(k) => (P256_CODEC, k.to_encoded_point(true).as_bytes().to_vec()), | ||
}; | ||
format!( | ||
"z{}", | ||
bs58::encode([&codec[..], &key].concat()).into_string() | ||
) | ||
} | ||
| ||
pub fn to_did_key(&self) -> String { | ||
format!("did:key:{}", self.to_multibase()) | ||
} | ||
| ||
/// The JWT `alg` this key signs with. | ||
pub fn alg(&self) -> &'static str { | ||
match self { | ||
PublicKey::K256(_) => "ES256K", | ||
PublicKey::P256(_) => "ES256", | ||
} | ||
} | ||
| ||
/// Checks a 64-byte `r||s` signature over SHA-256 of `message`. High-S | ||
/// signatures are refused, as atproto requires. | ||
pub fn verify(&self, message: &[u8], signature: &[u8]) -> Result<()> { | ||
use k256::ecdsa::signature::Verifier; | ||
let ok = match self { | ||
PublicKey::K256(key) => k256::ecdsa::Signature::from_slice(signature) | ||
.ok() | ||
.filter(|s| s.normalize_s().is_none()) | ||
.is_some_and(|s| key.verify(message, &s).is_ok()), | ||
PublicKey::P256(key) => p256::ecdsa::Signature::from_slice(signature) | ||
.ok() | ||
.filter(|s| s.normalize_s().is_none()) | ||
.is_some_and(|s| key.verify(message, &s).is_ok()), | ||
}; | ||
if ok { Ok(()) } else { bail!("bad signature") } | ||
} | ||
} | ||
| ||
/// An ephemeral P-256 key, for proving possession of a space credential. | ||
pub struct P256Key(p256::ecdsa::SigningKey); | ||
| ||
impl P256Key { | ||
pub fn generate() -> P256Key { | ||
P256Key(p256::ecdsa::SigningKey::random(&mut rand_core::OsRng)) | ||
} | ||
| ||
/// The 32-byte secret scalar, for a cache that keeps the key with what | ||
/// it is bound to. | ||
pub fn to_bytes(&self) -> Vec<u8> { | ||
self.0.to_bytes().to_vec() | ||
} | ||
| ||
pub fn from_bytes(bytes: &[u8]) -> Result<P256Key> { | ||
Ok(P256Key( | ||
p256::ecdsa::SigningKey::from_slice(bytes).context("bad P-256 secret key")?, | ||
)) | ||
} | ||
| ||
pub fn public(&self) -> PublicKey { | ||
PublicKey::P256(*self.0.verifying_key()) | ||
} | ||
| ||
/// A 64-byte, low-S `r||s` signature over SHA-256 of `message`. | ||
pub fn sign(&self, message: &[u8]) -> Vec<u8> { | ||
use p256::ecdsa::signature::Signer; | ||
let sig: p256::ecdsa::Signature = self.0.sign(message); | ||
sig.normalize_s().unwrap_or(sig).to_bytes().to_vec() | ||
} | ||
| ||
/// A compact JWT signed with this key, `alg` ES256. | ||
pub fn sign_jwt(&self, header: &Value, claims: &impl Serialize) -> String { | ||
let mut header = header.clone(); | ||
header["alg"] = "ES256".into(); | ||
let body = format!( | ||
"{}.{}", | ||
URL_SAFE_NO_PAD.encode(header.to_string()), | ||
URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).expect("claims serialize")) | ||
); | ||
let sig = URL_SAFE_NO_PAD.encode(self.sign(body.as_bytes())); | ||
format!("{body}.{sig}") | ||
} | ||
} | ||
| ||
/// A JWT's header and claims, unverified. | ||
pub fn jwt_parts(token: &str) -> Result<(Value, Value)> { | ||
let mut parts = token.split('.'); | ||
let mut decode = |what: &str| -> Result<Value> { | ||
let part = parts | ||
.next() | ||
.with_context(|| format!("a JWT without a {what}"))?; | ||
let bytes = URL_SAFE_NO_PAD | ||
.decode(part) | ||
.with_context(|| format!("bad base64 in a JWT {what}"))?; | ||
serde_json::from_slice(&bytes).with_context(|| format!("bad JSON in a JWT {what}")) | ||
}; | ||
Ok((decode("header")?, decode("payload")?)) | ||
} | ||
| ||
/// Verifies `token` against `key` and returns its claims. Checks the | ||
/// signature, that `alg` matches the key, and `exp` against the clock. | ||
pub fn verify_jwt<T: DeserializeOwned>(token: &str, key: &PublicKey) -> Result<T> { | ||
let (header, claims) = jwt_parts(token)?; | ||
if header["alg"].as_str() != Some(key.alg()) { | ||
bail!( | ||
"a JWT signed with {} where {} was expected", | ||
header["alg"], | ||
key.alg() | ||
); | ||
} | ||
let (signed, sig) = token | ||
.rsplit_once('.') | ||
.context("a JWT without a signature")?; | ||
let sig = URL_SAFE_NO_PAD | ||
.decode(sig) | ||
.context("bad base64 in a JWT signature")?; | ||
key.verify(signed.as_bytes(), &sig)?; | ||
let now = crate::keys::now_micros() / 1_000_000; | ||
match claims["exp"].as_u64() { | ||
Some(exp) if exp > now => {} | ||
_ => bail!("an expired JWT"), | ||
} | ||
Ok(serde_json::from_value(claims)?) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use serde_json::json; | ||
| ||
#[test] | ||
fn multikeys_round_trip() { | ||
let p = P256Key::generate().public(); | ||
assert_eq!(PublicKey::from_did_key(&p.to_did_key()).unwrap(), p); | ||
assert!(p.to_multibase().starts_with("zDn")); | ||
let k = PublicKey::K256( | ||
*k256::ecdsa::SigningKey::random(&mut rand_core::OsRng).verifying_key(), | ||
); | ||
assert_eq!(PublicKey::from_multibase(&k.to_multibase()).unwrap(), k); | ||
assert!(k.to_multibase().starts_with("zQ3s")); | ||
assert_eq!((k.alg(), p.alg()), ("ES256K", "ES256")); | ||
for bad in [ | ||
"x", | ||
"z0OIl", | ||
"z", | ||
"z2", | ||
&format!("z{}", bs58::encode([0xe7, 0x01, 1, 2]).into_string()), | ||
&format!("z{}", bs58::encode([0x80, 0x24, 1, 2]).into_string()), | ||
] { | ||
assert!(PublicKey::from_multibase(bad).is_err(), "{bad}"); | ||
} | ||
assert!(PublicKey::from_did_key("did:web:x").is_err()); | ||
| ||
// The older verification method types hold the bare key. | ||
let bare = |k: &PublicKey| match k { | ||
PublicKey::K256(k) => format!( | ||
"z{}", | ||
bs58::encode(k.to_encoded_point(false).as_bytes()).into_string() | ||
), | ||
PublicKey::P256(k) => format!( | ||
"z{}", | ||
bs58::encode(k.to_encoded_point(true).as_bytes()).into_string() | ||
), | ||
}; | ||
assert_eq!( | ||
PublicKey::from_method("EcdsaSecp256k1VerificationKey2019", &bare(&k)).unwrap(), | ||
k | ||
); | ||
assert_eq!( | ||
PublicKey::from_method("EcdsaSecp256r1VerificationKey2019", &bare(&p)).unwrap(), | ||
p | ||
); | ||
assert_eq!( | ||
PublicKey::from_method("Multikey", &p.to_multibase()).unwrap(), | ||
p | ||
); | ||
assert!(PublicKey::from_method("EcdsaSecp256k1VerificationKey2019", "x").is_err()); | ||
assert!(PublicKey::from_method("EcdsaSecp256k1VerificationKey2019", "z0").is_err()); | ||
assert!(PublicKey::from_method("EcdsaSecp256k1VerificationKey2019", "z2").is_err()); | ||
assert!(PublicKey::from_method("EcdsaSecp256r1VerificationKey2019", "z2").is_err()); | ||
} | ||
| ||
#[test] | ||
fn signatures() { | ||
use k256::ecdsa::signature::Signer; | ||
let key = P256Key::generate(); | ||
let again = P256Key::from_bytes(&key.to_bytes()).unwrap(); | ||
assert_eq!(again.public(), key.public()); | ||
assert!(P256Key::from_bytes(&[0; 32]).is_err()); | ||
let sig = key.sign(b"hello"); | ||
assert_eq!(sig.len(), 64); | ||
key.public().verify(b"hello", &sig).unwrap(); | ||
assert!(key.public().verify(b"other", &sig).is_err()); | ||
assert!(key.public().verify(b"hello", b"short").is_err()); | ||
| ||
let k = k256::ecdsa::SigningKey::random(&mut rand_core::OsRng); | ||
let s: k256::ecdsa::Signature = k.sign(b"m"); | ||
let low = s.normalize_s().unwrap_or(s); | ||
let public = PublicKey::K256(*k.verifying_key()); | ||
public.verify(b"m", &low.to_bytes()).unwrap(); | ||
// The same signature with s flipped high verifies in ECDSA, and is refused. | ||
let (r, s_low) = (low.r(), low.s()); | ||
let high = k256::ecdsa::Signature::from_scalars(r, -*s_low).unwrap(); | ||
assert!(public.verify(b"m", &high.to_bytes()).is_err()); | ||
} | ||
| ||
#[test] | ||
fn jwts() { | ||
let key = P256Key::generate(); | ||
let exp = crate::keys::now_micros() / 1_000_000 + 60; | ||
let token = key.sign_jwt(&json!({"typ": "JWT"}), &json!({"iss": "me", "exp": exp})); | ||
let claims: Value = verify_jwt(&token, &key.public()).unwrap(); | ||
assert_eq!(claims["iss"], "me"); | ||
let (header, _) = jwt_parts(&token).unwrap(); | ||
assert_eq!(header["alg"], "ES256"); | ||
| ||
let other = P256Key::generate(); | ||
assert!(verify_jwt::<Value>(&token, &other.public()).is_err()); | ||
let k = PublicKey::K256( | ||
*k256::ecdsa::SigningKey::random(&mut rand_core::OsRng).verifying_key(), | ||
); | ||
assert!(verify_jwt::<Value>(&token, &k).is_err()); | ||
let expired = key.sign_jwt(&json!({}), &json!({"exp": 1})); | ||
assert!(verify_jwt::<Value>(&expired, &key.public()).is_err()); | ||
let (signed, _) = token.rsplit_once('.').unwrap(); | ||
assert!(verify_jwt::<Value>(&format!("{signed}.!!"), &key.public()).is_err()); | ||
assert!(verify_jwt::<Value>(signed, &key.public()).is_err()); | ||
for bad in ["", "a", "e30", "!!.x", "e30.!!", "bm90IGpzb24.e30"] { | ||
assert!(jwt_parts(bad).is_err(), "{bad}"); | ||
} | ||
} | ||
} | ||
crates/knotless/src/env.rsnew+209−0
| @@ -0,0 +1,209 @@ | ||
//! Process configuration, read once from the environment. | ||
| ||
use std::path::PathBuf; | ||
use std::time::Duration; | ||
| ||
/// What a push does once the repository holds more pack records than | ||
/// [`Env::compact_threshold`]. | ||
#[derive(Clone, Copy, Debug, PartialEq, Eq)] | ||
pub enum CompactMode { | ||
Off, | ||
/// Compact before the push returns. | ||
Inline, | ||
/// Start `knotless compact` detached, so the push returns at once. | ||
Background, | ||
} | ||
| ||
/// How a handle becomes a DID, tried in order. | ||
#[derive(Clone, Debug, PartialEq, Eq)] | ||
pub enum HandleResolver { | ||
/// `<scheme>://<handle>/.well-known/atproto-did`. | ||
WellKnown { scheme: String }, | ||
/// `com.atproto.identity.resolveHandle` on a service. | ||
Xrpc { service: String }, | ||
} | ||
| ||
#[derive(Clone, Debug)] | ||
pub struct Env { | ||
pub config_dir: PathBuf, | ||
pub cache_dir: PathBuf, | ||
pub plc_url: String, | ||
pub handle_resolvers: Vec<HandleResolver>, | ||
/// Scheme for did:web documents; tests serve them over plain http. | ||
pub did_web_scheme: String, | ||
/// Skips DID resolution: every identity is served by this PDS. | ||
pub service: Option<String>, | ||
pub identifier: Option<String>, | ||
pub password: Option<String>, | ||
/// Packs up to this many bytes ride inside the pack record itself. | ||
pub inline_max: usize, | ||
/// Blob part size for larger packs; halved when a PDS refuses it. | ||
pub chunk_size: usize, | ||
pub compact_threshold: usize, | ||
pub compact_mode: CompactMode, | ||
/// Packs younger than this survive compaction, for readers mid-fetch. | ||
pub compact_grace: Duration, | ||
/// Claims older than this are dropped by compaction, and a push whose | ||
/// read of the refs is older than half of it is refused, so no writer | ||
/// can still hold a generation whose claim is gone. | ||
pub claim_grace: Duration, | ||
pub identity_ttl: Duration, | ||
pub timeout: Duration, | ||
/// The appview that indexes pulls and issues on public repositories. | ||
pub appview: Option<String>, | ||
/// The moderation service reports go to, when not the appview's. | ||
pub moderator: Option<String>, | ||
/// The appview's admin password, for its operator. | ||
pub admin_password: Option<String>, | ||
} | ||
| ||
impl Env { | ||
pub fn from_process() -> Env { | ||
Env::from_vars(|name| std::env::var(name).ok()) | ||
} | ||
| ||
pub fn from_vars(get: impl Fn(&str) -> Option<String>) -> Env { | ||
let home = get("HOME").unwrap_or_else(|| ".".into()); | ||
let dir = |own: &str, xdg: &str, fallback: &str| { | ||
get(own).map(PathBuf::from).unwrap_or_else(|| { | ||
get(xdg) | ||
.map(PathBuf::from) | ||
.unwrap_or_else(|| PathBuf::from(&home).join(fallback)) | ||
.join("knotless") | ||
}) | ||
}; | ||
let num = |name: &str, default: u64| { | ||
get(name) | ||
.and_then(|v| v.parse::<u64>().ok()) | ||
.unwrap_or(default) | ||
}; | ||
let handle_resolvers = match get("KNOTLESS_HANDLE_RESOLVER") { | ||
Some(list) => list | ||
.split(',') | ||
.filter(|s| !s.is_empty()) | ||
.map(|s| match s.strip_prefix("well-known:") { | ||
Some(scheme) => HandleResolver::WellKnown { | ||
scheme: scheme.into(), | ||
}, | ||
None => HandleResolver::Xrpc { service: s.into() }, | ||
}) | ||
.collect(), | ||
None => vec![ | ||
HandleResolver::WellKnown { | ||
scheme: "https".into(), | ||
}, | ||
HandleResolver::Xrpc { | ||
service: "https://public.api.bsky.app".into(), | ||
}, | ||
], | ||
}; | ||
Env { | ||
config_dir: dir("KNOTLESS_CONFIG_DIR", "XDG_CONFIG_HOME", ".config"), | ||
cache_dir: dir("KNOTLESS_CACHE_DIR", "XDG_CACHE_HOME", ".cache"), | ||
plc_url: get("KNOTLESS_PLC_URL").unwrap_or_else(|| "https://plc.directory".into()), | ||
handle_resolvers, | ||
did_web_scheme: get("KNOTLESS_DID_WEB_SCHEME").unwrap_or_else(|| "https".into()), | ||
service: get("KNOTLESS_SERVICE"), | ||
identifier: get("KNOTLESS_IDENTIFIER"), | ||
password: get("KNOTLESS_PASSWORD"), | ||
inline_max: num("KNOTLESS_INLINE_MAX", 192 * 1024) as usize, | ||
chunk_size: num("KNOTLESS_CHUNK_SIZE", 4_000_000) as usize, | ||
compact_threshold: num("KNOTLESS_COMPACT_THRESHOLD", 32) as usize, | ||
compact_mode: match get("KNOTLESS_COMPACT").as_deref() { | ||
Some("off") => CompactMode::Off, | ||
Some("inline") => CompactMode::Inline, | ||
_ => CompactMode::Background, | ||
}, | ||
compact_grace: Duration::from_secs(num("KNOTLESS_COMPACT_GRACE_SECS", 600)), | ||
claim_grace: Duration::from_secs(num("KNOTLESS_CLAIM_GRACE_SECS", 86_400)), | ||
identity_ttl: Duration::from_secs(num("KNOTLESS_IDENTITY_TTL_SECS", 3600)), | ||
timeout: Duration::from_secs(num("KNOTLESS_TIMEOUT_SECS", 60)), | ||
appview: get("KNOTLESS_APPVIEW").filter(|v| !v.is_empty()), | ||
moderator: get("KNOTLESS_MODERATOR").filter(|v| !v.is_empty()), | ||
admin_password: get("KNOTLESS_ADMIN_PASSWORD"), | ||
} | ||
} | ||
} | ||
| ||
/// Configuration plus the one HTTP client a process shares, so every request | ||
/// to a PDS reuses its connection. | ||
#[derive(Clone)] | ||
pub struct Ctx { | ||
pub env: Env, | ||
pub http: reqwest::Client, | ||
} | ||
| ||
impl Ctx { | ||
pub fn new(env: Env) -> Ctx { | ||
let http = reqwest::Client::builder() | ||
.user_agent(concat!("knotless/", env!("CARGO_PKG_VERSION"))) | ||
.connect_timeout(Duration::from_secs(10)) | ||
.timeout(env.timeout) | ||
.pool_idle_timeout(Duration::from_secs(90)) | ||
.tcp_nodelay(true) | ||
.build() | ||
.expect("the HTTP client configuration is static"); | ||
Ctx { env, http } | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use std::collections::HashMap; | ||
| ||
fn env(vars: &[(&str, &str)]) -> Env { | ||
let map: HashMap<String, String> = vars | ||
.iter() | ||
.map(|(k, v)| (k.to_string(), v.to_string())) | ||
.collect(); | ||
Env::from_vars(|k| map.get(k).cloned()) | ||
} | ||
| ||
#[test] | ||
fn defaults() { | ||
let e = env(&[("HOME", "/home/a")]); | ||
assert_eq!(e.config_dir, PathBuf::from("/home/a/.config/knotless")); | ||
assert_eq!(e.cache_dir, PathBuf::from("/home/a/.cache/knotless")); | ||
assert_eq!(e.plc_url, "https://plc.directory"); | ||
assert_eq!(e.handle_resolvers.len(), 2); | ||
assert_eq!(e.compact_mode, CompactMode::Background); | ||
assert_eq!(e.inline_max, 192 * 1024); | ||
assert!(e.service.is_none()); | ||
let bare = env(&[]); | ||
assert_eq!(bare.config_dir, PathBuf::from("./.config/knotless")); | ||
} | ||
| ||
#[test] | ||
fn overrides() { | ||
let e = env(&[ | ||
("XDG_CONFIG_HOME", "/x"), | ||
("KNOTLESS_CACHE_DIR", "/c"), | ||
("KNOTLESS_HANDLE_RESOLVER", "well-known:http,http://pds,"), | ||
("KNOTLESS_COMPACT", "off"), | ||
("KNOTLESS_INLINE_MAX", "10"), | ||
("KNOTLESS_CHUNK_SIZE", "nope"), | ||
]); | ||
assert_eq!(e.config_dir, PathBuf::from("/x/knotless")); | ||
assert_eq!(e.cache_dir, PathBuf::from("/c")); | ||
assert_eq!( | ||
e.handle_resolvers, | ||
vec![ | ||
HandleResolver::WellKnown { | ||
scheme: "http".into() | ||
}, | ||
HandleResolver::Xrpc { | ||
service: "http://pds".into() | ||
} | ||
] | ||
); | ||
assert_eq!(e.compact_mode, CompactMode::Off); | ||
assert_eq!(e.inline_max, 10); | ||
assert_eq!(e.chunk_size, 4_000_000); | ||
assert_eq!( | ||
env(&[("KNOTLESS_COMPACT", "inline")]).compact_mode, | ||
CompactMode::Inline | ||
); | ||
let _ = Ctx::new(e); | ||
} | ||
} | ||
crates/knotless/src/fetch.rsnew+98−0
| @@ -0,0 +1,98 @@ | ||
//! A fetch: walk the pack records newest first, download the ones whose tips | ||
//! are missing, and index them, until every wanted object is here. | ||
| ||
use crate::git::Git; | ||
use crate::records::PackRecord; | ||
use crate::remote::{Page, Remote, Stored}; | ||
use anyhow::{Result, bail}; | ||
use futures::{StreamExt, TryStreamExt}; | ||
use std::collections::BTreeSet; | ||
| ||
/// Downloads parallelism. Inline packs arrive with the listing and cost | ||
/// nothing here. | ||
const PARALLEL: usize = 8; | ||
| ||
/// Makes every object `wants` reaches present in `git`. `first` is the | ||
/// newest page of packs when the caller already has it. Returns the number | ||
/// of packs indexed. | ||
pub async fn fetch( | ||
remote: &Remote, | ||
key: &str, | ||
git: &Git, | ||
wants: &[String], | ||
first: Option<Page<PackRecord>>, | ||
) -> Result<usize> { | ||
let mut missing: BTreeSet<String> = git.missing(wants).await?.into_iter().collect(); | ||
if missing.is_empty() { | ||
return Ok(0); | ||
} | ||
let mut page = match first { | ||
Some(page) => page, | ||
None => remote.packs(key, None).await?, | ||
}; | ||
let mut indexed = 0; | ||
let mut restarted = false; | ||
loop { | ||
let tips: Vec<String> = page | ||
.items | ||
.iter() | ||
.flat_map(|p| p.value.tips.clone()) | ||
.collect(); | ||
let absent = git.missing(&tips).await?; | ||
let needed: Vec<&Stored<PackRecord>> = page | ||
.items | ||
.iter() | ||
.filter(|p| p.value.tips.iter().any(|t| absent.contains(t))) | ||
.collect(); | ||
let records: Vec<PackRecord> = needed.iter().map(|p| p.value.clone()).collect(); | ||
futures::stream::iter(records) | ||
.map(|record| async move { | ||
let bytes = download(remote, &record).await?; | ||
git.index_pack(&bytes).await | ||
}) | ||
.buffer_unordered(PARALLEL) | ||
.try_collect::<Vec<()>>() | ||
.await?; | ||
indexed += needed.len(); | ||
missing.extend(needed.iter().flat_map(|p| p.value.bases.iter().cloned())); | ||
let check: Vec<String> = missing.into_iter().collect(); | ||
missing = git.missing(&check).await?.into_iter().collect(); | ||
if missing.is_empty() { | ||
return Ok(indexed); | ||
} | ||
page = match page.next { | ||
Some(cursor) => remote.packs(key, Some(&cursor)).await?, | ||
// A compaction can replace packs between two pages of a walk; | ||
// one fresh walk sees its result. | ||
None if !restarted => { | ||
restarted = true; | ||
remote.packs(key, None).await? | ||
} | ||
None => bail!( | ||
"{}: the remote has no pack holding {}", | ||
remote.url, | ||
missing.into_iter().next().unwrap_or_default() | ||
), | ||
}; | ||
} | ||
} | ||
| ||
/// A pack's bytes, from the record itself or its blobs. | ||
pub async fn download(remote: &Remote, pack: &PackRecord) -> Result<Vec<u8>> { | ||
let bytes = match &pack.data { | ||
Some(data) => data.0.clone(), | ||
None => { | ||
let cids: Vec<String> = pack.parts.iter().map(|p| p.link.cid.clone()).collect(); | ||
let parts: Vec<Vec<u8>> = futures::stream::iter(cids) | ||
.map(|cid| async move { remote.get_blob(&cid).await }) | ||
.buffered(PARALLEL) | ||
.try_collect() | ||
.await?; | ||
parts.concat() | ||
} | ||
}; | ||
if bytes.len() as u64 != pack.size { | ||
bail!("a pack of {} bytes came back as {}", pack.size, bytes.len()); | ||
} | ||
Ok(bytes) | ||
} | ||
crates/knotless/src/forge.rsnew+1102−0
| @@ -0,0 +1,1102 @@ | ||
//! Forks, pulls, issues, comments and reviews. | ||
//! | ||
//! Each is a record in the account of whoever made it, naming the | ||
//! repository it is about. Nothing is written to the repository's own | ||
//! records, so anyone can open a pull or an issue on any repository they can | ||
//! read. Readers gather the records and work out each pull's and issue's | ||
//! state from [`StateRecord`]s, honouring only those from someone allowed to | ||
//! make the change. | ||
//! | ||
//! For a public repository the records are public, and an appview that | ||
//! follows the firehose indexes them (`KNOTLESS_APPVIEW`). For a private one | ||
//! they live in their authors' repos inside the repository's space, and a | ||
//! member gathers them straight from there. | ||
//! | ||
//! The owner moderates them: see [`crate::moderation`]. | ||
| ||
use crate::env::Ctx; | ||
use crate::fetch; | ||
use crate::gateway; | ||
use crate::git::Git; | ||
use crate::identity; | ||
use crate::keys; | ||
use crate::manage::{self, NewRepo}; | ||
use crate::mirror::Mirror; | ||
use crate::push::{self, Update}; | ||
use crate::records::{ | ||
self, BlockRecord, CommentRecord, GateRecord, IssueRecord, PullHead, PullRecord, RepoRecord, | ||
RepoRef, StateRecord, nsid, | ||
}; | ||
use crate::remote::{self, Remote, Snapshot}; | ||
use crate::session::{self, Session}; | ||
use crate::space::Space; | ||
use crate::url::RepoUrl; | ||
use crate::xrpc::Xrpc; | ||
use anyhow::{Context, Result, bail}; | ||
use serde::{Deserialize, Serialize}; | ||
use serde_json::Value; | ||
| ||
/// The `at://` URI of `did`'s repository record for `name`. | ||
pub fn repo_uri(did: &str, name: &str) -> String { | ||
format!("at://{did}/{}/{name}", nsid::REPO) | ||
} | ||
| ||
/// An `at://did/collection/rkey` URI's parts. | ||
pub fn parse_uri(uri: &str) -> Result<(&str, &str, &str)> { | ||
let parts: Vec<&str> = uri | ||
.strip_prefix("at://") | ||
.context("not an at:// URI")? | ||
.split('/') | ||
.collect(); | ||
match parts[..] { | ||
[did, collection, rkey] if did.starts_with("did:") => Ok((did, collection, rkey)), | ||
_ => bail!("{uri} is not at://<did>/<collection>/<rkey>"), | ||
} | ||
} | ||
| ||
/// The repository a `dev.knotless.repo` URI names. | ||
pub fn repo_url(uri: &str) -> Result<RepoUrl> { | ||
match parse_uri(uri)? { | ||
(did, nsid::REPO, name) => RepoUrl::parse(&format!("knotless://{did}/{name}")), | ||
_ => bail!("{uri} is not a repository"), | ||
} | ||
} | ||
| ||
/// Whether a forge record is about `did`'s repository `key`. | ||
pub fn is_about(value: &Value, did: &str, key: &str) -> bool { | ||
serde_json::from_value::<RepoRef>(value["repo"].clone()).is_ok_and(|r| r.is(did, key)) | ||
} | ||
| ||
/// Any forge record, with who wrote it. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
pub struct Entry { | ||
pub uri: String, | ||
pub author: String, | ||
pub value: Value, | ||
} | ||
| ||
impl Entry { | ||
fn collection(&self) -> &str { | ||
parse_uri(&self.uri).map_or("", |(_, c, _)| c) | ||
} | ||
} | ||
| ||
/// A pull or an issue, as it stands. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct Item { | ||
pub uri: String, | ||
pub author: String, | ||
/// `pull` or `issue`. | ||
pub kind: String, | ||
pub title: String, | ||
pub body: Option<String>, | ||
pub state: String, | ||
/// The merge, for a merged pull. | ||
pub merged: Option<String>, | ||
/// A pull's target branch, and the repository and branch it brings in. | ||
pub base: Option<String>, | ||
pub head: Option<PullHead>, | ||
pub comments: usize, | ||
pub approvals: usize, | ||
pub created_at: String, | ||
} | ||
| ||
/// An item and everything said about it, oldest first. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
pub struct Thread { | ||
pub item: Item, | ||
pub comments: Vec<Entry>, | ||
} | ||
| ||
/// `entries` without what the owner of `did`'s repository `key` keeps out: | ||
/// what accounts in `blocked`, or blocked from this repository alone, | ||
/// wrote, and what its gate hides, with what was said about that. | ||
pub fn moderate(did: &str, key: &str, blocked: &[String], entries: Vec<Entry>) -> Vec<Entry> { | ||
let owners = |collection: &'static str| { | ||
entries | ||
.iter() | ||
.filter(move |e| { | ||
e.author == did && e.collection() == collection && is_about(&e.value, did, key) | ||
}) | ||
.map(|e| e.value.clone()) | ||
}; | ||
let hidden: Vec<String> = owners(nsid::GATE) | ||
.filter_map(|v| serde_json::from_value::<GateRecord>(v).ok()) | ||
.flat_map(|g| g.hidden) | ||
.collect(); | ||
let blocked: Vec<String> = owners(nsid::BLOCK) | ||
.filter_map(|v| serde_json::from_value::<BlockRecord>(v).ok()) | ||
.map(|b| b.subject) | ||
.chain(blocked.iter().cloned()) | ||
.collect(); | ||
let out = |uri: &Value| uri.as_str().is_some_and(|u| hidden.iter().any(|h| h == u)); | ||
entries | ||
.into_iter() | ||
.filter(|e| !blocked.contains(&e.author)) | ||
.filter(|e| !hidden.contains(&e.uri) && !out(&e.value["subject"])) | ||
.collect() | ||
} | ||
| ||
/// Who may change an item's state: its author, and the repository's owner | ||
/// and collaborators. Only the latter may merge. | ||
/// The owner the repository had before it moved here counts too, for | ||
/// what they did before it moved: `at` is the state record's key. | ||
fn allowed( | ||
repo: &RepoRecord, | ||
owner: &str, | ||
item_author: &str, | ||
actor: &str, | ||
state: &str, | ||
at: &str, | ||
) -> bool { | ||
let moved_in = humantime::parse_rfc3339_weak(&repo.created_at) | ||
.ok() | ||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) | ||
.map(|d| d.as_micros() as u64); | ||
let before = repo.transferred_from.as_ref().is_some_and(|f| { | ||
f.did() == actor | ||
&& keys::tid_micros(at) | ||
.zip(moved_in) | ||
.is_some_and(|(t, m)| t < m) | ||
}); | ||
let maintainer = actor == owner || repo.is_collaborator(actor) || before; | ||
maintainer || (actor == item_author && state != "merged") | ||
} | ||
| ||
/// Pulls and issues from a pile of forge records about one repository, | ||
/// newest first. | ||
pub fn assemble(repo: &RepoRecord, owner: &str, entries: &[Entry]) -> Vec<Item> { | ||
let mut items: Vec<Item> = Vec::new(); | ||
for e in entries { | ||
let kind = match e.collection() { | ||
nsid::PULL => "pull", | ||
nsid::ISSUE => "issue", | ||
_ => continue, | ||
}; | ||
let pull: Option<PullRecord> = serde_json::from_value(e.value.clone()).ok(); | ||
let issue: Option<IssueRecord> = serde_json::from_value(e.value.clone()).ok(); | ||
let (title, body, created_at) = match (&pull, &issue) { | ||
(Some(p), _) => (p.title.clone(), p.body.clone(), p.created_at.clone()), | ||
(None, Some(i)) => (i.title.clone(), i.body.clone(), i.created_at.clone()), | ||
(None, None) => continue, | ||
}; | ||
items.push(Item { | ||
uri: e.uri.clone(), | ||
author: e.author.clone(), | ||
kind: kind.into(), | ||
title, | ||
body, | ||
state: "open".into(), | ||
merged: None, | ||
base: pull.as_ref().map(|p| p.base.clone()), | ||
head: pull.map(|p| p.head), | ||
comments: 0, | ||
approvals: 0, | ||
created_at, | ||
}); | ||
} | ||
// States in the order they were made: record keys are TIDs. | ||
let mut states: Vec<(&str, &Entry, StateRecord)> = entries | ||
.iter() | ||
.filter(|e| e.collection() == nsid::STATE) | ||
.filter_map(|e| { | ||
Some(( | ||
parse_uri(&e.uri).ok()?.2, | ||
e, | ||
serde_json::from_value(e.value.clone()).ok()?, | ||
)) | ||
}) | ||
.collect(); | ||
states.sort_by(|a, b| a.0.cmp(b.0)); | ||
for item in &mut items { | ||
for (at, e, s) in states.iter().filter(|(_, _, s)| s.subject == item.uri) { | ||
if allowed(repo, owner, &item.author, &e.author, &s.state, at) { | ||
item.state = s.state.clone(); | ||
item.merged = s.commit.clone().filter(|_| s.state == "merged"); | ||
if let (Some(base), Some(_)) = (&s.base, &item.base) { | ||
item.base = Some(base.clone()); | ||
} | ||
} | ||
} | ||
for e in entries.iter().filter(|e| e.collection() == nsid::COMMENT) { | ||
if let Ok(c) = serde_json::from_value::<CommentRecord>(e.value.clone()) | ||
&& c.subject == item.uri | ||
{ | ||
item.comments += 1; | ||
item.approvals += usize::from(c.verdict.as_deref() == Some("approve")); | ||
} | ||
} | ||
} | ||
items.sort_by(|a, b| b.created_at.cmp(&a.created_at)); | ||
items | ||
} | ||
| ||
/// The thread for `subject` among `entries`. | ||
pub fn thread(repo: &RepoRecord, owner: &str, entries: &[Entry], subject: &str) -> Result<Thread> { | ||
let item = assemble(repo, owner, entries) | ||
.into_iter() | ||
.find(|i| i.uri == subject) | ||
.with_context(|| format!("no pull or issue {subject}"))?; | ||
let mut comments: Vec<Entry> = entries | ||
.iter() | ||
.filter(|e| e.collection() == nsid::COMMENT && e.value["subject"] == subject) | ||
.cloned() | ||
.collect(); | ||
comments.sort_by(|a, b| a.uri.rsplit('/').next().cmp(&b.uri.rsplit('/').next())); | ||
Ok(Thread { item, comments }) | ||
} | ||
| ||
/// The repository `record`, named `name`, says it was moved from, if the | ||
/// old one's owner agreed: its offer, or the redirect left in its place, | ||
/// leads here. | ||
async fn previous( | ||
ctx: &Ctx, | ||
remote: &Remote, | ||
record: &RepoRecord, | ||
name: &str, | ||
) -> Result<Option<RepoRef>> { | ||
let Some(from) = record.transferred_from.clone() else { | ||
return Ok(None); | ||
}; | ||
let then = RepoUrl::parse(&format!("knotless://{}/{}", from.did(), from.name()))?; | ||
let old = Remote::connect(ctx, &then).await?; | ||
let old_snap = Snapshot { | ||
repo: old.repo().await?, | ||
..Default::default() | ||
}; | ||
let here = old.moved_to(ctx, &old_snap).await?; | ||
let agreed = here.is_some_and(|u| u.authority == remote.did() && u.name == name); | ||
Ok(agreed.then_some(from)) | ||
} | ||
| ||
/// A repository, opened, with its record. | ||
pub struct Target { | ||
pub url: RepoUrl, | ||
pub remote: Remote, | ||
pub snap: Snapshot, | ||
pub record: RepoRecord, | ||
/// The repository this one was moved from, when it says so and that | ||
/// one agrees: records about it are about this one. | ||
pub previous: Option<RepoRef>, | ||
} | ||
| ||
impl Target { | ||
pub async fn open(ctx: &Ctx, url: &RepoUrl) -> Result<Target> { | ||
let (remote, snap) = Remote::locate(ctx, url).await?; | ||
let repo = snap | ||
.repo | ||
.as_ref() | ||
.with_context(|| format!("{url}: no such repository"))?; | ||
let record = repo.value.clone(); | ||
let previous = previous(ctx, &remote, &record, &repo.rkey).await?; | ||
Ok(Target { | ||
previous, | ||
url: remote.url.clone(), | ||
remote, | ||
snap, | ||
record, | ||
}) | ||
} | ||
| ||
/// The repository as forge records name it, under the name it has now. | ||
pub fn reference(&self) -> RepoRef { | ||
self.snap | ||
.reference(self.remote.did()) | ||
.expect("an opened repository has a record") | ||
} | ||
| ||
fn is_about(&self, value: &Value) -> bool { | ||
is_about(value, self.remote.did(), &self.reference().key) | ||
|| self | ||
.previous | ||
.as_ref() | ||
.is_some_and(|p| is_about(value, p.did(), &p.key)) | ||
} | ||
| ||
/// Every forge record about the repository that its owner lets stand. | ||
/// The appview leaves out whom the owner blocks; in a space, nobody is | ||
/// between, so the owner's blocks are read here. | ||
pub async fn entries(&self, ctx: &Ctx) -> Result<Vec<Entry>> { | ||
let (entries, blocked) = match &self.remote.space { | ||
Some(_) => { | ||
let blocked = crate::moderation::blocked_by(ctx, self.remote.did()).await?; | ||
(self.space_entries(ctx).await?, blocked) | ||
} | ||
None => (self.indexed_entries(ctx).await?, Vec::new()), | ||
}; | ||
let key = self.reference().key; | ||
Ok(moderate(self.remote.did(), &key, &blocked, entries)) | ||
} | ||
| ||
async fn indexed_entries(&self, ctx: &Ctx) -> Result<Vec<Entry>> { | ||
#[derive(Deserialize)] | ||
struct Out { | ||
entries: Vec<Entry>, | ||
} | ||
let appview = ctx.env.appview.as_deref().context( | ||
"pulls and issues on public repositories come from an appview: set KNOTLESS_APPVIEW", | ||
)?; | ||
let repo = self.reference(); | ||
let out: Out = Xrpc::new(ctx.http.clone(), appview) | ||
.query( | ||
"dev.knotless.getForge", | ||
&[("repo", &repo.uri), ("key", &repo.key)], | ||
None, | ||
) | ||
.await?; | ||
Ok(out.entries) | ||
} | ||
| ||
/// The private repository's space, joined as whoever is logged in: | ||
/// reading others' repos there takes a credential, even for the owner. | ||
async fn joined(&self, ctx: &Ctx) -> Result<Space> { | ||
let me = session::me(ctx).await?; | ||
let uri = &self | ||
.remote | ||
.space | ||
.as_ref() | ||
.expect("a private repository") | ||
.uri; | ||
Space::join(ctx, &self.remote.xrpc, uri, self.remote.did(), &me).await | ||
} | ||
| ||
/// `did`'s repo, where its records about this repository are: in the | ||
/// space when there is one. | ||
async fn member(&self, ctx: &Ctx, did: &str, space: Option<&Space>) -> Result<Remote> { | ||
let id = identity::resolve(ctx, did).await?; | ||
Ok(Remote { | ||
xrpc: Xrpc::new(ctx.http.clone(), &id.pds), | ||
identity: id, | ||
url: self.url.clone(), | ||
space: space.map(|s| s.at(did)), | ||
}) | ||
} | ||
| ||
/// The pull or issue `subject` names, read from its author's repo, so | ||
/// it is there the moment it is written. | ||
async fn subject(&self, ctx: &Ctx, subject: &str) -> Result<(String, Value)> { | ||
let (did, collection, rkey) = parse_uri(subject)?; | ||
if collection != nsid::PULL && collection != nsid::ISSUE { | ||
bail!("{subject} is not a pull or an issue"); | ||
} | ||
let space = match self.remote.space { | ||
Some(_) => Some(self.joined(ctx).await?), | ||
None => None, | ||
}; | ||
let author = self.member(ctx, did, space.as_ref()).await?; | ||
let found = author.get_record::<Value>(collection, rkey).await?; | ||
let Some(found) = found.filter(|f| self.is_about(&f.value)) else { | ||
bail!("no pull or issue {subject} on {}", self.url); | ||
}; | ||
Ok((collection.to_string(), found.value)) | ||
} | ||
| ||
/// Reads every member's repo in the space. | ||
async fn space_entries(&self, ctx: &Ctx) -> Result<Vec<Entry>> { | ||
#[derive(Deserialize)] | ||
struct Repos { | ||
repos: Vec<SpaceRepo>, | ||
cursor: Option<String>, | ||
} | ||
#[derive(Deserialize)] | ||
struct SpaceRepo { | ||
did: String, | ||
} | ||
let space = self.joined(ctx).await?; | ||
let uri = space.uri.clone(); | ||
let mut dids = Vec::new(); | ||
let mut cursor: Option<String> = None; | ||
loop { | ||
let mut params = vec![("space", uri.as_str()), ("limit", "1000")]; | ||
if let Some(c) = &cursor { | ||
params.push(("cursor", c)); | ||
} | ||
let page: Repos = self | ||
.remote | ||
.xrpc | ||
.query_as( | ||
"com.atproto.space.listRepos", | ||
¶ms, | ||
&space.for_repo(self.remote.did()), | ||
) | ||
.await?; | ||
let done = page.repos.is_empty(); | ||
dids.extend(page.repos.into_iter().map(|r| r.did)); | ||
match page.cursor { | ||
Some(next) if !done => cursor = Some(next), | ||
_ => break, | ||
} | ||
} | ||
dids.sort(); | ||
dids.dedup(); | ||
let mut entries = Vec::new(); | ||
for did in dids { | ||
let member = self.member(ctx, &did, Some(&space)).await?; | ||
for collection in nsid::FORGE { | ||
for r in member.list_all::<Value>(collection).await? { | ||
if self.is_about(&r.value) { | ||
entries.push(Entry { | ||
uri: format!("at://{did}/{collection}/{}", r.rkey), | ||
author: did.clone(), | ||
value: r.value, | ||
}); | ||
} | ||
} | ||
} | ||
} | ||
Ok(entries) | ||
} | ||
| ||
/// Where `me` writes forge records: my public repo, or my repo in the | ||
/// repository's space. | ||
pub(crate) async fn mine(&self, ctx: &Ctx) -> Result<(Remote, Session)> { | ||
let me = session::me(ctx).await?; | ||
let id = identity::resolve(ctx, &me.did).await?; | ||
let mine = Remote { | ||
xrpc: Xrpc::new(ctx.http.clone(), &id.pds), | ||
identity: id, | ||
url: self.url.clone(), | ||
space: self | ||
.remote | ||
.space | ||
.as_ref() | ||
.map(|s| Space::owner(&s.uri, &me)), | ||
}; | ||
Ok((mine, me)) | ||
} | ||
| ||
/// Writes a forge record as `me`. Returns its URI. | ||
async fn write(&self, ctx: &Ctx, collection: &str, value: Value) -> Result<String> { | ||
let (mine, me) = self.mine(ctx).await?; | ||
let rkey = keys::tid(); | ||
mine.apply( | ||
vec![remote::create(collection, &rkey, value)], | ||
&me.access_jwt, | ||
) | ||
.await?; | ||
Ok(format!("at://{}/{collection}/{rkey}", me.did)) | ||
} | ||
| ||
/// The item `subject` names as it stands, with what its author's repo | ||
/// has now rather than what the index last saw. | ||
async fn item(&self, ctx: &Ctx, subject: &str) -> Result<(Item, Value)> { | ||
let (_, value) = self.subject(ctx, subject).await?; | ||
let mut entries = self.entries(ctx).await?; | ||
entries.retain(|e| e.uri != subject); | ||
entries.push(Entry { | ||
uri: subject.to_string(), | ||
author: parse_uri(subject)?.0.to_string(), | ||
value: value.clone(), | ||
}); | ||
let item = thread(&self.record, self.remote.did(), &entries, subject)?.item; | ||
Ok((item, value)) | ||
} | ||
} | ||
| ||
/// Hides `subject`, a pull, issue or comment, from the repository, or shows | ||
/// it again. Only its owner can. | ||
pub async fn hide(ctx: &Ctx, target: &RepoUrl, subject: &str, hidden: bool) -> Result<()> { | ||
let (_, collection, _) = parse_uri(subject)?; | ||
if ![nsid::PULL, nsid::ISSUE, nsid::COMMENT].contains(&collection) { | ||
bail!("{subject} is not a pull, an issue or a comment"); | ||
} | ||
let t = Target::open(ctx, target).await?; | ||
let (mine, me) = t.mine(ctx).await?; | ||
if me.did != t.remote.did() { | ||
bail!("only {}'s owner can hide what is said about it", t.url); | ||
} | ||
let reference = t.reference(); | ||
let found = mine | ||
.get_record::<GateRecord>(nsid::GATE, &reference.key) | ||
.await?; | ||
let mut gate = found.as_ref().map_or_else( | ||
|| GateRecord { | ||
repo: reference.clone(), | ||
hidden: Vec::new(), | ||
created_at: records::now(), | ||
}, | ||
|f| f.value.clone(), | ||
); | ||
gate.repo = reference.clone(); | ||
gate.hidden.retain(|h| h != subject); | ||
if hidden { | ||
gate.hidden.push(subject.to_string()); | ||
} | ||
let value = records::typed(nsid::GATE, &gate); | ||
let write = match found { | ||
Some(_) => remote::update(nsid::GATE, &reference.key, value), | ||
None => remote::create(nsid::GATE, &reference.key, value), | ||
}; | ||
mine.apply(vec![write], &me.access_jwt).await | ||
} | ||
| ||
pub async fn open_pull( | ||
ctx: &Ctx, | ||
target: &RepoUrl, | ||
base: &str, | ||
head: &RepoUrl, | ||
branch: &str, | ||
title: &str, | ||
body: Option<&str>, | ||
) -> Result<String> { | ||
let t = Target::open(ctx, target).await?; | ||
if t.snap.find_ref(&format!("refs/heads/{base}")).is_none() { | ||
bail!("{target} has no branch {base}"); | ||
} | ||
let source = Target::open(ctx, head).await?; | ||
if source | ||
.snap | ||
.find_ref(&format!("refs/heads/{branch}")) | ||
.is_none() | ||
{ | ||
bail!("{head} has no branch {branch}"); | ||
} | ||
let record = PullRecord { | ||
repo: t.reference(), | ||
base: base.to_string(), | ||
head: PullHead { | ||
repo: source.reference(), | ||
branch: branch.to_string(), | ||
}, | ||
title: title.to_string(), | ||
body: body.map(str::to_string), | ||
created_at: records::now(), | ||
}; | ||
t.write(ctx, nsid::PULL, records::typed(nsid::PULL, &record)) | ||
.await | ||
} | ||
| ||
pub async fn open_issue( | ||
ctx: &Ctx, | ||
target: &RepoUrl, | ||
title: &str, | ||
body: Option<&str>, | ||
) -> Result<String> { | ||
let t = Target::open(ctx, target).await?; | ||
let record = IssueRecord { | ||
repo: t.reference(), | ||
title: title.to_string(), | ||
body: body.map(str::to_string), | ||
created_at: records::now(), | ||
}; | ||
t.write(ctx, nsid::ISSUE, records::typed(nsid::ISSUE, &record)) | ||
.await | ||
} | ||
| ||
/// Comments on a pull or issue; with a verdict, reviews it. | ||
pub async fn comment( | ||
ctx: &Ctx, | ||
target: &RepoUrl, | ||
subject: &str, | ||
body: &str, | ||
verdict: Option<&str>, | ||
) -> Result<String> { | ||
if let Some(v) = verdict | ||
&& v != "approve" | ||
&& v != "changes" | ||
{ | ||
bail!("a verdict is approve or changes, not {v}"); | ||
} | ||
let t = Target::open(ctx, target).await?; | ||
let (collection, value) = t.subject(ctx, subject).await?; | ||
// A review looks at the head as it is now. | ||
let commit = match verdict { | ||
Some(_) if collection == nsid::PULL => { | ||
let head = serde_json::from_value::<PullRecord>(value)?.head; | ||
let source = Target::open(ctx, &repo_url(&head.repo.uri)?).await?; | ||
source | ||
.snap | ||
.find_ref(&format!("refs/heads/{}", head.branch)) | ||
.and_then(|r| r.value.target.clone()) | ||
} | ||
_ => None, | ||
}; | ||
let record = CommentRecord { | ||
repo: t.reference(), | ||
subject: subject.to_string(), | ||
body: body.to_string(), | ||
verdict: verdict.map(str::to_string), | ||
commit, | ||
created_at: records::now(), | ||
}; | ||
t.write(ctx, nsid::COMMENT, records::typed(nsid::COMMENT, &record)) | ||
.await | ||
} | ||
| ||
/// Closes or reopens a pull or issue. | ||
pub async fn set_state(ctx: &Ctx, target: &RepoUrl, subject: &str, state: &str) -> Result<String> { | ||
if state != "open" && state != "closed" { | ||
bail!("a pull or issue is open or closed; merging takes `merge`"); | ||
} | ||
let t = Target::open(ctx, target).await?; | ||
write_state(ctx, &t, subject, state, None).await | ||
} | ||
| ||
async fn write_state( | ||
ctx: &Ctx, | ||
t: &Target, | ||
subject: &str, | ||
state: &str, | ||
commit: Option<String>, | ||
) -> Result<String> { | ||
let record = StateRecord { | ||
repo: t.reference(), | ||
subject: subject.to_string(), | ||
state: state.to_string(), | ||
commit, | ||
base: None, | ||
created_at: records::now(), | ||
}; | ||
t.write(ctx, nsid::STATE, records::typed(nsid::STATE, &record)) | ||
.await | ||
} | ||
| ||
/// Merges a pull: a fast-forward when the base has not moved since the | ||
/// head branched off it, else a merge commit. Fails on conflicts. The owner | ||
/// pushes the result directly; a collaborator through the gateway. | ||
pub async fn merge(ctx: &Ctx, target: &RepoUrl, pull: &str) -> Result<String> { | ||
let t = Target::open(ctx, target).await?; | ||
// The pull as its author's repo has it now, not as the index last saw | ||
// it: the index may not have it yet. | ||
let (item, _) = t.item(ctx, pull).await?; | ||
let (Some(base), Some(head)) = (&item.base, &item.head) else { | ||
bail!("{pull} is an issue"); | ||
}; | ||
if item.state != "open" { | ||
bail!("{pull} is {}", item.state); | ||
} | ||
let key = t.snap.key().expect("an opened repository has a key"); | ||
let mirror = Mirror::open(ctx, t.remote.did(), key).await?; | ||
mirror.sync(&t.remote, &t.snap).await?; | ||
let git = mirror.git; | ||
let source = Target::open(ctx, &repo_url(&head.repo.uri)?).await?; | ||
let head_sha = source | ||
.snap | ||
.find_ref(&format!("refs/heads/{}", head.branch)) | ||
.and_then(|r| r.value.target.clone()) | ||
.with_context(|| format!("{} has no branch {}", source.url, head.branch))?; | ||
let source_key = source.snap.key().expect("an opened repository has a key"); | ||
fetch::fetch( | ||
&source.remote, | ||
source_key, | ||
&git, | ||
std::slice::from_ref(&head_sha), | ||
Some(source.snap.packs.clone()), | ||
) | ||
.await?; | ||
let refname = format!("refs/heads/{base}"); | ||
let base_sha = t | ||
.snap | ||
.find_ref(&refname) | ||
.and_then(|r| r.value.target.clone()) | ||
.with_context(|| format!("{target} has no branch {base}"))?; | ||
let me = session::me(ctx).await?; | ||
let merged = if git.is_ancestor(&base_sha, &head_sha).await? { | ||
head_sha | ||
} else { | ||
merge_commit(&git, &base_sha, &head_sha, &item, &me).await? | ||
}; | ||
let update = Update { | ||
src: Some(merged.clone()), | ||
dst: refname.clone(), | ||
force: false, | ||
}; | ||
let report = if me.did == t.remote.did() { | ||
push::push(ctx, &t.remote, &t.snap, &git, &me.access_jwt, &[update]).await? | ||
} else { | ||
gateway::push(ctx, &t.remote, &t.snap, &git, &me, &[update]).await? | ||
}; | ||
accepted(report)?; | ||
write_state(ctx, &t, pull, "merged", Some(merged.clone())).await?; | ||
Ok(merged) | ||
} | ||
| ||
/// Points a pull at other branches. The target branch changes for anyone | ||
/// who may change the pull's state, by a state record; the head branch only | ||
/// for the pull's author, who rewrites the pull record. | ||
pub async fn retarget( | ||
ctx: &Ctx, | ||
target: &RepoUrl, | ||
pull: &str, | ||
base: Option<&str>, | ||
head: Option<&str>, | ||
) -> Result<()> { | ||
let t = Target::open(ctx, target).await?; | ||
let (item, value) = t.item(ctx, pull).await?; | ||
if item.kind != "pull" { | ||
bail!("{pull} is an issue"); | ||
} | ||
if item.state == "merged" { | ||
bail!("{pull} is merged"); | ||
} | ||
if let Some(base) = base { | ||
if t.snap.find_ref(&format!("refs/heads/{base}")).is_none() { | ||
bail!("{target} has no branch {base}"); | ||
} | ||
let record = StateRecord { | ||
repo: t.reference(), | ||
subject: pull.to_string(), | ||
state: item.state.clone(), | ||
commit: None, | ||
base: Some(base.to_string()), | ||
created_at: records::now(), | ||
}; | ||
t.write(ctx, nsid::STATE, records::typed(nsid::STATE, &record)) | ||
.await?; | ||
} | ||
if let Some(head) = head { | ||
let (mine, me) = t.mine(ctx).await?; | ||
let (author, _, rkey) = parse_uri(pull)?; | ||
if author != me.did { | ||
bail!("only the author of {pull} can change its head"); | ||
} | ||
let mut record: PullRecord = serde_json::from_value(value)?; | ||
let source = Target::open(ctx, &repo_url(&record.head.repo.uri)?).await?; | ||
if source | ||
.snap | ||
.find_ref(&format!("refs/heads/{head}")) | ||
.is_none() | ||
{ | ||
bail!("{} has no branch {head}", source.url); | ||
} | ||
record.head.branch = head.to_string(); | ||
let value = records::typed(nsid::PULL, &record); | ||
mine.apply( | ||
vec![remote::update(nsid::PULL, rkey, value)], | ||
&me.access_jwt, | ||
) | ||
.await?; | ||
} | ||
Ok(()) | ||
} | ||
| ||
/// Fails on the first ref the push refused. | ||
fn accepted(report: push::Report) -> Result<()> { | ||
match report.into_iter().find(|(_, r)| r.is_err()) { | ||
Some((name, Err(reason))) => bail!("{name}: {reason}"), | ||
_ => Ok(()), | ||
} | ||
} | ||
| ||
async fn merge_commit( | ||
git: &Git, | ||
base: &str, | ||
head: &str, | ||
item: &Item, | ||
me: &Session, | ||
) -> Result<String> { | ||
let out = git | ||
.output_of(&["merge-tree", "--write-tree", "--name-only", base, head]) | ||
.await?; | ||
let text = String::from_utf8_lossy(&out.stdout); | ||
let mut lines = text.lines(); | ||
let tree = lines.next().unwrap_or_default().to_string(); | ||
if !out.status.success() { | ||
let files: Vec<&str> = lines.take_while(|l| !l.is_empty()).collect(); | ||
bail!("merging conflicts in {}", files.join(", ")); | ||
} | ||
let mut git = git.clone(); | ||
let email = format!("{}@knotless.invalid", me.did); | ||
git.envs.extend([ | ||
("GIT_AUTHOR_NAME".into(), me.handle.clone()), | ||
("GIT_AUTHOR_EMAIL".into(), email.clone()), | ||
("GIT_COMMITTER_NAME".into(), me.handle.clone()), | ||
("GIT_COMMITTER_EMAIL".into(), email), | ||
]); | ||
let message = format!("Merge {}\n\n{}", item.title, item.uri); | ||
git.text( | ||
&["commit-tree", &tree, "-p", base, "-p", head, "-m", &message], | ||
None, | ||
) | ||
.await | ||
} | ||
| ||
/// Forks a repository into my account, under `name` or its own. A private | ||
/// repository forks into a private one. | ||
pub async fn fork(ctx: &Ctx, source: &RepoUrl, name: Option<&str>) -> Result<RepoUrl> { | ||
let src = Target::open(ctx, source).await?; | ||
let me = session::me(ctx).await?; | ||
let dest = RepoUrl::parse(&format!( | ||
"knotless://{}/{}", | ||
me.did, | ||
name.unwrap_or(src.reference().name()) | ||
))?; | ||
let new = NewRepo { | ||
private: src.remote.space.is_some(), | ||
fork_of: Some(src.reference()), | ||
..Default::default() | ||
}; | ||
copy(ctx, &src, &dest, new, &me).await?; | ||
Ok(dest) | ||
} | ||
| ||
/// Takes up a repository its owner has offered to my account: copies it | ||
/// in, under the name the offer gives and with the same key, naming it as | ||
/// where it came from. From then on it is here: readers of the old one | ||
/// follow, and its pulls, issues and forks come along. | ||
pub async fn accept(ctx: &Ctx, source: &RepoUrl) -> Result<RepoUrl> { | ||
let src = Target::open(ctx, source).await?; | ||
let me = session::me(ctx).await?; | ||
let Some(to) = &src.record.transfer else { | ||
bail!("{source} has not been offered to anyone"); | ||
}; | ||
if to.did() != me.did { | ||
bail!("{source} is offered to {}, not to you", to.did()); | ||
} | ||
let dest = RepoUrl::parse(&format!("knotless://{}/{}", me.did, to.name()))?; | ||
let new = NewRepo { | ||
fork_of: src.record.fork_of.clone(), | ||
key: Some(src.reference().key), | ||
transferred_from: Some(src.reference()), | ||
..Default::default() | ||
}; | ||
copy(ctx, &src, &dest, new, &me).await?; | ||
Ok(dest) | ||
} | ||
| ||
/// Creates `dest` as `new` says, with `src`'s description and default | ||
/// branch, and pushes it every ref `src` has. | ||
async fn copy(ctx: &Ctx, src: &Target, dest: &RepoUrl, new: NewRepo, me: &Session) -> Result<()> { | ||
let key = src.snap.key().expect("an opened repository has a key"); | ||
let mirror = Mirror::open(ctx, src.remote.did(), key).await?; | ||
mirror.sync(&src.remote, &src.snap).await?; | ||
let new = NewRepo { | ||
description: src.record.description.clone(), | ||
default_branch: src.record.default_branch.clone(), | ||
..new | ||
}; | ||
manage::create(ctx, dest, &new).await?; | ||
let copy = Remote::open(ctx, dest).await?; | ||
let snap = copy.snapshot(ctx).await?; | ||
let updates: Vec<Update> = src | ||
.snap | ||
.live_refs() | ||
.into_iter() | ||
.map(|(name, sha)| Update { | ||
src: Some(sha), | ||
dst: name, | ||
force: false, | ||
}) | ||
.collect(); | ||
accepted(push::push(ctx, ©, &snap, &mirror.git, &me.access_jwt, &updates).await?) | ||
} | ||
| ||
/// Pulls or issues on a repository, newest first. | ||
pub async fn list(ctx: &Ctx, target: &RepoUrl) -> Result<Vec<Item>> { | ||
let t = Target::open(ctx, target).await?; | ||
let entries = t.entries(ctx).await?; | ||
Ok(assemble(&t.record, t.remote.did(), &entries)) | ||
} | ||
| ||
pub async fn show(ctx: &Ctx, target: &RepoUrl, subject: &str) -> Result<Thread> { | ||
let t = Target::open(ctx, target).await?; | ||
let entries = t.entries(ctx).await?; | ||
thread(&t.record, t.remote.did(), &entries, subject) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use serde_json::json; | ||
| ||
#[test] | ||
fn refused_pushes_fail() { | ||
let ok = vec![("refs/heads/a".to_string(), Ok(()))]; | ||
accepted(ok.clone()).unwrap(); | ||
let mut refused = ok; | ||
refused.push(("refs/heads/b".into(), Err("fetch first".into()))); | ||
let err = accepted(refused).unwrap_err(); | ||
assert_eq!(err.to_string(), "refs/heads/b: fetch first"); | ||
} | ||
| ||
/// The repository the records are about. | ||
fn r() -> Value { | ||
json!({"uri": "at://did:o/dev.knotless.repo/r", "key": "k"}) | ||
} | ||
| ||
fn entry(author: &str, collection: &str, rkey: &str, value: Value) -> Entry { | ||
Entry { | ||
uri: format!("at://{author}/{collection}/{rkey}"), | ||
author: author.into(), | ||
value, | ||
} | ||
} | ||
| ||
#[test] | ||
fn records_about_a_repository() { | ||
assert!(is_about(&json!({"repo": r()}), "did:o", "k")); | ||
assert!(!is_about(&json!({"repo": r()}), "did:o", "k2")); | ||
assert!(!is_about( | ||
&json!({"repo": "at://did:o/dev.knotless.repo/r"}), | ||
"did:o", | ||
"k" | ||
)); | ||
} | ||
| ||
#[test] | ||
fn what_the_owner_keeps_out() { | ||
let issue = | ||
|author: &str, rkey: &str| entry(author, nsid::ISSUE, rkey, json!({"repo": r()})); | ||
let comment = |rkey: &str, on: &str| { | ||
entry( | ||
"did:a", | ||
nsid::COMMENT, | ||
rkey, | ||
json!({"repo": r(), "subject": on}), | ||
) | ||
}; | ||
let gate = |author: &str, hidden: &[&str]| { | ||
entry( | ||
author, | ||
nsid::GATE, | ||
"k", | ||
json!({"repo": r(), "hidden": hidden, "createdAt": "t"}), | ||
) | ||
}; | ||
let block = |author: &str, subject: &str| { | ||
entry( | ||
author, | ||
nsid::BLOCK, | ||
subject, | ||
json!({"subject": subject, "repo": r(), "createdAt": "t"}), | ||
) | ||
}; | ||
let shown = issue("did:a", "1"); | ||
let hidden = issue("did:a", "2"); | ||
let blocked = issue("did:b", "3"); | ||
let kept_out = issue("did:c", "6"); | ||
let all = vec![ | ||
shown.clone(), | ||
hidden.clone(), | ||
blocked.clone(), | ||
kept_out, | ||
comment("4", &hidden.uri), | ||
comment("5", &shown.uri), | ||
// Only the owner's gate and blocks count. | ||
gate("did:a", &[&shown.uri]), | ||
gate("did:o", &[&hidden.uri]), | ||
block("did:a", "did:o"), | ||
block("did:o", "did:c"), | ||
]; | ||
let kept: Vec<String> = moderate("did:o", "k", &["did:b".into()], all) | ||
.into_iter() | ||
.map(|e| e.uri) | ||
.collect(); | ||
assert_eq!( | ||
kept, | ||
[ | ||
shown.uri.clone(), | ||
"at://did:a/dev.knotless.comment/5".into(), | ||
"at://did:a/dev.knotless.gate/k".into(), | ||
"at://did:o/dev.knotless.gate/k".into(), | ||
"at://did:a/dev.knotless.block/did:o".into(), | ||
"at://did:o/dev.knotless.block/did:c".into(), | ||
] | ||
); | ||
} | ||
| ||
#[test] | ||
fn uris() { | ||
assert_eq!( | ||
repo_uri("did:plc:a", "r"), | ||
"at://did:plc:a/dev.knotless.repo/r" | ||
); | ||
assert_eq!( | ||
repo_url("at://did:plc:a/dev.knotless.repo/r").unwrap().name, | ||
"r" | ||
); | ||
assert!(repo_url("at://did:plc:a/dev.knotless.pull/r").is_err()); | ||
assert!(parse_uri("https://x").is_err()); | ||
assert!(parse_uri("at://did:plc:a/c").is_err()); | ||
assert!(parse_uri("at://handle/c/r").is_err()); | ||
} | ||
| ||
#[test] | ||
fn states_from_those_allowed_to_set_them() { | ||
let repo = RepoRecord { | ||
collaborators: vec![records::Collaborator { | ||
did: "did:c".into(), | ||
}], | ||
..Default::default() | ||
}; | ||
let pull = |rkey: &str, at: &str| { | ||
entry( | ||
"did:a", | ||
nsid::PULL, | ||
rkey, | ||
json!({ | ||
"repo": r(), "base": "main", "head": {"repo": r(), "branch": "x"}, | ||
"title": format!("pull {rkey}"), "createdAt": at, | ||
}), | ||
) | ||
}; | ||
let state = |author: &str, rkey: &str, subject: &str, state: &str| { | ||
entry( | ||
author, | ||
nsid::STATE, | ||
rkey, | ||
json!({ | ||
"repo": r(), "subject": subject, "state": state, "commit": "c", "createdAt": "t", | ||
}), | ||
) | ||
}; | ||
let p1 = "at://did:a/dev.knotless.pull/1"; | ||
let p2 = "at://did:a/dev.knotless.pull/2"; | ||
let i3 = "at://did:a/dev.knotless.issue/3"; | ||
let entries = vec![ | ||
pull("1", "2026-01-01"), | ||
pull("2", "2026-01-02"), | ||
entry( | ||
"did:a", | ||
nsid::ISSUE, | ||
"3", | ||
json!({"repo": r(), "title": "bug", "createdAt": "2026-01-03"}), | ||
), | ||
entry("did:a", nsid::ISSUE, "4", json!({"repo": r()})), | ||
entry("did:a", "dev.knotless.other", "5", json!({})), | ||
entry("did:a", nsid::STATE, "6", json!({"junk": true})), | ||
// The author may close, a stranger may not reopen, the owner merges. | ||
state("did:a", "a1", p1, "closed"), | ||
state("did:x", "a2", p1, "open"), | ||
state("did:a", "a3", p2, "merged"), | ||
state("did:o", "a4", p2, "merged"), | ||
state("did:c", "a5", i3, "closed"), | ||
// A collaborator points the pull at another branch. | ||
entry( | ||
"did:c", | ||
nsid::STATE, | ||
"a6", | ||
json!({"repo": r(), "subject": p1, "state": "closed", "base": "next", "createdAt": "t"}), | ||
), | ||
entry( | ||
"did:x", | ||
nsid::COMMENT, | ||
"c1", | ||
json!({"repo": r(), "subject": p2, "body": "lgtm", "verdict": "approve", "createdAt": "t"}), | ||
), | ||
entry( | ||
"did:x", | ||
nsid::COMMENT, | ||
"c0", | ||
json!({"repo": r(), "subject": p2, "body": "hm", "createdAt": "t"}), | ||
), | ||
entry("did:x", nsid::COMMENT, "c2", json!({"junk": 1})), | ||
]; | ||
let items = assemble(&repo, "did:o", &entries); | ||
let titles: Vec<&str> = items.iter().map(|i| i.title.as_str()).collect(); | ||
assert_eq!(titles, ["bug", "pull 2", "pull 1"]); | ||
assert_eq!(items[0].state, "closed"); | ||
assert_eq!( | ||
(items[1].state.as_str(), items[1].merged.as_deref()), | ||
("merged", Some("c")) | ||
); | ||
assert_eq!((items[1].comments, items[1].approvals), (2, 1)); | ||
assert_eq!(items[2].state, "closed"); | ||
assert_eq!(items[2].base.as_deref(), Some("next")); | ||
let t = thread(&repo, "did:o", &entries, p2).unwrap(); | ||
let bodies: Vec<&Value> = t.comments.iter().map(|c| &c.value["body"]).collect(); | ||
assert_eq!(bodies, [&json!("hm"), &json!("lgtm")]); | ||
assert!(thread(&repo, "did:o", &entries, "at://did:a/dev.knotless.pull/9").is_err()); | ||
} | ||
} | ||
crates/knotless/src/gateway.rsnew+391−0
| @@ -0,0 +1,391 @@ | ||
//! Writing through a gateway. | ||
//! | ||
//! Only an account can write its own records, so a collaborator cannot push | ||
//! to someone else's repository directly. The owner delegates writes to a | ||
//! gateway (`knotless delegate`), and names collaborators on the repository | ||
//! record. A collaborator's push or commit goes to the gateway with a | ||
//! service token from their own PDS; the gateway checks it and the list, | ||
//! and writes as the owner. | ||
//! | ||
//! The gateway also batches: commits to one branch that arrive together | ||
//! become one chain of commits and one write, where pushing each would have | ||
//! raced the others for the branch. | ||
| ||
use crate::auth; | ||
use crate::commit::{Change, Committed, Spec}; | ||
use crate::env::Ctx; | ||
use crate::git::Git; | ||
use crate::identity; | ||
use crate::push::{Report, Update}; | ||
use crate::records::Bytes; | ||
use crate::remote::{Remote, Snapshot}; | ||
use crate::session::Session; | ||
use crate::store; | ||
use crate::xrpc::Xrpc; | ||
use anyhow::{Context, Result, bail}; | ||
use serde::{Deserialize, Serialize}; | ||
use serde_json::json; | ||
use std::collections::BTreeSet; | ||
| ||
pub const PUSH: &str = "dev.knotless.pushRefs"; | ||
pub const COMMIT: &str = "dev.knotless.createCommit"; | ||
pub const DELEGATE: &str = "dev.knotless.putDelegation"; | ||
/// The DID document service a gateway is reached at. | ||
pub const SERVICE: &str = "#knotless_gateway"; | ||
pub const PUSH_MIME: &str = "application/x-knotless-push"; | ||
| ||
/// Where the gateway `did` takes requests. | ||
pub async fn endpoint(ctx: &Ctx, did: &str) -> Result<String> { | ||
identity::service(&identity::document(ctx, did).await?, SERVICE) | ||
.with_context(|| format!("{did} names no {SERVICE} service")) | ||
} | ||
| ||
/// The gateway a repository writes through. | ||
fn gateway_of<'a>(remote: &Remote, snap: &'a Snapshot) -> Result<&'a str> { | ||
match snap.repo.as_ref().and_then(|r| r.value.gateway.as_deref()) { | ||
Some(gateway) => Ok(gateway), | ||
None => bail!( | ||
"{} belongs to {}, and names no gateway to write through", | ||
remote.url, | ||
remote.did() | ||
), | ||
} | ||
} | ||
| ||
async fn client(ctx: &Ctx, me: &Session, gateway: &str, lxm: &str) -> Result<(Xrpc, String)> { | ||
let (endpoint, token) = tokio::try_join!( | ||
endpoint(ctx, gateway), | ||
auth::service_token(ctx, me, gateway, lxm) | ||
)?; | ||
Ok((Xrpc::new(ctx.http.clone(), &endpoint), token)) | ||
} | ||
| ||
/// What precedes the pack in a push to a gateway, on a line of its own. | ||
#[derive(Serialize, Deserialize, Debug, PartialEq)] | ||
pub struct PushHeader { | ||
pub repo: String, | ||
pub name: String, | ||
pub updates: Vec<WireUpdate>, | ||
} | ||
| ||
/// An update with its source resolved to a hash. | ||
#[derive(Serialize, Deserialize, Debug, PartialEq)] | ||
pub struct WireUpdate { | ||
pub dst: String, | ||
/// Absent for a deletion. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub src: Option<String>, | ||
#[serde(default, skip_serializing_if = "std::ops::Not::not")] | ||
pub force: bool, | ||
} | ||
| ||
#[derive(Serialize, Deserialize, Debug, PartialEq)] | ||
pub struct RefResult { | ||
#[serde(rename = "ref")] | ||
pub name: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub error: Option<String>, | ||
} | ||
| ||
#[derive(Serialize, Deserialize)] | ||
pub struct PushResults { | ||
pub results: Vec<RefResult>, | ||
} | ||
| ||
/// Splits a push body into its header and pack. | ||
pub fn split_push(body: &[u8]) -> Result<(PushHeader, &[u8])> { | ||
let end = body | ||
.iter() | ||
.position(|b| *b == b'\n') | ||
.context("a push without a header line")?; | ||
Ok((serde_json::from_slice(&body[..end])?, &body[end + 1..])) | ||
} | ||
| ||
/// Pushes through the repository's gateway, as `me`. | ||
pub async fn push( | ||
ctx: &Ctx, | ||
remote: &Remote, | ||
snap: &Snapshot, | ||
git: &Git, | ||
me: &Session, | ||
updates: &[Update], | ||
) -> Result<Report> { | ||
let gateway = gateway_of(remote, snap)?; | ||
let mut wire = Vec::new(); | ||
for update in updates { | ||
let src = match &update.src { | ||
Some(src) => Some(git.rev_parse(src).await?), | ||
None => None, | ||
}; | ||
wire.push(WireUpdate { | ||
dst: update.dst.clone(), | ||
src, | ||
force: update.force, | ||
}); | ||
} | ||
let tips: Vec<String> = wire | ||
.iter() | ||
.filter_map(|u| u.src.clone()) | ||
.collect::<BTreeSet<_>>() | ||
.into_iter() | ||
.collect(); | ||
let candidates: Vec<String> = snap.live_refs().into_iter().map(|(_, sha)| sha).collect(); | ||
let absent = git.missing(&candidates).await?; | ||
let bases: Vec<String> = candidates | ||
.into_iter() | ||
.filter(|s| !absent.contains(s)) | ||
.collect(); | ||
let pack = if tips.is_empty() { | ||
Vec::new() | ||
} else { | ||
git.pack_objects(&tips, &bases).await? | ||
}; | ||
let header = PushHeader { | ||
repo: remote.did().to_string(), | ||
name: remote.url.name.clone(), | ||
updates: wire, | ||
}; | ||
let mut body = serde_json::to_vec(&header)?; | ||
body.push(b'\n'); | ||
body.extend(pack); | ||
let (xrpc, token) = client(ctx, me, gateway, PUSH).await?; | ||
let out: PushResults = xrpc | ||
.procedure_bytes(PUSH, body, PUSH_MIME, Some(&token)) | ||
.await?; | ||
Ok(out | ||
.results | ||
.into_iter() | ||
.map(|r| (r.name, r.error.map_or(Ok(()), Err))) | ||
.collect()) | ||
} | ||
| ||
/// A change as it travels to a gateway. | ||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] | ||
pub struct WireChange { | ||
pub path: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub data: Option<Bytes>, | ||
#[serde(default, skip_serializing_if = "std::ops::Not::not")] | ||
pub executable: bool, | ||
} | ||
| ||
/// Who a commit names as its author. | ||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] | ||
pub struct WireAuthor { | ||
pub name: String, | ||
pub email: String, | ||
} | ||
| ||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] | ||
pub struct CommitRequest { | ||
pub repo: String, | ||
pub name: String, | ||
pub branch: String, | ||
pub message: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub author: Option<WireAuthor>, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub base: Option<String>, | ||
pub changes: Vec<WireChange>, | ||
} | ||
| ||
impl CommitRequest { | ||
/// The request for `spec`, to `owner`'s repository `name`. | ||
pub fn new(owner: &str, name: &str, spec: &Spec) -> CommitRequest { | ||
CommitRequest { | ||
repo: owner.to_string(), | ||
name: name.to_string(), | ||
branch: spec.branch.clone(), | ||
message: spec.message.clone(), | ||
author: spec | ||
.author | ||
.clone() | ||
.map(|(name, email)| WireAuthor { name, email }), | ||
base: spec.base.clone(), | ||
changes: spec | ||
.changes | ||
.iter() | ||
.map(|c| match c { | ||
Change::Put { | ||
path, | ||
data, | ||
executable, | ||
} => WireChange { | ||
path: path.clone(), | ||
data: Some(Bytes(data.clone())), | ||
executable: *executable, | ||
}, | ||
Change::Delete { path } => WireChange { | ||
path: path.clone(), | ||
data: None, | ||
executable: false, | ||
}, | ||
}) | ||
.collect(), | ||
} | ||
} | ||
| ||
pub fn spec(&self) -> Spec { | ||
Spec { | ||
branch: self.branch.clone(), | ||
message: self.message.clone(), | ||
changes: self | ||
.changes | ||
.iter() | ||
.map(|c| match &c.data { | ||
Some(data) => Change::Put { | ||
path: c.path.clone(), | ||
data: data.0.clone(), | ||
executable: c.executable, | ||
}, | ||
None => Change::Delete { | ||
path: c.path.clone(), | ||
}, | ||
}) | ||
.collect(), | ||
author: self.author.clone().map(|a| (a.name, a.email)), | ||
base: self.base.clone(), | ||
retries: 0, | ||
via_gateway: false, | ||
} | ||
} | ||
} | ||
| ||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] | ||
pub struct CommitResponse { | ||
pub sha: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub parent: Option<String>, | ||
pub changed: bool, | ||
/// How many commits went out in the write that carried this one. | ||
pub batched: usize, | ||
} | ||
| ||
/// Commits through the repository's gateway, as `me`. | ||
pub async fn commit(ctx: &Ctx, remote: &Remote, me: &Session, spec: &Spec) -> Result<Committed> { | ||
let snap = remote.snapshot(ctx).await?; | ||
let gateway = gateway_of(remote, &snap)?; | ||
let req = CommitRequest::new(remote.did(), &remote.url.name, spec); | ||
let (xrpc, token) = client(ctx, me, gateway, COMMIT).await?; | ||
let out: CommitResponse = xrpc | ||
.procedure(COMMIT, Some(&serde_json::to_value(&req)?), Some(&token)) | ||
.await?; | ||
Ok(Committed { | ||
sha: out.sha, | ||
parent: out.parent, | ||
attempts: 1, | ||
changed: out.changed, | ||
}) | ||
} | ||
| ||
/// Delegates `me`'s writes to the gateway `did`: mints an app password for | ||
/// it, hands it over, and remembers the gateway for `share --write`. | ||
pub async fn delegate(ctx: &Ctx, me: &Session, did: &str) -> Result<()> { | ||
#[derive(Deserialize)] | ||
struct AppPassword { | ||
password: String, | ||
} | ||
let created: AppPassword = Xrpc::new(ctx.http.clone(), &me.service) | ||
.procedure( | ||
"com.atproto.server.createAppPassword", | ||
Some(&json!({ "name": format!("knotless gateway {}", crate::keys::tid()) })), | ||
Some(&me.access_jwt), | ||
) | ||
.await?; | ||
let (xrpc, token) = client(ctx, me, did, DELEGATE).await?; | ||
xrpc.procedure::<serde::de::IgnoredAny>( | ||
DELEGATE, | ||
Some(&json!({ "password": created.password })), | ||
Some(&token), | ||
) | ||
.await?; | ||
store::write_bytes(&gateway_path(ctx, &me.did), did.as_bytes(), false) | ||
} | ||
| ||
fn gateway_path(ctx: &Ctx, did: &str) -> std::path::PathBuf { | ||
ctx.env | ||
.config_dir | ||
.join("gateways") | ||
.join(did.replace(':', "_")) | ||
} | ||
| ||
/// The gateway `did` last delegated to. | ||
pub fn delegated(ctx: &Ctx, did: &str) -> Option<String> { | ||
std::fs::read_to_string(gateway_path(ctx, did)).ok() | ||
} | ||
| ||
/// The paths among `changes` that changed between `base` and `tip`: a | ||
/// commit made against `base` would silently undo those. | ||
pub async fn stale_paths( | ||
git: &Git, | ||
base: &str, | ||
tip: &str, | ||
changes: &[Change], | ||
) -> Result<Vec<String>> { | ||
if base == tip { | ||
return Ok(Vec::new()); | ||
} | ||
let changed = git | ||
.text( | ||
&["diff", "--name-only", "-z", "--no-renames", base, tip, "--"], | ||
None, | ||
) | ||
.await?; | ||
let changed: BTreeSet<&str> = changed.split('\0').collect(); | ||
Ok(changes | ||
.iter() | ||
.map(|c| match c { | ||
Change::Put { path, .. } | Change::Delete { path } => path.clone(), | ||
}) | ||
.filter(|p| changed.contains(p.as_str())) | ||
.collect()) | ||
} | ||
| ||
/// Whether a commit's tip is a hash git can name, as a gateway requires. | ||
pub fn is_sha(text: &str) -> bool { | ||
text.len() == 40 && text.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn push_bodies() { | ||
assert!(split_push(b"no header").is_err()); | ||
assert!(split_push(b"{}\n").is_err()); | ||
let (header, pack) = | ||
split_push(b"{\"repo\":\"r\",\"name\":\"n\",\"updates\":[]}\nPACK").unwrap(); | ||
assert_eq!(header.repo, "r"); | ||
assert_eq!(pack, b"PACK"); | ||
assert!(is_sha(&"a".repeat(40)) && !is_sha("main") && !is_sha(&"A".repeat(40))); | ||
} | ||
| ||
#[test] | ||
fn commit_requests_round_trip() { | ||
let spec = Spec { | ||
branch: "main".into(), | ||
message: "m".into(), | ||
changes: vec![ | ||
Change::Put { | ||
path: "a".into(), | ||
data: b"x".to_vec(), | ||
executable: true, | ||
}, | ||
Change::Delete { path: "b".into() }, | ||
], | ||
author: Some(("A".into(), "a@x".into())), | ||
base: None, | ||
retries: 0, | ||
via_gateway: false, | ||
}; | ||
let req = CommitRequest::new("did:a", "r", &spec); | ||
let wire = serde_json::to_value(&req).unwrap(); | ||
assert_eq!(wire["author"], json!({"name": "A", "email": "a@x"})); | ||
assert_eq!(wire["changes"][1], json!({"path": "b"})); | ||
let back: CommitRequest = serde_json::from_value(wire).unwrap(); | ||
assert_eq!(back, req); | ||
let again = back.spec(); | ||
assert_eq!((again.author, again.changes.len()), (spec.author, 2)); | ||
} | ||
} | ||
crates/knotless/src/git.rsnew+308−0
| @@ -0,0 +1,308 @@ | ||
//! The local repository, through the `git` binary. Pack building and | ||
//! indexing are C git's fastest paths; everything here is a thin wrapper. | ||
| ||
use anyhow::{Context, Result, bail}; | ||
use std::collections::HashSet; | ||
use std::path::{Path, PathBuf}; | ||
use std::process::Stdio; | ||
use tokio::io::AsyncWriteExt; | ||
use tokio::process::Command; | ||
| ||
#[derive(Clone, Debug)] | ||
pub struct Git { | ||
pub dir: PathBuf, | ||
/// Extra environment for every command, such as `GIT_INDEX_FILE`. | ||
pub envs: Vec<(String, String)>, | ||
} | ||
| ||
pub const ZERO: &str = "0000000000000000000000000000000000000000"; | ||
| ||
impl Git { | ||
pub fn new(dir: impl Into<PathBuf>) -> Git { | ||
Git { | ||
dir: dir.into(), | ||
envs: Vec::new(), | ||
} | ||
} | ||
| ||
/// The repository git runs a remote helper for. | ||
pub fn from_env() -> Git { | ||
Git::new(std::env::var_os("GIT_DIR").map_or_else(|| PathBuf::from(".git"), PathBuf::from)) | ||
} | ||
| ||
/// Opens the bare repository at `dir`, creating it first if need be. | ||
/// Creation happens beside it and lands with a rename, so processes | ||
/// racing to create the same repository all end up using one. | ||
pub async fn init_bare(dir: &Path) -> Result<Git> { | ||
if !dir.join("HEAD").exists() { | ||
let parent = dir.parent().context("a repository path has a parent")?; | ||
std::fs::create_dir_all(parent)?; | ||
let tmp = parent.join(format!(".init-{}", crate::keys::tid())); | ||
let git = Git::new(&tmp); | ||
git.run(&["init", "--bare", "-q", &tmp.to_string_lossy()], None) | ||
.await?; | ||
for (k, v) in [ | ||
("uploadpack.allowFilter", "true"), | ||
("uploadpack.allowAnySHA1InWant", "true"), | ||
("gc.auto", "0"), | ||
] { | ||
git.run(&["config", k, v], None).await?; | ||
} | ||
if std::fs::rename(&tmp, dir).is_err() { | ||
// Another process created it first. | ||
std::fs::remove_dir_all(&tmp)?; | ||
} | ||
} | ||
Ok(Git::new(dir)) | ||
} | ||
| ||
fn command(&self, args: &[&str]) -> Command { | ||
let mut cmd = Command::new("git"); | ||
cmd.arg("--git-dir").arg(&self.dir).args(args); | ||
cmd.env_remove("GIT_DIR"); | ||
for (k, v) in &self.envs { | ||
cmd.env(k, v); | ||
} | ||
cmd.stdin(Stdio::piped()) | ||
.stdout(Stdio::piped()) | ||
.stderr(Stdio::piped()); | ||
cmd.kill_on_drop(true); | ||
cmd | ||
} | ||
| ||
async fn output(&self, args: &[&str], input: Option<&[u8]>) -> Result<std::process::Output> { | ||
let mut child = self | ||
.command(args) | ||
.spawn() | ||
.context("running git: is it installed?")?; | ||
let mut stdin = child.stdin.take().expect("stdin is piped"); | ||
let input = input.map(<[u8]>::to_vec); | ||
let writer = tokio::spawn(async move { | ||
if let Some(bytes) = input { | ||
// A command that exits without reading reports its own error. | ||
let _ = stdin.write_all(&bytes).await; | ||
} | ||
}); | ||
let out = child.wait_with_output().await?; | ||
writer.await?; | ||
Ok(out) | ||
} | ||
| ||
/// Runs git and returns its exit status and output, whatever they are. | ||
pub async fn output_of(&self, args: &[&str]) -> Result<std::process::Output> { | ||
self.output(args, None).await | ||
} | ||
| ||
/// Runs git and returns stdout, or fails with its stderr. | ||
pub async fn run(&self, args: &[&str], input: Option<&[u8]>) -> Result<Vec<u8>> { | ||
let out = self.output(args, input).await?; | ||
if !out.status.success() { | ||
bail!( | ||
"git {}: {}", | ||
args.first().copied().unwrap_or_default(), | ||
String::from_utf8_lossy(&out.stderr).trim() | ||
); | ||
} | ||
Ok(out.stdout) | ||
} | ||
| ||
pub async fn text(&self, args: &[&str], input: Option<&[u8]>) -> Result<String> { | ||
Ok(String::from_utf8_lossy(&self.run(args, input).await?) | ||
.trim() | ||
.to_string()) | ||
} | ||
| ||
/// Of `shas`, the ones this repository does not have. | ||
pub async fn missing(&self, shas: &[String]) -> Result<HashSet<String>> { | ||
if shas.is_empty() { | ||
return Ok(HashSet::new()); | ||
} | ||
let input = shas.join("\n") + "\n"; | ||
let out = self | ||
.text( | ||
&["cat-file", "--batch-check=%(objectname)"], | ||
Some(input.as_bytes()), | ||
) | ||
.await?; | ||
Ok(out | ||
.lines() | ||
.filter_map(|l| l.strip_suffix(" missing")) | ||
.map(str::to_string) | ||
.collect()) | ||
} | ||
| ||
pub async fn has(&self, sha: &str) -> Result<bool> { | ||
Ok(self.missing(&[sha.to_string()]).await?.is_empty()) | ||
} | ||
| ||
pub async fn rev_parse(&self, rev: &str) -> Result<String> { | ||
self.text(&["rev-parse", "--verify", "--end-of-options", rev], None) | ||
.await | ||
} | ||
| ||
pub async fn is_ancestor(&self, ancestor: &str, descendant: &str) -> Result<bool> { | ||
let out = self | ||
.output(&["merge-base", "--is-ancestor", ancestor, descendant], None) | ||
.await?; | ||
match out.status.code() { | ||
Some(0) => Ok(true), | ||
Some(1) => Ok(false), | ||
_ => bail!( | ||
"git merge-base: {}", | ||
String::from_utf8_lossy(&out.stderr).trim() | ||
), | ||
} | ||
} | ||
| ||
/// A self-contained pack of what `tips` reach and `bases` do not. | ||
pub async fn pack_objects(&self, tips: &[String], bases: &[String]) -> Result<Vec<u8>> { | ||
let mut input = String::new(); | ||
for tip in tips { | ||
input.push_str(tip); | ||
input.push('\n'); | ||
} | ||
for base in bases { | ||
input.push('^'); | ||
input.push_str(base); | ||
input.push('\n'); | ||
} | ||
self.run( | ||
&[ | ||
"pack-objects", | ||
"--revs", | ||
"--stdout", | ||
"-q", | ||
"--delta-base-offset", | ||
], | ||
Some(input.as_bytes()), | ||
) | ||
.await | ||
} | ||
| ||
pub async fn index_pack(&self, pack: &[u8]) -> Result<()> { | ||
self.run(&["index-pack", "--stdin", "--fix-thin"], Some(pack)) | ||
.await?; | ||
Ok(()) | ||
} | ||
| ||
/// Applies `update-ref --stdin` instructions as one transaction. | ||
pub async fn update_refs(&self, script: &str) -> Result<()> { | ||
if !script.is_empty() { | ||
self.run(&["update-ref", "--stdin"], Some(script.as_bytes())) | ||
.await?; | ||
} | ||
Ok(()) | ||
} | ||
| ||
/// Every ref under `refs/`, as `(name, sha)`. | ||
pub async fn refs(&self) -> Result<Vec<(String, String)>> { | ||
let out = self | ||
.text( | ||
&["for-each-ref", "--format=%(refname) %(objectname)", "refs/"], | ||
None, | ||
) | ||
.await?; | ||
Ok(out | ||
.lines() | ||
.filter_map(|l| l.split_once(' ')) | ||
.map(|(n, s)| (n.to_string(), s.to_string())) | ||
.collect()) | ||
} | ||
} | ||
| ||
/// The object count in a pack's header. | ||
pub fn pack_objects_count(pack: &[u8]) -> u64 { | ||
pack.get(8..12).map_or(0, |b| { | ||
u64::from(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) | ||
}) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
pub async fn repo_with_commits(n: usize) -> (tempfile::TempDir, Git, Vec<String>) { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let work = dir.path().join("work"); | ||
std::fs::create_dir_all(&work).unwrap(); | ||
let sh = |args: &[&str]| { | ||
let st = std::process::Command::new("git") | ||
.current_dir(&work) | ||
.args(args) | ||
.env("GIT_AUTHOR_NAME", "t") | ||
.env("GIT_AUTHOR_EMAIL", "t@t") | ||
.env("GIT_COMMITTER_NAME", "t") | ||
.env("GIT_COMMITTER_EMAIL", "t@t") | ||
.output() | ||
.unwrap(); | ||
assert!(st.status.success(), "{st:?}"); | ||
String::from_utf8(st.stdout).unwrap().trim().to_string() | ||
}; | ||
sh(&["init", "-q", "-b", "main"]); | ||
let mut shas = vec![]; | ||
for i in 0..n { | ||
std::fs::write(work.join("f"), format!("{i}\n")).unwrap(); | ||
sh(&["add", "f"]); | ||
sh(&["commit", "-q", "-m", &format!("c{i}")]); | ||
shas.push(sh(&["rev-parse", "HEAD"])); | ||
} | ||
let git = Git::new(work.join(".git")); | ||
(dir, git, shas) | ||
} | ||
| ||
#[tokio::test] | ||
async fn pack_round_trip() { | ||
let (_d, git, shas) = repo_with_commits(3).await; | ||
assert!(git.has(&shas[2]).await.unwrap()); | ||
assert_eq!(git.rev_parse("refs/heads/main").await.unwrap(), shas[2]); | ||
assert!(git.rev_parse("nope").await.is_err()); | ||
assert!(git.is_ancestor(&shas[0], &shas[2]).await.unwrap()); | ||
assert!(!git.is_ancestor(&shas[2], &shas[0]).await.unwrap()); | ||
assert!(git.is_ancestor(ZERO, &shas[0]).await.is_err()); | ||
let pack = git | ||
.pack_objects(&[shas[2].clone()], &[shas[0].clone()]) | ||
.await | ||
.unwrap(); | ||
assert_eq!(pack_objects_count(&pack), 6); | ||
assert_eq!(pack_objects_count(b"PACK"), 0); | ||
| ||
let other = tempfile::tempdir().unwrap(); | ||
let path = other.path().join("bare.git"); | ||
let bare = Git::init_bare(&path).await.unwrap(); | ||
Git::init_bare(&path).await.unwrap(); | ||
// Losing a creation race: the winner's repository is kept. | ||
let racing = other.path().join("racing.git"); | ||
std::fs::create_dir_all(racing.join("occupied")).unwrap(); | ||
Git::init_bare(&racing).await.unwrap(); | ||
assert!(racing.join("occupied").exists()); | ||
let full = git.pack_objects(&[shas[2].clone()], &[]).await.unwrap(); | ||
bare.index_pack(&full).await.unwrap(); | ||
assert!( | ||
bare.missing(&[shas[1].clone(), ZERO.into()]) | ||
.await | ||
.unwrap() | ||
.contains(ZERO) | ||
); | ||
assert!(bare.missing(&[]).await.unwrap().is_empty()); | ||
bare.update_refs(&format!("update refs/heads/x {}\n", shas[2])) | ||
.await | ||
.unwrap(); | ||
bare.update_refs("").await.unwrap(); | ||
assert_eq!( | ||
bare.refs().await.unwrap(), | ||
vec![("refs/heads/x".into(), shas[2].clone())] | ||
); | ||
assert!(bare.index_pack(b"garbage").await.is_err()); | ||
} | ||
| ||
#[tokio::test] | ||
async fn env_and_spawn_failures() { | ||
let git = Git::from_env(); | ||
assert!(git.dir.ends_with(".git") || std::env::var_os("GIT_DIR").is_some()); | ||
let mut git = Git::new("/nonexistent/knotless"); | ||
git.envs.push(("GIT_INDEX_FILE".into(), "/tmp/x".into())); | ||
assert!(git.refs().await.is_err()); | ||
assert!(Git::init_bare(Path::new("/dev/null/x")).await.is_err()); | ||
assert!(Git::init_bare(Path::new("/")).await.is_err()); | ||
} | ||
} | ||
crates/knotless/src/helper.rsnew+331−0
| @@ -0,0 +1,331 @@ | ||
//! The git remote helper protocol (`gitremote-helpers(7)`), over any async | ||
//! reader and writer, against a [`Backend`]. | ||
| ||
use crate::compact; | ||
use crate::env::{CompactMode, Ctx}; | ||
use crate::fetch; | ||
use crate::gateway; | ||
use crate::git::Git; | ||
use crate::push::{self, Report, Update}; | ||
use crate::remote::{Remote, Snapshot}; | ||
use crate::session; | ||
use crate::url::RepoUrl; | ||
use anyhow::{Result, bail}; | ||
use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncWrite, AsyncWriteExt}; | ||
| ||
pub struct Listing { | ||
/// `(sha, refname)`. | ||
pub refs: Vec<(String, String)>, | ||
/// The ref HEAD points at. | ||
pub head: Option<String>, | ||
} | ||
| ||
pub trait Backend { | ||
fn list(&mut self) -> impl Future<Output = Result<Listing>>; | ||
fn fetch(&mut self, wants: Vec<String>) -> impl Future<Output = Result<()>>; | ||
fn push(&mut self, updates: Vec<Update>) -> impl Future<Output = Result<Report>>; | ||
} | ||
| ||
pub async fn run<B, R, W>(backend: &mut B, input: R, mut output: W) -> Result<()> | ||
where | ||
B: Backend, | ||
R: AsyncBufRead + Unpin, | ||
W: AsyncWrite + Unpin, | ||
{ | ||
let mut lines = input.lines(); | ||
while let Some(line) = lines.next_line().await? { | ||
let reply = if line == "capabilities" { | ||
"option\nfetch\npush\n\n".to_string() | ||
} else if line.starts_with("option ") { | ||
"unsupported\n".to_string() | ||
} else if line == "list" || line == "list for-push" { | ||
let listing = backend.list().await?; | ||
let mut out = String::new(); | ||
for (sha, name) in &listing.refs { | ||
out.push_str(&format!("{sha} {name}\n")); | ||
} | ||
if let Some(head) = listing | ||
.head | ||
.filter(|h| listing.refs.iter().any(|(_, n)| n == h)) | ||
{ | ||
out.push_str(&format!("@{head} HEAD\n")); | ||
} | ||
out + "\n" | ||
} else if let Some(first) = line.strip_prefix("fetch ") { | ||
let mut wants = vec![sha_of(first)]; | ||
for line in batch(&mut lines).await? { | ||
match line.strip_prefix("fetch ") { | ||
Some(rest) => wants.push(sha_of(rest)), | ||
None => bail!("expected a fetch line, got {line:?}"), | ||
} | ||
} | ||
backend.fetch(wants).await?; | ||
"\n".to_string() | ||
} else if let Some(first) = line.strip_prefix("push ") { | ||
let mut updates = vec![Update::parse(first)?]; | ||
for line in batch(&mut lines).await? { | ||
match line.strip_prefix("push ") { | ||
Some(rest) => updates.push(Update::parse(rest)?), | ||
None => bail!("expected a push line, got {line:?}"), | ||
} | ||
} | ||
let mut out = String::new(); | ||
for (name, result) in backend.push(updates).await? { | ||
match result { | ||
Ok(()) => out.push_str(&format!("ok {name}\n")), | ||
Err(reason) => out.push_str(&format!("error {name} {reason}\n")), | ||
} | ||
} | ||
out + "\n" | ||
} else if line.is_empty() { | ||
return Ok(()); | ||
} else { | ||
bail!("unknown remote helper command {line:?}"); | ||
}; | ||
output.write_all(reply.as_bytes()).await?; | ||
output.flush().await?; | ||
} | ||
Ok(()) | ||
} | ||
| ||
fn sha_of(rest: &str) -> String { | ||
rest.split(' ').next().unwrap_or_default().to_string() | ||
} | ||
| ||
/// The rest of a batch, up to the blank line that ends it. | ||
async fn batch<R: AsyncBufRead + Unpin>(lines: &mut tokio::io::Lines<R>) -> Result<Vec<String>> { | ||
let mut out = Vec::new(); | ||
while let Some(line) = lines.next_line().await? { | ||
if line.is_empty() { | ||
return Ok(out); | ||
} | ||
out.push(line); | ||
} | ||
Ok(out) | ||
} | ||
| ||
/// Ties a named remote to the account it first reached. A handle can | ||
/// change hands, and the URL keeps naming the handle, so a remote whose | ||
/// handle now leads to another account would fetch someone else's refs. The | ||
/// first contact records the account's DID in the remote's config; a later | ||
/// one that resolves elsewhere is refused. | ||
pub async fn pin(git: &Git, remote: &str, url: &RepoUrl, did: &str) -> Result<()> { | ||
let key = format!("remote.{remote}.knotlessdid"); | ||
let pinned = git | ||
.text(&["config", "--default", "", "--get", &key], None) | ||
.await?; | ||
match pinned.as_str() { | ||
"" => { | ||
git.text(&["config", &key, did], None).await?; | ||
} | ||
same if same == did => {} | ||
other => bail!( | ||
"{url} now leads to {did}, not {other} as when this remote was set up: \ | ||
if the account changed its handle, point the remote at its new one; \ | ||
to trust the new account, run `git config --unset {key}`" | ||
), | ||
} | ||
Ok(()) | ||
} | ||
| ||
/// The backend `git-remote-knotless` runs. | ||
pub struct Knotless { | ||
pub ctx: Ctx, | ||
/// The URL git was given. | ||
asked: RepoUrl, | ||
/// Where it leads: elsewhere, once the repository has moved. | ||
pub remote: Remote, | ||
pub git: Git, | ||
snap: Option<Snapshot>, | ||
} | ||
| ||
impl Knotless { | ||
pub async fn new(ctx: Ctx, url: &RepoUrl, git: Git) -> Result<Knotless> { | ||
let remote = Remote::open(&ctx, url).await?; | ||
Ok(Knotless { | ||
ctx, | ||
asked: url.clone(), | ||
remote, | ||
git, | ||
snap: None, | ||
}) | ||
} | ||
| ||
async fn snapshot(&mut self) -> Result<Snapshot> { | ||
if let Some(snap) = &self.snap { | ||
return Ok(snap.clone()); | ||
} | ||
let snap = self.remote.snapshot(&self.ctx).await?; | ||
let (remote, snap) = self.remote.clone().follow(&self.ctx, snap).await?; | ||
self.remote = remote; | ||
if let Some(repo) = &snap.repo { | ||
let now = RepoUrl { | ||
authority: self.remote.url.authority.clone(), | ||
name: repo.rkey.clone(), | ||
}; | ||
if now != self.asked { | ||
eprintln!( | ||
"knotless: {} has moved to {now}; point the remote at it with `git remote set-url <remote> {now}`", | ||
self.asked | ||
); | ||
} | ||
} | ||
self.snap = Some(snap.clone()); | ||
Ok(snap) | ||
} | ||
| ||
async fn after_push(&self, snap: &Snapshot, auth: &session::Session) -> Result<()> { | ||
let count = snap.packs.items.len() | ||
+ 1 | ||
+ usize::from(snap.packs.next.is_some()) * crate::remote::PAGE; | ||
if count < self.ctx.env.compact_threshold { | ||
return Ok(()); | ||
} | ||
match self.ctx.env.compact_mode { | ||
CompactMode::Off => {} | ||
CompactMode::Inline => { | ||
compact::compact(&self.ctx, &self.remote, auth).await?; | ||
} | ||
CompactMode::Background => { | ||
let exe = std::env::current_exe()?.with_file_name("knotless"); | ||
// Detached: the push has already landed, and git is waiting. | ||
let _ = std::process::Command::new(exe) | ||
.arg("compact") | ||
.arg(self.remote.url.to_string()) | ||
.stdin(std::process::Stdio::null()) | ||
.stdout(std::process::Stdio::null()) | ||
.stderr(std::process::Stdio::null()) | ||
.spawn(); | ||
} | ||
} | ||
Ok(()) | ||
} | ||
} | ||
| ||
impl Backend for Knotless { | ||
async fn list(&mut self) -> Result<Listing> { | ||
self.snap = None; | ||
let snap = self.snapshot().await?; | ||
Ok(Listing { | ||
refs: snap | ||
.live_refs() | ||
.into_iter() | ||
.map(|(name, sha)| (sha, name)) | ||
.collect(), | ||
head: snap | ||
.repo | ||
.as_ref() | ||
.and_then(|r| r.value.default_branch.as_ref()) | ||
.map(|b| format!("refs/heads/{b}")), | ||
}) | ||
} | ||
| ||
async fn fetch(&mut self, wants: Vec<String>) -> Result<()> { | ||
let snap = self.snapshot().await?; | ||
let Some(key) = snap.key() else { | ||
bail!("{}: no such repository", self.remote.url); | ||
}; | ||
fetch::fetch( | ||
&self.remote, | ||
key, | ||
&self.git, | ||
&wants, | ||
Some(snap.packs.clone()), | ||
) | ||
.await?; | ||
Ok(()) | ||
} | ||
| ||
async fn push(&mut self, updates: Vec<Update>) -> Result<Report> { | ||
let snap = self.snapshot().await?; | ||
if let Some(me) = session::reader(&self.ctx, &self.remote.identity).await | ||
&& me.did != self.remote.did() | ||
{ | ||
self.snap = None; | ||
return gateway::push(&self.ctx, &self.remote, &snap, &self.git, &me, &updates).await; | ||
} | ||
let auth = session::ensure(&self.ctx, &self.remote.identity).await?; | ||
let report = push::push( | ||
&self.ctx, | ||
&self.remote, | ||
&snap, | ||
&self.git, | ||
&auth.access_jwt, | ||
&updates, | ||
) | ||
.await?; | ||
self.snap = None; | ||
if report.iter().any(|(_, r)| r.is_ok()) { | ||
self.after_push(&snap, &auth).await?; | ||
} | ||
Ok(report) | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[derive(Default)] | ||
struct Fake { | ||
fetched: Vec<String>, | ||
pushed: Vec<Update>, | ||
} | ||
| ||
impl Backend for Fake { | ||
async fn list(&mut self) -> Result<Listing> { | ||
Ok(Listing { | ||
refs: vec![("a".repeat(40), "refs/heads/main".into())], | ||
head: Some("refs/heads/main".into()), | ||
}) | ||
} | ||
async fn fetch(&mut self, wants: Vec<String>) -> Result<()> { | ||
self.fetched = wants; | ||
Ok(()) | ||
} | ||
async fn push(&mut self, updates: Vec<Update>) -> Result<Report> { | ||
self.pushed = updates.clone(); | ||
Ok(vec![ | ||
(updates[0].dst.clone(), Ok(())), | ||
("refs/heads/x".into(), Err("non-fast-forward".into())), | ||
]) | ||
} | ||
} | ||
| ||
async fn session(input: &str) -> (Result<()>, String, Fake) { | ||
let mut fake = Fake::default(); | ||
let mut out = Vec::new(); | ||
let res = run(&mut fake, input.as_bytes(), &mut out).await; | ||
(res, String::from_utf8(out).unwrap(), fake) | ||
} | ||
| ||
#[tokio::test] | ||
async fn conversation() { | ||
let a = "a".repeat(40); | ||
let (res, out, fake) = session(&format!( | ||
"capabilities\noption progress true\nlist\nfetch {a} refs/heads/main\nfetch {a} refs/heads/x\n\npush +refs/heads/main:refs/heads/main\npush :refs/heads/x\n\n\n" | ||
)) | ||
.await; | ||
res.unwrap(); | ||
assert_eq!( | ||
out, | ||
format!( | ||
"option\nfetch\npush\n\nunsupported\n{a} refs/heads/main\n@refs/heads/main HEAD\n\n\nok refs/heads/main\nerror refs/heads/x non-fast-forward\n\n" | ||
) | ||
); | ||
assert_eq!(fake.fetched, vec![a.clone(), a]); | ||
assert_eq!(fake.pushed.len(), 2); | ||
} | ||
| ||
#[tokio::test] | ||
async fn eof_and_errors() { | ||
assert!(session("list for-push").await.0.is_ok()); | ||
assert!(session("fetch x y").await.0.is_ok()); | ||
assert!(session("push a:b").await.0.is_ok()); | ||
assert!(session("bogus\n").await.0.is_err()); | ||
assert!(session("fetch x y\npush a:b\n\n").await.0.is_err()); | ||
assert!(session("push a:b\nfetch x y\n\n").await.0.is_err()); | ||
assert!(session("push nope\n\n").await.0.is_err()); | ||
assert!(session("push a:b\nnope\n\n").await.0.is_err()); | ||
} | ||
} | ||
crates/knotless/src/identity.rsnew+208−0
| @@ -0,0 +1,208 @@ | ||
//! Handle → DID → PDS, cached on disk so a warm push spends no round trip | ||
//! on it. | ||
| ||
use crate::crypto::PublicKey; | ||
use crate::env::{Ctx, HandleResolver}; | ||
use crate::store; | ||
use crate::xrpc::Xrpc; | ||
use anyhow::{Context, Result, anyhow, bail}; | ||
use serde::{Deserialize, Serialize}; | ||
use serde_json::Value; | ||
use std::collections::BTreeMap; | ||
| ||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] | ||
pub struct Identity { | ||
pub did: String, | ||
pub pds: String, | ||
} | ||
| ||
#[derive(Serialize, Deserialize)] | ||
struct Cached { | ||
did: String, | ||
pds: String, | ||
at: u64, | ||
} | ||
| ||
fn now_secs() -> u64 { | ||
crate::keys::now_micros() / 1_000_000 | ||
} | ||
| ||
pub async fn resolve(ctx: &Ctx, authority: &str) -> Result<Identity> { | ||
let path = ctx.env.cache_dir.join("identity.json"); | ||
let mut cache: BTreeMap<String, Cached> = store::read(&path).unwrap_or_default(); | ||
if let Some(hit) = cache.get(authority) | ||
&& now_secs().saturating_sub(hit.at) < ctx.env.identity_ttl.as_secs() | ||
{ | ||
return Ok(Identity { | ||
did: hit.did.clone(), | ||
pds: hit.pds.clone(), | ||
}); | ||
} | ||
let did = if authority.starts_with("did:") { | ||
authority.to_string() | ||
} else { | ||
resolve_handle(ctx, authority).await? | ||
}; | ||
let pds = match &ctx.env.service { | ||
Some(service) => service.clone(), | ||
None => resolve_pds(ctx, &did).await?, | ||
}; | ||
cache.insert( | ||
authority.to_string(), | ||
Cached { | ||
did: did.clone(), | ||
pds: pds.clone(), | ||
at: now_secs(), | ||
}, | ||
); | ||
// A cache that cannot be written only costs the next lookup. | ||
let _ = store::write(&path, &cache, false); | ||
Ok(Identity { did, pds }) | ||
} | ||
| ||
/// Drops `authority` from the cache, for a caller that found it stale. | ||
pub fn forget(ctx: &Ctx, authority: &str) { | ||
let path = ctx.env.cache_dir.join("identity.json"); | ||
if let Some(mut cache) = store::read::<BTreeMap<String, Cached>>(&path) | ||
&& cache.remove(authority).is_some() | ||
{ | ||
let _ = store::write(&path, &cache, false); | ||
} | ||
} | ||
| ||
pub async fn resolve_handle(ctx: &Ctx, handle: &str) -> Result<String> { | ||
let mut failures = Vec::new(); | ||
for resolver in &ctx.env.handle_resolvers { | ||
let attempt = match resolver { | ||
HandleResolver::WellKnown { scheme } => well_known(ctx, scheme, handle).await, | ||
HandleResolver::Xrpc { service } => { | ||
#[derive(Deserialize)] | ||
struct Out { | ||
did: String, | ||
} | ||
Xrpc::new(ctx.http.clone(), service) | ||
.query::<Out>( | ||
"com.atproto.identity.resolveHandle", | ||
&[("handle", handle)], | ||
None, | ||
) | ||
.await | ||
.map(|o| o.did) | ||
} | ||
}; | ||
match attempt { | ||
Ok(did) if did.starts_with("did:") => return Ok(did), | ||
Ok(other) => failures.push(format!("not a DID: {other}")), | ||
Err(err) => failures.push(err.to_string()), | ||
} | ||
} | ||
bail!("cannot resolve handle {handle}: {}", failures.join("; ")) | ||
} | ||
| ||
async fn well_known(ctx: &Ctx, scheme: &str, handle: &str) -> Result<String> { | ||
let res = ctx | ||
.http | ||
.get(format!("{scheme}://{handle}/.well-known/atproto-did")) | ||
.send() | ||
.await? | ||
.error_for_status()?; | ||
Ok(res.text().await?.trim().to_string()) | ||
} | ||
| ||
pub async fn resolve_pds(ctx: &Ctx, did: &str) -> Result<String> { | ||
let doc = document(ctx, did).await?; | ||
service(&doc, "#atproto_pds").ok_or_else(|| anyhow!("{did} names no #atproto_pds service")) | ||
} | ||
| ||
/// `did`'s DID document, from the PLC directory or its `did:web` host. | ||
pub async fn document(ctx: &Ctx, did: &str) -> Result<Value> { | ||
let url = if let Some(host) = did.strip_prefix("did:web:") { | ||
format!( | ||
"{}://{}/.well-known/did.json", | ||
ctx.env.did_web_scheme, | ||
host.replace("%3A", ":") | ||
) | ||
} else if did.starts_with("did:plc:") { | ||
format!("{}/{did}", ctx.env.plc_url.trim_end_matches('/')) | ||
} else { | ||
bail!("{did}: unsupported DID method"); | ||
}; | ||
Ok(ctx | ||
.http | ||
.get(&url) | ||
.send() | ||
.await? | ||
.error_for_status() | ||
.context(format!("resolving {did}"))? | ||
.json() | ||
.await?) | ||
} | ||
| ||
/// The endpoint of the service whose id ends in `id`, such as `#atproto_pds`. | ||
pub fn service(doc: &Value, id: &str) -> Option<String> { | ||
doc.get("service")? | ||
.as_array()? | ||
.iter() | ||
.find(|s| { | ||
s.get("id") | ||
.and_then(Value::as_str) | ||
.is_some_and(|i| i.ends_with(id)) | ||
})? | ||
.get("serviceEndpoint")? | ||
.as_str() | ||
.map(|s| s.trim_end_matches('/').to_string()) | ||
} | ||
| ||
/// The key the account signs with: its `#atproto` verification method. | ||
pub fn signing_key(doc: &Value) -> Result<PublicKey> { | ||
let method = doc | ||
.get("verificationMethod") | ||
.and_then(Value::as_array) | ||
.and_then(|methods| { | ||
methods.iter().find(|m| { | ||
m.get("id") | ||
.and_then(Value::as_str) | ||
.is_some_and(|id| id.ends_with("#atproto")) | ||
}) | ||
}) | ||
.context("the DID document has no #atproto key")?; | ||
let multibase = method | ||
.get("publicKeyMultibase") | ||
.and_then(Value::as_str) | ||
.context("the #atproto key has no publicKeyMultibase")?; | ||
let kind = method | ||
.get("type") | ||
.and_then(Value::as_str) | ||
.unwrap_or("Multikey"); | ||
PublicKey::from_method(kind, multibase) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use serde_json::json; | ||
| ||
#[test] | ||
fn service_lookup() { | ||
let doc = json!({"service": [ | ||
{"id": "#other", "serviceEndpoint": "x"}, | ||
{"id": "did:plc:a#atproto_pds", "serviceEndpoint": "https://pds.example/"} | ||
]}); | ||
let pds = |doc: &Value| service(doc, "#atproto_pds"); | ||
assert_eq!(pds(&doc).as_deref(), Some("https://pds.example")); | ||
assert_eq!(pds(&json!({"service": []})), None); | ||
assert_eq!(pds(&json!({})), None); | ||
} | ||
| ||
#[test] | ||
fn signing_keys() { | ||
let key = crate::crypto::P256Key::generate().public(); | ||
let doc = json!({"verificationMethod": [ | ||
{"id": "#other", "publicKeyMultibase": "zjunk"}, | ||
{"id": "did:plc:a#atproto", "publicKeyMultibase": key.to_multibase()} | ||
]}); | ||
assert_eq!(signing_key(&doc).unwrap(), key); | ||
assert!(signing_key(&json!({})).is_err()); | ||
assert!(signing_key(&json!({"verificationMethod": [{"id": "#atproto"}]})).is_err()); | ||
} | ||
} | ||
crates/knotless/src/keys.rsnew+129−0
| @@ -0,0 +1,129 @@ | ||
//! Record keys. | ||
//! | ||
//! Every record that belongs to a repository is keyed `<key>.<suffix>`, where | ||
//! `<key>` is the TID the repository was minted with. A collection holds the | ||
//! records of every repository in the account, and the reference PDS lists a | ||
//! collection in rkey order from a cursor, so a scan from `<key>/` (`/` sorts | ||
//! just after `.`) walks one repository's records, newest suffix first, and | ||
//! stops at the first key outside it. | ||
| ||
use crate::base32; | ||
use sha2::{Digest, Sha256}; | ||
use std::sync::atomic::{AtomicU64, Ordering}; | ||
use std::time::{SystemTime, UNIX_EPOCH}; | ||
| ||
static LAST_MICROS: AtomicU64 = AtomicU64::new(0); | ||
| ||
pub fn now_micros() -> u64 { | ||
SystemTime::now() | ||
.duration_since(UNIX_EPOCH) | ||
.expect("the clock is after 1970") | ||
.as_micros() as u64 | ||
} | ||
| ||
/// A TID that sorts after every TID this process made before it. | ||
pub fn tid() -> String { | ||
let now = now_micros(); | ||
let micros = LAST_MICROS | ||
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |last| { | ||
Some(now.max(last + 1)) | ||
}) | ||
.map(|last| now.max(last + 1)) | ||
.expect("the update closure always returns Some"); | ||
tid_from(micros, u64::from(std::process::id())) | ||
} | ||
| ||
pub fn tid_from(micros: u64, clock: u64) -> String { | ||
let micros = micros & ((1 << 53) - 1); | ||
base32::encode_u64(base32::SORTABLE, (micros << 10) | (clock & 1023)) | ||
} | ||
| ||
/// The timestamp in a TID, or `None` if `tid` is not one. | ||
pub fn tid_micros(tid: &str) -> Option<u64> { | ||
base32::decode_u64(base32::SORTABLE, tid).map(|v| v >> 10) | ||
} | ||
| ||
/// 26 symbols of SHA-256: a ref name of any spelling, as a key fragment. | ||
pub fn ref_hash(name: &str) -> String { | ||
let digest = Sha256::digest(name.as_bytes()); | ||
let mut text = base32::encode(base32::LOWER, &digest); | ||
text.truncate(26); | ||
text | ||
} | ||
| ||
pub fn ref_rkey(key: &str, name: &str) -> String { | ||
format!("{key}.{}", ref_hash(name)) | ||
} | ||
| ||
/// The claim a push creates to move `name` off generation `generation`. | ||
pub fn claim_rkey(key: &str, name: &str, generation: u64) -> String { | ||
format!("{key}.{}.{generation}", ref_hash(name)) | ||
} | ||
| ||
pub fn pack_rkey(key: &str) -> String { | ||
format!("{key}.{}", tid()) | ||
} | ||
| ||
/// The suffix of a key under a repository: a pack's TID, a ref's hash. | ||
pub fn suffix<'a>(key: &str, rkey: &'a str) -> Option<&'a str> { | ||
rkey.strip_prefix(key)?.strip_prefix('.') | ||
} | ||
| ||
/// Where a newest-first scan of one repository's records starts. | ||
pub fn scan_cursor(key: &str) -> String { | ||
format!("{key}/") | ||
} | ||
| ||
/// A repository name doubles as the `repo` record's key, so it has to be a | ||
/// valid record key, and short. | ||
pub fn valid_name(name: &str) -> bool { | ||
!name.is_empty() | ||
&& name.len() <= 100 | ||
&& name != "." | ||
&& name != ".." | ||
&& name | ||
.bytes() | ||
.all(|b| b.is_ascii_alphanumeric() || b"._~-".contains(&b)) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn tids_increase_and_decode() { | ||
let a = tid(); | ||
let b = tid(); | ||
assert!(b > a); | ||
assert_eq!(a.len(), 13); | ||
let micros = tid_micros(&a).unwrap(); | ||
assert!(micros.abs_diff(now_micros()) < 10_000_000); | ||
assert_eq!(tid_micros(&tid_from(42, 7)), Some(42)); | ||
assert_eq!(tid_micros("nope"), None); | ||
} | ||
| ||
#[test] | ||
fn key_shapes() { | ||
let key = "3kabc2def3ghi"; | ||
let h = ref_hash("refs/heads/main"); | ||
assert_eq!(h.len(), 26); | ||
assert_eq!(ref_rkey(key, "refs/heads/main"), format!("{key}.{h}")); | ||
assert_eq!( | ||
claim_rkey(key, "refs/heads/main", 4), | ||
format!("{key}.{h}.4") | ||
); | ||
let pack = pack_rkey(key); | ||
assert!(tid_micros(suffix(key, &pack).unwrap()).is_some()); | ||
assert_eq!(suffix(key, "other.x"), None); | ||
assert_eq!(suffix(key, &format!("{key}x")), None); | ||
assert!(scan_cursor(key).as_str() > pack.as_str()); | ||
} | ||
| ||
#[test] | ||
fn names() { | ||
assert!(valid_name("my-project_v2.0~x")); | ||
for bad in ["", ".", "..", "a/b", "sp ace", "é", &"a".repeat(101)] { | ||
assert!(!valid_name(bad), "{bad}"); | ||
} | ||
} | ||
} | ||
crates/knotless/src/lib.rsnew+38−0
| @@ -0,0 +1,38 @@ | ||
//! Git repositories that live in an atproto account and work against any | ||
//! stock PDS. | ||
//! | ||
//! A repository is a handful of record types in the owner's repo (see | ||
//! `lexicons/`): a `repo` record keyed by name, one `ref` record per ref, an | ||
//! immutable `pack` record per push, and a `claim` record per ref move. The | ||
//! claim's key names the ref and the generation it moved from, so two pushes | ||
//! racing from the same generation both try to create the same key and the | ||
//! PDS lets only one through. That makes every push a single `applyWrites` | ||
//! with per-ref compare-and-swap, without `swapCommit` contention against | ||
//! anything else the account writes. | ||
| ||
pub mod auth; | ||
pub mod base32; | ||
pub mod cli; | ||
pub mod commit; | ||
pub mod compact; | ||
pub mod crypto; | ||
pub mod env; | ||
pub mod fetch; | ||
pub mod forge; | ||
pub mod gateway; | ||
pub mod git; | ||
pub mod helper; | ||
pub mod identity; | ||
pub mod keys; | ||
pub mod manage; | ||
pub mod mirror; | ||
pub mod moderation; | ||
pub mod push; | ||
pub mod read; | ||
pub mod records; | ||
pub mod remote; | ||
pub mod session; | ||
pub mod space; | ||
pub mod store; | ||
pub mod url; | ||
pub mod xrpc; | ||
crates/knotless/src/manage.rsnew+425−0
| @@ -0,0 +1,425 @@ | ||
//! Creating repositories and deciding who may read and push them: what the | ||
//! `knotless` command and the MCP server both do. | ||
| ||
use crate::env::Ctx; | ||
use crate::gateway; | ||
use crate::identity; | ||
use crate::keys; | ||
use crate::records::{self, RedirectRecord, RepoRecord, RepoRef, nsid}; | ||
use crate::remote::{self, Remote, Snapshot, Stored}; | ||
use crate::session; | ||
use crate::session::Session; | ||
use crate::space::{Admin, Space}; | ||
use crate::url::RepoUrl; | ||
use anyhow::{Result, bail}; | ||
| ||
#[derive(Clone, Debug, Default)] | ||
pub struct NewRepo { | ||
pub private: bool, | ||
pub description: Option<String>, | ||
pub default_branch: Option<String>, | ||
/// The repository this one forks. | ||
pub fork_of: Option<records::RepoRef>, | ||
/// The key to give it; a new one when absent. A repository moved from | ||
/// another account keeps its key. | ||
pub key: Option<String>, | ||
/// The repository this one is moved from. | ||
pub transferred_from: Option<records::RepoRef>, | ||
} | ||
| ||
/// Creates `url`'s repository record, in a space of its own when private. | ||
pub async fn create(ctx: &Ctx, url: &RepoUrl, new: &NewRepo) -> Result<()> { | ||
let mut remote = Remote::connect(ctx, url).await?; | ||
let auth = session::ensure(ctx, &remote.identity).await?; | ||
if new.private { | ||
let admin = Admin { | ||
xrpc: &remote.xrpc, | ||
session: &auth, | ||
}; | ||
let (uri, made) = admin.create(&url.name).await?; | ||
remote.space = Some(Space::owner(&uri, &auth)); | ||
if !made { | ||
unclaimed(&remote, url).await?; | ||
} | ||
} | ||
let record = RepoRecord { | ||
key: new.key.clone().unwrap_or_else(keys::tid), | ||
name: url.name.clone(), | ||
description: new.description.clone(), | ||
default_branch: new.default_branch.clone().or(Some("main".into())), | ||
fork_of: new.fork_of.clone(), | ||
transferred_from: new.transferred_from.clone(), | ||
created_at: records::now(), | ||
..Default::default() | ||
}; | ||
let mut writes = vec![remote::create( | ||
nsid::REPO, | ||
&url.name, | ||
records::typed(nsid::REPO, &record), | ||
)]; | ||
// A repository renamed away from this name left a redirect here, which | ||
// the new one takes the place of. | ||
if remote | ||
.get_record::<RedirectRecord>(nsid::REDIRECT, &url.name) | ||
.await? | ||
.is_some() | ||
{ | ||
writes.push(remote::delete(nsid::REDIRECT, &url.name)); | ||
} | ||
if let Err(err) = remote.apply(writes, &auth.access_jwt).await { | ||
if remote.repo().await?.is_some() { | ||
bail!("{url} already exists"); | ||
} | ||
return Err(err); | ||
} | ||
Ok(()) | ||
} | ||
| ||
/// Refuses a space that already exists under `url`'s name for a new | ||
/// private repository, if it belongs to another: one renamed since, which | ||
/// keeps its records in the space of its first name and points to them from | ||
/// each later one. Members of that repository hold credentials for the | ||
/// space's address, so it never changes hands. | ||
async fn unclaimed(remote: &Remote, url: &RepoUrl) -> Result<()> { | ||
let redirect = remote | ||
.get_record::<RedirectRecord>(nsid::REDIRECT, &url.name) | ||
.await?; | ||
if let Some(r) = redirect { | ||
bail!( | ||
"{url} was a name of a private repository now named {}: its members can still reach it by that name", | ||
r.value.to.name() | ||
); | ||
} | ||
Ok(()) | ||
} | ||
| ||
/// The spaces named for later names of a private repository, which only | ||
/// point to the one its records are in, named for its first name. There is | ||
/// one for each name it has had since: its old names, which are the | ||
/// redirects in its space, and the one it has now. | ||
async fn pointers(remote: &Remote, repo: &Stored<RepoRecord>) -> Result<Vec<String>> { | ||
let space = remote.space.as_ref().expect("a private repository"); | ||
let first = space.uri.rsplit('/').next().unwrap_or_default(); | ||
let mut names: Vec<String> = remote | ||
.list_all::<RedirectRecord>(nsid::REDIRECT) | ||
.await? | ||
.into_iter() | ||
.filter(|r| r.value.to.is(remote.did(), &repo.value.key)) | ||
.map(|r| r.rkey) | ||
.collect(); | ||
names.push(repo.rkey.clone()); | ||
names.retain(|n| n != first); | ||
Ok(names | ||
.iter() | ||
.map(|n| crate::space::uri(remote.did(), n)) | ||
.collect()) | ||
} | ||
| ||
/// Lets `who` read a private repository, and with `write`, push to any | ||
/// repository through its gateway. | ||
pub async fn share(ctx: &Ctx, url: &RepoUrl, who: &str, write: bool) -> Result<()> { | ||
set_access(ctx, url, who, Some(write)).await | ||
} | ||
| ||
/// Takes back whatever `share` gave `who`. | ||
pub async fn unshare(ctx: &Ctx, url: &RepoUrl, who: &str) -> Result<()> { | ||
set_access(ctx, url, who, None).await | ||
} | ||
| ||
async fn set_access(ctx: &Ctx, url: &RepoUrl, who: &str, access: Option<bool>) -> Result<()> { | ||
let who = identity::resolve(ctx, who).await?; | ||
let remote = Remote::open(ctx, url).await?; | ||
let auth = session::ensure(ctx, &remote.identity).await?; | ||
let Some(repo) = remote.repo().await? else { | ||
bail!("{url}: no such repository"); | ||
}; | ||
if let Some(space) = &remote.space { | ||
let admin = Admin { | ||
xrpc: &remote.xrpc, | ||
session: &auth, | ||
}; | ||
// Members of the spaces that point to it too, to find it by | ||
// those names. | ||
let mut spaces = vec![space.uri.clone()]; | ||
spaces.extend(pointers(&remote, &repo).await?); | ||
for uri in &spaces { | ||
match access { | ||
Some(_) => admin.put_member(uri, &who.did).await?, | ||
None => admin.remove_member(uri, &who.did).await?, | ||
} | ||
} | ||
} else if access == Some(false) { | ||
bail!("{url} is public: anyone can read it, so share it for writing to let them push"); | ||
} | ||
let name = repo.rkey; | ||
let mut record = repo.value; | ||
record.collaborators.retain(|c| c.did != who.did); | ||
if access == Some(true) { | ||
record.collaborators.push(records::Collaborator { | ||
did: who.did.clone(), | ||
}); | ||
match gateway::delegated(ctx, remote.did()) { | ||
Some(gateway) => record.gateway = Some(gateway), | ||
None if record.gateway.is_some() => {} | ||
None => bail!("collaborators push through a gateway: run `knotless delegate` first"), | ||
} | ||
} | ||
remote | ||
.apply( | ||
vec![remote::update( | ||
nsid::REPO, | ||
&name, | ||
records::typed(nsid::REPO, &record), | ||
)], | ||
&auth.access_jwt, | ||
) | ||
.await | ||
} | ||
| ||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] | ||
pub struct Member { | ||
pub did: String, | ||
/// `owner`, `write` or `read`. | ||
pub access: &'static str, | ||
} | ||
| ||
/// Who may do what: the owner, collaborators, and a private repository's | ||
/// readers. | ||
pub async fn members(ctx: &Ctx, url: &RepoUrl) -> Result<Vec<Member>> { | ||
let remote = Remote::open(ctx, url).await?; | ||
let Some(repo) = remote.repo().await? else { | ||
bail!("{url}: no such repository"); | ||
}; | ||
let writers: Vec<String> = repo | ||
.value | ||
.collaborators | ||
.into_iter() | ||
.map(|c| c.did) | ||
.collect(); | ||
let mut readers = Vec::new(); | ||
if let Some(space) = &remote.space { | ||
let auth = session::ensure(ctx, &remote.identity).await?; | ||
let admin = Admin { | ||
xrpc: &remote.xrpc, | ||
session: &auth, | ||
}; | ||
readers = admin.members(&space.uri).await?; | ||
} | ||
let mut all = vec![Member { | ||
did: remote.did().to_string(), | ||
access: "owner", | ||
}]; | ||
all.extend(writers.iter().map(|did| Member { | ||
did: did.clone(), | ||
access: "write", | ||
})); | ||
all.extend( | ||
readers | ||
.into_iter() | ||
.filter(|m| !writers.contains(&m.did)) | ||
.map(|m| Member { | ||
did: m.did, | ||
access: "read", | ||
}), | ||
); | ||
Ok(all) | ||
} | ||
| ||
/// Moves a public repository to a new name. Its records are keyed by its | ||
/// key, not its name, so only the repository record moves. The old name | ||
/// keeps a redirect to the new one, and every redirect already pointing at | ||
/// the repository is repointed, so each old name is one hop from it. | ||
pub async fn rename(ctx: &Ctx, url: &RepoUrl, name: &str) -> Result<RepoUrl> { | ||
let to = RepoUrl::parse(&format!("knotless://{}/{name}", url.authority))?; | ||
let remote = Remote::open(ctx, url).await?; | ||
let auth = session::ensure(ctx, &remote.identity).await?; | ||
let Some(repo) = remote.repo().await? else { | ||
bail!("{url}: no such repository"); | ||
}; | ||
if repo.rkey == to.name { | ||
bail!("{url} is already named {}", to.name); | ||
} | ||
// A private repository's records stay in the space of its first name; | ||
// a space under the new name points there, for members to find it by. | ||
let pointer = match &remote.space { | ||
Some(space) if space.uri != crate::space::uri(remote.did(), &to.name) => { | ||
Some(point(&remote, &auth, &space.uri, &to, &repo.value.key).await?) | ||
} | ||
_ => None, | ||
}; | ||
let mut record = repo.value; | ||
record.name = to.name.clone(); | ||
let target = RepoRef::new(remote.did(), &to.name, &record.key); | ||
let redirect = |at: &str| { | ||
let value = RedirectRecord { | ||
to: target.clone(), | ||
space: None, | ||
created_at: records::now(), | ||
}; | ||
(at.to_string(), records::typed(nsid::REDIRECT, &value)) | ||
}; | ||
let mut writes = vec![ | ||
remote::create(nsid::REPO, &to.name, records::typed(nsid::REPO, &record)), | ||
remote::delete(nsid::REPO, &repo.rkey), | ||
]; | ||
let (at, value) = redirect(&repo.rkey); | ||
writes.push(remote::create(nsid::REDIRECT, &at, value)); | ||
for old in remote.list_all::<RedirectRecord>(nsid::REDIRECT).await? { | ||
if old.rkey == to.name { | ||
// The new name was an old name of this or another repository. | ||
writes.push(remote::delete(nsid::REDIRECT, &old.rkey)); | ||
} else if old.value.to.is(remote.did(), &record.key) { | ||
let (at, value) = redirect(&old.rkey); | ||
writes.push(remote::update(nsid::REDIRECT, &at, value)); | ||
} | ||
} | ||
remote.apply(writes, &auth.access_jwt).await?; | ||
if let Some((there, write)) = pointer { | ||
there.apply(vec![write], &auth.access_jwt).await?; | ||
} | ||
Ok(to) | ||
} | ||
| ||
/// Makes the space under `to`'s name point to the private repository whose | ||
/// records are in `home`, with the same members: a new space, or the one | ||
/// it had from a time it had this name before. Returns where the redirect | ||
/// goes and its write, for once the repository has the name. | ||
async fn point( | ||
remote: &Remote, | ||
auth: &Session, | ||
home: &str, | ||
to: &RepoUrl, | ||
key: &str, | ||
) -> Result<(Remote, serde_json::Value)> { | ||
let admin = Admin { | ||
xrpc: &remote.xrpc, | ||
session: auth, | ||
}; | ||
let (uri, made) = admin.create(&to.name).await?; | ||
let there = Remote { | ||
url: to.clone(), | ||
space: Some(Space::owner(&uri, auth)), | ||
..remote.clone() | ||
}; | ||
if !made && there.home().await?.as_deref() != Some(home) { | ||
bail!("{to} already exists"); | ||
} | ||
for member in admin.members(home).await? { | ||
admin.put_member(&uri, &member.did).await?; | ||
} | ||
let value = RedirectRecord { | ||
to: RepoRef::new(remote.did(), &to.name, key), | ||
space: Some(home.to_string()), | ||
created_at: records::now(), | ||
}; | ||
let value = records::typed(nsid::REDIRECT, &value); | ||
let write = match made { | ||
true => remote::create(nsid::REDIRECT, &to.name, value), | ||
false => remote::update(nsid::REDIRECT, &to.name, value), | ||
}; | ||
Ok((there, write)) | ||
} | ||
| ||
/// Offers a public repository to another account, under `to`'s name, or | ||
/// with `None` withdraws the offer. The repository moves when that account | ||
/// accepts ([`crate::forge::accept`]). | ||
pub async fn transfer(ctx: &Ctx, url: &RepoUrl, to: Option<&RepoUrl>) -> Result<()> { | ||
let remote = Remote::open(ctx, url).await?; | ||
if remote.space.is_some() { | ||
bail!( | ||
"{url} is private: its members' pulls and issues are in its space, which cannot move" | ||
); | ||
} | ||
let auth = session::ensure(ctx, &remote.identity).await?; | ||
let Some(repo) = remote.repo().await? else { | ||
bail!("{url}: no such repository"); | ||
}; | ||
let mut record = repo.value; | ||
record.transfer = match to { | ||
Some(to) => { | ||
let them = identity::resolve(ctx, &to.authority).await?; | ||
if them.did == remote.did() { | ||
bail!("{url} is already in that account: rename it instead"); | ||
} | ||
Some(RepoRef::new(&them.did, &to.name, &record.key)) | ||
} | ||
None => None, | ||
}; | ||
remote | ||
.apply( | ||
vec![remote::update( | ||
nsid::REPO, | ||
&repo.rkey, | ||
records::typed(nsid::REPO, &record), | ||
)], | ||
&auth.access_jwt, | ||
) | ||
.await | ||
} | ||
| ||
/// Deletes a repository and everything in it. | ||
pub async fn delete(ctx: &Ctx, url: &RepoUrl) -> Result<()> { | ||
let remote = Remote::open(ctx, url).await?; | ||
let auth = session::ensure(ctx, &remote.identity).await?; | ||
let Some(repo) = remote.repo().await? else { | ||
bail!("{url}: no such repository"); | ||
}; | ||
if let Some(space) = &remote.space { | ||
// Deleting the space deletes the owner's repo in it, and with it | ||
// go the spaces that point to it. | ||
let admin = Admin { | ||
xrpc: &remote.xrpc, | ||
session: &auth, | ||
}; | ||
for uri in pointers(&remote, &repo).await? { | ||
admin.delete(&uri).await?; | ||
} | ||
return admin.delete(&space.uri).await; | ||
} | ||
// A repository another account has taken up leaves redirects to it, | ||
// under its name and every old one; any other leaves nothing. | ||
let snap = Snapshot { | ||
repo: Some(repo.clone()), | ||
..Default::default() | ||
}; | ||
let moved = match remote.moved_to(ctx, &snap).await? { | ||
Some(_) => repo.value.transfer.clone(), | ||
None => None, | ||
}; | ||
let key = repo.value.key; | ||
let mut writes = vec![remote::delete(nsid::REPO, &repo.rkey)]; | ||
let redirect = |to: &RepoRef| { | ||
let value = RedirectRecord { | ||
to: to.clone(), | ||
space: None, | ||
created_at: records::now(), | ||
}; | ||
records::typed(nsid::REDIRECT, &value) | ||
}; | ||
if let Some(to) = &moved { | ||
writes.push(remote::create(nsid::REDIRECT, &repo.rkey, redirect(to))); | ||
} | ||
for r in remote.list_all::<RedirectRecord>(nsid::REDIRECT).await? { | ||
match &moved { | ||
Some(to) if r.value.to.is(remote.did(), &key) => { | ||
writes.push(remote::update(nsid::REDIRECT, &r.rkey, redirect(to))); | ||
} | ||
None if r.value.to.is(remote.did(), &key) => { | ||
writes.push(remote::delete(nsid::REDIRECT, &r.rkey)); | ||
} | ||
_ => {} | ||
} | ||
} | ||
for collection in [nsid::REF, nsid::CLAIM, nsid::PACK] { | ||
for r in remote | ||
.scan_all::<serde_json::Value>(collection, &key) | ||
.await? | ||
{ | ||
writes.push(remote::delete(collection, &r.rkey)); | ||
} | ||
} | ||
for chunk in writes.chunks(200) { | ||
remote.apply(chunk.to_vec(), &auth.access_jwt).await?; | ||
} | ||
Ok(()) | ||
} | ||
crates/knotless/src/mirror.rsnew+145−0
| @@ -0,0 +1,145 @@ | ||
//! A bare copy of a repository in the cache directory, kept in step with | ||
//! the records. Compaction, checkout-less commits and `knotless-serve` all | ||
//! work from one. | ||
| ||
use crate::env::Ctx; | ||
use crate::fetch; | ||
use crate::git::Git; | ||
use crate::remote::{Remote, Snapshot}; | ||
use anyhow::Result; | ||
use std::path::PathBuf; | ||
use std::time::Duration; | ||
| ||
/// A lock file. One left behind by a process that died is taken over once | ||
/// it is older than any sync could be. | ||
struct Lock(PathBuf); | ||
| ||
const STALE: Duration = Duration::from_secs(120); | ||
| ||
impl Lock { | ||
async fn acquire(path: PathBuf) -> Result<Lock> { | ||
loop { | ||
match std::fs::OpenOptions::new() | ||
.write(true) | ||
.create_new(true) | ||
.open(&path) | ||
{ | ||
Ok(_) => return Ok(Lock(path)), | ||
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { | ||
let age = std::fs::metadata(&path) | ||
.and_then(|m| m.modified()) | ||
.ok() | ||
.and_then(|t| t.elapsed().ok()) | ||
.unwrap_or_default(); | ||
if age > STALE { | ||
let _ = std::fs::remove_file(&path); | ||
} else { | ||
tokio::time::sleep(Duration::from_millis(10)).await; | ||
} | ||
} | ||
Err(e) => return Err(e.into()), | ||
} | ||
} | ||
} | ||
} | ||
| ||
impl Drop for Lock { | ||
fn drop(&mut self) { | ||
let _ = std::fs::remove_file(&self.0); | ||
} | ||
} | ||
| ||
pub struct Mirror { | ||
pub git: Git, | ||
} | ||
| ||
pub fn path(ctx: &Ctx, did: &str, key: &str) -> PathBuf { | ||
ctx.env | ||
.cache_dir | ||
.join("mirrors") | ||
.join(did.replace(':', "_")) | ||
.join(format!("{key}.git")) | ||
} | ||
| ||
impl Mirror { | ||
pub async fn open(ctx: &Ctx, did: &str, key: &str) -> Result<Mirror> { | ||
Ok(Mirror { | ||
git: Git::init_bare(&path(ctx, did, key)).await?, | ||
}) | ||
} | ||
| ||
/// Fetches what `snap` names and sets the mirror's refs and HEAD to match | ||
/// it exactly. Processes sharing the cache take turns at the refs. | ||
pub async fn sync(&self, remote: &Remote, snap: &Snapshot) -> Result<()> { | ||
let Some(key) = snap.key() else { | ||
return Ok(()); | ||
}; | ||
let live = snap.live_refs(); | ||
let wants: Vec<String> = live.iter().map(|(_, sha)| sha.clone()).collect(); | ||
// Indexing packs needs no lock: each lands under its own name. | ||
fetch::fetch(remote, key, &self.git, &wants, Some(snap.packs.clone())).await?; | ||
let _lock = Lock::acquire(self.git.dir.with_extension("lock")).await?; | ||
let mut want = live; | ||
want.sort(); | ||
let local = self.git.refs().await?; | ||
if local != want { | ||
let mut script = String::new(); | ||
for (name, sha) in &want { | ||
script.push_str(&format!("update {name} {sha}\n")); | ||
} | ||
for (name, _) in &local { | ||
if !want.iter().any(|(n, _)| n == name) { | ||
script.push_str(&format!("delete {name}\n")); | ||
} | ||
} | ||
self.git.update_refs(&script).await?; | ||
} | ||
if let Some(branch) = snap | ||
.repo | ||
.as_ref() | ||
.and_then(|r| r.value.default_branch.as_deref()) | ||
{ | ||
let head = format!("ref: refs/heads/{branch}\n"); | ||
if std::fs::read_to_string(self.git.dir.join("HEAD")) | ||
.ok() | ||
.as_deref() | ||
!= Some(head.as_str()) | ||
{ | ||
self.git | ||
.run( | ||
&["symbolic-ref", "HEAD", &format!("refs/heads/{branch}")], | ||
None, | ||
) | ||
.await?; | ||
} | ||
} | ||
Ok(()) | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[tokio::test] | ||
async fn locks() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let path = dir.path().join("m.lock"); | ||
let held = Lock::acquire(path.clone()).await.unwrap(); | ||
let waiter = tokio::spawn(Lock::acquire(path.clone())); | ||
tokio::time::sleep(Duration::from_millis(50)).await; | ||
assert!(!waiter.is_finished()); | ||
drop(held); | ||
drop(waiter.await.unwrap().unwrap()); | ||
// Abandoned long ago: taken over. | ||
let file = std::fs::File::create(&path).unwrap(); | ||
file.set_modified(std::time::SystemTime::now() - STALE * 2) | ||
.unwrap(); | ||
drop(Lock::acquire(path.clone()).await.unwrap()); | ||
assert!( | ||
Lock::acquire(dir.path().join("no/such/dir.lock")) | ||
.await | ||
.is_err() | ||
); | ||
} | ||
} | ||
crates/knotless/src/moderation.rsnew+282−0
| @@ -0,0 +1,282 @@ | ||
//! Moderation, as atproto does it. | ||
//! | ||
//! A repository's owner moderates what others write about it: accounts they | ||
//! block ([`BlockRecord`](crate::records::BlockRecord)) from all their | ||
//! repositories or from that one are left out, and what they hide in the | ||
//! repository's [`GateRecord`](crate::records::GateRecord) is too, as a | ||
//! thread's author hides replies with a threadgate. | ||
//! | ||
//! Anyone can report an account, a repository or what someone wrote about | ||
//! one to a moderation service (`com.atproto.moderation.createReport`, with | ||
//! a service token). That service's operator takes things down | ||
//! (`com.atproto.admin.updateSubjectStatus`), and an appview honours its | ||
//! takedowns and the `!takedown` and `!hide` labels of the labelers it | ||
//! follows. | ||
| ||
use crate::auth; | ||
use crate::env::Ctx; | ||
use crate::forge::{Target, parse_uri}; | ||
use crate::identity; | ||
use crate::records::{BlockRecord, nsid}; | ||
use crate::remote::Remote; | ||
use crate::session::{self, Session}; | ||
use crate::url::RepoUrl; | ||
use crate::xrpc::Xrpc; | ||
use anyhow::{Context, Result, bail}; | ||
use base64::Engine; | ||
use serde::Deserialize; | ||
use serde_json::{Value, json}; | ||
| ||
pub const REPORT: &str = "com.atproto.moderation.createReport"; | ||
pub const UPDATE_STATUS: &str = "com.atproto.admin.updateSubjectStatus"; | ||
pub const GET_STATUS: &str = "com.atproto.admin.getSubjectStatus"; | ||
pub const LIST_REPORTS: &str = "dev.knotless.listReports"; | ||
/// The DID document service a moderation service takes reports at. | ||
pub const LABELER: &str = "#atproto_labeler"; | ||
/// The labels that take something out of view. | ||
pub const TAKEDOWN: &str = "!takedown"; | ||
pub const HIDE: &str = "!hide"; | ||
| ||
/// The full name of a report's reason: `spam`, `violation`, `misleading`, | ||
/// `sexual`, `rude` or `other`. | ||
pub fn reason_type(short: &str) -> Result<String> { | ||
let name = match short { | ||
"spam" => "Spam", | ||
"violation" => "Violation", | ||
"misleading" => "Misleading", | ||
"sexual" => "Sexual", | ||
"rude" => "Rude", | ||
"other" => "Other", | ||
_ => bail!("{short}: a reason is spam, violation, misleading, sexual, rude or other"), | ||
}; | ||
Ok(format!("com.atproto.moderation.defs#reason{name}")) | ||
} | ||
| ||
/// What a report or a takedown is about, as atproto names it: an account | ||
/// (`com.atproto.admin.defs#repoRef`) or a record as it is now | ||
/// (`com.atproto.repo.strongRef`). `target` is a handle or DID, a | ||
/// `knotless://` URL for the repository's record, or an `at://` URI. | ||
pub async fn subject(ctx: &Ctx, target: &str) -> Result<Value> { | ||
if target.starts_with("knotless://") { | ||
let url = RepoUrl::parse(target)?; | ||
let (remote, snap) = Remote::locate(ctx, &url).await?; | ||
let repo = snap | ||
.repo | ||
.with_context(|| format!("{url}: no such repository"))?; | ||
let uri = format!("at://{}/{}/{}", remote.did(), nsid::REPO, repo.rkey); | ||
return Ok(strong_ref(&uri, &repo.cid)); | ||
} | ||
if target.starts_with("at://") { | ||
let (did, collection, rkey) = parse_uri(target)?; | ||
let url = RepoUrl::parse(&format!("knotless://{did}/_"))?; | ||
let found = Remote::connect(ctx, &url) | ||
.await? | ||
.get_record::<Value>(collection, rkey) | ||
.await? | ||
.with_context(|| format!("no record {target}"))?; | ||
return Ok(strong_ref(target, &found.cid)); | ||
} | ||
let id = identity::resolve(ctx, target).await?; | ||
Ok(json!({"$type": "com.atproto.admin.defs#repoRef", "did": id.did})) | ||
} | ||
| ||
fn strong_ref(uri: &str, cid: &str) -> Value { | ||
json!({"$type": "com.atproto.repo.strongRef", "uri": uri, "cid": cid}) | ||
} | ||
| ||
/// The moderation service reports go to, and where it takes them: | ||
/// `KNOTLESS_MODERATOR`, else the appview's own. | ||
async fn moderator(ctx: &Ctx) -> Result<(String, String)> { | ||
let doc = match (&ctx.env.moderator, &ctx.env.appview) { | ||
(Some(did), _) => identity::document(ctx, did).await?, | ||
(None, Some(appview)) => { | ||
let url = format!("{}/.well-known/did.json", appview.trim_end_matches('/')); | ||
ctx.http | ||
.get(url) | ||
.send() | ||
.await? | ||
.error_for_status()? | ||
.json() | ||
.await? | ||
} | ||
(None, None) => bail!("reports go to a moderation service: set KNOTLESS_MODERATOR"), | ||
}; | ||
let did = doc["id"].as_str().context("a DID document names its DID")?; | ||
let endpoint = identity::service(&doc, LABELER) | ||
.with_context(|| format!("{did} names no {LABELER} service"))?; | ||
Ok((did.to_string(), endpoint)) | ||
} | ||
| ||
/// Reports `target` (see [`subject`]) for `reason` (see [`reason_type`]). | ||
pub async fn report(ctx: &Ctx, target: &str, reason: &str, text: Option<&str>) -> Result<Value> { | ||
let reason_type = reason_type(reason)?; | ||
let subject = subject(ctx, target).await?; | ||
let me = session::me(ctx).await?; | ||
let (did, endpoint) = moderator(ctx).await?; | ||
let token = auth::service_token(ctx, &me, &did, REPORT).await?; | ||
let mut body = json!({"reasonType": reason_type, "subject": subject}); | ||
if let Some(text) = text { | ||
body["reason"] = text.into(); | ||
} | ||
Xrpc::new(ctx.http.clone(), &endpoint) | ||
.procedure(REPORT, Some(&body), Some(&token)) | ||
.await | ||
} | ||
| ||
/// Blocks `who`, or stops blocking them, from `me`'s repositories, or from | ||
/// `repo` alone. Returns their DID. | ||
pub async fn block(ctx: &Ctx, who: &str, repo: Option<&RepoUrl>, on: bool) -> Result<String> { | ||
let did = identity::resolve(ctx, who).await?.did; | ||
let (account, me, scope) = match repo { | ||
Some(url) => { | ||
let t = Target::open(ctx, url).await?; | ||
let (mine, me) = t.mine(ctx).await?; | ||
if me.did != t.remote.did() { | ||
bail!("only {}'s owner can keep accounts out of it", t.url); | ||
} | ||
(mine, me, Some(t.reference())) | ||
} | ||
None => { | ||
let me = session::me(ctx).await?; | ||
(mine(ctx, &me).await?, me, None) | ||
} | ||
}; | ||
if did == me.did { | ||
bail!("that is you"); | ||
} | ||
let key = scope.as_ref().map(|r| &r.key); | ||
let existing: Vec<String> = blocks_of(&account) | ||
.await? | ||
.into_iter() | ||
.filter(|(_, b)| b.subject == did && b.repo.as_ref().map(|r| &r.key) == key) | ||
.map(|(rkey, _)| rkey) | ||
.collect(); | ||
let record = BlockRecord { | ||
subject: did.clone(), | ||
repo: scope, | ||
created_at: crate::records::now(), | ||
}; | ||
let writes: Vec<Value> = match (on, existing.is_empty()) { | ||
(true, true) => vec![crate::remote::create( | ||
nsid::BLOCK, | ||
&crate::keys::tid(), | ||
crate::records::typed(nsid::BLOCK, &record), | ||
)], | ||
(true, false) => Vec::new(), | ||
(false, _) => existing | ||
.iter() | ||
.map(|rkey| crate::remote::delete(nsid::BLOCK, rkey)) | ||
.collect(), | ||
}; | ||
if !writes.is_empty() { | ||
account.apply(writes, &me.access_jwt).await?; | ||
} | ||
Ok(did) | ||
} | ||
| ||
async fn mine(ctx: &Ctx, me: &Session) -> Result<Remote> { | ||
Remote::connect(ctx, &RepoUrl::parse(&format!("knotless://{}/_", me.did))?).await | ||
} | ||
| ||
/// The blocks in `account`, by their keys. | ||
async fn blocks_of(account: &Remote) -> Result<Vec<(String, BlockRecord)>> { | ||
Ok(account | ||
.list_all::<BlockRecord>(nsid::BLOCK) | ||
.await? | ||
.into_iter() | ||
.map(|b| (b.rkey, b.value)) | ||
.collect()) | ||
} | ||
| ||
/// Whom `did` keeps out of all their repositories, from their own repo. | ||
pub async fn blocked_by(ctx: &Ctx, did: &str) -> Result<Vec<String>> { | ||
let account = Remote::connect(ctx, &RepoUrl::parse(&format!("knotless://{did}/_"))?).await?; | ||
Ok(blocks_of(&account) | ||
.await? | ||
.into_iter() | ||
.filter(|(_, b)| b.repo.is_none()) | ||
.map(|(_, b)| b.subject) | ||
.collect()) | ||
} | ||
| ||
/// A moderation service's operator, by its admin password | ||
/// (`KNOTLESS_ADMIN_PASSWORD`), at the appview. | ||
pub struct Admin { | ||
xrpc: Xrpc, | ||
headers: Vec<(String, String)>, | ||
} | ||
| ||
impl Admin { | ||
pub fn new(ctx: &Ctx) -> Result<Admin> { | ||
let service = ctx | ||
.env | ||
.appview | ||
.as_deref() | ||
.context("moderation happens at an appview: set KNOTLESS_APPVIEW")?; | ||
let password = ctx | ||
.env | ||
.admin_password | ||
.as_deref() | ||
.context("set KNOTLESS_ADMIN_PASSWORD to the appview's admin password")?; | ||
let basic = base64::engine::general_purpose::STANDARD.encode(format!("admin:{password}")); | ||
Ok(Admin { | ||
xrpc: Xrpc::new(ctx.http.clone(), service), | ||
headers: vec![("authorization".into(), format!("Basic {basic}"))], | ||
}) | ||
} | ||
| ||
/// Takes `subject` (see [`subject`]) down, or puts it back. | ||
pub async fn takedown(&self, ctx: &Ctx, target: &str, applied: bool) -> Result<Value> { | ||
let subject = subject(ctx, target).await?; | ||
let body = json!({"subject": subject, "takedown": {"applied": applied}}); | ||
self.xrpc | ||
.procedure_as(UPDATE_STATUS, Some(&body), &self.headers) | ||
.await | ||
} | ||
| ||
/// Whether `target` is taken down. | ||
pub async fn status(&self, ctx: &Ctx, target: &str) -> Result<Value> { | ||
let subject = subject(ctx, target).await?; | ||
let param = match subject["did"].as_str() { | ||
Some(did) => ("did", did.to_string()), | ||
None => ( | ||
"uri", | ||
subject["uri"].as_str().unwrap_or_default().to_string(), | ||
), | ||
}; | ||
self.xrpc | ||
.query_as(GET_STATUS, &[(param.0, ¶m.1)], &self.headers) | ||
.await | ||
} | ||
| ||
/// Reports, newest first. | ||
pub async fn reports(&self, limit: usize) -> Result<Vec<Value>> { | ||
#[derive(Deserialize)] | ||
struct Out { | ||
reports: Vec<Value>, | ||
} | ||
let limit = limit.to_string(); | ||
let out: Out = self | ||
.xrpc | ||
.query_as(LIST_REPORTS, &[("limit", &limit)], &self.headers) | ||
.await?; | ||
Ok(out.reports) | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn reasons() { | ||
assert_eq!( | ||
reason_type("spam").unwrap(), | ||
"com.atproto.moderation.defs#reasonSpam" | ||
); | ||
for r in ["violation", "misleading", "sexual", "rude", "other"] { | ||
assert!(reason_type(r).unwrap().ends_with(&r[1..])); | ||
} | ||
assert!(reason_type("boring").is_err()); | ||
} | ||
} | ||
crates/knotless/src/push.rsnew+405−0
| @@ -0,0 +1,405 @@ | ||
//! A push: one pack record, and per ref a claim plus the ref itself, in one | ||
//! `applyWrites`. | ||
| ||
use crate::env::Ctx; | ||
use crate::git::{self, Git}; | ||
use crate::keys; | ||
use crate::records::{ | ||
self, BlobRef, Bytes, ClaimRecord, PACK_MIME, PackRecord, RefRecord, RepoRecord, nsid, | ||
}; | ||
use crate::remote::{self, Remote, Snapshot, Stored}; | ||
use crate::xrpc::XrpcError; | ||
use anyhow::{Result, bail}; | ||
use futures::{StreamExt, TryStreamExt}; | ||
use serde_json::Value; | ||
use std::collections::BTreeSet; | ||
| ||
/// One refspec of a push. `src: None` deletes `dst`. | ||
#[derive(Clone, Debug, PartialEq, Eq)] | ||
pub struct Update { | ||
pub src: Option<String>, | ||
pub dst: String, | ||
pub force: bool, | ||
} | ||
| ||
impl Update { | ||
/// Parses `[+]<src>:<dst>`, the form git hands a remote helper. | ||
pub fn parse(spec: &str) -> Result<Update> { | ||
let (force, spec) = match spec.strip_prefix('+') { | ||
Some(rest) => (true, rest), | ||
None => (false, spec), | ||
}; | ||
let Some((src, dst)) = spec.split_once(':') else { | ||
bail!("bad refspec {spec}"); | ||
}; | ||
Ok(Update { | ||
src: (!src.is_empty()).then(|| src.to_string()), | ||
dst: dst.to_string(), | ||
force, | ||
}) | ||
} | ||
} | ||
| ||
fn is_sha(text: &str) -> bool { | ||
(text.len() == 40 || text.len() == 64) | ||
&& text.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) | ||
} | ||
| ||
/// Per ref, `Ok` or the reason it was refused, in git's words. | ||
pub type Report = Vec<(String, std::result::Result<(), String>)>; | ||
| ||
/// The most writes the reference PDS takes in one `applyWrites`. | ||
const MAX_WRITES: usize = 200; | ||
| ||
struct Accepted { | ||
name: String, | ||
from: Option<String>, | ||
to: Option<String>, | ||
previous: Option<Stored<RefRecord>>, | ||
} | ||
| ||
pub async fn push( | ||
ctx: &Ctx, | ||
remote: &Remote, | ||
snap: &Snapshot, | ||
git: &Git, | ||
auth: &str, | ||
updates: &[Update], | ||
) -> Result<Report> { | ||
let mut report = Report::new(); | ||
let mut accepted = Vec::new(); | ||
for update in updates { | ||
let previous = snap.find_ref(&update.dst).cloned(); | ||
let from = previous.as_ref().and_then(|r| r.value.target.clone()); | ||
let to = match &update.src { | ||
Some(src) if is_sha(src) => Some(src.clone()), | ||
Some(src) => Some(git.rev_parse(src).await?), | ||
None => None, | ||
}; | ||
if to == from { | ||
report.push((update.dst.clone(), Ok(()))); | ||
continue; | ||
} | ||
if let (Some(old), Some(new), false) = (&from, &to, update.force) { | ||
if !git.has(old).await? { | ||
report.push((update.dst.clone(), Err("fetch first".into()))); | ||
continue; | ||
} | ||
if !git.is_ancestor(old, new).await? { | ||
report.push((update.dst.clone(), Err("non-fast-forward".into()))); | ||
continue; | ||
} | ||
} | ||
accepted.push(Accepted { | ||
name: update.dst.clone(), | ||
from, | ||
to, | ||
previous, | ||
}); | ||
} | ||
if accepted.is_empty() { | ||
return Ok(report); | ||
} | ||
| ||
let mut writes = Vec::new(); | ||
let key = match &snap.repo { | ||
Some(repo) => repo.value.key.clone(), | ||
None => { | ||
let key = keys::tid(); | ||
let heads: Vec<&str> = accepted | ||
.iter() | ||
.filter(|a| a.to.is_some()) | ||
.filter_map(|a| a.name.strip_prefix("refs/heads/")) | ||
.collect(); | ||
let default_branch = heads | ||
.iter() | ||
.find(|n| **n == "main") | ||
.or(heads.first()) | ||
.map(|n| n.to_string()); | ||
let repo = RepoRecord { | ||
key: key.clone(), | ||
name: remote.url.name.clone(), | ||
description: None, | ||
default_branch, | ||
created_at: records::now(), | ||
..Default::default() | ||
}; | ||
writes.push(remote::create( | ||
nsid::REPO, | ||
&remote.url.name, | ||
records::typed(nsid::REPO, &repo), | ||
)); | ||
key | ||
} | ||
}; | ||
| ||
let tips: Vec<String> = accepted | ||
.iter() | ||
.filter_map(|a| a.to.clone()) | ||
.collect::<BTreeSet<_>>() | ||
.into_iter() | ||
.collect(); | ||
let pack_rkey = if tips.is_empty() { | ||
None | ||
} else { | ||
let candidates: Vec<String> = snap | ||
.live_refs() | ||
.into_iter() | ||
.map(|(_, sha)| sha) | ||
.collect::<BTreeSet<_>>() | ||
.into_iter() | ||
.collect(); | ||
let absent = git.missing(&candidates).await?; | ||
let bases: Vec<String> = candidates | ||
.into_iter() | ||
.filter(|s| !absent.contains(s)) | ||
.collect(); | ||
let pack = git.pack_objects(&tips, &bases).await?; | ||
let objects = git::pack_objects_count(&pack); | ||
if objects == 0 { | ||
None | ||
} else { | ||
let rkey = keys::pack_rkey(&key); | ||
let record = place(ctx, remote, auth, &key, pack, tips, bases, objects).await?; | ||
writes.push(remote::create( | ||
nsid::PACK, | ||
&rkey, | ||
records::typed(nsid::PACK, &record), | ||
)); | ||
Some(rkey) | ||
} | ||
}; | ||
| ||
let now = records::now(); | ||
let mut ref_writes = Vec::new(); | ||
for a in &accepted { | ||
let generation = a.previous.as_ref().map_or(0, |r| r.value.generation); | ||
let claim = ClaimRecord { | ||
repo: key.clone(), | ||
name: a.name.clone(), | ||
generation, | ||
from: a.from.clone(), | ||
to: a.to.clone(), | ||
pack: pack_rkey.clone(), | ||
created_at: now.clone(), | ||
}; | ||
let record = RefRecord { | ||
repo: key.clone(), | ||
name: a.name.clone(), | ||
target: a.to.clone(), | ||
generation: generation + 1, | ||
updated_at: now.clone(), | ||
}; | ||
let rkey = keys::ref_rkey(&key, &a.name); | ||
let value = records::typed(nsid::REF, &record); | ||
ref_writes.push(vec![ | ||
remote::create( | ||
nsid::CLAIM, | ||
&keys::claim_rkey(&key, &a.name, generation), | ||
records::typed(nsid::CLAIM, &claim), | ||
), | ||
if a.previous.is_some() { | ||
remote::update(nsid::REF, &rkey, value) | ||
} else { | ||
remote::create(nsid::REF, &rkey, value) | ||
}, | ||
]); | ||
} | ||
| ||
// Compaction drops claims a day old: a read older than half that could | ||
// re-create a dropped claim and move a ref back. | ||
if snap | ||
.read_at | ||
.is_some_and(|at| at.elapsed() > ctx.env.claim_grace / 2) | ||
{ | ||
bail!("the refs were read too long ago to push against: fetch, then push again"); | ||
} | ||
let batches = batch(writes, ref_writes); | ||
let mut done = 0; | ||
for (i, writes) in batches.iter().enumerate() { | ||
let refs_in_batch = writes | ||
.iter() | ||
.filter(|w| w["collection"] == nsid::REF) | ||
.count(); | ||
if let Err(err) = remote.apply(writes.clone(), auth).await { | ||
let pending = &accepted[done..done + refs_in_batch]; | ||
match conflicts(remote, &key, snap, pending, i == 0).await? { | ||
Some(moved) => { | ||
for a in &accepted[done..] { | ||
let reason = if moved.contains(&a.name) { | ||
"fetch first".to_string() | ||
} else { | ||
"not pushed: another ref in the push moved concurrently".to_string() | ||
}; | ||
report.push((a.name.clone(), Err(reason))); | ||
} | ||
for a in &accepted[..done] { | ||
report.push((a.name.clone(), Ok(()))); | ||
} | ||
return Ok(report); | ||
} | ||
None => return Err(err), | ||
} | ||
} | ||
done += refs_in_batch; | ||
} | ||
report.extend(accepted.iter().map(|a| (a.name.clone(), Ok(())))); | ||
Ok(report) | ||
} | ||
| ||
/// Splits the writes into `applyWrites` calls: the first carries the repo | ||
/// and pack records, and a ref's claim always travels with the ref. | ||
fn batch(mut first: Vec<Value>, refs: Vec<Vec<Value>>) -> Vec<Vec<Value>> { | ||
let mut batches = Vec::new(); | ||
for pair in refs { | ||
if first.len() + pair.len() > MAX_WRITES { | ||
batches.push(std::mem::take(&mut first)); | ||
} | ||
first.extend(pair); | ||
} | ||
batches.push(first); | ||
batches | ||
} | ||
| ||
/// After a refused write, which refs moved under the push. `None` when | ||
/// nothing did, so the refusal was something else. | ||
async fn conflicts( | ||
remote: &Remote, | ||
key: &str, | ||
snap: &Snapshot, | ||
pending: &[Accepted], | ||
includes_repo: bool, | ||
) -> Result<Option<BTreeSet<String>>> { | ||
if includes_repo && snap.repo.is_none() && remote.repo().await?.is_some() { | ||
return Ok(Some(pending.iter().map(|a| a.name.clone()).collect())); | ||
} | ||
let now = remote.refs(key).await?; | ||
let moved: BTreeSet<String> = pending | ||
.iter() | ||
.filter(|a| { | ||
let then = a.previous.as_ref().map(|r| r.value.generation); | ||
let current = now | ||
.iter() | ||
.find(|r| r.value.name == a.name) | ||
.map(|r| r.value.generation); | ||
then != current | ||
}) | ||
.map(|a| a.name.clone()) | ||
.collect(); | ||
Ok((!moved.is_empty()).then_some(moved)) | ||
} | ||
| ||
/// Builds the pack record: inline when small, else over blobs. | ||
#[allow(clippy::too_many_arguments)] | ||
pub async fn place( | ||
ctx: &Ctx, | ||
remote: &Remote, | ||
auth: &str, | ||
key: &str, | ||
pack: Vec<u8>, | ||
tips: Vec<String>, | ||
bases: Vec<String>, | ||
objects: u64, | ||
) -> Result<PackRecord> { | ||
let size = pack.len() as u64; | ||
let (data, parts) = if pack.len() <= ctx.env.inline_max { | ||
(Some(Bytes(pack)), Vec::new()) | ||
} else { | ||
(None, upload(ctx, remote, auth, &pack).await?) | ||
}; | ||
Ok(PackRecord { | ||
repo: key.to_string(), | ||
tips, | ||
bases, | ||
size, | ||
objects, | ||
data, | ||
parts, | ||
supersedes: Vec::new(), | ||
created_at: records::now(), | ||
}) | ||
} | ||
| ||
/// Uploads `pack` as blobs of at most the chunk size, four at a time, | ||
/// halving the size while the PDS calls a part too large. | ||
pub async fn upload(ctx: &Ctx, remote: &Remote, auth: &str, pack: &[u8]) -> Result<Vec<BlobRef>> { | ||
let mut chunk = ctx.env.chunk_size.max(1); | ||
loop { | ||
// Owned parts, so the stream's future is Send for any caller. | ||
let parts: Vec<Vec<u8>> = pack.chunks(chunk).map(<[u8]>::to_vec).collect(); | ||
let res: Result<Vec<BlobRef>> = futures::stream::iter(parts) | ||
.map(|part| { | ||
let xrpc = remote.xrpc.clone(); | ||
let auth = auth.to_string(); | ||
async move { xrpc.upload_blob(part, PACK_MIME, &auth).await } | ||
}) | ||
.buffered(4) | ||
.try_collect() | ||
.await; | ||
match res { | ||
Err(err) if chunk > 64 * 1024 && too_large(&err) => chunk /= 2, | ||
other => return other, | ||
} | ||
} | ||
} | ||
| ||
fn too_large(err: &anyhow::Error) -> bool { | ||
XrpcError::of(err).is_some_and(|e| { | ||
e.status == 413 || e.error == "BlobTooLarge" || e.error == "PayloadTooLargeError" | ||
}) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use serde_json::json; | ||
| ||
#[test] | ||
fn refspecs() { | ||
assert_eq!( | ||
Update::parse("+refs/heads/a:refs/heads/b").unwrap(), | ||
Update { | ||
src: Some("refs/heads/a".into()), | ||
dst: "refs/heads/b".into(), | ||
force: true | ||
} | ||
); | ||
assert_eq!( | ||
Update::parse(":refs/heads/b").unwrap(), | ||
Update { | ||
src: None, | ||
dst: "refs/heads/b".into(), | ||
force: false | ||
} | ||
); | ||
assert!(Update::parse("nope").is_err()); | ||
} | ||
| ||
#[test] | ||
fn batching() { | ||
let pair = || vec![json!(1), json!(2)]; | ||
let b = batch(vec![json!(0)], vec![pair(), pair()]); | ||
assert_eq!(b.len(), 1); | ||
assert_eq!(b[0].len(), 5); | ||
let many: Vec<Vec<Value>> = (0..150).map(|_| pair()).collect(); | ||
let b = batch(vec![json!(0)], many); | ||
assert_eq!(b.len(), 2); | ||
assert!(b.iter().all(|w| w.len() <= MAX_WRITES)); | ||
assert_eq!(b[0].len(), 199); | ||
} | ||
| ||
#[test] | ||
fn too_large_errors() { | ||
let e = |status, error: &str| { | ||
anyhow::Error::new(XrpcError { | ||
status, | ||
error: error.into(), | ||
message: String::new(), | ||
}) | ||
}; | ||
assert!(too_large(&e(413, "x"))); | ||
assert!(too_large(&e(400, "BlobTooLarge"))); | ||
assert!(!too_large(&e(400, "InvalidRequest"))); | ||
assert!(!too_large(&anyhow::anyhow!("io"))); | ||
} | ||
} | ||
crates/knotless/src/read.rsnew+284−0
| @@ -0,0 +1,284 @@ | ||
//! Reading a repository without a working copy: files, trees, history, | ||
//! diffs and search, from any local git directory (usually a mirror). | ||
//! `knotless-serve` answers its HTTP API with these, and the MCP server its | ||
//! tools. | ||
| ||
use crate::git::Git; | ||
use anyhow::Result; | ||
use serde::Serialize; | ||
use std::fmt; | ||
| ||
/// Input refused before git saw it. | ||
#[derive(Debug, Clone, PartialEq)] | ||
pub struct BadInput(pub String); | ||
| ||
impl fmt::Display for BadInput { | ||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | ||
f.write_str(&self.0) | ||
} | ||
} | ||
| ||
impl std::error::Error for BadInput {} | ||
| ||
/// `rev`, or `HEAD`, refusing anything git could read as an option. | ||
pub fn rev(value: Option<&str>) -> Result<String> { | ||
let rev = value.unwrap_or("HEAD"); | ||
if rev.is_empty() || rev.starts_with('-') || rev.contains(['\0', '\n', ' ', ':']) { | ||
return Err(BadInput(format!("bad revision {rev:?}")).into()); | ||
} | ||
Ok(rev.to_string()) | ||
} | ||
| ||
/// A path inside the repository, without leading or trailing slashes. | ||
pub fn path(value: Option<&str>) -> String { | ||
value.unwrap_or_default().trim_matches('/').to_string() | ||
} | ||
| ||
/// Whether `rev` names a commit by its full hash, so anything read at it can | ||
/// be cached forever. | ||
pub fn is_pinned(rev: &str) -> bool { | ||
(rev.len() == 40 || rev.len() == 64) && rev.bytes().all(|b| b.is_ascii_hexdigit()) | ||
} | ||
| ||
#[derive(Serialize, Debug, Clone, PartialEq)] | ||
pub struct Entry { | ||
pub name: String, | ||
pub mode: String, | ||
#[serde(rename = "type")] | ||
pub kind: String, | ||
pub sha: String, | ||
pub size: Option<u64>, | ||
} | ||
| ||
pub async fn tree(git: &Git, rev: &str, path: &str) -> Result<Vec<Entry>> { | ||
let out = git | ||
.run(&["ls-tree", "-l", "-z", &format!("{rev}:{path}")], None) | ||
.await?; | ||
Ok(String::from_utf8_lossy(&out) | ||
.split('\0') | ||
.filter_map(|line| { | ||
let (meta, name) = line.split_once('\t')?; | ||
let mut f = meta.split_whitespace(); | ||
let (mode, kind, sha, size) = (f.next()?, f.next()?, f.next()?, f.next()?); | ||
Some(Entry { | ||
name: name.into(), | ||
mode: mode.into(), | ||
kind: kind.into(), | ||
sha: sha.into(), | ||
size: size.parse().ok(), | ||
}) | ||
}) | ||
.collect()) | ||
} | ||
| ||
/// The blob at `path`, with its object id. | ||
pub async fn blob(git: &Git, rev: &str, path: &str) -> Result<(String, Vec<u8>)> { | ||
let sha = git | ||
.text(&["rev-parse", "--verify", &format!("{rev}:{path}")], None) | ||
.await?; | ||
let bytes = git.run(&["cat-file", "blob", &sha], None).await?; | ||
Ok((sha, bytes)) | ||
} | ||
| ||
#[derive(Serialize, Debug, Clone, PartialEq)] | ||
pub struct Author { | ||
pub name: String, | ||
pub email: String, | ||
pub date: String, | ||
} | ||
| ||
#[derive(Serialize, Debug, Clone, PartialEq)] | ||
pub struct Commit { | ||
pub sha: String, | ||
pub parents: Vec<String>, | ||
pub author: Author, | ||
pub message: String, | ||
} | ||
| ||
pub async fn log(git: &Git, rev: &str, path: &str, limit: usize) -> Result<Vec<Commit>> { | ||
log_page(git, rev, path, 0, limit).await | ||
} | ||
| ||
/// The history after its first `skip` commits. | ||
pub async fn log_page( | ||
git: &Git, | ||
rev: &str, | ||
path: &str, | ||
skip: usize, | ||
limit: usize, | ||
) -> Result<Vec<Commit>> { | ||
let limit = limit.clamp(1, 1000).to_string(); | ||
let skip = format!("--skip={skip}"); | ||
let mut args = vec![ | ||
"log", | ||
"-z", | ||
"--format=%H%x1f%P%x1f%an%x1f%ae%x1f%aI%x1f%B", | ||
"-n", | ||
&limit, | ||
&skip, | ||
rev, | ||
"--", | ||
]; | ||
if !path.is_empty() { | ||
args.push(path); | ||
} | ||
let out = git.run(&args, None).await?; | ||
Ok(String::from_utf8_lossy(&out) | ||
.split('\0') | ||
.filter(|s| !s.is_empty()) | ||
.map(|record| { | ||
let mut f = record.splitn(6, '\x1f'); | ||
let mut field = || f.next().unwrap_or_default().to_string(); | ||
let (sha, parents, name, email, date, message) = | ||
(field(), field(), field(), field(), field(), field()); | ||
Commit { | ||
sha, | ||
parents: parents.split_whitespace().map(str::to_string).collect(), | ||
author: Author { name, email, date }, | ||
message: message.trim_end().to_string(), | ||
} | ||
}) | ||
.collect()) | ||
} | ||
| ||
/// The change `head` makes since it left `base`, as a unified diff. | ||
pub async fn diff(git: &Git, base: &str, head: &str) -> Result<String> { | ||
git.text( | ||
&[ | ||
"diff", | ||
"--no-color", | ||
"--no-ext-diff", | ||
&format!("{base}...{head}"), | ||
"--", | ||
], | ||
None, | ||
) | ||
.await | ||
} | ||
| ||
#[derive(Serialize, Debug, Clone, PartialEq)] | ||
pub struct Match { | ||
pub path: String, | ||
pub line: u64, | ||
pub text: String, | ||
} | ||
| ||
/// Lines matching `pattern` (a fixed string, case-insensitive) at `rev`. | ||
pub async fn grep(git: &Git, rev: &str, pattern: &str, limit: usize) -> Result<Vec<Match>> { | ||
if pattern.is_empty() { | ||
return Err(BadInput("an empty search".into()).into()); | ||
} | ||
let out = git | ||
.output_of(&[ | ||
"grep", "-I", "-n", "-i", "-F", "--null", "-e", pattern, rev, "--", | ||
]) | ||
.await?; | ||
// 1 is "no match"; anything else past 0 is a real failure. | ||
if !matches!(out.status.code(), Some(0 | 1)) { | ||
anyhow::bail!("git grep: {}", String::from_utf8_lossy(&out.stderr).trim()); | ||
} | ||
let prefix = format!("{rev}:"); | ||
Ok(String::from_utf8_lossy(&out.stdout) | ||
.lines() | ||
.filter_map(|line| { | ||
let mut f = line.splitn(3, '\0'); | ||
let (file, number, text) = (f.next()?, f.next()?, f.next()?); | ||
Some(Match { | ||
path: file.strip_prefix(&prefix).unwrap_or(file).to_string(), | ||
line: number.parse().ok()?, | ||
text: text.to_string(), | ||
}) | ||
}) | ||
.take(limit.max(1)) | ||
.collect()) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn inputs() { | ||
assert_eq!(rev(None).unwrap(), "HEAD"); | ||
for bad in ["", "--output=x", "a b", "a:b"] { | ||
let err = rev(Some(bad)).unwrap_err(); | ||
assert!(err.downcast_ref::<BadInput>().is_some(), "{bad}"); | ||
} | ||
assert_eq!(path(Some("/a/b/")), "a/b"); | ||
assert!(is_pinned(&"a".repeat(40))); | ||
assert!(!is_pinned("main")); | ||
assert_eq!(BadInput("x".into()).to_string(), "x"); | ||
} | ||
| ||
#[tokio::test] | ||
async fn reads() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let work = dir.path(); | ||
let sh = |args: &[&str]| { | ||
let out = std::process::Command::new("git") | ||
.current_dir(work) | ||
.args(args) | ||
.env("GIT_AUTHOR_NAME", "Ada") | ||
.env("GIT_AUTHOR_EMAIL", "ada@x") | ||
.env("GIT_COMMITTER_NAME", "Ada") | ||
.env("GIT_COMMITTER_EMAIL", "ada@x") | ||
.output() | ||
.unwrap(); | ||
assert!(out.status.success(), "{out:?}"); | ||
String::from_utf8(out.stdout).unwrap().trim().to_string() | ||
}; | ||
sh(&["init", "-q", "-b", "main"]); | ||
std::fs::create_dir(work.join("src")).unwrap(); | ||
std::fs::write(work.join("src/lib.rs"), "fn Main() {}\n").unwrap(); | ||
sh(&["add", "."]); | ||
sh(&["commit", "-q", "-m", "one\n\nbody"]); | ||
let first = sh(&["rev-parse", "HEAD"]); | ||
std::fs::write(work.join("README"), "hello main\n").unwrap(); | ||
sh(&["add", "."]); | ||
sh(&["commit", "-q", "-m", "two"]); | ||
let git = Git::new(work.join(".git")); | ||
| ||
let root = tree(&git, "HEAD", "").await.unwrap(); | ||
assert_eq!( | ||
root.iter().map(|e| e.name.as_str()).collect::<Vec<_>>(), | ||
["README", "src"] | ||
); | ||
assert_eq!(root[1].kind, "tree"); | ||
assert_eq!(root[1].size, None); | ||
assert_eq!(tree(&git, "HEAD", "src").await.unwrap()[0].size, Some(13)); | ||
assert!(tree(&git, "HEAD", "nope").await.is_err()); | ||
| ||
let (sha, bytes) = blob(&git, &first, "src/lib.rs").await.unwrap(); | ||
assert_eq!(bytes, b"fn Main() {}\n"); | ||
assert_eq!(sha.len(), 40); | ||
| ||
let commits = log(&git, "HEAD", "", 0).await.unwrap(); | ||
assert_eq!(commits.len(), 1); | ||
let commits = log(&git, "HEAD", "src", 10).await.unwrap(); | ||
assert_eq!(commits[0].message, "one\n\nbody"); | ||
assert_eq!(commits[0].author.name, "Ada"); | ||
assert!(commits[0].parents.is_empty()); | ||
| ||
assert!( | ||
diff(&git, &first, "HEAD") | ||
.await | ||
.unwrap() | ||
.contains("+hello main") | ||
); | ||
| ||
let hits = grep(&git, "HEAD", "MAIN", 10).await.unwrap(); | ||
assert_eq!(hits.len(), 2); | ||
assert_eq!( | ||
hits[0], | ||
Match { | ||
path: "README".into(), | ||
line: 1, | ||
text: "hello main".into() | ||
} | ||
); | ||
assert_eq!(grep(&git, "HEAD", "main", 1).await.unwrap().len(), 1); | ||
assert!(grep(&git, "HEAD", "absent", 10).await.unwrap().is_empty()); | ||
assert!(grep(&git, "HEAD", "", 10).await.is_err()); | ||
assert!(grep(&git, "nope", "x", 10).await.is_err()); | ||
} | ||
} | ||
crates/knotless/src/records.rsnew+383−0
| @@ -0,0 +1,383 @@ | ||
//! The records a repository is made of. `lexicons/` holds their schemas. | ||
| ||
use base64::Engine; | ||
use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}; | ||
use serde::de::Error as _; | ||
use serde::ser::SerializeMap; | ||
use serde::{Deserialize, Deserializer, Serialize, Serializer}; | ||
use serde_json::Value; | ||
| ||
pub mod nsid { | ||
pub const REPO: &str = "dev.knotless.repo"; | ||
pub const REF: &str = "dev.knotless.ref"; | ||
pub const CLAIM: &str = "dev.knotless.claim"; | ||
pub const PACK: &str = "dev.knotless.pack"; | ||
pub const REDIRECT: &str = "dev.knotless.redirect"; | ||
/// Every collection a repository writes, for readers that watch them. | ||
pub const ALL: [&str; 5] = [REPO, REF, CLAIM, PACK, REDIRECT]; | ||
pub const PULL: &str = "dev.knotless.pull"; | ||
pub const ISSUE: &str = "dev.knotless.issue"; | ||
pub const COMMENT: &str = "dev.knotless.comment"; | ||
pub const STATE: &str = "dev.knotless.state"; | ||
pub const GATE: &str = "dev.knotless.gate"; | ||
pub const BLOCK: &str = "dev.knotless.block"; | ||
/// What people write about a repository, in their own accounts, and | ||
/// what its owner hides of it and whom they keep out. A block names no | ||
/// repository when it keeps someone out of all of them. | ||
pub const FORGE: [&str; 6] = [PULL, ISSUE, COMMENT, STATE, GATE, BLOCK]; | ||
} | ||
| ||
pub const PACK_MIME: &str = "application/x-git-packed-objects"; | ||
| ||
/// A repository, keyed by its name. `key` prefixes the keys of everything | ||
/// else the repository holds, so a rename writes this record and nothing else. | ||
#[derive(Serialize, Deserialize, Clone, Default, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct RepoRecord { | ||
pub key: String, | ||
pub name: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub description: Option<String>, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub default_branch: Option<String>, | ||
/// Accounts that may push, through the gateway. | ||
#[serde(default, skip_serializing_if = "Vec::is_empty")] | ||
pub collaborators: Vec<Collaborator>, | ||
/// The repository this one was forked from. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub fork_of: Option<RepoRef>, | ||
/// The DID of a gateway the owner has delegated writes to, which takes | ||
/// pushes and commits from collaborators and batches commits. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub gateway: Option<String>, | ||
/// Where the owner has offered to move the repository: another | ||
/// account's record, named before it exists. Once that account holds a | ||
/// repository there that names this one as `transferredFrom`, this one | ||
/// has moved. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub transfer: Option<RepoRef>, | ||
/// The repository this one was moved from, at the owner's offer. Its | ||
/// pulls, issues and forks are this one's. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub transferred_from: Option<RepoRef>, | ||
pub created_at: String, | ||
} | ||
| ||
impl RepoRecord { | ||
/// Whether `did` is among the collaborators. | ||
pub fn is_collaborator(&self, did: &str) -> bool { | ||
self.collaborators.iter().any(|c| c.did == did) | ||
} | ||
} | ||
| ||
/// A repository, as other records name it: the `at://` URI of its record | ||
/// when they were written, and its key. The name in the URI may since have | ||
/// changed, or been given to another repository; the owner and the key | ||
/// still say which repository it is. | ||
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)] | ||
pub struct RepoRef { | ||
pub uri: String, | ||
pub key: String, | ||
} | ||
| ||
impl RepoRef { | ||
pub fn new(did: &str, name: &str, key: &str) -> RepoRef { | ||
RepoRef { | ||
uri: format!("at://{did}/{}/{name}", nsid::REPO), | ||
key: key.to_string(), | ||
} | ||
} | ||
| ||
/// The owner's DID, from the URI. | ||
pub fn did(&self) -> &str { | ||
let rest = self.uri.strip_prefix("at://").unwrap_or_default(); | ||
rest.split('/').next().unwrap_or_default() | ||
} | ||
| ||
/// The name the repository had when this was written. | ||
pub fn name(&self) -> &str { | ||
self.uri.rsplit('/').next().unwrap_or_default() | ||
} | ||
| ||
/// Whether this names `did`'s repository with `key`. | ||
pub fn is(&self, did: &str, key: &str) -> bool { | ||
self.key == key && self.did() == did | ||
} | ||
} | ||
| ||
/// Left under a repository's old name when it is renamed, so that clones, | ||
/// remotes and links made with the old name still find it. Keyed by the | ||
/// old name; a repository created under that name replaces it. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct RedirectRecord { | ||
pub to: RepoRef, | ||
/// For a private repository renamed since it was made: the space its | ||
/// records are in, which keeps the name it was made with. This redirect | ||
/// is then in the space named for a later name, which only points there. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub space: Option<String>, | ||
pub created_at: String, | ||
} | ||
| ||
/// An account that may push to a repository. An object rather than a bare | ||
/// DID, so what a collaborator may do can be said beside it later. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
pub struct Collaborator { | ||
pub did: String, | ||
} | ||
| ||
/// One ref. A deleted ref keeps its record with no target, so its generation | ||
/// never repeats and an old claim can never be replayed. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct RefRecord { | ||
pub repo: String, | ||
pub name: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub target: Option<String>, | ||
pub generation: u64, | ||
pub updated_at: String, | ||
} | ||
| ||
/// The move of a ref off one generation. Its key is the lock; its body is | ||
/// the reflog entry. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct ClaimRecord { | ||
pub repo: String, | ||
pub name: String, | ||
pub generation: u64, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub from: Option<String>, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub to: Option<String>, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub pack: Option<String>, | ||
pub created_at: String, | ||
} | ||
| ||
/// A self-contained git pack: every object reachable from `tips` and not | ||
/// from `bases`. Small packs ride in `data`; large ones are split over blobs. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct PackRecord { | ||
pub repo: String, | ||
pub tips: Vec<String>, | ||
#[serde(default, skip_serializing_if = "Vec::is_empty")] | ||
pub bases: Vec<String>, | ||
pub size: u64, | ||
pub objects: u64, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub data: Option<Bytes>, | ||
#[serde(default, skip_serializing_if = "Vec::is_empty")] | ||
pub parts: Vec<BlobRef>, | ||
#[serde(default, skip_serializing_if = "Vec::is_empty")] | ||
pub supersedes: Vec<String>, | ||
pub created_at: String, | ||
} | ||
| ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct BlobRef { | ||
#[serde(rename = "$type")] | ||
pub kind: String, | ||
#[serde(rename = "ref")] | ||
pub link: Link, | ||
pub mime_type: String, | ||
pub size: u64, | ||
} | ||
| ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
pub struct Link { | ||
#[serde(rename = "$link")] | ||
pub cid: String, | ||
} | ||
| ||
/// A request to merge `head` into `base`, in the requester's account. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct PullRecord { | ||
/// The target repository. | ||
pub repo: RepoRef, | ||
pub base: String, | ||
pub head: PullHead, | ||
pub title: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub body: Option<String>, | ||
pub created_at: String, | ||
} | ||
| ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
pub struct PullHead { | ||
/// The repository the branch is in: the target or a fork. | ||
pub repo: RepoRef, | ||
pub branch: String, | ||
} | ||
| ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct IssueRecord { | ||
pub repo: RepoRef, | ||
pub title: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub body: Option<String>, | ||
pub created_at: String, | ||
} | ||
| ||
/// A comment on a pull or issue; with a verdict, a review. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct CommentRecord { | ||
pub repo: RepoRef, | ||
/// The `at://` URI of the pull or issue. | ||
pub subject: String, | ||
pub body: String, | ||
/// `approve` or `changes`, for a review. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub verdict: Option<String>, | ||
/// The commit a review looked at. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub commit: Option<String>, | ||
pub created_at: String, | ||
} | ||
| ||
/// A pull or issue opened, closed or merged. The latest from someone who | ||
/// may make that change wins: its author, the owner or a collaborator, and | ||
/// for `merged`, only the latter two. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct StateRecord { | ||
pub repo: RepoRef, | ||
pub subject: String, | ||
pub state: String, | ||
/// The merge, for `merged`. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub commit: Option<String>, | ||
/// The branch a pull targets from here on, as when the branch it | ||
/// targeted was renamed. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub base: Option<String>, | ||
pub created_at: String, | ||
} | ||
| ||
/// What a repository's owner hides of what others wrote about it, as | ||
/// `app.bsky.feed.threadgate` hides replies: one per repository, keyed by its | ||
/// key, in the owner's account. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct GateRecord { | ||
pub repo: RepoRef, | ||
/// Pulls, issues and comments, by `at://` URI. | ||
#[serde(default)] | ||
pub hidden: Vec<String>, | ||
pub created_at: String, | ||
} | ||
| ||
/// An account a repository's owner keeps out: what it writes about their | ||
/// repositories, or about `repo` alone, does not show. As | ||
/// `app.bsky.graph.block` is, keyed by a TID, in the owner's account; one | ||
/// naming a private repository is in their repo inside its space. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct BlockRecord { | ||
pub subject: String, | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub repo: Option<RepoRef>, | ||
pub created_at: String, | ||
} | ||
| ||
/// Bytes in the atproto JSON encoding, `{"$bytes": "<base64>"}`. | ||
#[derive(Clone, Debug, PartialEq)] | ||
pub struct Bytes(pub Vec<u8>); | ||
| ||
const B64: GeneralPurpose = GeneralPurpose::new( | ||
&base64::alphabet::STANDARD, | ||
GeneralPurposeConfig::new() | ||
.with_encode_padding(false) | ||
.with_decode_padding_mode(DecodePaddingMode::Indifferent), | ||
); | ||
| ||
impl Serialize for Bytes { | ||
fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { | ||
let mut map = s.serialize_map(Some(1))?; | ||
map.serialize_entry("$bytes", &B64.encode(&self.0))?; | ||
map.end() | ||
} | ||
} | ||
| ||
impl<'de> Deserialize<'de> for Bytes { | ||
fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> { | ||
#[derive(Deserialize)] | ||
struct Raw { | ||
#[serde(rename = "$bytes")] | ||
bytes: String, | ||
} | ||
let raw = Raw::deserialize(d)?; | ||
B64.decode(raw.bytes).map(Bytes).map_err(D::Error::custom) | ||
} | ||
} | ||
| ||
/// `value` as a record of type `nsid`. | ||
pub fn typed<T: Serialize>(nsid: &str, value: &T) -> Value { | ||
let mut v = serde_json::to_value(value).expect("records serialize to JSON objects"); | ||
v["$type"] = nsid.into(); | ||
v | ||
} | ||
| ||
pub fn now() -> String { | ||
humantime::format_rfc3339_millis(std::time::SystemTime::now()).to_string() | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use serde_json::json; | ||
| ||
#[test] | ||
fn bytes_round_trip() { | ||
let v = serde_json::to_value(Bytes(vec![1, 2, 3, 4])).unwrap(); | ||
assert_eq!(v, json!({"$bytes": "AQIDBA"})); | ||
let back: Bytes = serde_json::from_value(json!({"$bytes": "AQIDBA=="})).unwrap(); | ||
assert_eq!(back.0, vec![1, 2, 3, 4]); | ||
assert!(serde_json::from_value::<Bytes>(json!({"$bytes": "!!"})).is_err()); | ||
assert!(serde_json::from_value::<Bytes>(json!(4)).is_err()); | ||
} | ||
| ||
#[test] | ||
fn typed_records() { | ||
let pack = PackRecord { | ||
repo: "k".into(), | ||
tips: vec!["a".into()], | ||
bases: vec![], | ||
size: 1, | ||
objects: 1, | ||
data: Some(Bytes(vec![0])), | ||
parts: vec![], | ||
supersedes: vec![], | ||
created_at: now(), | ||
}; | ||
let v = typed(nsid::PACK, &pack); | ||
assert_eq!(v["$type"], "dev.knotless.pack"); | ||
assert!(v.get("bases").is_none()); | ||
assert!(v.get("parts").is_none()); | ||
let back: PackRecord = serde_json::from_value(v).unwrap(); | ||
assert_eq!(back, pack); | ||
let blob: BlobRef = serde_json::from_value(json!({ | ||
"$type": "blob", "ref": {"$link": "bafy"}, "mimeType": PACK_MIME, "size": 3 | ||
})) | ||
.unwrap(); | ||
assert_eq!(blob.link.cid, "bafy"); | ||
assert!(now().ends_with('Z')); | ||
} | ||
| ||
#[test] | ||
fn repo_refs() { | ||
let r = RepoRef::new("did:plc:a", "proj", "3kabc"); | ||
assert_eq!(r.uri, "at://did:plc:a/dev.knotless.repo/proj"); | ||
assert_eq!((r.did(), r.name()), ("did:plc:a", "proj")); | ||
assert!(r.is("did:plc:a", "3kabc")); | ||
assert!(!r.is("did:plc:a", "3kxyz") && !r.is("did:plc:b", "3kabc")); | ||
assert_eq!(RepoRef::default().did(), ""); | ||
} | ||
} | ||
crates/knotless/src/remote.rsnew+535−0
| @@ -0,0 +1,535 @@ | ||
//! One repository's records on its owner's PDS: in their public repo, or, | ||
//! for a private repository, in a space. | ||
| ||
use crate::env::Ctx; | ||
use crate::identity::{self, Identity}; | ||
use crate::keys; | ||
use crate::records::{PackRecord, RedirectRecord, RefRecord, RepoRecord, RepoRef, nsid}; | ||
use crate::session; | ||
use crate::space::{self, Space}; | ||
use crate::store; | ||
use crate::url::RepoUrl; | ||
use crate::xrpc::{Xrpc, XrpcError}; | ||
use anyhow::Result; | ||
use serde::Deserialize; | ||
use serde::de::DeserializeOwned; | ||
use serde_json::{Value, json}; | ||
use std::collections::BTreeMap; | ||
| ||
pub const PAGE: usize = 100; | ||
| ||
#[derive(Clone, Debug, PartialEq)] | ||
pub struct Stored<T> { | ||
pub rkey: String, | ||
pub cid: String, | ||
pub value: T, | ||
} | ||
| ||
#[derive(Clone, Debug, PartialEq)] | ||
pub struct Page<T> { | ||
pub items: Vec<Stored<T>>, | ||
/// Where the next page starts, when the scan may continue. | ||
pub next: Option<String>, | ||
} | ||
| ||
impl<T> Default for Page<T> { | ||
fn default() -> Self { | ||
Page { | ||
items: Vec::new(), | ||
next: None, | ||
} | ||
} | ||
} | ||
| ||
/// Everything a `list` needs, read in one round trip when the repository's | ||
/// key is cached. | ||
#[derive(Clone, Debug, Default)] | ||
pub struct Snapshot { | ||
pub repo: Option<Stored<RepoRecord>>, | ||
pub refs: Vec<Stored<RefRecord>>, | ||
/// The newest page of packs. | ||
pub packs: Page<PackRecord>, | ||
/// When the refs were read, for a push to tell whether its read is too | ||
/// old to write against. | ||
pub read_at: Option<std::time::Instant>, | ||
} | ||
| ||
impl Snapshot { | ||
pub fn key(&self) -> Option<&str> { | ||
self.repo.as_ref().map(|r| r.value.key.as_str()) | ||
} | ||
| ||
/// Refs that point somewhere, as `(name, sha)`. | ||
pub fn live_refs(&self) -> Vec<(String, String)> { | ||
self.refs | ||
.iter() | ||
.filter_map(|r| Some((r.value.name.clone(), r.value.target.clone()?))) | ||
.collect() | ||
} | ||
| ||
/// The repository as other records name it, if there is one. | ||
pub fn reference(&self, did: &str) -> Option<RepoRef> { | ||
let repo = self.repo.as_ref()?; | ||
Some(RepoRef::new(did, &repo.rkey, &repo.value.key)) | ||
} | ||
| ||
pub fn find_ref(&self, name: &str) -> Option<&Stored<RefRecord>> { | ||
self.refs.iter().find(|r| r.value.name == name) | ||
} | ||
} | ||
| ||
#[derive(Clone)] | ||
pub struct Remote { | ||
pub xrpc: Xrpc, | ||
pub identity: Identity, | ||
pub url: RepoUrl, | ||
/// The space a private repository lives in. | ||
pub space: Option<Space>, | ||
} | ||
| ||
#[derive(Deserialize)] | ||
struct ListOut { | ||
records: Vec<RawRecord>, | ||
cursor: Option<String>, | ||
} | ||
| ||
/// A record as either API lists it: the public one names it by URI, the | ||
/// space one by rkey. | ||
#[derive(Deserialize)] | ||
struct RawRecord { | ||
#[serde(default)] | ||
uri: String, | ||
rkey: Option<String>, | ||
cid: String, | ||
value: Value, | ||
} | ||
| ||
impl RawRecord { | ||
fn rkey(&self) -> &str { | ||
match &self.rkey { | ||
Some(rkey) => rkey, | ||
None => self.uri.rsplit('/').next().unwrap_or(&self.uri), | ||
} | ||
} | ||
} | ||
| ||
fn parse<T: DeserializeOwned>(raw: RawRecord) -> Result<Stored<T>> { | ||
let rkey = raw.rkey().to_string(); | ||
let value = match serde_json::from_value(raw.value) { | ||
Ok(value) => value, | ||
Err(err) => anyhow::bail!("reading record {rkey}: {err}"), | ||
}; | ||
Ok(Stored { | ||
rkey, | ||
cid: raw.cid, | ||
value, | ||
}) | ||
} | ||
| ||
pub fn create(collection: &str, rkey: &str, value: Value) -> Value { | ||
json!({"$type": "com.atproto.repo.applyWrites#create", "collection": collection, "rkey": rkey, "value": value}) | ||
} | ||
| ||
pub fn update(collection: &str, rkey: &str, value: Value) -> Value { | ||
json!({"$type": "com.atproto.repo.applyWrites#update", "collection": collection, "rkey": rkey, "value": value}) | ||
} | ||
| ||
pub fn delete(collection: &str, rkey: &str) -> Value { | ||
json!({"$type": "com.atproto.repo.applyWrites#delete", "collection": collection, "rkey": rkey}) | ||
} | ||
| ||
fn key_cache_path(ctx: &Ctx) -> std::path::PathBuf { | ||
ctx.env.cache_dir.join("keys.json") | ||
} | ||
| ||
impl Remote { | ||
/// The repository's public records, without looking for a space. | ||
pub async fn connect(ctx: &Ctx, url: &RepoUrl) -> Result<Remote> { | ||
let identity = identity::resolve(ctx, &url.authority).await?; | ||
Ok(Remote { | ||
xrpc: Xrpc::new(ctx.http.clone(), &identity.pds), | ||
identity, | ||
url: url.clone(), | ||
space: None, | ||
}) | ||
} | ||
| ||
/// The repository wherever it lives. A name with no public repository | ||
/// record is looked for in a space, as whoever is logged in; one nobody | ||
/// can find stays public, for a first push to create. Which it was is | ||
/// cached with the repository's key. | ||
pub async fn open(ctx: &Ctx, url: &RepoUrl) -> Result<Remote> { | ||
let public = Remote::connect(ctx, url).await?; | ||
let cache: BTreeMap<String, String> = store::read(&key_cache_path(ctx)).unwrap_or_default(); | ||
if cache.contains_key(&public.cache_key()) { | ||
return Ok(public); | ||
} | ||
let known = cache.contains_key(&format!("{}#space", public.cache_key())); | ||
if !known && public.repo().await?.is_some() { | ||
return Ok(public); | ||
} | ||
let Some(me) = session::reader(ctx, &public.identity).await else { | ||
return Ok(public); | ||
}; | ||
let uri = space::uri(public.did(), &url.name); | ||
let found = if me.did == public.did() { | ||
Space::owner(&uri, &me) | ||
} else { | ||
match Space::join(ctx, &public.xrpc, &uri, public.did(), &me).await { | ||
Ok(found) => found, | ||
Err(err) if !known && space::refused(&err) => return Ok(public), | ||
Err(err) => return Err(err), | ||
} | ||
}; | ||
let private = Remote { | ||
space: Some(found), | ||
..public.clone() | ||
}; | ||
let found = private.repo().await; | ||
// A private repository renamed since it was made keeps its records | ||
// in the space named for its first name; the space named for this | ||
// one only points there. | ||
if let Ok(None) = found | ||
&& let Some(home) = private.home().await? | ||
{ | ||
let space = if me.did == public.did() { | ||
Space::owner(&home, &me) | ||
} else { | ||
Space::join(ctx, &public.xrpc, &home, public.did(), &me).await? | ||
}; | ||
return Ok(Remote { | ||
space: Some(space), | ||
..public | ||
}); | ||
} | ||
match found { | ||
Ok(Some(_)) => Ok(private), | ||
_ if known => Ok(private), | ||
Ok(None) => Ok(public), | ||
Err(err) if space::refused(&err) => Ok(public), | ||
Err(err) => Err(err), | ||
} | ||
} | ||
| ||
pub fn did(&self) -> &str { | ||
&self.identity.did | ||
} | ||
| ||
/// The space this one points to, for a space named for a later name of | ||
/// a private repository. | ||
pub async fn home(&self) -> Result<Option<String>> { | ||
let redirect = self | ||
.get_record::<RedirectRecord>(nsid::REDIRECT, &self.url.name) | ||
.await?; | ||
Ok(redirect.and_then(|r| r.value.space)) | ||
} | ||
| ||
fn cache_key(&self) -> String { | ||
let key = format!("{}/{}", self.did(), self.url.name); | ||
match self.space { | ||
Some(_) => format!("{key}#space"), | ||
None => key, | ||
} | ||
} | ||
| ||
fn headers(&self) -> &[(String, String)] { | ||
self.space.as_ref().map_or(&[], |s| &s.headers) | ||
} | ||
| ||
/// A read from the public repo or the space, with `params` naming the | ||
/// repo and, for a space, the space. | ||
async fn read<T: DeserializeOwned>(&self, method: &str, params: &[(&str, &str)]) -> Result<T> { | ||
let mut all = vec![("repo", self.did())]; | ||
let nsid = match &self.space { | ||
Some(space) => { | ||
all.push(("space", &space.uri)); | ||
format!("com.atproto.space.{method}") | ||
} | ||
None => format!("com.atproto.repo.{method}"), | ||
}; | ||
all.extend_from_slice(params); | ||
self.xrpc.query_as(&nsid, &all, self.headers()).await | ||
} | ||
| ||
pub async fn get_blob(&self, cid: &str) -> Result<Vec<u8>> { | ||
match &self.space { | ||
Some(space) => { | ||
self.xrpc | ||
.get_space_blob(&space.uri, self.did(), cid, &space.headers) | ||
.await | ||
} | ||
None => self.xrpc.get_blob(self.did(), cid).await, | ||
} | ||
} | ||
| ||
/// Where a scan of `collection` resumes after `rkey`: the rkey itself | ||
/// for the public API, the record's full URI for a space. | ||
fn cursor(&self, collection: &str, rkey: &str) -> String { | ||
match &self.space { | ||
Some(space) => format!("{}/{}/{collection}/{rkey}", space.uri, self.did()), | ||
None => rkey.to_string(), | ||
} | ||
} | ||
| ||
pub async fn get_record<T: DeserializeOwned>( | ||
&self, | ||
collection: &str, | ||
rkey: &str, | ||
) -> Result<Option<Stored<T>>> { | ||
let res = self | ||
.read::<RawRecord>("getRecord", &[("collection", collection), ("rkey", rkey)]) | ||
.await; | ||
match res { | ||
Ok(raw) => parse(raw).map(Some), | ||
Err(err) if XrpcError::of(&err).is_some_and(|e| e.error == "RecordNotFound") => { | ||
Ok(None) | ||
} | ||
Err(err) => Err(err), | ||
} | ||
} | ||
| ||
/// The repository record under the URL's name, or, for a repository | ||
/// renamed since, under the name the old one's redirect leads to: the | ||
/// record's `rkey` is the name it has now. A rename repoints every | ||
/// redirect to the repository, so one hop is the rule; a few more let a | ||
/// read race a rename. A redirect to another account is not followed. | ||
pub async fn repo(&self) -> Result<Option<Stored<RepoRecord>>> { | ||
let mut name = self.url.name.clone(); | ||
let mut key: Option<String> = None; | ||
for _ in 0..4 { | ||
if let Some(repo) = self.get_record::<RepoRecord>(nsid::REPO, &name).await? { | ||
// A name given to another repository since is not this one. | ||
return Ok(Some(repo).filter(|r| key.is_none_or(|k| r.value.key == k))); | ||
} | ||
let redirect = self | ||
.get_record::<RedirectRecord>(nsid::REDIRECT, &name) | ||
.await?; | ||
match redirect { | ||
Some(r) if r.value.to.did() == self.did() => { | ||
name = r.value.to.name().to_string(); | ||
key = Some(r.value.to.key); | ||
} | ||
_ => return Ok(None), | ||
} | ||
} | ||
Ok(None) | ||
} | ||
| ||
/// Where the repository has gone in another account, if it has: an | ||
/// offer of its owner's that the other account took up, or the | ||
/// redirect left when the old copy was deleted. Either way the new | ||
/// repository has to name this one as `transferredFrom`, so no account | ||
/// can claim to be another's repository. | ||
pub async fn moved_to(&self, ctx: &Ctx, snap: &Snapshot) -> Result<Option<RepoUrl>> { | ||
let (to, key) = match &snap.repo { | ||
Some(repo) => match &repo.value.transfer { | ||
Some(to) => (to.clone(), repo.value.key.clone()), | ||
None => return Ok(None), | ||
}, | ||
None => match self | ||
.get_record::<RedirectRecord>(nsid::REDIRECT, &self.url.name) | ||
.await? | ||
{ | ||
Some(r) if r.value.to.did() != self.did() => { | ||
let key = r.value.to.key.clone(); | ||
(r.value.to, key) | ||
} | ||
_ => return Ok(None), | ||
}, | ||
}; | ||
let url = RepoUrl { | ||
authority: to.did().to_string(), | ||
name: to.name().to_string(), | ||
}; | ||
let there = Remote::connect(ctx, &url).await?; | ||
let Some(found) = there.repo().await? else { | ||
return Ok(None); | ||
}; | ||
let back = found | ||
.value | ||
.transferred_from | ||
.is_some_and(|f| f.is(self.did(), &key)); | ||
Ok(back.then_some(RepoUrl { | ||
authority: url.authority, | ||
name: found.rkey, | ||
})) | ||
} | ||
| ||
/// The repository wherever it now is: [`Remote::open`] and a snapshot, | ||
/// following moves to other accounts. | ||
pub async fn locate(ctx: &Ctx, url: &RepoUrl) -> Result<(Remote, Snapshot)> { | ||
let remote = Remote::open(ctx, url).await?; | ||
let snap = remote.snapshot(ctx).await?; | ||
remote.follow(ctx, snap).await | ||
} | ||
| ||
/// From a snapshot of where the repository was, the remote and snapshot | ||
/// of where it is, a few moves on at most. | ||
pub async fn follow(self, ctx: &Ctx, snap: Snapshot) -> Result<(Remote, Snapshot)> { | ||
let (mut remote, mut snap) = (self, snap); | ||
let mut hops = 0; | ||
while hops < 4 | ||
&& let Some(url) = remote.moved_to(ctx, &snap).await? | ||
{ | ||
remote = Remote::open(ctx, &url).await?; | ||
snap = remote.snapshot(ctx).await?; | ||
hops += 1; | ||
} | ||
Ok((remote, snap)) | ||
} | ||
| ||
/// One page of `key`'s records in `collection`, newest key first. | ||
pub async fn scan<T: DeserializeOwned>( | ||
&self, | ||
collection: &str, | ||
key: &str, | ||
cursor: Option<&str>, | ||
) -> Result<Page<T>> { | ||
let start = self.cursor(collection, cursor.unwrap_or(&keys::scan_cursor(key))); | ||
let limit = PAGE.to_string(); | ||
let out: ListOut = self | ||
.read( | ||
"listRecords", | ||
&[ | ||
("collection", collection), | ||
("limit", &limit), | ||
("cursor", &start), | ||
], | ||
) | ||
.await?; | ||
let full = out.records.len() == PAGE; | ||
let mut page = Page::default(); | ||
for raw in out.records { | ||
let rkey = raw.rkey(); | ||
if keys::suffix(key, rkey).is_none() { | ||
page.next = None; | ||
return Ok(page); | ||
} | ||
page.next = Some(rkey.to_string()); | ||
page.items.push(parse(raw)?); | ||
} | ||
if !full { | ||
page.next = None; | ||
} | ||
Ok(page) | ||
} | ||
| ||
/// Every record of `key`'s in `collection`. | ||
pub async fn scan_all<T: DeserializeOwned>( | ||
&self, | ||
collection: &str, | ||
key: &str, | ||
) -> Result<Vec<Stored<T>>> { | ||
let mut all = Vec::new(); | ||
let mut cursor = None; | ||
loop { | ||
let page = self.scan(collection, key, cursor.as_deref()).await?; | ||
all.extend(page.items); | ||
match page.next { | ||
Some(next) => cursor = Some(next), | ||
None => return Ok(all), | ||
} | ||
} | ||
} | ||
| ||
pub async fn refs(&self, key: &str) -> Result<Vec<Stored<RefRecord>>> { | ||
self.scan_all(nsid::REF, key).await | ||
} | ||
| ||
pub async fn packs(&self, key: &str, cursor: Option<&str>) -> Result<Page<PackRecord>> { | ||
self.scan(nsid::PACK, key, cursor).await | ||
} | ||
| ||
/// The repository record, its refs, and its newest packs. With the key | ||
/// cached this is one round trip of three parallel reads; the repository | ||
/// record still comes along to catch a name that now means another key. | ||
pub async fn snapshot(&self, ctx: &Ctx) -> Result<Snapshot> { | ||
let cache_key = self.cache_key(); | ||
let path = key_cache_path(ctx); | ||
let mut cache: BTreeMap<String, String> = store::read(&path).unwrap_or_default(); | ||
let read_at = Some(std::time::Instant::now()); | ||
let repo = match cache.get(&cache_key) { | ||
Some(key) => { | ||
let (repo, refs, packs) = | ||
tokio::try_join!(self.repo(), self.refs(key), self.packs(key, None))?; | ||
if repo.as_ref().is_some_and(|r| &r.value.key == key) { | ||
return Ok(Snapshot { | ||
repo, | ||
refs, | ||
packs, | ||
read_at, | ||
}); | ||
} | ||
repo | ||
} | ||
None => self.repo().await?, | ||
}; | ||
let Some(repo) = repo else { | ||
return Ok(Snapshot::default()); | ||
}; | ||
let key = repo.value.key.clone(); | ||
cache.insert(cache_key, key.clone()); | ||
let _ = store::write(&path, &cache, false); | ||
let (refs, packs) = tokio::try_join!(self.refs(&key), self.packs(&key, None))?; | ||
Ok(Snapshot { | ||
repo: Some(repo), | ||
refs, | ||
packs, | ||
read_at, | ||
}) | ||
} | ||
| ||
/// Applies writes made with [`create`], [`update`] and [`delete`], | ||
/// atomically, to the public repo or the space. | ||
pub async fn apply(&self, mut writes: Vec<Value>, auth: &str) -> Result<()> { | ||
let (nsid, body) = match &self.space { | ||
Some(space) => { | ||
for write in &mut writes { | ||
let kind = write["$type"].as_str().unwrap_or_default(); | ||
write["$type"] = kind | ||
.replace("com.atproto.repo.", "com.atproto.space.") | ||
.into(); | ||
} | ||
( | ||
"com.atproto.space.applyWrites", | ||
json!({"space": space.uri, "repo": self.did(), "writes": writes}), | ||
) | ||
} | ||
None => ( | ||
"com.atproto.repo.applyWrites", | ||
json!({"repo": self.did(), "writes": writes}), | ||
), | ||
}; | ||
self.xrpc | ||
.procedure::<serde::de::IgnoredAny>(nsid, Some(&body), Some(auth)) | ||
.await?; | ||
Ok(()) | ||
} | ||
| ||
/// The account's repositories. | ||
pub async fn list_repos(&self) -> Result<Vec<Stored<RepoRecord>>> { | ||
self.list_all(nsid::REPO).await | ||
} | ||
| ||
/// Every record in `collection`, from the public repo or the space. | ||
pub async fn list_all<T: DeserializeOwned>(&self, collection: &str) -> Result<Vec<Stored<T>>> { | ||
let mut all = Vec::new(); | ||
let mut cursor: Option<String> = None; | ||
let limit = PAGE.to_string(); | ||
loop { | ||
let mut params = vec![("collection", collection), ("limit", limit.as_str())]; | ||
if let Some(c) = &cursor { | ||
params.push(("cursor", c)); | ||
} | ||
let out: ListOut = self.read("listRecords", ¶ms).await?; | ||
let full = out.records.len() == PAGE; | ||
for raw in out.records { | ||
all.push(parse(raw)?); | ||
} | ||
match out.cursor { | ||
Some(next) if full => cursor = Some(next), | ||
_ => return Ok(all), | ||
} | ||
} | ||
} | ||
} | ||
crates/knotless/src/session.rsnew+163−0
| @@ -0,0 +1,163 @@ | ||
//! App password sessions, stored per DID and refreshed in place. | ||
| ||
use crate::env::Ctx; | ||
use crate::identity::{self, Identity}; | ||
use crate::store; | ||
use crate::xrpc::Xrpc; | ||
use anyhow::{Result, bail}; | ||
use base64::Engine; | ||
use serde::{Deserialize, Serialize}; | ||
use serde_json::json; | ||
use std::path::PathBuf; | ||
| ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct Session { | ||
pub did: String, | ||
pub handle: String, | ||
#[serde(default)] | ||
pub service: String, | ||
pub access_jwt: String, | ||
pub refresh_jwt: String, | ||
} | ||
| ||
fn path(ctx: &Ctx, did: &str) -> PathBuf { | ||
ctx.env | ||
.config_dir | ||
.join("sessions") | ||
.join(format!("{}.json", did.replace(':', "_"))) | ||
} | ||
| ||
pub fn load(ctx: &Ctx, did: &str) -> Option<Session> { | ||
store::read(&path(ctx, did)) | ||
} | ||
| ||
fn save(ctx: &Ctx, session: &Session) -> Result<()> { | ||
store::write(&path(ctx, &session.did), session, true) | ||
} | ||
| ||
pub async fn login(ctx: &Ctx, identifier: &str, password: &str) -> Result<Session> { | ||
let id = identity::resolve(ctx, identifier).await?; | ||
let mut session: Session = Xrpc::new(ctx.http.clone(), &id.pds) | ||
.procedure( | ||
"com.atproto.server.createSession", | ||
Some(&json!({ "identifier": identifier, "password": password })), | ||
None, | ||
) | ||
.await?; | ||
session.service = id.pds; | ||
save(ctx, &session)?; | ||
Ok(session) | ||
} | ||
| ||
/// Removes the stored session. Returns whether there was one. | ||
pub async fn logout(ctx: &Ctx, identifier: &str) -> Result<bool> { | ||
let id = identity::resolve(ctx, identifier).await?; | ||
Ok(std::fs::remove_file(path(ctx, &id.did)).is_ok()) | ||
} | ||
| ||
/// A session for `id` with an access token that will last the call: the | ||
/// stored one, refreshed if it is about to expire, or a fresh login from | ||
/// `KNOTLESS_PASSWORD`. | ||
pub async fn ensure(ctx: &Ctx, id: &Identity) -> Result<Session> { | ||
if let Some(stored) = load(ctx, &id.did) { | ||
if !expiring(&stored.access_jwt) { | ||
return Ok(stored); | ||
} | ||
if let Ok(fresh) = refresh(ctx, &stored).await { | ||
return Ok(fresh); | ||
} | ||
} | ||
if let Some(password) = &ctx.env.password { | ||
let identifier = ctx.env.identifier.clone().unwrap_or_else(|| id.did.clone()); | ||
return login(ctx, &identifier, password).await; | ||
} | ||
bail!( | ||
"not logged in as {}: run `knotless login <handle>` or set KNOTLESS_PASSWORD", | ||
id.did | ||
) | ||
} | ||
| ||
/// Marks `session` as the account `knotless login` last logged in. | ||
pub fn remember(ctx: &Ctx, session: &Session) -> Result<()> { | ||
store::write_bytes( | ||
&ctx.env.config_dir.join("me"), | ||
session.did.as_bytes(), | ||
false, | ||
) | ||
} | ||
| ||
/// The account this machine acts as: `KNOTLESS_IDENTIFIER`, else the one | ||
/// `knotless login` last logged in. | ||
pub async fn me(ctx: &Ctx) -> Result<Session> { | ||
let identifier = match &ctx.env.identifier { | ||
Some(identifier) => identifier.clone(), | ||
None => std::fs::read_to_string(ctx.env.config_dir.join("me")) | ||
.map_err(|_| anyhow::anyhow!("not logged in: run `knotless login <handle>`"))?, | ||
}; | ||
let id = identity::resolve(ctx, identifier.trim()).await?; | ||
ensure(ctx, &id).await | ||
} | ||
| ||
/// Whoever is reading `owner`'s repositories: the owner, when logged in as | ||
/// them, else [`me`]. None when nobody is. | ||
pub async fn reader(ctx: &Ctx, owner: &Identity) -> Option<Session> { | ||
if let Ok(session) = ensure(ctx, owner).await { | ||
return Some(session); | ||
} | ||
me(ctx).await.ok() | ||
} | ||
| ||
async fn refresh(ctx: &Ctx, stored: &Session) -> Result<Session> { | ||
let mut session: Session = Xrpc::new(ctx.http.clone(), &stored.service) | ||
.procedure( | ||
"com.atproto.server.refreshSession", | ||
None, | ||
Some(&stored.refresh_jwt), | ||
) | ||
.await?; | ||
session.service = stored.service.clone(); | ||
save(ctx, &session)?; | ||
Ok(session) | ||
} | ||
| ||
/// Whether a JWT expires within a minute, or cannot be read. | ||
pub fn expiring(jwt: &str) -> bool { | ||
#[derive(Deserialize)] | ||
struct Claims { | ||
exp: u64, | ||
} | ||
let claims = jwt | ||
.split('.') | ||
.nth(1) | ||
.and_then(|p| { | ||
base64::engine::general_purpose::URL_SAFE_NO_PAD | ||
.decode(p) | ||
.ok() | ||
}) | ||
.and_then(|b| serde_json::from_slice::<Claims>(&b).ok()); | ||
match claims { | ||
Some(c) => c.exp < crate::keys::now_micros() / 1_000_000 + 60, | ||
None => true, | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
fn jwt(exp: u64) -> String { | ||
let body = | ||
base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(format!("{{\"exp\":{exp}}}")); | ||
format!("h.{body}.s") | ||
} | ||
| ||
#[test] | ||
fn expiry() { | ||
let now = crate::keys::now_micros() / 1_000_000; | ||
assert!(!expiring(&jwt(now + 3600))); | ||
assert!(expiring(&jwt(now + 10))); | ||
assert!(expiring("garbage")); | ||
assert!(expiring("a.!!.b")); | ||
} | ||
} | ||
crates/knotless/src/space.rsnew+381−0
| @@ -0,0 +1,381 @@ | ||
//! Permissioned spaces, where private repositories live. | ||
//! | ||
//! A private repository is a space of type [`SPACE_TYPE`] keyed by its name, | ||
//! with its records in the owner's repo inside it. The owner reads with their | ||
//! own session. Anyone else exchanges a delegation token from their own PDS | ||
//! for a space credential from the owner's, bound to an ephemeral P-256 key | ||
//! that signs each request. | ||
| ||
use crate::crypto::{self, P256Key}; | ||
use crate::env::Ctx; | ||
use crate::keys; | ||
use crate::session::Session; | ||
use crate::store; | ||
use crate::xrpc::{Xrpc, XrpcError}; | ||
use anyhow::{Context, Result}; | ||
use base64::Engine; | ||
use base64::engine::general_purpose::STANDARD; | ||
use serde::{Deserialize, Serialize}; | ||
use serde_json::{Value, json}; | ||
use std::sync::Arc; | ||
| ||
pub const SPACE_TYPE: &str = "dev.knotless.space"; | ||
| ||
const LABEL: &str = "atproto-space"; | ||
| ||
/// The space holding `did`'s private repository `name`. | ||
pub fn uri(did: &str, name: &str) -> String { | ||
format!("at://{did}/space/{SPACE_TYPE}/{name}") | ||
} | ||
| ||
/// A space, and the headers that let requests read its owner's repo in it. | ||
#[derive(Clone)] | ||
pub struct Space { | ||
pub uri: String, | ||
pub headers: Vec<(String, String)>, | ||
/// A credential and its key, to read any member's repo in the space. | ||
credential: Option<(String, Arc<P256Key>)>, | ||
} | ||
| ||
#[derive(Serialize, Deserialize)] | ||
struct Cached { | ||
credential: String, | ||
key: String, | ||
} | ||
| ||
/// The errors that mean "this space is not yours to read", which a reader | ||
/// cannot tell apart from "there is no such space", or that the PDS has no | ||
/// spaces at all: one without them passes `com.atproto.space.*` on to an | ||
/// appview it may not have. | ||
pub fn refused(err: &anyhow::Error) -> bool { | ||
XrpcError::of(err).is_some_and(|e| { | ||
matches!( | ||
e.error.as_str(), | ||
"SpaceNotFound" | ||
| "SpaceDeleted" | ||
| "UserNotAuthorized" | ||
| "NotAuthorized" | ||
| "MethodNotImplemented" | ||
| "UpstreamFailure" | ||
) | ||
}) | ||
} | ||
| ||
impl Space { | ||
/// The owner reads their own repo in the space with their session. | ||
pub fn owner(uri: &str, session: &Session) -> Space { | ||
Space { | ||
uri: uri.to_string(), | ||
headers: crate::xrpc::bearer(Some(&session.access_jwt)), | ||
credential: None, | ||
} | ||
} | ||
| ||
/// A credential holder reads `audience`'s repo in the space. | ||
pub fn member(uri: &str, audience: &str, credential: &str, key: P256Key) -> Space { | ||
Space { | ||
uri: uri.to_string(), | ||
headers: signed(&key, &format!("Atproto-Space {credential}"), Some(audience)), | ||
credential: Some((credential.to_string(), Arc::new(key))), | ||
} | ||
} | ||
| ||
/// The space as read from `did`'s repo in it. | ||
pub fn at(&self, did: &str) -> Space { | ||
Space { | ||
headers: self.for_repo(did), | ||
..self.clone() | ||
} | ||
} | ||
| ||
/// Headers to read `did`'s repo in the space, or to ask the authority | ||
/// (`did` the space's) which repos it holds. Only a credential reaches | ||
/// past its holder's own repo. | ||
pub fn for_repo(&self, did: &str) -> Vec<(String, String)> { | ||
match &self.credential { | ||
Some((credential, key)) => { | ||
signed(key, &format!("Atproto-Space {credential}"), Some(did)) | ||
} | ||
None => self.headers.clone(), | ||
} | ||
} | ||
| ||
/// Joins `uri` as `me`, a member reading `authority`'s repo there: | ||
/// a cached credential while it lasts, else a fresh one. `host` is the | ||
/// authority's PDS. | ||
pub async fn join( | ||
ctx: &Ctx, | ||
host: &Xrpc, | ||
uri: &str, | ||
authority: &str, | ||
me: &Session, | ||
) -> Result<Space> { | ||
let path = ctx.env.config_dir.join("credentials").join(format!( | ||
"{}.{}.json", | ||
me.did.replace(':', "_"), | ||
keys::ref_hash(uri) | ||
)); | ||
if let Some(cached) = store::read::<Cached>(&path) | ||
&& let Ok(bytes) = STANDARD.decode(&cached.key) | ||
&& let Ok(key) = P256Key::from_bytes(&bytes) | ||
&& lasts(&cached.credential) | ||
{ | ||
return Ok(Space::member(uri, authority, &cached.credential, key)); | ||
} | ||
let key = P256Key::generate(); | ||
let credential = credential(ctx, host, uri, me, &key).await?; | ||
let cached = Cached { | ||
credential: credential.clone(), | ||
key: STANDARD.encode(key.to_bytes()), | ||
}; | ||
// A credential that cannot be cached only costs the next exchange. | ||
let _ = store::write(&path, &cached, true); | ||
Ok(Space::member(uri, authority, &credential, key)) | ||
} | ||
} | ||
| ||
/// Whether a credential has a minute left in it. | ||
fn lasts(credential: &str) -> bool { | ||
let now = keys::now_micros() / 1_000_000; | ||
crypto::jwt_parts(credential) | ||
.ok() | ||
.and_then(|(_, claims)| claims["exp"].as_u64()) | ||
.is_some_and(|exp| exp > now + 60) | ||
} | ||
| ||
/// Exchanges a delegation token from `me`'s PDS for a space credential | ||
/// from `host`, bound to `key`. | ||
async fn credential( | ||
ctx: &Ctx, | ||
host: &Xrpc, | ||
uri: &str, | ||
me: &Session, | ||
key: &P256Key, | ||
) -> Result<String> { | ||
#[derive(Deserialize)] | ||
struct Token { | ||
token: String, | ||
} | ||
#[derive(Deserialize)] | ||
struct Credential { | ||
credential: String, | ||
} | ||
let token: Token = Xrpc::new(ctx.http.clone(), &me.service) | ||
.query( | ||
"com.atproto.space.getDelegationToken", | ||
&[("space", uri)], | ||
Some(&me.access_jwt), | ||
) | ||
.await | ||
.context( | ||
"getting a delegation token, which takes a full login: an app password cannot \ | ||
read another account's private repository", | ||
)?; | ||
let headers = signed(key, &format!("Bearer {}", token.token), None); | ||
let out: Credential = host | ||
.procedure_as( | ||
"com.atproto.space.getSpaceCredential", | ||
Some(&json!({ "space": uri })), | ||
&headers, | ||
) | ||
.await?; | ||
Ok(out.credential) | ||
} | ||
| ||
/// RFC 9421 message signature headers over `authorization`, and over the | ||
/// audience when a credential is in use. Without an audience the signature | ||
/// names its key, which the authority binds the credential to. | ||
pub fn signed(key: &P256Key, authorization: &str, audience: Option<&str>) -> Vec<(String, String)> { | ||
let params = match audience { | ||
Some(_) => r#"("authorization" "atproto-space-audience")"#.to_string(), | ||
None => format!(r#"("authorization");keyid="{}""#, key.public().to_did_key()), | ||
}; | ||
let mut base = format!("\"authorization\": {authorization}\n"); | ||
if let Some(audience) = audience { | ||
base.push_str(&format!("\"atproto-space-audience\": {audience}\n")); | ||
} | ||
base.push_str(&format!("\"@signature-params\": {params}")); | ||
let signature = STANDARD.encode(key.sign(base.as_bytes())); | ||
let mut headers = vec![("authorization".to_string(), authorization.to_string())]; | ||
if let Some(audience) = audience { | ||
headers.push(("atproto-space-audience".to_string(), audience.to_string())); | ||
} | ||
headers.push(("signature-input".to_string(), format!("{LABEL}={params}"))); | ||
headers.push(("signature".to_string(), format!("{LABEL}=:{signature}:"))); | ||
headers | ||
} | ||
| ||
/// The owner's administration of the spaces their private repos live in. | ||
pub struct Admin<'a> { | ||
pub xrpc: &'a Xrpc, | ||
pub session: &'a Session, | ||
} | ||
| ||
#[derive(Deserialize, Debug, Clone, PartialEq)] | ||
pub struct Member { | ||
pub did: String, | ||
#[serde(default)] | ||
pub read: bool, | ||
#[serde(default)] | ||
pub write: bool, | ||
} | ||
| ||
impl Admin<'_> { | ||
fn auth(&self) -> Option<&str> { | ||
Some(&self.session.access_jwt) | ||
} | ||
| ||
/// Creates the space for private repository `name`, readable and | ||
/// writable by its member list. One that already exists is kept: the | ||
/// second value says whether it was made now. | ||
pub async fn create(&self, name: &str) -> Result<(String, bool)> { | ||
let member_list = json!({"$type": "com.atproto.simplespace.defs#memberListPolicy"}); | ||
let res = self | ||
.xrpc | ||
.procedure::<serde::de::IgnoredAny>( | ||
"com.atproto.simplespace.createSpace", | ||
Some(&json!({ | ||
"spaceType": SPACE_TYPE, | ||
"skey": name, | ||
"readPolicy": member_list, | ||
"writePolicy": member_list, | ||
"appAccess": {"$type": "com.atproto.simplespace.defs#open"}, | ||
})), | ||
self.auth(), | ||
) | ||
.await; | ||
let made = match res { | ||
Ok(_) => true, | ||
Err(err) if XrpcError::of(&err).is_some_and(|e| e.error == "SpaceAlreadyExists") => { | ||
false | ||
} | ||
Err(err) => return Err(err), | ||
}; | ||
Ok((uri(&self.session.did, name), made)) | ||
} | ||
| ||
pub async fn delete(&self, uri: &str) -> Result<()> { | ||
self.call( | ||
"com.atproto.simplespace.deleteSpace", | ||
json!({ "space": uri }), | ||
) | ||
.await | ||
} | ||
| ||
/// Adds `did` to the space. Every member writes to their own repo | ||
/// there, which is where their issues and comments live; pushing to the | ||
/// repository is a separate grant, the repo record's collaborators. | ||
pub async fn put_member(&self, uri: &str, did: &str) -> Result<()> { | ||
self.call( | ||
"com.atproto.simplespace.putMember", | ||
json!({ "space": uri, "did": did, "read": true, "write": true }), | ||
) | ||
.await | ||
} | ||
| ||
pub async fn remove_member(&self, uri: &str, did: &str) -> Result<()> { | ||
self.call( | ||
"com.atproto.simplespace.removeMember", | ||
json!({ "space": uri, "did": did }), | ||
) | ||
.await | ||
} | ||
| ||
pub async fn members(&self, uri: &str) -> Result<Vec<Member>> { | ||
#[derive(Deserialize)] | ||
struct Out { | ||
members: Vec<Member>, | ||
cursor: Option<String>, | ||
} | ||
let mut all = Vec::new(); | ||
let mut cursor = None::<String>; | ||
loop { | ||
let mut params = vec![("space", uri), ("limit", "1000")]; | ||
if let Some(c) = &cursor { | ||
params.push(("cursor", c)); | ||
} | ||
let out: Out = self | ||
.xrpc | ||
.query("com.atproto.simplespace.listMembers", ¶ms, self.auth()) | ||
.await?; | ||
let done = out.members.is_empty(); | ||
all.extend(out.members); | ||
match out.cursor { | ||
Some(next) if !done => cursor = Some(next), | ||
_ => return Ok(all), | ||
} | ||
} | ||
} | ||
| ||
async fn call(&self, nsid: &str, body: Value) -> Result<()> { | ||
self.xrpc | ||
.procedure::<serde::de::IgnoredAny>(nsid, Some(&body), self.auth()) | ||
.await | ||
.map(drop) | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn signature_bases() { | ||
let key = P256Key::generate(); | ||
let headers = signed(&key, "Bearer t", None); | ||
let did = key.public().to_did_key(); | ||
assert_eq!(headers[0], ("authorization".into(), "Bearer t".into())); | ||
assert_eq!( | ||
headers[1].1, | ||
format!(r#"atproto-space=("authorization");keyid="{did}""#) | ||
); | ||
let sig = headers[2] | ||
.1 | ||
.strip_prefix("atproto-space=:") | ||
.unwrap() | ||
.strip_suffix(':') | ||
.unwrap(); | ||
let base = format!( | ||
"\"authorization\": Bearer t\n\"@signature-params\": (\"authorization\");keyid=\"{did}\"" | ||
); | ||
key.public() | ||
.verify(base.as_bytes(), &STANDARD.decode(sig).unwrap()) | ||
.unwrap(); | ||
| ||
let own = Space { | ||
uri: "at://x/space/t/k".into(), | ||
headers: vec![("authorization".into(), "Bearer t".into())], | ||
credential: None, | ||
}; | ||
assert_eq!(own.for_repo("did:plc:m"), own.headers); | ||
| ||
let space = Space::member("at://x/space/t/k", "did:plc:o", "cred", key); | ||
assert_eq!(space.headers.len(), 4); | ||
assert_eq!(space.for_repo("did:plc:o")[1], space.headers[1]); | ||
assert_eq!(space.for_repo("did:plc:m")[1].1, "did:plc:m"); | ||
let key = P256Key::from_bytes(&space.credential.as_ref().unwrap().1.to_bytes()).unwrap(); | ||
assert_eq!( | ||
space.headers[1], | ||
("atproto-space-audience".into(), "did:plc:o".into()) | ||
); | ||
let sig = space.headers[3] | ||
.1 | ||
.strip_prefix("atproto-space=:") | ||
.unwrap() | ||
.strip_suffix(':') | ||
.unwrap(); | ||
let base = "\"authorization\": Atproto-Space cred\n\"atproto-space-audience\": did:plc:o\n\"@signature-params\": (\"authorization\" \"atproto-space-audience\")"; | ||
key.public() | ||
.verify(base.as_bytes(), &STANDARD.decode(sig).unwrap()) | ||
.unwrap(); | ||
} | ||
| ||
#[test] | ||
fn credential_lifetimes() { | ||
let key = P256Key::generate(); | ||
let now = keys::now_micros() / 1_000_000; | ||
assert!(lasts(&key.sign_jwt(&json!({}), &json!({"exp": now + 600})))); | ||
assert!(!lasts(&key.sign_jwt(&json!({}), &json!({"exp": now + 30})))); | ||
assert!(!lasts("junk")); | ||
} | ||
} | ||
crates/knotless/src/store.rsnew+77−0
| @@ -0,0 +1,77 @@ | ||
//! Small JSON files under the config and cache directories. | ||
| ||
use anyhow::{Context, Result}; | ||
use serde::Serialize; | ||
use serde::de::DeserializeOwned; | ||
use std::path::Path; | ||
| ||
/// The file's contents, or `None` when it is missing or unreadable: every | ||
/// file here is a cache or can be recreated by logging in again. | ||
pub fn read<T: DeserializeOwned>(path: &Path) -> Option<T> { | ||
let bytes = std::fs::read(path).ok()?; | ||
serde_json::from_slice(&bytes).ok() | ||
} | ||
| ||
/// Writes through a temporary file and a rename, so a reader never sees half | ||
/// a file. `private` restricts it to the owner. | ||
pub fn write<T: Serialize>(path: &Path, value: &T, private: bool) -> Result<()> { | ||
write_bytes(path, &serde_json::to_vec_pretty(value)?, private) | ||
} | ||
| ||
pub fn write_bytes(path: &Path, bytes: &[u8], private: bool) -> Result<()> { | ||
let dir = path.parent().context("a store path has a parent")?; | ||
std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?; | ||
let tmp = dir.join(format!( | ||
".{}.{}.tmp", | ||
path.file_name().and_then(|n| n.to_str()).unwrap_or("store"), | ||
crate::keys::tid() | ||
)); | ||
std::fs::write(&tmp, bytes).with_context(|| format!("writing {}", tmp.display()))?; | ||
#[cfg(unix)] | ||
if private { | ||
use std::os::unix::fs::PermissionsExt; | ||
std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600))?; | ||
} | ||
#[cfg(not(unix))] | ||
let _ = private; | ||
std::fs::rename(&tmp, path).with_context(|| format!("writing {}", path.display()))?; | ||
Ok(()) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use std::collections::BTreeMap; | ||
| ||
#[test] | ||
fn round_trip_and_failures() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let path = dir.path().join("a/b.json"); | ||
assert_eq!(read::<BTreeMap<String, u8>>(&path), None); | ||
let mut m = BTreeMap::new(); | ||
m.insert("x".to_string(), 1u8); | ||
write(&path, &m, true).unwrap(); | ||
write(&path, &m, false).unwrap(); | ||
assert_eq!(read::<BTreeMap<String, u8>>(&path), Some(m.clone())); | ||
std::fs::write(&path, b"{").unwrap(); | ||
assert_eq!(read::<BTreeMap<String, u8>>(&path), None); | ||
assert!(write(Path::new("/"), &m, false).is_err()); | ||
let blocked = dir.path().join("file"); | ||
std::fs::write(&blocked, b"").unwrap(); | ||
assert!(write(&blocked.join("x.json"), &m, false).is_err()); | ||
// A directory where the file should go: the rename fails. | ||
let occupied = dir.path().join("occupied"); | ||
std::fs::create_dir_all(occupied.join("inside")).unwrap(); | ||
assert!(write(&occupied, &m, false).is_err()); | ||
// A directory it cannot write into. | ||
#[cfg(unix)] | ||
{ | ||
use std::os::unix::fs::PermissionsExt; | ||
let locked = dir.path().join("locked"); | ||
std::fs::create_dir_all(&locked).unwrap(); | ||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o500)).unwrap(); | ||
assert!(write(&locked.join("x.json"), &m, false).is_err()); | ||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700)).unwrap(); | ||
} | ||
} | ||
} | ||
crates/knotless/src/url.rsnew+69−0
| @@ -0,0 +1,69 @@ | ||
//! `knotless://<handle or did>/<name>` remote URLs. | ||
| ||
use crate::keys; | ||
use anyhow::{Result, bail}; | ||
use std::fmt; | ||
| ||
pub const SCHEME: &str = "knotless://"; | ||
| ||
#[derive(Clone, Debug, PartialEq, Eq)] | ||
pub struct RepoUrl { | ||
pub authority: String, | ||
pub name: String, | ||
} | ||
| ||
impl RepoUrl { | ||
pub fn parse(url: &str) -> Result<RepoUrl> { | ||
let Some(rest) = url.strip_prefix(SCHEME) else { | ||
bail!("{url}: a knotless URL starts with {SCHEME}"); | ||
}; | ||
let rest = rest.trim_end_matches('/'); | ||
let Some((authority, name)) = rest.split_once('/') else { | ||
bail!("{url}: expected {SCHEME}<handle or did>/<name>"); | ||
}; | ||
let name = name.strip_suffix(".git").unwrap_or(name); | ||
if authority.is_empty() || !keys::valid_name(name) { | ||
bail!( | ||
"{url}: expected {SCHEME}<handle or did>/<name>, a name of letters, digits and ._~-" | ||
); | ||
} | ||
Ok(RepoUrl { | ||
authority: authority.into(), | ||
name: name.into(), | ||
}) | ||
} | ||
} | ||
| ||
impl fmt::Display for RepoUrl { | ||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | ||
write!(f, "{SCHEME}{}/{}", self.authority, self.name) | ||
} | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn parses() { | ||
let u = RepoUrl::parse("knotless://alice.test/proj.git/").unwrap(); | ||
assert_eq!(u.authority, "alice.test"); | ||
assert_eq!(u.name, "proj"); | ||
assert_eq!(u.to_string(), "knotless://alice.test/proj"); | ||
let d = RepoUrl::parse("knotless://did:plc:abc/x").unwrap(); | ||
assert_eq!(d.authority, "did:plc:abc"); | ||
} | ||
| ||
#[test] | ||
fn rejects() { | ||
for bad in [ | ||
"https://a/b", | ||
"knotless://alice", | ||
"knotless:///x", | ||
"knotless://a/b/c", | ||
"knotless://a/", | ||
] { | ||
assert!(RepoUrl::parse(bad).is_err(), "{bad}"); | ||
} | ||
} | ||
} | ||
crates/knotless/src/xrpc.rsnew+200−0
| @@ -0,0 +1,200 @@ | ||
//! A minimal XRPC client over the shared HTTP connection pool. | ||
| ||
use crate::records::BlobRef; | ||
use anyhow::Result; | ||
use serde::Deserialize; | ||
use serde::de::DeserializeOwned; | ||
use serde_json::Value; | ||
use std::fmt; | ||
| ||
/// An XRPC call that reached the server and was refused. | ||
#[derive(Debug, Clone, PartialEq)] | ||
pub struct XrpcError { | ||
pub status: u16, | ||
pub error: String, | ||
pub message: String, | ||
} | ||
| ||
impl fmt::Display for XrpcError { | ||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | ||
write!(f, "{} ({}): {}", self.error, self.status, self.message) | ||
} | ||
} | ||
| ||
impl std::error::Error for XrpcError {} | ||
| ||
impl XrpcError { | ||
/// The XRPC error inside `err`, if that is what it is. | ||
pub fn of(err: &anyhow::Error) -> Option<&XrpcError> { | ||
err.downcast_ref() | ||
} | ||
| ||
fn from_body(status: u16, body: &[u8]) -> XrpcError { | ||
#[derive(Deserialize, Default)] | ||
struct Body { | ||
error: Option<String>, | ||
message: Option<String>, | ||
} | ||
let parsed: Body = serde_json::from_slice(body).unwrap_or_default(); | ||
XrpcError { | ||
status, | ||
error: parsed.error.unwrap_or_else(|| format!("HTTP{status}")), | ||
message: parsed | ||
.message | ||
.unwrap_or_else(|| String::from_utf8_lossy(body).chars().take(200).collect()), | ||
} | ||
} | ||
} | ||
| ||
#[derive(Clone)] | ||
pub struct Xrpc { | ||
http: reqwest::Client, | ||
pub base: String, | ||
} | ||
| ||
impl Xrpc { | ||
pub fn new(http: reqwest::Client, base: &str) -> Xrpc { | ||
Xrpc { | ||
http, | ||
base: base.trim_end_matches('/').to_string(), | ||
} | ||
} | ||
| ||
fn url(&self, nsid: &str) -> String { | ||
format!("{}/xrpc/{nsid}", self.base) | ||
} | ||
| ||
pub async fn query<T: DeserializeOwned>( | ||
&self, | ||
nsid: &str, | ||
params: &[(&str, &str)], | ||
auth: Option<&str>, | ||
) -> Result<T> { | ||
self.query_as(nsid, params, &bearer(auth)).await | ||
} | ||
| ||
/// A query sent with `headers`, such as a space credential's. | ||
pub async fn query_as<T: DeserializeOwned>( | ||
&self, | ||
nsid: &str, | ||
params: &[(&str, &str)], | ||
headers: &[(String, String)], | ||
) -> Result<T> { | ||
let req = self.http.get(self.url(nsid)).query(params); | ||
json(with(req, headers).send().await?).await | ||
} | ||
| ||
/// Calls a procedure. `body: None` sends none, which a procedure that | ||
/// takes no input requires: the reference PDS refuses even `{}`. | ||
pub async fn procedure<T: DeserializeOwned>( | ||
&self, | ||
nsid: &str, | ||
body: Option<&Value>, | ||
auth: Option<&str>, | ||
) -> Result<T> { | ||
self.procedure_as(nsid, body, &bearer(auth)).await | ||
} | ||
| ||
pub async fn procedure_as<T: DeserializeOwned>( | ||
&self, | ||
nsid: &str, | ||
body: Option<&Value>, | ||
headers: &[(String, String)], | ||
) -> Result<T> { | ||
let mut req = self.http.post(self.url(nsid)); | ||
if let Some(body) = body { | ||
req = req.json(body); | ||
} | ||
json(with(req, headers).send().await?).await | ||
} | ||
| ||
/// A procedure whose input is raw bytes of `mime`. | ||
pub async fn procedure_bytes<T: DeserializeOwned>( | ||
&self, | ||
nsid: &str, | ||
body: Vec<u8>, | ||
mime: &str, | ||
auth: Option<&str>, | ||
) -> Result<T> { | ||
let req = self | ||
.http | ||
.post(self.url(nsid)) | ||
.header(reqwest::header::CONTENT_TYPE, mime) | ||
.body(body); | ||
json(with(req, &bearer(auth)).send().await?).await | ||
} | ||
| ||
pub async fn upload_blob(&self, bytes: Vec<u8>, mime: &str, auth: &str) -> Result<BlobRef> { | ||
#[derive(Deserialize)] | ||
struct Out { | ||
blob: BlobRef, | ||
} | ||
let res = self | ||
.http | ||
.post(self.url("com.atproto.repo.uploadBlob")) | ||
.bearer_auth(auth) | ||
.header(reqwest::header::CONTENT_TYPE, mime) | ||
.body(bytes) | ||
.send() | ||
.await?; | ||
Ok(json::<Out>(res).await?.blob) | ||
} | ||
| ||
pub async fn get_blob(&self, did: &str, cid: &str) -> Result<Vec<u8>> { | ||
let res = self | ||
.http | ||
.get(self.url("com.atproto.sync.getBlob")) | ||
.query(&[("did", did), ("cid", cid)]) | ||
.send() | ||
.await?; | ||
Ok(ok(res).await?.to_vec()) | ||
} | ||
| ||
/// A blob a record in `space` references. | ||
pub async fn get_space_blob( | ||
&self, | ||
space: &str, | ||
did: &str, | ||
cid: &str, | ||
headers: &[(String, String)], | ||
) -> Result<Vec<u8>> { | ||
let req = self | ||
.http | ||
.get(self.url("com.atproto.space.getBlob")) | ||
.query(&[("space", space), ("repo", did), ("cid", cid)]); | ||
Ok(ok(with(req, headers).send().await?).await?.to_vec()) | ||
} | ||
} | ||
| ||
/// The headers that carry a bearer token, if there is one. | ||
pub fn bearer(token: Option<&str>) -> Vec<(String, String)> { | ||
token | ||
.map(|t| ("authorization".to_string(), format!("Bearer {t}"))) | ||
.into_iter() | ||
.collect() | ||
} | ||
| ||
fn with(mut req: reqwest::RequestBuilder, headers: &[(String, String)]) -> reqwest::RequestBuilder { | ||
for (name, value) in headers { | ||
req = req.header(name, value); | ||
} | ||
req | ||
} | ||
| ||
async fn ok(res: reqwest::Response) -> Result<bytes::Bytes> { | ||
let status = res.status().as_u16(); | ||
let body = res.bytes().await?; | ||
if (200..300).contains(&status) { | ||
Ok(body) | ||
} else { | ||
Err(XrpcError::from_body(status, &body).into()) | ||
} | ||
} | ||
| ||
/// The response as JSON. A procedure with no output answers with an empty | ||
/// body, which reads as `null`. | ||
async fn json<T: DeserializeOwned>(res: reqwest::Response) -> Result<T> { | ||
let body = ok(res).await?; | ||
let body: &[u8] = if body.is_empty() { b"null" } else { &body }; | ||
Ok(serde_json::from_slice(body)?) | ||
} | ||
crates/knotless/tests/cli.rsnew+306−0
| @@ -0,0 +1,306 @@ | ||
//! The `knotless` command. | ||
| ||
mod common; | ||
use common::*; | ||
| ||
#[test] | ||
fn sessions() { | ||
let mut s = Sandbox::new(); | ||
let handle = s.account.handle.clone(); | ||
let password = s.account.password.clone(); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
let (code, out) = s.cli(&["login", &handle, "--password-stdin"], password.as_bytes()); | ||
assert_eq!(code, 0); | ||
assert!(out.contains(&s.account.did), "{out}"); | ||
assert_eq!(s.cli(&["login", &handle, "--bogus"], b"").0, 1); | ||
assert_eq!(s.cli(&["login", &handle], b"").0, 1); | ||
assert_eq!(s.cli(&["login"], b"").0, 1); | ||
// The stored session pushes with no password around. | ||
let work = s.work("w", 1); | ||
s.git(&work, &["push", "-q", &s.url("logged-in"), "main"]); | ||
assert_eq!(s.cli(&["logout", &handle], b""), (0, "logged out".into())); | ||
assert_eq!(s.cli(&["logout", &handle], b""), (0, "no session".into())); | ||
let (ok, _, err) = s.git_raw(&work, &["push", &s.url("logged-in"), "main:other"]); | ||
assert!(!ok); | ||
assert!(err.contains("not logged in"), "{err}"); | ||
s.set("KNOTLESS_PASSWORD", &password); | ||
assert_eq!(s.cli(&["login", &handle], b"").0, 0); | ||
} | ||
| ||
#[test] | ||
fn repository_lifecycle() { | ||
let s = Sandbox::new(); | ||
let url = s.url("life"); | ||
assert_eq!( | ||
s.cli( | ||
&[ | ||
"create", | ||
&url, | ||
"--description", | ||
"a thing", | ||
"--default-branch", | ||
"trunk" | ||
], | ||
b"" | ||
) | ||
.0, | ||
0 | ||
); | ||
let (code, _) = s.cli(&["create", &url], b""); | ||
assert_eq!(code, 1); | ||
assert_eq!(s.cli(&["create", &url, "--bogus"], b"").0, 1); | ||
let (_, out) = s.cli(&["ls", &s.account.handle], b""); | ||
assert!(out.contains("life\ta thing"), "{out}"); | ||
| ||
let file = s.path("hello.txt"); | ||
std::fs::write(&file, "hi\n").unwrap(); | ||
let (code, out) = s.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"trunk", | ||
"-m", | ||
"hello", | ||
"--put", | ||
"hello.txt", | ||
file.to_str().unwrap(), | ||
"--put-exec", | ||
"run.sh", | ||
"-", | ||
"--author", | ||
"Ada", | ||
"ada@example.com", | ||
"--retries", | ||
"2", | ||
"--json", | ||
], | ||
b"#!/bin/sh\n", | ||
); | ||
assert_eq!(code, 0, "{out}"); | ||
let v: serde_json::Value = serde_json::from_str(&out).unwrap(); | ||
assert_eq!(v["changed"], true); | ||
assert_eq!(v["attempts"], 1); | ||
let sha = v["sha"].as_str().unwrap().to_string(); | ||
| ||
let (_, refs) = s.cli(&["refs", &url], b""); | ||
assert_eq!(refs, format!("{sha}\trefs/heads/trunk")); | ||
| ||
let (code, out) = s.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"--branch", | ||
"trunk", | ||
"--message", | ||
"bye", | ||
"--delete", | ||
"run.sh", | ||
], | ||
b"", | ||
); | ||
assert_eq!(code, 0); | ||
assert_eq!(out.len(), 40); | ||
// The same content again changes nothing and pushes nothing. | ||
let (_, out) = s.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"trunk", | ||
"-m", | ||
"again", | ||
"--put", | ||
"hello.txt", | ||
file.to_str().unwrap(), | ||
"--json", | ||
], | ||
b"", | ||
); | ||
let v: serde_json::Value = serde_json::from_str(&out).unwrap(); | ||
assert_eq!(v["changed"], false); | ||
for bad in [ | ||
vec!["commit", &url, "-m", "no branch"], | ||
vec!["commit", &url, "-b", "x", "-m", "y", "--retries", "many"], | ||
vec!["commit", &url, "--bogus"], | ||
vec![ | ||
"commit", | ||
&url, | ||
"-b", | ||
"x", | ||
"-m", | ||
"y", | ||
"--put", | ||
"a", | ||
"/nonexistent", | ||
], | ||
] { | ||
assert_eq!(s.cli(&bad, b"").0, 1, "{bad:?}"); | ||
} | ||
| ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone.to_str().unwrap()], | ||
); | ||
assert_eq!(s.git(&clone, &["symbolic-ref", "HEAD"]), "refs/heads/trunk"); | ||
assert_eq!( | ||
s.git(&clone, &["log", "-1", "--format=%an", "HEAD~1"]), | ||
"Ada" | ||
); | ||
assert!(!clone.join("run.sh").exists()); | ||
// The remote remembers whose it is, and will not follow its handle to | ||
// another account. | ||
let pin = "remote.origin.knotlessdid"; | ||
assert_eq!(s.git(&clone, &["config", pin]), s.account.did); | ||
s.git(&clone, &["config", pin, "did:plc:someoneelse"]); | ||
let (fetched, _, err) = s.git_raw(&clone, &["fetch", "-q", "origin"]); | ||
assert!(!fetched && err.contains("not did:plc:someoneelse"), "{err}"); | ||
s.git(&clone, &["config", "--unset", pin]); | ||
s.git(&clone, &["fetch", "-q", "origin"]); | ||
| ||
let (code, out) = s.cli(&["mirror", &url], b""); | ||
assert_eq!(code, 0); | ||
assert!(std::path::Path::new(&out).join("HEAD").exists()); | ||
assert_eq!( | ||
s.cli(&["compact", &url], b"").1, | ||
"2 packs, 2 folded, 0 old claims dropped" | ||
); | ||
| ||
let renamed = s.url("life2"); | ||
let before = s.cli(&["refs", &url], b"").1; | ||
assert_eq!( | ||
s.cli(&["rename", &url, "life2"], b""), | ||
(0, format!("renamed to {renamed}")) | ||
); | ||
assert_eq!(s.cli(&["refs", &renamed], b"").1, before); | ||
// The old name leads to it still. | ||
assert_eq!(s.cli(&["refs", &url], b"").1, before); | ||
assert_eq!(s.cli(&["rename", &renamed, "life2"], b"").0, 1); | ||
// A push through the old name lands in the renamed repository, and git | ||
// is told where it went. | ||
let (pushed, _, err) = s.git_raw(&clone, &["push", "-q", &url, "HEAD:refs/heads/moved"]); | ||
assert!( | ||
pushed && err.contains(&format!("has moved to {renamed}")), | ||
"{err}" | ||
); | ||
assert!( | ||
s.cli(&["refs", &renamed], b"") | ||
.1 | ||
.contains("refs/heads/moved") | ||
); | ||
// Renamed again, every old name points straight at the new one. | ||
let third = s.url("life3"); | ||
assert_eq!(s.cli(&["rename", &url, "life3"], b"").0, 0); | ||
for old in ["life", "life2"] { | ||
let to = &s.record("dev.knotless.redirect", old)["to"]["uri"]; | ||
assert!(to.as_str().unwrap().ends_with("/life3"), "{old}: {to}"); | ||
} | ||
// Back to its first name, whose redirect gives way. | ||
assert_eq!(s.cli(&["rename", &third, "life"], b"").0, 0); | ||
assert!(s.record("dev.knotless.redirect", "life").is_null()); | ||
assert!(s.cli(&["refs", &third], b"").1.contains("refs/heads/moved")); | ||
// A new repository under an old name takes the name over. | ||
assert_eq!(s.cli(&["create", &renamed], b"").0, 0); | ||
assert_eq!(s.cli(&["refs", &renamed], b""), (0, String::new())); | ||
assert!(s.record("dev.knotless.redirect", "life2").is_null()); | ||
| ||
let gone = s.url("gone"); | ||
for missing in [ | ||
vec!["refs", &gone], | ||
vec!["mirror", &gone], | ||
vec!["rename", &gone, "x"], | ||
vec!["delete", &gone, "--yes"], | ||
] { | ||
assert_eq!(s.cli(&missing, b"").0, 1, "{missing:?}"); | ||
} | ||
assert_eq!(s.cli(&["delete", &url], b"").0, 1); | ||
assert_eq!(s.cli(&["delete", &url, "--yes"], b"").0, 0); | ||
assert_eq!(s.cli(&["refs", &url], b"").0, 1); | ||
// Its redirects went with it. | ||
assert_eq!(s.cli(&["refs", &third], b"").0, 1); | ||
assert!(s.record("dev.knotless.redirect", "life3").is_null()); | ||
assert_eq!(s.cli(&["delete", &renamed, "--yes"], b"").0, 0); | ||
assert_eq!(s.cli(&["ls", &s.account.handle], b"").1, ""); | ||
} | ||
| ||
#[test] | ||
fn redirects_followed_only_to_this_repository() { | ||
let s = Sandbox::new(); | ||
assert_eq!(s.cli(&["create", &s.url("here")], b"").0, 0); | ||
let key = s.record("dev.knotless.repo", "here")["key"].clone(); | ||
let redirect = |rkey: &str, did: &str, to: &str, key: &serde_json::Value| { | ||
let to = | ||
serde_json::json!({"uri": format!("at://{did}/dev.knotless.repo/{to}"), "key": key}); | ||
s.put_record( | ||
"dev.knotless.redirect", | ||
rkey, | ||
serde_json::json!({"$type": "dev.knotless.redirect", "to": to, "createdAt": "2026-10-21T00:00:00.000Z"}), | ||
); | ||
}; | ||
let me = s.account.did.clone(); | ||
redirect("ok", &me, "here", &key); | ||
// Into another account, to a name another repository has taken, or | ||
// round in a circle, a redirect leads nowhere. | ||
redirect("away", "did:plc:someoneelse", "here", &key); | ||
redirect("stale", &me, "here", &serde_json::json!("3zzzzzzzzzzzz")); | ||
redirect("loop1", &me, "loop2", &key); | ||
redirect("loop2", &me, "loop1", &key); | ||
assert_eq!(s.cli(&["refs", &s.url("ok")], b""), (0, String::new())); | ||
for nowhere in ["away", "stale", "loop1"] { | ||
assert_eq!( | ||
s.cli(&["rename", &s.url(nowhere), "x"], b"").0, | ||
1, | ||
"{nowhere}" | ||
); | ||
} | ||
} | ||
| ||
#[test] | ||
fn usage() { | ||
let s = Sandbox::new(); | ||
assert_eq!(s.cli(&[], b"").0, 1); | ||
let (code, out) = s.cli(&["help"], b""); | ||
assert_eq!(code, 0); | ||
assert!(out.starts_with("usage")); | ||
assert_eq!(s.cli(&["frobnicate"], b"").0, 1); | ||
assert_eq!(s.cli(&["refs", "https://nope"], b"").0, 1); | ||
let (ok, out, _) = s.knotless(&["--help"], b""); | ||
assert!(ok && out.starts_with("usage")); | ||
} | ||
| ||
#[test] | ||
fn rejected_credentials() { | ||
let s = Sandbox::new(); | ||
let url = s.url("creds"); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
let ctx = ctx(&s); | ||
rt.block_on(async { | ||
let id = knotless::identity::resolve(&ctx, &s.account.handle) | ||
.await | ||
.unwrap(); | ||
let mut session = knotless::session::ensure(&ctx, &id).await.unwrap(); | ||
// A token that reads as fresh but the PDS does not accept. | ||
let body = base64_url(&format!("{{\"exp\":{}}}", 4_000_000_000u64)); | ||
session.access_jwt = format!("x.{body}.y"); | ||
let path = s | ||
.path("config/sessions") | ||
.join(format!("{}.json", s.account.did.replace(':', "_"))); | ||
std::fs::write(path, serde_json::to_vec(&session).unwrap()).unwrap(); | ||
}); | ||
assert_eq!(s.cli(&["create", &url], b"").0, 1); | ||
} | ||
| ||
pub fn base64_url(text: &str) -> String { | ||
const A: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; | ||
let bytes = text.as_bytes(); | ||
let mut out = String::new(); | ||
for chunk in bytes.chunks(3) { | ||
let n = | ||
chunk.iter().fold(0u32, |acc, b| (acc << 8) | u32::from(*b)) << (8 * (3 - chunk.len())); | ||
for i in 0..=chunk.len() { | ||
out.push(A[((n >> (18 - 6 * i)) & 63) as usize] as char); | ||
} | ||
} | ||
out | ||
} | ||
crates/knotless/tests/common/mod.rsnew+401−0
| @@ -0,0 +1,401 @@ | ||
//! A reference PDS for the process, and sandboxes that point at it. | ||
#![allow(dead_code)] | ||
| ||
use knotless::env::Env; | ||
use serde::Deserialize; | ||
use std::io::{BufRead, BufReader}; | ||
use std::path::{Path, PathBuf}; | ||
use std::process::{Child, Command, Stdio}; | ||
use std::sync::{Mutex, OnceLock}; | ||
use std::time::Duration; | ||
| ||
#[derive(Deserialize, Clone, Debug)] | ||
pub struct Account { | ||
pub handle: String, | ||
pub did: String, | ||
pub password: String, | ||
} | ||
| ||
#[derive(Deserialize, Clone, Debug)] | ||
pub struct Net { | ||
pub pds: String, | ||
pub plc: String, | ||
/// Whether the PDS is on the permissioned spaces alpha. | ||
#[serde(default)] | ||
pub spaces: bool, | ||
pub accounts: Vec<Account>, | ||
} | ||
| ||
static NET: OnceLock<Net> = OnceLock::new(); | ||
static CHILD: Mutex<Option<Child>> = Mutex::new(None); | ||
| ||
fn root() -> PathBuf { | ||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") | ||
} | ||
| ||
/// Set when the harness is one `just pds` left running, which other test | ||
/// binaries share: accounts are then made as they are needed. | ||
static SHARED: OnceLock<bool> = OnceLock::new(); | ||
| ||
/// The harness `KNOTLESS_TEST_NET` names, if it is there and answering. | ||
fn shared() -> Option<Net> { | ||
let file = std::env::var_os("KNOTLESS_TEST_NET")?; | ||
let net: Net = serde_json::from_str(&std::fs::read_to_string(file).ok()?).ok()?; | ||
let url = format!("{}/xrpc/com.atproto.server.describeServer", net.pds); | ||
let up = reqwest::blocking::get(url).is_ok_and(|r| r.status().is_success()); | ||
up.then_some(net) | ||
} | ||
| ||
/// The reference PDS (`harness/`): the one `KNOTLESS_TEST_NET` names, if | ||
/// it is up, else one started for this test binary, from npm or the build | ||
/// `KNOTLESS_DEV_ENV` names, which exits when this process does and its | ||
/// stdin closes. | ||
pub fn net() -> &'static Net { | ||
NET.get_or_init(|| { | ||
if let Some(net) = shared() { | ||
SHARED.set(true).unwrap(); | ||
return net; | ||
} | ||
// The dev-env picks free ports and then binds them, so harnesses | ||
// starting side by side can collide: try again on a fresh one. | ||
let mut failures = Vec::new(); | ||
for _ in 0..3 { | ||
match start() { | ||
Ok((net, child)) => { | ||
*CHILD.lock().unwrap() = Some(child); | ||
return net; | ||
} | ||
Err(log) => failures.push(log), | ||
} | ||
} | ||
panic!("the harness exited:\n{}", failures.join("\n---\n")); | ||
}) | ||
} | ||
| ||
/// One harness, up and reporting, or why it died. | ||
fn start() -> Result<(Net, Child), String> { | ||
let names: Vec<String> = (0..24).map(|i| format!("user{i}")).collect(); | ||
let mut child = Command::new("node") | ||
.arg(root().join("harness/pds.mjs")) | ||
.arg("--accounts") | ||
.arg(names.join(",")) | ||
.stdin(Stdio::piped()) | ||
.stdout(Stdio::piped()) | ||
.stderr(Stdio::piped()) | ||
.spawn() | ||
.expect("node runs harness/pds.mjs (run `npm install` in harness/)"); | ||
// Kept to explain a harness that dies before it is up. | ||
let log = std::sync::Arc::new(Mutex::new(String::new())); | ||
let stderr = BufReader::new(child.stderr.take().unwrap()); | ||
let sink = log.clone(); | ||
let drained = std::thread::spawn(move || { | ||
for line in stderr.lines().map_while(Result::ok) { | ||
let mut log = sink.lock().unwrap(); | ||
if log.len() < 1 << 16 { | ||
log.push_str(&line); | ||
log.push('\n'); | ||
} | ||
} | ||
}); | ||
let mut lines = BufReader::new(child.stdout.take().unwrap()).lines(); | ||
let net: Net = loop { | ||
let Some(Ok(line)) = lines.next() else { | ||
let _ = child.wait(); | ||
let _ = drained.join(); | ||
return Err(log.lock().unwrap().clone()); | ||
}; | ||
if line.starts_with("{\"pds\"") { | ||
break serde_json::from_str(&line).unwrap(); | ||
} | ||
}; | ||
std::thread::spawn(move || for _ in lines {}); | ||
Ok((net, child)) | ||
} | ||
| ||
static NEXT: Mutex<usize> = Mutex::new(0); | ||
| ||
/// A new account on a shared harness, named so that no other test binary, | ||
/// in this run or an earlier one, has it: a handle's first label is at | ||
/// most 18 characters. | ||
fn created() -> Account { | ||
let n = { | ||
let mut next = NEXT.lock().unwrap(); | ||
*next += 1; | ||
*next | ||
}; | ||
let nanos = std::time::SystemTime::now() | ||
.duration_since(std::time::UNIX_EPOCH) | ||
.unwrap() | ||
.as_nanos(); | ||
let tag = (nanos as u64 ^ (u64::from(std::process::id()) << 32)) as u32; | ||
let handle = format!("k{tag:08x}{n}.test"); | ||
let password = format!("{handle}-password"); | ||
let res: serde_json::Value = reqwest::blocking::Client::new() | ||
.post(format!( | ||
"{}/xrpc/com.atproto.server.createAccount", | ||
net().pds | ||
)) | ||
.json(&serde_json::json!({ | ||
"email": format!("{handle}@test.com"), | ||
"handle": handle, | ||
"password": password, | ||
})) | ||
.send() | ||
.and_then(|r| r.error_for_status()) | ||
.and_then(|r| r.json()) | ||
.expect("the shared harness makes an account"); | ||
Account { | ||
handle, | ||
did: res["did"].as_str().unwrap().to_string(), | ||
password, | ||
} | ||
} | ||
| ||
/// An account no other test uses. | ||
pub fn account() -> Account { | ||
net(); | ||
if SHARED.get() == Some(&true) { | ||
return created(); | ||
} | ||
let mut next = NEXT.lock().unwrap(); | ||
let a = net().accounts[*next].clone(); | ||
*next += 1; | ||
a | ||
} | ||
| ||
/// Where cargo put the binaries, for git to find `git-remote-knotless` on: | ||
/// beside this test binary's `deps/` directory, which holds this run's | ||
/// build, instrumented or not. | ||
pub fn bin_dir() -> PathBuf { | ||
if let Some(dir) = std::env::var_os("KNOTLESS_BIN_DIR") { | ||
return dir.into(); | ||
} | ||
let exe = std::env::current_exe().unwrap(); | ||
let dir = exe.parent().unwrap().parent().unwrap().to_path_buf(); | ||
assert!( | ||
dir.join("git-remote-knotless").exists(), | ||
"no git-remote-knotless in {}: build the workspace first", | ||
dir.display() | ||
); | ||
dir | ||
} | ||
| ||
/// Config and cache directories of its own, and an account. | ||
pub struct Sandbox { | ||
pub dir: tempfile::TempDir, | ||
pub account: Account, | ||
pub extra: Vec<(String, String)>, | ||
} | ||
| ||
impl Sandbox { | ||
pub fn new() -> Sandbox { | ||
Sandbox::with(account()) | ||
} | ||
| ||
pub fn with(account: Account) -> Sandbox { | ||
Sandbox { | ||
dir: tempfile::tempdir().unwrap(), | ||
account, | ||
extra: vec![], | ||
} | ||
} | ||
| ||
pub fn set(&mut self, k: &str, v: &str) -> &mut Sandbox { | ||
self.extra.retain(|(key, _)| key != k); | ||
self.extra.push((k.into(), v.into())); | ||
self | ||
} | ||
| ||
pub fn path(&self, p: &str) -> PathBuf { | ||
self.dir.path().join(p) | ||
} | ||
| ||
pub fn vars(&self) -> Vec<(String, String)> { | ||
let net = net(); | ||
let mut v: Vec<(String, String)> = vec![ | ||
( | ||
"KNOTLESS_CONFIG_DIR".into(), | ||
self.path("config").to_string_lossy().into(), | ||
), | ||
( | ||
"KNOTLESS_CACHE_DIR".into(), | ||
self.path("cache").to_string_lossy().into(), | ||
), | ||
("KNOTLESS_HANDLE_RESOLVER".into(), net.pds.clone()), | ||
("KNOTLESS_PLC_URL".into(), net.plc.clone()), | ||
("KNOTLESS_PASSWORD".into(), self.account.password.clone()), | ||
("KNOTLESS_COMPACT".into(), "off".into()), | ||
("KNOTLESS_COMPACT_GRACE_SECS".into(), "0".into()), | ||
]; | ||
for (k, val) in &self.extra { | ||
v.retain(|(key, _)| key != k); | ||
v.push((k.clone(), val.clone())); | ||
} | ||
// An empty value unsets the variable. | ||
v.retain(|(_, val)| !val.is_empty()); | ||
v | ||
} | ||
| ||
pub fn env(&self) -> Env { | ||
let vars = self.vars(); | ||
Env::from_vars(|k| { | ||
vars.iter() | ||
.find(|(key, _)| key == k) | ||
.map(|(_, v)| v.clone()) | ||
}) | ||
} | ||
| ||
pub fn url(&self, name: &str) -> String { | ||
format!("knotless://{}/{name}", self.account.handle) | ||
} | ||
| ||
pub fn command(&self, program: &str, cwd: &Path) -> Command { | ||
let mut cmd = Command::new(program); | ||
cmd.current_dir(cwd); | ||
for (k, _) in std::env::vars() { | ||
if k.starts_with("KNOTLESS_") { | ||
cmd.env_remove(k); | ||
} | ||
} | ||
let path = format!("{}:{}", bin_dir().display(), std::env::var("PATH").unwrap()); | ||
cmd.env("PATH", path); | ||
cmd.envs(self.vars()); | ||
for (k, v) in [ | ||
("GIT_AUTHOR_NAME", "Test"), | ||
("GIT_AUTHOR_EMAIL", "t@example.com"), | ||
("GIT_COMMITTER_NAME", "Test"), | ||
("GIT_COMMITTER_EMAIL", "t@example.com"), | ||
("GIT_CONFIG_NOSYSTEM", "1"), | ||
("GIT_CONFIG_GLOBAL", "/dev/null"), | ||
] { | ||
cmd.env(k, v); | ||
} | ||
cmd | ||
} | ||
| ||
/// Runs git in `cwd`, returning (success, stdout, stderr). | ||
pub fn git_raw(&self, cwd: &Path, args: &[&str]) -> (bool, String, String) { | ||
std::fs::create_dir_all(cwd).unwrap(); | ||
let out = self.command("git", cwd).args(args).output().unwrap(); | ||
( | ||
out.status.success(), | ||
String::from_utf8_lossy(&out.stdout).trim().to_string(), | ||
String::from_utf8_lossy(&out.stderr).trim().to_string(), | ||
) | ||
} | ||
| ||
pub fn git(&self, cwd: &Path, args: &[&str]) -> String { | ||
let (ok, out, err) = self.git_raw(cwd, args); | ||
assert!(ok, "git {args:?} failed: {err}"); | ||
out | ||
} | ||
| ||
/// Runs the `knotless` binary. | ||
pub fn knotless(&self, args: &[&str], stdin: &[u8]) -> (bool, String, String) { | ||
use std::io::Write; | ||
let mut child = self | ||
.command( | ||
&bin_dir().join("knotless").to_string_lossy(), | ||
self.dir.path(), | ||
) | ||
.args(args) | ||
.stdin(Stdio::piped()) | ||
.stdout(Stdio::piped()) | ||
.stderr(Stdio::piped()) | ||
.spawn() | ||
.unwrap(); | ||
child.stdin.take().unwrap().write_all(stdin).unwrap(); | ||
let out = child.wait_with_output().unwrap(); | ||
( | ||
out.status.success(), | ||
String::from_utf8_lossy(&out.stdout).trim().to_string(), | ||
String::from_utf8_lossy(&out.stderr).trim().to_string(), | ||
) | ||
} | ||
| ||
/// Runs `knotless` in process. | ||
pub fn cli(&self, args: &[&str], stdin: &[u8]) -> (i32, String) { | ||
let mut argv = vec!["knotless".to_string()]; | ||
argv.extend(args.iter().map(|s| s.to_string())); | ||
let mut out = Vec::new(); | ||
let mut input = stdin; | ||
let code = knotless::cli::main_with(argv, self.env(), &mut input, &mut out); | ||
(code, String::from_utf8(out).unwrap().trim().to_string()) | ||
} | ||
| ||
/// A working repository with `n` commits on main, each touching `f`. | ||
pub fn work(&self, name: &str, n: usize) -> PathBuf { | ||
let dir = self.path(name); | ||
self.git(&dir, &["init", "-q", "-b", "main"]); | ||
for i in 0..n { | ||
self.commit_file(&dir, "f", &format!("{name} {i}\n")); | ||
} | ||
dir | ||
} | ||
| ||
/// A record of the account's, read straight from the PDS: null if | ||
/// there is none. | ||
pub fn record(&self, collection: &str, rkey: &str) -> serde_json::Value { | ||
let res: serde_json::Value = reqwest::blocking::Client::new() | ||
.get(format!("{}/xrpc/com.atproto.repo.getRecord", net().pds)) | ||
.query(&[ | ||
("repo", self.account.did.as_str()), | ||
("collection", collection), | ||
("rkey", rkey), | ||
]) | ||
.send() | ||
.and_then(|r| r.json()) | ||
.unwrap(); | ||
res["value"].clone() | ||
} | ||
| ||
/// Writes a record into the account directly, as any client could. | ||
pub fn put_record(&self, collection: &str, rkey: &str, record: serde_json::Value) { | ||
let client = reqwest::blocking::Client::new(); | ||
let session: serde_json::Value = client | ||
.post(format!( | ||
"{}/xrpc/com.atproto.server.createSession", | ||
net().pds | ||
)) | ||
.json(&serde_json::json!({ | ||
"identifier": self.account.did, | ||
"password": self.account.password, | ||
})) | ||
.send() | ||
.and_then(|r| r.json()) | ||
.unwrap(); | ||
client | ||
.post(format!("{}/xrpc/com.atproto.repo.putRecord", net().pds)) | ||
.bearer_auth(session["accessJwt"].as_str().unwrap()) | ||
.json(&serde_json::json!({ | ||
"repo": self.account.did, | ||
"collection": collection, | ||
"rkey": rkey, | ||
"record": record, | ||
})) | ||
.send() | ||
.and_then(|r| r.error_for_status()) | ||
.unwrap(); | ||
} | ||
| ||
pub fn commit_file(&self, dir: &Path, file: &str, content: &str) -> String { | ||
std::fs::write(dir.join(file), content).unwrap(); | ||
self.git(dir, &["add", file]); | ||
self.git(dir, &["commit", "-q", "-m", &format!("write {file}")]); | ||
self.git(dir, &["rev-parse", "HEAD"]) | ||
} | ||
} | ||
| ||
pub fn ctx(sandbox: &Sandbox) -> knotless::env::Ctx { | ||
knotless::env::Ctx::new(sandbox.env()) | ||
} | ||
| ||
pub fn wait_until(mut f: impl FnMut() -> bool) { | ||
for _ in 0..200 { | ||
if f() { | ||
return; | ||
} | ||
std::thread::sleep(Duration::from_millis(100)); | ||
} | ||
panic!("condition never held"); | ||
} | ||
crates/knotless/tests/git_flows.rsnew+300−0
| @@ -0,0 +1,300 @@ | ||
//! git itself against a reference PDS, through `git-remote-knotless`. | ||
| ||
mod common; | ||
use common::*; | ||
| ||
#[test] | ||
fn push_clone_fetch() { | ||
let s = Sandbox::new(); | ||
let work = s.work("work", 3); | ||
let url = s.url("proj"); | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
| ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone.to_str().unwrap()], | ||
); | ||
assert_eq!( | ||
s.git(&clone, &["rev-parse", "HEAD"]), | ||
s.git(&work, &["rev-parse", "HEAD"]) | ||
); | ||
assert_eq!(s.git(&clone, &["rev-list", "--count", "HEAD"]), "3"); | ||
assert_eq!(s.git(&clone, &["symbolic-ref", "HEAD"]), "refs/heads/main"); | ||
| ||
s.commit_file(&work, "g", "more\n"); | ||
s.git(&work, &["checkout", "-q", "-b", "feature"]); | ||
let tip = s.commit_file(&work, "h", "feature\n"); | ||
s.git(&work, &["push", "-q", &url, "main", "feature"]); | ||
s.git(&clone, &["fetch", "-q", "origin"]); | ||
assert_eq!(s.git(&clone, &["rev-parse", "origin/feature"]), tip); | ||
assert_eq!(s.git(&clone, &["rev-list", "--count", "origin/main"]), "4"); | ||
| ||
// A second repository in the account: each one's records stay apart. | ||
let other = s.url("proj-two"); | ||
s.git(&work, &["push", "-q", &other, "feature"]); | ||
assert_eq!(s.git(&work, &["ls-remote", &other]).lines().count(), 2); | ||
assert_eq!(s.git(&work, &["ls-remote", &url]).lines().count(), 3); | ||
| ||
// Up to date: nothing to push, nothing to fetch. | ||
s.git(&work, &["push", "-q", &url, "feature"]); | ||
s.git(&clone, &["fetch", "-q", "origin"]); | ||
s.git(&clone, &["fsck", "--connectivity-only"]); | ||
} | ||
| ||
#[test] | ||
fn refuses_what_git_refuses() { | ||
let s = Sandbox::new(); | ||
let url = s.url("guarded"); | ||
let a = s.work("a", 2); | ||
s.git(&a, &["push", "-q", &url, "main"]); | ||
let b = s.path("b"); | ||
s.git(s.dir.path(), &["clone", "-q", &url, b.to_str().unwrap()]); | ||
| ||
s.commit_file(&a, "x", "a\n"); | ||
s.git(&a, &["push", "-q", &url, "main"]); | ||
s.commit_file(&b, "y", "b\n"); | ||
// b has not seen a's commit: git asks it to fetch first. | ||
let (ok, _, err) = s.git_raw(&b, &["push", &url, "main"]); | ||
assert!(!ok); | ||
assert!(err.contains("fetch first"), "{err}"); | ||
| ||
// Rewritten history is not a fast-forward, unless forced. | ||
s.git(&b, &["fetch", "-q", "origin"]); | ||
let (ok, _, err) = s.git_raw(&b, &["push", &url, "main"]); | ||
assert!(!ok); | ||
assert!(err.contains("non-fast-forward"), "{err}"); | ||
s.git(&b, &["push", "-q", "--force", &url, "main"]); | ||
s.git(&a, &["fetch", "-q", &url, "main"]); | ||
assert_eq!( | ||
s.git(&a, &["rev-parse", "FETCH_HEAD"]), | ||
s.git(&b, &["rev-parse", "HEAD"]) | ||
); | ||
| ||
// Delete a branch, then bring it back. | ||
s.git(&b, &["push", "-q", &url, "main:side"]); | ||
s.git(&b, &["push", "-q", &url, ":side"]); | ||
let (_, refs, _) = s.git_raw(&b, &["ls-remote", &url]); | ||
assert!(!refs.contains("refs/heads/side"), "{refs}"); | ||
s.git(&b, &["push", "-q", &url, "main:side"]); | ||
assert!(s.git(&b, &["ls-remote", &url]).contains("refs/heads/side")); | ||
} | ||
| ||
#[test] | ||
fn tags_and_empty_repositories() { | ||
let s = Sandbox::new(); | ||
let url = s.url("tagged"); | ||
let (code, _) = s.cli(&["create", &url], b""); | ||
assert_eq!(code, 0); | ||
let empty = s.path("empty"); | ||
let (ok, _, err) = s.git_raw(s.dir.path(), &["clone", &url, empty.to_str().unwrap()]); | ||
assert!(ok, "{err}"); | ||
assert!(err.contains("empty repository"), "{err}"); | ||
| ||
let work = s.work("work", 1); | ||
s.git(&work, &["tag", "-a", "v1", "-m", "release"]); | ||
s.git(&work, &["push", "-q", &url, "main", "--tags"]); | ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone.to_str().unwrap()], | ||
); | ||
assert_eq!(s.git(&clone, &["describe"]), "v1"); | ||
} | ||
| ||
#[test] | ||
fn large_packs_go_over_blobs() { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_INLINE_MAX", "1024"); | ||
// Larger than the reference PDS's 5 MB blob limit: the push halves it. | ||
s.set("KNOTLESS_CHUNK_SIZE", "6000000"); | ||
let work = s.work("work", 1); | ||
// Incompressible, so the pack stays larger than one blob. | ||
let mut x: u64 = 0x9e3779b97f4a7c15; | ||
let noise: Vec<u8> = (0..7_000_000) | ||
.map(|_| { | ||
x ^= x << 13; | ||
x ^= x >> 7; | ||
x ^= x << 17; | ||
x as u8 | ||
}) | ||
.collect(); | ||
std::fs::write(work.join("blob.bin"), &noise).unwrap(); | ||
s.git(&work, &["add", "blob.bin"]); | ||
s.git(&work, &["commit", "-q", "-m", "big"]); | ||
let url = s.url("big"); | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone.to_str().unwrap()], | ||
); | ||
assert_eq!(std::fs::read(clone.join("blob.bin")).unwrap(), noise); | ||
} | ||
| ||
#[test] | ||
fn concurrent_agents_on_their_own_branches() { | ||
let s = Sandbox::new(); | ||
let url = s.url("swarm"); | ||
let base = s.work("base", 1); | ||
s.git(&base, &["push", "-q", &url, "main"]); | ||
let commands: Vec<_> = (0..8) | ||
.map(|i| { | ||
let dir = s.path(&format!("agent{i}")); | ||
s.git(s.dir.path(), &["clone", "-q", &url, dir.to_str().unwrap()]); | ||
s.git(&dir, &["checkout", "-q", "-b", &format!("agent/{i}")]); | ||
s.commit_file(&dir, "work", &format!("{i}\n")); | ||
let mut cmd = s.command("git", &dir); | ||
cmd.args(["push", "-q", "origin", &format!("agent/{i}")]); | ||
cmd | ||
}) | ||
.collect(); | ||
let handles: Vec<_> = commands | ||
.into_iter() | ||
.map(|mut cmd| std::thread::spawn(move || cmd.output().unwrap())) | ||
.collect(); | ||
for h in handles { | ||
let out = h.join().unwrap(); | ||
assert!( | ||
out.status.success(), | ||
"{}", | ||
String::from_utf8_lossy(&out.stderr) | ||
); | ||
} | ||
let refs = s.git(&base, &["ls-remote", &url]); | ||
assert_eq!( | ||
refs.lines() | ||
.filter(|l| l.contains("refs/heads/agent/")) | ||
.count(), | ||
8 | ||
); | ||
} | ||
| ||
#[test] | ||
fn racing_pushes_to_one_branch_land_exactly_one() { | ||
let s = Sandbox::new(); | ||
let url = s.url("race"); | ||
let base = s.work("base", 1); | ||
s.git(&base, &["push", "-q", &url, "main"]); | ||
let commands: Vec<_> = (0..6) | ||
.map(|i| { | ||
let dir = s.path(&format!("racer{i}")); | ||
s.git(s.dir.path(), &["clone", "-q", &url, dir.to_str().unwrap()]); | ||
s.commit_file(&dir, "work", &format!("{i}\n")); | ||
let mut cmd = s.command("git", &dir); | ||
cmd.args(["push", "origin", "main"]); | ||
cmd | ||
}) | ||
.collect(); | ||
let handles: Vec<_> = commands | ||
.into_iter() | ||
.map(|mut cmd| std::thread::spawn(move || cmd.output().unwrap())) | ||
.collect(); | ||
let landed = handles | ||
.into_iter() | ||
.map(|h| h.join().unwrap()) | ||
.filter(|o| o.status.success()) | ||
.count(); | ||
assert!(landed >= 1); | ||
// Each racer started from the same tip, so only one can have moved it. | ||
let refs = s.git(&base, &["ls-remote", &url, "main"]); | ||
let tip = refs.split_whitespace().next().unwrap(); | ||
assert_eq!(s.git(&base, &["fetch", "-q", &url, "main"]), ""); | ||
assert_eq!(s.git(&base, &["rev-list", "--count", tip]), "2"); | ||
assert_eq!(landed, 1); | ||
} | ||
| ||
#[test] | ||
fn background_compaction_after_push() { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_COMPACT", "background") | ||
.set("KNOTLESS_COMPACT_THRESHOLD", "3"); | ||
let url = s.url("compacting"); | ||
let work = s.work("work", 1); | ||
for i in 0..4 { | ||
s.commit_file(&work, "f", &format!("{i}\n")); | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
} | ||
let ctx = ctx(&s); | ||
let rt = tokio::runtime::Runtime::new().unwrap(); | ||
wait_until(|| { | ||
rt.block_on(async { | ||
let remote = knotless::remote::Remote::connect( | ||
&ctx, | ||
&knotless::url::RepoUrl::parse(&url).unwrap(), | ||
) | ||
.await | ||
.unwrap(); | ||
let snap = remote.snapshot(&ctx).await.unwrap(); | ||
snap.packs.items.len() <= 2 | ||
}) | ||
}); | ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone.to_str().unwrap()], | ||
); | ||
assert_eq!(s.git(&clone, &["rev-list", "--count", "HEAD"]), "5"); | ||
} | ||
| ||
#[test] | ||
fn inline_compaction_after_push() { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_COMPACT", "inline") | ||
.set("KNOTLESS_COMPACT_THRESHOLD", "2"); | ||
let url = s.url("inline-compact"); | ||
let work = s.work("work", 1); | ||
for i in 0..3 { | ||
s.commit_file(&work, "f", &format!("{i}\n")); | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
} | ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone.to_str().unwrap()], | ||
); | ||
assert_eq!(s.git(&clone, &["rev-list", "--count", "HEAD"]), "4"); | ||
} | ||
| ||
#[test] | ||
fn helper_usage() { | ||
let s = Sandbox::new(); | ||
let out = s | ||
.command(env!("CARGO_BIN_EXE_git-remote-knotless"), s.dir.path()) | ||
.output() | ||
.unwrap(); | ||
assert!(!out.status.success()); | ||
assert!(String::from_utf8_lossy(&out.stderr).contains("usage")); | ||
} | ||
| ||
#[test] | ||
fn compaction_off_past_threshold() { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_COMPACT", "off") | ||
.set("KNOTLESS_COMPACT_THRESHOLD", "1"); | ||
let url = s.url("uncompacted"); | ||
let work = s.work("work", 1); | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
s.commit_file(&work, "f", "again\n"); | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
assert_eq!( | ||
s.cli(&["compact", &url], b"").1, | ||
"2 packs, 2 folded, 0 old claims dropped" | ||
); | ||
} | ||
| ||
#[test] | ||
fn tags_only_repository() { | ||
let s = Sandbox::new(); | ||
let url = s.url("tags-only"); | ||
let work = s.work("work", 1); | ||
s.git(&work, &["tag", "v0"]); | ||
s.git(&work, &["push", "-q", &url, "v0"]); | ||
let (code, path) = s.cli(&["mirror", &url], b""); | ||
assert_eq!(code, 0); | ||
let mirror = std::path::Path::new(&path); | ||
assert_eq!( | ||
s.git(mirror, &["--git-dir", &path, "rev-parse", "v0"]), | ||
s.git(&work, &["rev-parse", "HEAD"]) | ||
); | ||
} | ||
crates/knotless/tests/library.rsnew+700−0
| @@ -0,0 +1,700 @@ | ||
//! The library's edges, against the reference PDS and small mock servers. | ||
| ||
mod common; | ||
use common::*; | ||
| ||
use axum::Router; | ||
use axum::routing::get; | ||
use knotless::commit::{Change, Spec}; | ||
use knotless::env::{Ctx, HandleResolver}; | ||
use knotless::git::Git; | ||
use knotless::push::{self, Update}; | ||
use knotless::records::{self, Bytes, PackRecord, nsid}; | ||
use knotless::remote::{self, Remote}; | ||
use knotless::url::RepoUrl; | ||
use knotless::{compact, fetch, identity, keys, session}; | ||
use serde_json::json; | ||
| ||
fn rt() -> tokio::runtime::Runtime { | ||
tokio::runtime::Runtime::new().unwrap() | ||
} | ||
| ||
/// Serves `router` on a free port; returns `host:port`. | ||
fn mock(rt: &tokio::runtime::Runtime, router: Router) -> String { | ||
let listener = rt | ||
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) | ||
.unwrap(); | ||
let addr = listener.local_addr().unwrap().to_string(); | ||
rt.spawn(async move { axum::serve(listener, router).await.unwrap() }); | ||
addr | ||
} | ||
| ||
async fn connect(ctx: &Ctx, url: &str) -> Remote { | ||
Remote::connect(ctx, &RepoUrl::parse(url).unwrap()) | ||
.await | ||
.unwrap() | ||
} | ||
| ||
async fn auth(ctx: &Ctx, remote: &Remote) -> String { | ||
session::ensure(ctx, &remote.identity) | ||
.await | ||
.unwrap() | ||
.access_jwt | ||
} | ||
| ||
fn put(path: &str, data: &str) -> Change { | ||
Change::Put { | ||
path: path.into(), | ||
data: data.as_bytes().to_vec(), | ||
executable: false, | ||
} | ||
} | ||
| ||
fn spec(changes: Vec<Change>, retries: u32) -> Spec { | ||
Spec { | ||
branch: "main".into(), | ||
message: "m".into(), | ||
changes, | ||
author: None, | ||
retries, | ||
..Default::default() | ||
} | ||
} | ||
| ||
#[test] | ||
fn identity_resolution() { | ||
let rt = rt(); | ||
let s = Sandbox::new(); | ||
let did = s.account.did.clone(); | ||
let doc = json!({"service": [{"id": "#atproto_pds", "serviceEndpoint": net().pds}]}); | ||
let host = mock( | ||
&rt, | ||
Router::new() | ||
.route( | ||
"/.well-known/atproto-did", | ||
get(move || { | ||
let did = did.clone(); | ||
async move { did } | ||
}), | ||
) | ||
.route( | ||
"/.well-known/did.json", | ||
get(move || { | ||
let doc = doc.clone(); | ||
async move { axum::Json(doc) } | ||
}), | ||
), | ||
); | ||
let not_a_did = mock( | ||
&rt, | ||
Router::new().route("/.well-known/atproto-did", get(|| async { "nope" })), | ||
); | ||
let bare = mock_doc_without_pds(&rt); | ||
rt.block_on(async { | ||
let mut env = s.env(); | ||
env.handle_resolvers = vec![ | ||
HandleResolver::WellKnown { | ||
scheme: "http".into(), | ||
}, | ||
HandleResolver::Xrpc { | ||
service: net().pds.clone(), | ||
}, | ||
]; | ||
env.did_web_scheme = "http".into(); | ||
let ctx = Ctx::new(env.clone()); | ||
let id = identity::resolve(&ctx, &host).await.unwrap(); | ||
assert_eq!(id.did, s.account.did); | ||
assert_eq!(id.pds, net().pds); | ||
// Cached: a second lookup needs no resolver at all. | ||
let mut offline = env.clone(); | ||
offline.handle_resolvers = vec![]; | ||
assert_eq!( | ||
identity::resolve(&Ctx::new(offline.clone()), &host) | ||
.await | ||
.unwrap(), | ||
id | ||
); | ||
identity::forget(&ctx, &host); | ||
identity::forget(&ctx, &host); | ||
assert!(identity::resolve(&Ctx::new(offline), &host).await.is_err()); | ||
| ||
let web = format!("did:web:{}", host.replace(':', "%3A")); | ||
assert_eq!(identity::resolve_pds(&ctx, &web).await.unwrap(), net().pds); | ||
assert!(identity::resolve_pds(&ctx, "did:key:z6Mk").await.is_err()); | ||
assert!( | ||
identity::resolve_pds(&ctx, &format!("did:web:{}", bare.replace(':', "%3A"))) | ||
.await | ||
.is_err() | ||
); | ||
assert!( | ||
identity::resolve_pds(&ctx, "did:web:127.0.0.1%3A9") | ||
.await | ||
.is_err() | ||
); | ||
let err = identity::resolve_pds(&ctx, "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa") | ||
.await | ||
.unwrap_err(); | ||
assert!(format!("{err:#}").contains("resolving did:plc:"), "{err:#}"); | ||
| ||
let mut picky = env.clone(); | ||
picky.handle_resolvers = vec![HandleResolver::WellKnown { | ||
scheme: "http".into(), | ||
}]; | ||
let err = identity::resolve_handle(&Ctx::new(picky), ¬_a_did) | ||
.await | ||
.unwrap_err(); | ||
assert!(err.to_string().contains("not a DID"), "{err}"); | ||
| ||
let mut fixed = env.clone(); | ||
fixed.service = Some("http://pinned.example".into()); | ||
fixed.cache_dir = s.path("other-cache"); | ||
assert_eq!( | ||
identity::resolve(&Ctx::new(fixed), &s.account.did) | ||
.await | ||
.unwrap() | ||
.pds, | ||
"http://pinned.example" | ||
); | ||
}); | ||
} | ||
| ||
fn mock_doc_without_pds(rt: &tokio::runtime::Runtime) -> String { | ||
mock( | ||
rt, | ||
Router::new().route( | ||
"/.well-known/did.json", | ||
get(|| async { axum::Json(json!({"service": []})) }), | ||
), | ||
) | ||
} | ||
| ||
#[test] | ||
fn xrpc_failures() { | ||
let rt = rt(); | ||
let addr = mock( | ||
&rt, | ||
Router::new() | ||
.route( | ||
"/xrpc/com.atproto.sync.getBlob", | ||
get(|| async { (axum::http::StatusCode::NOT_FOUND, "gone") }), | ||
) | ||
.route("/xrpc/x.y", get(|| async { "not json" })) | ||
.route( | ||
"/xrpc/com.atproto.repo.getRecord", | ||
get(|| async { (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "{}") }), | ||
), | ||
); | ||
rt.block_on(async { | ||
let x = knotless::xrpc::Xrpc::new(reqwest::Client::new(), &format!("http://{addr}/")); | ||
let err = x.get_blob("did:plc:a", "bafy").await.unwrap_err(); | ||
let e = knotless::xrpc::XrpcError::of(&err).unwrap(); | ||
assert_eq!( | ||
(e.status, e.error.as_str(), e.message.as_str()), | ||
(404, "HTTP404", "gone") | ||
); | ||
assert_eq!(e.to_string(), "HTTP404 (404): gone"); | ||
assert!( | ||
x.query::<serde_json::Value>("x.y", &[], Some("t")) | ||
.await | ||
.is_err() | ||
); | ||
let r = Remote { | ||
xrpc: x, | ||
identity: identity::Identity { | ||
did: "did:plc:a".into(), | ||
pds: String::new(), | ||
}, | ||
url: RepoUrl::parse("knotless://a/b").unwrap(), | ||
space: None, | ||
}; | ||
let err = r.repo().await.unwrap_err(); | ||
assert_eq!(knotless::xrpc::XrpcError::of(&err).unwrap().status, 500); | ||
}); | ||
} | ||
| ||
#[test] | ||
fn session_refresh_and_fallbacks() { | ||
let mut s = Sandbox::new(); | ||
let rt = rt(); | ||
rt.block_on(async { | ||
let ctx = ctx(&s); | ||
let id = identity::resolve(&ctx, &s.account.handle).await.unwrap(); | ||
let first = session::ensure(&ctx, &id).await.unwrap(); | ||
// An access token about to lapse is refreshed with the refresh token. | ||
let path = s | ||
.path("config/sessions") | ||
.join(format!("{}.json", s.account.did.replace(':', "_"))); | ||
let mut stale = first.clone(); | ||
stale.access_jwt = "a.e30.b".into(); | ||
std::fs::write(&path, serde_json::to_vec(&stale).unwrap()).unwrap(); | ||
let refreshed = session::ensure(&ctx, &id).await.unwrap(); | ||
assert_ne!(refreshed.access_jwt, stale.access_jwt); | ||
assert_eq!(refreshed.service, net().pds); | ||
// A refresh the PDS refuses falls back to the password. | ||
stale.refresh_jwt = "bad".into(); | ||
std::fs::write(&path, serde_json::to_vec(&stale).unwrap()).unwrap(); | ||
assert!(session::ensure(&ctx, &id).await.is_ok()); | ||
}); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
s.set( | ||
"KNOTLESS_CONFIG_DIR", | ||
s.path("empty-config").to_str().unwrap(), | ||
); | ||
rt.block_on(async { | ||
let ctx = ctx(&s); | ||
let id = identity::resolve(&ctx, &s.account.handle).await.unwrap(); | ||
assert!(session::ensure(&ctx, &id).await.is_err()); | ||
assert!( | ||
session::login(&ctx, &s.account.handle, "wrong") | ||
.await | ||
.is_err() | ||
); | ||
}); | ||
} | ||
| ||
#[test] | ||
fn snapshots_follow_a_name_to_its_current_key() { | ||
let s = Sandbox::new(); | ||
let rt = rt(); | ||
let url = s.url("reborn"); | ||
let ctx = ctx(&s); | ||
let r = rt.block_on(connect(&ctx, &url)); | ||
let key1 = rt.block_on(async { | ||
assert!(r.snapshot(&ctx).await.unwrap().repo.is_none()); | ||
knotless::commit::commit(&ctx, &r.url, &spec(vec![put("a", "1")], 0)) | ||
.await | ||
.unwrap(); | ||
r.snapshot(&ctx).await.unwrap().key().unwrap().to_string() | ||
}); | ||
assert_eq!(s.cli(&["delete", &url, "--yes"], b"").0, 0); | ||
rt.block_on(async { | ||
// Deleted: the cached key finds no repository. | ||
assert!(r.snapshot(&ctx).await.unwrap().repo.is_none()); | ||
// Recreated under the same name: a new key, found through the record. | ||
knotless::commit::commit(&ctx, &r.url, &spec(vec![put("a", "2")], 0)) | ||
.await | ||
.unwrap(); | ||
let again = r.snapshot(&ctx).await.unwrap(); | ||
assert_ne!(again.key().unwrap(), key1); | ||
assert_eq!(again.refs.len(), 1); | ||
}); | ||
} | ||
| ||
#[test] | ||
fn many_refs_many_packs_many_repos() { | ||
let s = Sandbox::new(); | ||
let rt = rt(); | ||
let url = s.url("wide"); | ||
let work = s.work("w", 1); | ||
for i in 0..105 { | ||
s.git(&work, &["tag", &format!("t{i:03}")]); | ||
} | ||
// 105 refs is more than one applyWrites holds: two batches. | ||
s.git(&work, &["push", "-q", &url, "main", "--tags"]); | ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone.to_str().unwrap()], | ||
); | ||
assert_eq!(s.git(&clone, &["tag"]).lines().count(), 105); | ||
| ||
rt.block_on(async { | ||
let ctx = ctx(&s); | ||
let r = connect(&ctx, &url).await; | ||
let auth = auth(&ctx, &r).await; | ||
let snap = r.snapshot(&ctx).await.unwrap(); | ||
let key = snap.key().unwrap().to_string(); | ||
assert_eq!(snap.refs.len(), 106); | ||
| ||
// 105 more packs, each one commit, written straight as records. | ||
let git = Git::new(work.join(".git")); | ||
let mut parent = git.rev_parse("HEAD").await.unwrap(); | ||
let mut writes = Vec::new(); | ||
for i in 0..105 { | ||
let tree = git.text(&["rev-parse", "HEAD^{tree}"], None).await.unwrap(); | ||
let sha = git | ||
.text( | ||
&["commit-tree", &tree, "-p", &parent, "-m", &format!("p{i}")], | ||
None, | ||
) | ||
.await | ||
.unwrap(); | ||
let pack = git | ||
.pack_objects(std::slice::from_ref(&sha), std::slice::from_ref(&parent)) | ||
.await | ||
.unwrap(); | ||
let record = push::place( | ||
&ctx, | ||
&r, | ||
&auth, | ||
&key, | ||
pack, | ||
vec![sha.clone()], | ||
vec![parent.clone()], | ||
1, | ||
) | ||
.await | ||
.unwrap(); | ||
writes.push(remote::create( | ||
nsid::PACK, | ||
&keys::pack_rkey(&key), | ||
records::typed(nsid::PACK, &record), | ||
)); | ||
parent = sha; | ||
} | ||
for chunk in writes.chunks(100) { | ||
r.apply(chunk.to_vec(), &auth).await.unwrap(); | ||
} | ||
let page = r.packs(&key, None).await.unwrap(); | ||
assert_eq!(page.items.len(), 100); | ||
assert!(page.next.is_some()); | ||
// The oldest pack is on the second page; the walk follows it. | ||
let fresh = Git::init_bare(&s.path("fresh.git")).await.unwrap(); | ||
let n = fetch::fetch(&r, &key, &fresh, &[parent.clone()], None) | ||
.await | ||
.unwrap(); | ||
assert_eq!(n, 106); | ||
| ||
// A want no pack holds: one fresh walk, then an error. | ||
let err = fetch::fetch(&r, &key, &fresh, &["1".repeat(40)], None) | ||
.await | ||
.unwrap_err(); | ||
assert!(err.to_string().contains("no pack holding"), "{err}"); | ||
| ||
// A record whose size disagrees with its bytes. | ||
let bad = PackRecord { | ||
size: 99, | ||
data: Some(Bytes(vec![1])), | ||
..page.items[0].value.clone() | ||
}; | ||
assert!(fetch::download(&r, &bad).await.is_err()); | ||
| ||
// Fold the 106 packs into one; a fresh clone still works. | ||
let c = compact::compact(&ctx, &r, &session::ensure(&ctx, &r.identity).await.unwrap()) | ||
.await | ||
.unwrap(); | ||
assert_eq!(c.before, 106); | ||
assert_eq!(c.removed, 106); | ||
assert_eq!(r.packs(&key, None).await.unwrap().items.len(), 1); | ||
| ||
// More than a page of repositories in one account. | ||
let mut repos = Vec::new(); | ||
for i in 0..101 { | ||
let rec = records::RepoRecord { | ||
key: keys::tid(), | ||
name: format!("bulk{i}"), | ||
description: None, | ||
default_branch: None, | ||
created_at: records::now(), | ||
..Default::default() | ||
}; | ||
repos.push(remote::create( | ||
nsid::REPO, | ||
&rec.name.clone(), | ||
records::typed(nsid::REPO, &rec), | ||
)); | ||
} | ||
r.apply(repos, &auth).await.unwrap(); | ||
assert_eq!(r.list_repos().await.unwrap().len(), 102); | ||
}); | ||
let clone2 = s.path("clone2"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &url, clone2.to_str().unwrap()], | ||
); | ||
s.git(&clone2, &["fsck", "--connectivity-only"]); | ||
} | ||
| ||
#[test] | ||
fn push_edges() { | ||
let s = Sandbox::new(); | ||
let rt = rt(); | ||
let url = s.url("edges"); | ||
let work = s.work("w", 2); | ||
let git = Git::new(work.join(".git")); | ||
rt.block_on(async { | ||
let ctx = ctx(&s); | ||
let r = connect(&ctx, &url).await; | ||
let auth = auth(&ctx, &r).await; | ||
let up = |src: &str, dst: &str| Update { | ||
src: Some(src.into()), | ||
dst: dst.into(), | ||
force: false, | ||
}; | ||
| ||
// Two first pushes from the same empty view: one creates the | ||
// repository, the other is told to fetch. | ||
let empty = r.snapshot(&ctx).await.unwrap(); | ||
let ok = push::push( | ||
&ctx, | ||
&r, | ||
&empty, | ||
&git, | ||
&auth, | ||
&[up("HEAD", "refs/heads/main")], | ||
) | ||
.await | ||
.unwrap(); | ||
assert_eq!(ok[0].1, Ok(())); | ||
let lost = push::push( | ||
&ctx, | ||
&r, | ||
&empty, | ||
&git, | ||
&auth, | ||
&[up("HEAD", "refs/heads/other")], | ||
) | ||
.await | ||
.unwrap(); | ||
assert_eq!(lost[0].1, Err("fetch first".into())); | ||
| ||
// Already there: nothing to write. | ||
let snap = r.snapshot(&ctx).await.unwrap(); | ||
assert_eq!( | ||
push::push( | ||
&ctx, | ||
&r, | ||
&snap, | ||
&git, | ||
&auth, | ||
&[up("HEAD", "refs/heads/main")] | ||
) | ||
.await | ||
.unwrap()[0] | ||
.1, | ||
Ok(()) | ||
); | ||
// Backwards without force. | ||
let back = push::push( | ||
&ctx, | ||
&r, | ||
&snap, | ||
&git, | ||
&auth, | ||
&[up("HEAD~1", "refs/heads/main")], | ||
) | ||
.await | ||
.unwrap(); | ||
assert_eq!(back[0].1, Err("non-fast-forward".into())); | ||
| ||
// A commit the remote already has needs no pack. | ||
let snap = r.snapshot(&ctx).await.unwrap(); | ||
let same = push::push( | ||
&ctx, | ||
&r, | ||
&snap, | ||
&git, | ||
&auth, | ||
&[up("HEAD~1", "refs/heads/old")], | ||
) | ||
.await | ||
.unwrap(); | ||
assert_eq!(same[0].1, Ok(())); | ||
// Nor does a delete. | ||
let snap = r.snapshot(&ctx).await.unwrap(); | ||
let del = Update { | ||
src: None, | ||
dst: "refs/heads/old".into(), | ||
force: false, | ||
}; | ||
assert_eq!( | ||
push::push(&ctx, &r, &snap, &git, &auth, &[del]) | ||
.await | ||
.unwrap()[0] | ||
.1, | ||
Ok(()) | ||
); | ||
| ||
// A refusal that is not a race comes back as the error it was. | ||
let snap = r.snapshot(&ctx).await.unwrap(); | ||
assert!( | ||
push::push( | ||
&ctx, | ||
&r, | ||
&snap, | ||
&git, | ||
"bad-token", | ||
&[up("HEAD", "refs/heads/n")] | ||
) | ||
.await | ||
.is_err() | ||
); | ||
| ||
// A later batch can lose while an earlier one landed. | ||
let tags: Vec<Update> = (0..120) | ||
.map(|i| up("HEAD", &format!("refs/tags/b{i:03}"))) | ||
.collect(); | ||
let stale = r.snapshot(&ctx).await.unwrap(); | ||
let racer = push::push(&ctx, &r, &stale, &git, &auth, &tags[110..111]) | ||
.await | ||
.unwrap(); | ||
assert_eq!(racer[0].1, Ok(())); | ||
let report = push::push(&ctx, &r, &stale, &git, &auth, &tags) | ||
.await | ||
.unwrap(); | ||
let landed = report.iter().filter(|(_, r)| r.is_ok()).count(); | ||
assert!(landed > 0 && landed < 120, "{landed}"); | ||
assert!( | ||
report | ||
.iter() | ||
.any(|(n, r)| n == "refs/tags/b110" && *r == Err("fetch first".into())) | ||
); | ||
| ||
// A record that is not what its collection says: an error, not a guess. | ||
let key = r.snapshot(&ctx).await.unwrap().key().unwrap().to_string(); | ||
let junk = remote::create( | ||
nsid::REF, | ||
&format!("{key}.zzzz"), | ||
json!({"$type": nsid::REF, "name": 5}), | ||
); | ||
r.apply(vec![junk], &auth).await.unwrap(); | ||
let err = r.refs(&key).await.unwrap_err(); | ||
assert!(err.to_string().contains("reading record"), "{err}"); | ||
| ||
// An empty repository: a mirror has nothing to sync, compaction | ||
// nothing to fold, and a fetch has nothing to read. | ||
let none = connect(&ctx, &s.url("never")).await; | ||
let snap = none.snapshot(&ctx).await.unwrap(); | ||
let mirror = knotless::mirror::Mirror::open(&ctx, none.did(), "k") | ||
.await | ||
.unwrap(); | ||
mirror.sync(&none, &snap).await.unwrap(); | ||
let session = session::ensure(&ctx, &none.identity).await.unwrap(); | ||
assert_eq!( | ||
compact::compact(&ctx, &none, &session).await.unwrap(), | ||
compact::Compacted::default() | ||
); | ||
let mut backend = knotless::helper::Knotless::new(ctx.clone(), &none.url, git.clone()) | ||
.await | ||
.unwrap(); | ||
use knotless::helper::Backend; | ||
assert!(backend.fetch(vec!["a".repeat(40)]).await.is_err()); | ||
}); | ||
} | ||
| ||
#[test] | ||
fn compaction_keeps_young_packs_and_drops_dead_refs() { | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_COMPACT_GRACE_SECS", "3600"); | ||
let rt = rt(); | ||
let url = s.url("young"); | ||
rt.block_on(async { | ||
let ctx = ctx(&s); | ||
let r = connect(&ctx, &url).await; | ||
knotless::commit::commit(&ctx, &r.url, &spec(vec![put("a", "1")], 0)) | ||
.await | ||
.unwrap(); | ||
knotless::commit::commit(&ctx, &r.url, &spec(vec![put("a", "2")], 0)) | ||
.await | ||
.unwrap(); | ||
let session = session::ensure(&ctx, &r.identity).await.unwrap(); | ||
// Everything is younger than the grace period. | ||
assert_eq!( | ||
compact::compact(&ctx, &r, &session).await.unwrap().removed, | ||
0 | ||
); | ||
// With no grace for claims, a read is stale at once, and every | ||
// claim is old enough to drop. | ||
let mut aged = s.env(); | ||
aged.claim_grace = std::time::Duration::ZERO; | ||
let aged = Ctx::new(aged); | ||
let snap = r.snapshot(&aged).await.unwrap(); | ||
let mirror = knotless::mirror::Mirror::open(&aged, r.did(), snap.key().unwrap()) | ||
.await | ||
.unwrap(); | ||
let drop_main = Update { | ||
src: None, | ||
dst: "refs/heads/main".into(), | ||
force: false, | ||
}; | ||
let err = push::push( | ||
&aged, | ||
&r, | ||
&snap, | ||
&mirror.git, | ||
&session.access_jwt, | ||
&[drop_main], | ||
) | ||
.await | ||
.unwrap_err(); | ||
assert!(err.to_string().contains("too long ago"), "{err}"); | ||
let c = compact::compact(&aged, &r, &session).await.unwrap(); | ||
assert_eq!((c.removed, c.claims), (0, 2)); | ||
let mut now = s.env(); | ||
now.compact_grace = std::time::Duration::ZERO; | ||
let ctx0 = Ctx::new(now); | ||
// No live refs left: compaction only deletes. | ||
let snap = r.snapshot(&ctx0).await.unwrap(); | ||
let git = knotless::mirror::Mirror::open(&ctx0, r.did(), snap.key().unwrap()) | ||
.await | ||
.unwrap() | ||
.git; | ||
let del = Update { | ||
src: None, | ||
dst: "refs/heads/main".into(), | ||
force: false, | ||
}; | ||
push::push(&ctx0, &r, &snap, &git, &session.access_jwt, &[del]) | ||
.await | ||
.unwrap(); | ||
let c = compact::compact(&ctx0, &r, &session).await.unwrap(); | ||
assert_eq!((c.before, c.removed), (2, 2)); | ||
let key = snap.key().unwrap(); | ||
assert!(r.packs(key, None).await.unwrap().items.is_empty()); | ||
// The mirror drops the deleted ref too. | ||
let snap = r.snapshot(&ctx0).await.unwrap(); | ||
let mirror = knotless::mirror::Mirror::open(&ctx0, r.did(), key) | ||
.await | ||
.unwrap(); | ||
mirror.sync(&r, &snap).await.unwrap(); | ||
assert!(mirror.git.refs().await.unwrap().is_empty()); | ||
}); | ||
} | ||
| ||
#[test] | ||
fn concurrent_commits_replay_onto_the_new_tip() { | ||
let s = Sandbox::new(); | ||
let url = RepoUrl::parse(&s.url("busy")).unwrap(); | ||
let ctx = ctx(&s); | ||
rt().block_on(knotless::commit::commit( | ||
&ctx, | ||
&url, | ||
&spec(vec![put("seed", "0")], 0), | ||
)) | ||
.unwrap(); | ||
let env = s.env(); | ||
let run = |retries: u32| { | ||
let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); | ||
let threads: Vec<_> = (0..8) | ||
.map(|i| { | ||
// A client per runtime: pooled connections belong to one. | ||
let (ctx, url, barrier) = (Ctx::new(env.clone()), url.clone(), barrier.clone()); | ||
std::thread::spawn(move || { | ||
barrier.wait(); | ||
rt().block_on(knotless::commit::commit( | ||
&ctx, | ||
&url, | ||
&spec(vec![put(&format!("f{retries}-{i}"), "x")], retries), | ||
)) | ||
}) | ||
}) | ||
.collect::<Vec<_>>(); | ||
threads | ||
.into_iter() | ||
.map(|t| t.join().unwrap()) | ||
.collect::<Vec<_>>() | ||
}; | ||
let patient = run(50); | ||
assert!(patient.iter().all(|r| r.is_ok())); | ||
assert!(patient.iter().any(|r| r.as_ref().unwrap().attempts > 1)); | ||
let impatient = run(0); | ||
assert!(impatient.iter().any(|r| r.is_err())); | ||
let clone = s.path("clone"); | ||
s.git( | ||
s.dir.path(), | ||
&["clone", "-q", &s.url("busy"), clone.to_str().unwrap()], | ||
); | ||
for i in 0..8 { | ||
assert!(clone.join(format!("f50-{i}")).exists()); | ||
} | ||
} | ||
crates/knotless/tests/spaces.rsnew+323−0
| @@ -0,0 +1,323 @@ | ||
//! Private repositories, in permissioned spaces. These need a PDS on the | ||
//! spaces alpha (`KNOTLESS_DEV_ENV`), and pass trivially without one. | ||
| ||
mod common; | ||
use common::*; | ||
| ||
fn spaces() -> bool { | ||
let on = net().spaces; | ||
if !on { | ||
eprintln!("skipped: the PDS is not on the spaces alpha"); | ||
} | ||
on | ||
} | ||
| ||
/// A sandbox for `account` that reads as itself, with its password logged | ||
/// in up front rather than kept in the environment. | ||
fn member(account: Account) -> Sandbox { | ||
let mut s = Sandbox::with(account); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
let password = s.account.password.clone(); | ||
let handle = s.account.handle.clone(); | ||
assert_eq!( | ||
s.cli(&["login", &handle, "--password-stdin"], password.as_bytes()) | ||
.0, | ||
0 | ||
); | ||
s | ||
} | ||
| ||
#[test] | ||
fn private_repositories() { | ||
if !spaces() { | ||
return; | ||
} | ||
let mut owner = Sandbox::new(); | ||
// Small enough that a pack goes over blobs, which a space holds too. | ||
owner.set("KNOTLESS_INLINE_MAX", "1000"); | ||
let url = owner.url("secret"); | ||
assert_eq!(owner.cli(&["create", &url, "--private"], b"").0, 0); | ||
let (code, _) = owner.cli(&["create", &url, "--private"], b""); | ||
assert_eq!(code, 1, "the repository record is create-only"); | ||
| ||
let work = owner.work("w", 3); | ||
let noise: String = (0..4000) | ||
.map(|i| char::from(b'a' + (i * 7 % 26) as u8)) | ||
.collect(); | ||
owner.commit_file(&work, "big", &noise); | ||
owner.git(&work, &["push", "-q", &url, "main"]); | ||
let head = owner.git(&work, &["rev-parse", "HEAD"]); | ||
let (_, refs) = owner.cli(&["refs", &url], b""); | ||
assert_eq!(refs, format!("{head}\trefs/heads/main")); | ||
| ||
// Nothing of it is public. | ||
let (_, listed) = owner.cli(&["ls", &owner.account.handle], b""); | ||
assert!(!listed.contains("secret"), "{listed}"); | ||
let public = std::process::Command::new("curl") | ||
.args([ | ||
"-s", | ||
&format!( | ||
"{}/xrpc/com.atproto.repo.listRecords?repo={}&collection=dev.knotless.pack", | ||
net().pds, | ||
owner.account.did | ||
), | ||
]) | ||
.output() | ||
.unwrap(); | ||
assert_eq!(String::from_utf8_lossy(&public.stdout), r#"{"records":[]}"#); | ||
| ||
// An outsider, logged in or not, finds no such repository. | ||
let outsider = member(account()); | ||
let (code, _) = outsider.cli(&["refs", &url], b""); | ||
assert_eq!(code, 1); | ||
let mut anonymous = Sandbox::with(account()); | ||
anonymous.set("KNOTLESS_PASSWORD", ""); | ||
assert_eq!(anonymous.cli(&["refs", &url], b"").0, 1); | ||
| ||
// A member reads it with a space credential, cached between runs. | ||
let reader = member(account()); | ||
assert_eq!( | ||
owner.cli(&["share", &url, &reader.account.handle], b"").0, | ||
0 | ||
); | ||
let clone = reader.path("clone"); | ||
reader.git(reader.dir.path(), &["clone", "-q", &url, "clone"]); | ||
assert_eq!(reader.git(&clone, &["rev-parse", "HEAD"]), head); | ||
assert_eq!(std::fs::read_to_string(clone.join("big")).unwrap(), noise); | ||
let creds = std::fs::read_dir(reader.path("config/credentials")) | ||
.unwrap() | ||
.count(); | ||
assert_eq!(creds, 1); | ||
reader.git(&clone, &["fetch", "-q"]); | ||
| ||
// The owner commits without a checkout, and compacts, in the space. | ||
let file = owner.path("note"); | ||
std::fs::write(&file, "n\n").unwrap(); | ||
let (code, sha) = owner.cli( | ||
&[ | ||
"commit", | ||
&url, | ||
"-b", | ||
"main", | ||
"-m", | ||
"note", | ||
"--put", | ||
"note", | ||
&file.to_string_lossy(), | ||
], | ||
b"", | ||
); | ||
assert_eq!(code, 0, "{sha}"); | ||
let (code, out) = owner.cli(&["compact", &url], b""); | ||
assert_eq!(code, 0, "{out}"); | ||
assert!( | ||
out.contains("folded") && !out.contains(" 0 folded"), | ||
"{out}" | ||
); | ||
reader.git(&clone, &["pull", "-q", "--ff-only"]); | ||
assert_eq!(reader.git(&clone, &["rev-parse", "HEAD"]), sha); | ||
let (_, mirror) = owner.cli(&["mirror", &url], b""); | ||
assert!(mirror.ends_with(".git"), "{mirror}"); | ||
| ||
// A reader cannot push: writes go to the owner's repo. | ||
let (ok, _, _) = reader.git_raw(&clone, &["push", "-q", "origin", "HEAD:refs/heads/mine"]); | ||
assert!(!ok); | ||
| ||
let (_, members) = owner.cli(&["members", &url], b""); | ||
assert_eq!( | ||
members, | ||
format!("{}\towner\n{}\tread", owner.account.did, reader.account.did) | ||
); | ||
| ||
// An app password reads its own account's spaces, not other accounts'. | ||
let app = app_password(&reader); | ||
let mut via_app = Sandbox::with(reader.account.clone()); | ||
via_app.set("KNOTLESS_PASSWORD", &app); | ||
via_app.set("KNOTLESS_IDENTIFIER", &reader.account.handle); | ||
let (ok, _, err) = via_app.git_raw(&via_app.path("x"), &["ls-remote", &url]); | ||
assert!(!ok); | ||
assert!(err.contains("takes a full login"), "{err}"); | ||
// Pushing takes a gateway to write through, which needs delegating. | ||
let (code, _) = owner.cli(&["share", &url, &reader.account.handle, "--write"], b""); | ||
assert_eq!(code, 1); | ||
assert_eq!( | ||
owner | ||
.cli(&["share", &url, &reader.account.handle, "--bogus"], b"") | ||
.0, | ||
1 | ||
); | ||
assert_eq!( | ||
owner | ||
.cli(&["unshare", &url, &reader.account.handle, "--write"], b"") | ||
.0, | ||
1 | ||
); | ||
assert_eq!( | ||
owner.cli(&["unshare", &url, &reader.account.handle], b"").0, | ||
0 | ||
); | ||
let (_, members) = owner.cli(&["members", &url], b""); | ||
assert_eq!(members, format!("{}\towner", owner.account.did)); | ||
// Without a cached credential, the former member is an outsider again. | ||
let again = member(reader.account.clone()); | ||
assert_eq!(again.cli(&["refs", &url], b"").0, 1); | ||
| ||
assert_eq!(owner.cli(&["delete", &url, "--yes"], b"").0, 0); | ||
assert_eq!(owner.cli(&["refs", &url], b"").0, 1); | ||
for cmd in ["share", "members"] { | ||
let (code, _) = owner.cli(&[cmd, &url, &reader.account.handle], b""); | ||
assert_eq!(code, 1, "{cmd}"); | ||
} | ||
} | ||
| ||
#[test] | ||
fn renaming_a_private_repository() { | ||
if !spaces() { | ||
return; | ||
} | ||
let owner = member(account()); | ||
let ok = |s: &Sandbox, args: &[&str]| { | ||
let (code, out) = s.cli(args, b""); | ||
assert_eq!(code, 0, "{args:?}: {out}"); | ||
out | ||
}; | ||
let fails = |s: &Sandbox, args: &[&str]| { | ||
let (code, out) = s.cli(args, b""); | ||
assert_eq!(code, 1, "{args:?} should fail: {out}"); | ||
}; | ||
let first = owner.url("first"); | ||
ok(&owner, &["create", &first, "--private"]); | ||
let work = owner.work("w", 1); | ||
owner.git(&work, &["push", "-q", &first, "main"]); | ||
let before = ok(&owner, &["refs", &first]); | ||
let reader = member(account()); | ||
ok(&owner, &["share", &first, &reader.account.handle]); | ||
| ||
// Its records stay in the space of its first name; the space of the | ||
// new one points there, with the same members. | ||
let second = owner.url("second"); | ||
ok(&owner, &["rename", &first, "second"]); | ||
for url in [&first, &second] { | ||
assert_eq!(ok(&reader, &["refs", url]), before, "{url}"); | ||
} | ||
let issue = ok(&reader, &["issue", &second, "Found by its new name"]); | ||
assert!(ok(&owner, &["list", &first]).contains(&issue)); | ||
// Shared later, a member finds it by every name. | ||
let late = member(account()); | ||
ok(&owner, &["share", &second, &late.account.handle]); | ||
let third = owner.url("third"); | ||
ok(&owner, &["rename", &second, "third"]); | ||
for url in [&first, &second, &third] { | ||
assert_eq!(ok(&late, &["refs", url]), before, "{url}"); | ||
} | ||
// Back to a name it had, then to its first. | ||
ok(&owner, &["rename", &third, "second"]); | ||
assert_eq!(ok(&late, &["refs", &third]), before); | ||
ok(&owner, &["rename", &second, "first"]); | ||
assert_eq!(ok(&reader, &["refs", &second]), before); | ||
fails(&owner, &["rename", &first, "first"]); | ||
| ||
// Its names stay its own: its members can reach it by any of them. | ||
for url in [&first, &second, &third] { | ||
fails(&owner, &["create", url, "--private"]); | ||
} | ||
let fourth = owner.url("fourth"); | ||
ok(&owner, &["create", &owner.url("taken"), "--private"]); | ||
fails(&owner, &["rename", &first, "taken"]); | ||
ok(&owner, &["rename", &first, "fourth"]); | ||
assert_eq!(ok(&reader, &["refs", &first]), before); | ||
| ||
// Unshared, a member loses every name; deleted, every name goes. | ||
ok(&owner, &["unshare", &fourth, &late.account.handle]); | ||
let late = member(late.account.clone()); | ||
fails(&late, &["refs", &third]); | ||
ok(&owner, &["delete", &fourth, "--yes"]); | ||
for url in [&first, &second, &third, &fourth] { | ||
fails(&owner, &["refs", url]); | ||
} | ||
} | ||
| ||
fn app_password(s: &Sandbox) -> String { | ||
let ctx = ctx(s); | ||
tokio::runtime::Runtime::new().unwrap().block_on(async { | ||
let id = knotless::identity::resolve(&ctx, &s.account.handle) | ||
.await | ||
.unwrap(); | ||
let session = knotless::session::ensure(&ctx, &id).await.unwrap(); | ||
let out: serde_json::Value = knotless::xrpc::Xrpc::new(ctx.http.clone(), &id.pds) | ||
.procedure( | ||
"com.atproto.server.createAppPassword", | ||
Some(&serde_json::json!({"name": knotless::keys::tid()})), | ||
Some(&session.access_jwt), | ||
) | ||
.await | ||
.unwrap(); | ||
out["password"].as_str().unwrap().to_string() | ||
}) | ||
} | ||
| ||
#[test] | ||
fn space_administration_failures() { | ||
if !spaces() { | ||
return; | ||
} | ||
let s = Sandbox::new(); | ||
let ctx = ctx(&s); | ||
tokio::runtime::Runtime::new().unwrap().block_on(async { | ||
let xrpc = knotless::xrpc::Xrpc::new(ctx.http.clone(), &net().pds); | ||
let session = knotless::session::Session { | ||
did: s.account.did.clone(), | ||
handle: s.account.handle.clone(), | ||
service: net().pds.clone(), | ||
access_jwt: "bogus".into(), | ||
refresh_jwt: "bogus".into(), | ||
}; | ||
let admin = knotless::space::Admin { | ||
xrpc: &xrpc, | ||
session: &session, | ||
}; | ||
assert!(admin.create("nope").await.is_err()); | ||
}); | ||
} | ||
| ||
#[test] | ||
fn sharing_public_repositories() { | ||
let s = Sandbox::new(); | ||
let url = s.url("open"); | ||
let other = account(); | ||
assert_eq!(s.cli(&["create", &url], b"").0, 0); | ||
let (code, _) = s.cli(&["share", &url, &other.handle], b""); | ||
assert_eq!(code, 1, "anyone can already read it"); | ||
let (code, _) = s.cli(&["share", &url, &other.handle, "--write"], b""); | ||
assert_eq!(code, 1, "pushing takes a gateway"); | ||
let (_, members) = s.cli(&["members", &url], b""); | ||
assert_eq!(members, format!("{}\towner", s.account.did)); | ||
} | ||
| ||
#[test] | ||
fn a_bad_owner_token_is_an_error_not_a_public_repository() { | ||
if !spaces() { | ||
return; | ||
} | ||
let mut s = Sandbox::new(); | ||
s.set("KNOTLESS_PASSWORD", ""); | ||
// A session that will not expire for an hour, signed by nobody. | ||
let exp = knotless::keys::now_micros() / 1_000_000 + 3600; | ||
let claims = format!("{{\"exp\":{exp}}}"); | ||
use base64::Engine; | ||
let b64 = |t: &str| base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(t); | ||
let jwt = format!("{}.{}.sig", b64("{}"), b64(&claims)); | ||
let session = knotless::session::Session { | ||
did: s.account.did.clone(), | ||
handle: s.account.handle.clone(), | ||
service: net().pds.clone(), | ||
access_jwt: jwt.clone(), | ||
refresh_jwt: jwt, | ||
}; | ||
let path = s | ||
.path("config/sessions") | ||
.join(format!("{}.json", s.account.did.replace(':', "_"))); | ||
knotless::store::write(&path, &session, true).unwrap(); | ||
let (code, _) = s.cli(&["refs", &s.url("anything")], b""); | ||
assert_eq!(code, 1); | ||
} | ||
docs/design.mdnew+399−0
| @@ -0,0 +1,399 @@ | ||
# Design | ||
| ||
## Constraints | ||
| ||
knotless has to run against the reference PDS (`@atproto/pds`) as shipped, | ||
so public repositories use only `com.atproto.repo.*`, | ||
`com.atproto.sync.getBlob` and `com.atproto.server.*`. Private repositories | ||
add `com.atproto.space.*` and `com.atproto.simplespace.*` from the | ||
permissioned-data alpha of the same PDS. What that PDS allows shapes every | ||
choice below. | ||
| ||
| Limit (reference PDS defaults) | Consequence | | ||
| --- | --- | | ||
| Blob upload ≤ 5 MB (`PDS_BLOB_UPLOAD_LIMIT`) | Large packs are split into 4 MB parts; a part refused as too large is retried at half the size. | | ||
| JSON request body ≈ 1 MB; the firehose carries every record | Packs ride inline only up to 192 KiB (`KNOTLESS_INLINE_MAX`). | | ||
| `applyWrites` ≤ 200 writes | A push of more than ~99 refs spans batches. Each ref's claim travels in the same batch as the ref. | | ||
| Rate limits, when enabled (as on bsky.social): 5000 points/hour per account (create 3, update 2, delete 1); `uploadBlob` 1000/day | A small push costs 8 points (pack 3, claim 3, ref 2), so about 600 pushes an hour per account. Inline packs keep small pushes off the blob quota. | | ||
| `applyWrites` has no per-write swap, only `swapCommit` | Compare-and-swap comes from create-only claim keys (below), not from `swapCommit`, which would conflict with every other write to the account. | | ||
| A create on an existing key fails the whole batch, with a 500 | A refused batch is followed by a reread of the refs, to tell a lost race from any other failure. | | ||
| Only an account writes to its own repo | Collaborators push through a gateway the owner has delegated to (below). | | ||
| `listRecords` filters by collection, not by prefix | Keys share the repository's prefix, and scans start from the cursor `<key>/`. The reference PDS orders by rkey and pages with `rkey < cursor`, so the scan stops at the first key outside the prefix. | | ||
| ||
## Records | ||
| ||
``` | ||
dev.knotless.repo <name> key, name, description?, defaultBranch?, | ||
collaborators?, gateway?, forkOf?, | ||
transfer?, transferredFrom? | ||
dev.knotless.ref <key>.<h(ref)> name, target?, generation | ||
dev.knotless.claim <key>.<h(ref)>.<generation> name, generation, from?, to?, pack? | ||
dev.knotless.pack <key>.<tid> tips, bases, size, objects, data | parts, supersedes? | ||
dev.knotless.redirect <old name> to, space? | ||
| ||
dev.knotless.pull <tid> repo, base, head {repo, branch}, title, body? | ||
dev.knotless.issue <tid> repo, title, body? | ||
dev.knotless.comment <tid> repo, subject, body, verdict?, commit? | ||
dev.knotless.state <tid> repo, subject, state, commit?, base? | ||
dev.knotless.gate <key> repo, hidden | ||
dev.knotless.block <tid> subject, repo? | ||
``` | ||
| ||
`h(ref)` is the first 26 characters of the lowercase base32 SHA-256 of the | ||
ref name, so any ref git allows becomes a valid record key. `key` is a TID | ||
minted when the repository is created. Every record carries it, so a rename | ||
rewrites only the `repo` record. | ||
| ||
The first five live in the owner's repo. The pull, issue, comment and | ||
state records live in each author's repo, and the gate in the owner's; they | ||
are described under [Pulls and issues](#pulls-and-issues). | ||
| ||
### Identity | ||
| ||
A repository is its owner's DID and its `key`. The name is for people: it | ||
can change, and once a repository is deleted or renamed away, another can | ||
take it. So a record that names a repository (a pull, issue, comment or | ||
state, a pull's head, a fork's `forkOf`) holds a `repoRef`: the `at://` URI | ||
of the `repo` record as it was named when written, and the key. Readers | ||
match on the URI's owner and the key, never the name, so those records | ||
follow a rename and never attach to a later repository of the same name. | ||
| ||
A rename, in one `applyWrites`, creates the `repo` record under the new | ||
name, deletes the old one, and leaves a `redirect` under the old name. It | ||
also repoints every redirect already leading to the repository, so an old | ||
name is always one hop from the current one, and deletes a redirect under | ||
the new name if there was one. A lookup by name that finds no `repo` record | ||
reads the `redirect` under that name and follows it, so clone URLs, | ||
remotes and links made with the old name keep working; `git` is told the | ||
new name. A redirect is followed only within the account and only to a | ||
record with the key it names. Creating a repository under a name that holds | ||
a redirect replaces it; deleting a repository deletes its redirects. | ||
| ||
A private repository's records stay in the space named for its first name, | ||
its home. A rename creates a space under the new name, with the same | ||
members, holding only a redirect whose `space` names the home; a member who | ||
opens the repository by its new name finds the redirect there and joins the | ||
home. Sharing and unsharing change every such space, and deleting the | ||
repository deletes them all. Members' cached credentials are bound to a | ||
space's address, so a space's name is never taken by another repository: | ||
creating one where a renamed repository's space is refuses. | ||
| ||
A repository moves to another account in two steps, since neither account | ||
can write to the other. The owner offers it, setting `transfer` on the | ||
`repo` record to a `repoRef` in the new account. The recipient accepts: | ||
the repository is copied into their account with the same key and | ||
`transferredFrom` naming the old place. A move holds only when both sides | ||
say so: a reader following the old name goes to the new place only if the | ||
new `repo` record names the old one, and records about the old repository | ||
count for the new one only if the old owner's offer, or the redirect they | ||
leave when they delete the old copy, leads there. State changes the old | ||
owner made before the new `repo` record was created still count; later | ||
ones do not. When the old copy is deleted, its name and redirects become | ||
redirects into the new account. | ||
| ||
A `knotless://handle/name` remote resolves the handle on every use. The | ||
helper records the DID the remote first resolved to | ||
(`remote.<name>.knotlessdid`) and refuses to push or fetch if the handle | ||
later names another account. | ||
| ||
### Push | ||
| ||
1. Read the `repo` record, the repository's refs and its newest page of | ||
packs, in parallel. | ||
2. For each ref: refuse a non-fast-forward unless forced, and refuse an old | ||
target that is not available locally ("fetch first"). | ||
3. `git pack-objects --revs` of the new tips, excluding every remote ref | ||
target the local repository has. The pack is self-contained: no thin | ||
deltas, so it can be indexed alone. | ||
4. One `applyWrites`: create the `repo` record if this is the first push, | ||
create the pack, and for each ref create | ||
`claim(<key>.<h>.<generation>)` and put | ||
`ref(<key>.<h>, generation + 1)`. | ||
| ||
Two writers that read generation *n* both create `claim(...n)`, and the PDS | ||
accepts only the first. The ref update travels in the same batch, so it | ||
lands only with its claim. A deleted ref keeps its record, without a | ||
target, so its generation keeps counting and an old claim can never be | ||
replayed. | ||
| ||
### Fetch | ||
| ||
Every wanted object present already: done. Otherwise, walk the pack pages | ||
newest first. Download, in parallel, each pack with a tip that is missing | ||
locally, and index it. Add the pack's `bases` to the wanted set, and stop | ||
once everything wanted is present. If a walk ends with something still | ||
missing, walk once more from the top: a compaction may have replaced packs | ||
between two pages. If it is still missing after that, report it. | ||
| ||
### Compaction | ||
| ||
Repack every object the live refs reach into one pack, written with | ||
`supersedes` listing the packs it replaces, and delete those in the same | ||
`applyWrites`. Claims older than a day go in the same writes: a claim | ||
guards only the generation it names while a writer may still hold a read | ||
of it, and a push refuses to write from a read older than half that long. Only packs older than the grace period (ten minutes) are | ||
replaced. A reader that listed them before the delete either finishes | ||
inside that window or restarts its walk once, as above. Objects that no | ||
ref reaches are dropped here, and their blobs are dereferenced and removed | ||
by the PDS. | ||
| ||
### Commits without a checkout | ||
| ||
`knotless commit` syncs a bare mirror in the cache directory. It builds | ||
the commit there through a temporary index (`read-tree`, `hash-object`, | ||
`update-index --index-info`, `write-tree`, `commit-tree`) and pushes it. | ||
Put and delete name paths, so replaying them onto a tip that moved cannot | ||
conflict. A lost race backs off with exponential jitter and replays. The | ||
claim pins the parent's generation, so these pushes skip git's | ||
fast-forward check. | ||
| ||
Mirrors are shared by every process using the cache. Packs are indexed | ||
without a lock, because each lands under its own name. The ref transaction | ||
runs under a lock file, and a lock older than two minutes is treated as | ||
abandoned. | ||
| ||
## Private repositories | ||
| ||
A private repository is the space | ||
`at://<owner>/space/dev.knotless.space/<name>`, created with open app access | ||
and member-list read and write policies. Its records are the public ones, | ||
written with the space's `applyWrites` into the owner's repo inside the | ||
space. A create on an existing key fails there too, so the claim | ||
compare-and-swap is unchanged. | ||
| ||
The owner reads their own repo in the space with their session. Anyone else, | ||
the owner included when reading another member's repo, exchanges a | ||
delegation token from their own PDS for a space credential from the | ||
owner's, signing both requests with a fresh P-256 key (RFC 9421, as the | ||
alpha specifies). Reads then carry the credential and an audience naming | ||
the repo being read. App passwords cannot mint delegation tokens, so | ||
reading someone else's private repository takes a full login. The | ||
credential and its key are cached, readable only by the user, and reused | ||
until a minute before they expire. | ||
| ||
A repository is looked for publicly first, then in a space as whoever is | ||
logged in; which one it was is cached with its key. A PDS without spaces | ||
answers the space methods as unknown, and the repository is public. | ||
| ||
Every member is a space writer: the alpha lists a member's repo in the | ||
space only if their writes are allowed, and members write their issues and | ||
comments there. Pushing is a separate grant, the `collaborators` of the | ||
`repo` record. Deleting a private repository deletes the space. | ||
| ||
## Gateway | ||
| ||
Only an account can write to its own repo, so a collaborator cannot push | ||
directly. A gateway is a `knotless-serve` with `--public-url`, identified as | ||
`did:web` of its host and serving its own DID document with a | ||
`#knotless_gateway` service. | ||
| ||
- **Delegation.** `knotless delegate` creates an app password named for | ||
the gateway and posts it to `dev.knotless.putDelegation`, authenticated with a | ||
service token. The gateway logs in with it and keeps the session; the | ||
owner revokes it like any app password. Sharing a repository for writing | ||
records the gateway in its `repo` record. | ||
- **Authentication.** Every call carries a service token from | ||
`com.atproto.server.getServiceAuth`, with the gateway as audience and the | ||
method as `lxm`. The gateway checks it against the signing key in the | ||
issuer's DID document (Multikey, or the legacy secp256k1 type PLC still | ||
returns), then checks the issuer is the owner or a collaborator. | ||
- **Push.** `dev.knotless.pushRefs` takes a JSON header line (the repository and | ||
ref updates) and then the pack. The gateway indexes it into its mirror and | ||
pushes as the owner, with the same compare-and-swap as any push. | ||
- **Commit.** `dev.knotless.createCommit` takes path changes and an optional | ||
`base`. Commits to one branch queue; whichever request takes the branch | ||
turn drains the queue, builds the commits as a chain and lands them in | ||
one push. If the branch moved, the chain is rebuilt on the new tip, up to | ||
eight times. A change to a path that also changed since its `base` is a | ||
conflict (409) for that commit alone. | ||
| ||
The gateway holds the owner's app password, so it is trusted with write | ||
access to the owner's account: run your own, or use one you trust. | ||
| ||
## Pulls and issues | ||
| ||
Items and everything said about them are records in their authors' | ||
accounts, naming the repository as under [Identity](#identity). A pull names a target branch and a head repository and branch, | ||
which may be a fork. A comment with a verdict is a review and records the | ||
head commit it saw. An item's state is the latest `state` record, ordered | ||
by key, from someone allowed to set it: the item's author, the owner, or a | ||
collaborator. Only the owner and collaborators can set `merged`. A `state` | ||
with a `base` moves an open pull to another target branch, as after a | ||
branch is renamed; the pull's author changes its head branch by rewriting | ||
the pull record. | ||
| ||
To list them, a client needs every author's records about one repository. | ||
For public repositories an appview indexes them from the firehose | ||
(`dev.knotless.getForge`); for private ones the client enumerates the | ||
space's member repos (`com.atproto.space.listRepos`) and reads each. A | ||
comment reads the item it is about straight from its author's repo, so it | ||
does not wait for an index. | ||
| ||
`knotless merge` syncs the target's mirror, fetches the head from the | ||
source's packs, and fast-forwards when it can. Otherwise it builds a merge | ||
commit with `git merge-tree --write-tree`, and stops on conflicts. It pushes | ||
directly as the owner or through the gateway as a collaborator, then writes | ||
a `merged` state naming the commit. A fork copies every ref into a new | ||
repository whose `repo` record names its source; a fork of a private | ||
repository is private. | ||
| ||
### Moderation | ||
| ||
Moderation is atproto's. A repository's owner keeps out what accounts they | ||
block write about it, and hides pulls, issues and comments by listing them | ||
in the gate, a record keyed by the repository's key, as a threadgate's | ||
`hiddenReplies` hides replies. What was said about a hidden item goes with | ||
it. A block is the forge's own record, shaped as `app.bsky.graph.block`: | ||
blocking someone on Bluesky says nothing about their pulls, and a block | ||
can name one repository, keeping its subject out of that one alone. A | ||
block naming a private repository is kept in the space with it. Readers | ||
apply both: the appview indexes owners' blocks and gates; in a space the | ||
client reads the owner's blocks from their repo and the space. | ||
| ||
Anyone can report an account, a repository or a record with | ||
`com.atproto.moderation.createReport`, sent with a service token to the | ||
moderation service the DID document of `KNOTLESS_MODERATOR` names | ||
(`#atproto_labeler`), or by default to the appview's. The service's | ||
operator reads reports with `dev.knotless.listReports` and takes subjects | ||
down with `com.atproto.admin.updateSubjectStatus`, both with the admin | ||
password as a PDS takes it. | ||
| ||
## knotless-serve | ||
| ||
A stateless cache in front of PDSes. Per repository it keeps a bare | ||
mirror, synced on demand when older than `--ttl`. With `--firehose` it | ||
also follows `subscribeRepos`: a commit touching a `dev.knotless.*` | ||
collection marks that account's mirrors stale and refreshes them in the | ||
background. While the subscription is up, a mirror with no new events for | ||
its account is served without asking the PDS; while it is down, the TTL | ||
applies. | ||
| ||
With `--jetstream` the same watching runs over | ||
`network.bsky.jetstream.subscribeEvents`, asking only for `dev.knotless.*` | ||
commits (identity, account and sync events arrive for every account, since | ||
Jetstream filters collections on commits alone). Jetstream re-encodes the | ||
firehose as JSON and includes each record decoded, so the index takes the | ||
record from the event and writes it in event order, with no `getRecord`; it | ||
resumes from the sequence after the last one seen, as the cursor is | ||
inclusive. Jetstream frames are unsigned: the record is what the Jetstream | ||
operator says the commit held. That is the trust a server already gives the | ||
relay it follows, and the index stays a cache, rebuildable by `--backfill`. | ||
A cursor means something only on the host that issued it. | ||
| ||
Clones run through `git upload-pack --stateless-rpc`, so protocol v2, | ||
`--filter` and `--depth` work as they do anywhere. The API answers from | ||
the same mirror. Revisions are rejected if git could read them as options, | ||
and responses named by a full hash are marked immutable. | ||
| ||
With `--index` it is also an appview. Firehose events for knotless | ||
collections are indexed into SQLite by fetching the record as it now | ||
stands, or removing it; `--backfill` walks `com.atproto.sync.listRepos` | ||
on a PDS or relay once at start. The index answers | ||
`dev.knotless.listRepos` (newest first, matched on name and description, paged by cursor) | ||
and `dev.knotless.getForge`, which takes the repository's URI and key and | ||
returns the records naming that owner and key under any name. The index is | ||
a cache: one built for another version of its tables is dropped and rebuilt | ||
from the firehose and `--backfill`. | ||
| ||
An appview with `--public-url` is also a moderation service. It keeps the | ||
reports it is sent and its takedowns in the same file, outside the cache, | ||
so a rebuild keeps them. A takedown of an account or record leaves the | ||
index, the pages, the API and clones at once. Each goes out as a | ||
`!takedown` label (or its negation) on `com.atproto.label.subscribeLabels` | ||
and `queryLabels`, and with `--labeler <did>` an appview follows another | ||
labeler's stream and honours its `!takedown` and `!hide` labels the same | ||
way. Labels go out unsigned; a follower trusts the endpoint the labeler's | ||
DID document names. | ||
| ||
### Web interface | ||
| ||
Read-only HTML over the same mirrors and index. Pages read objects in | ||
process through `gix`, opened once per mirror, because starting a git | ||
process costs more than the reads themselves; only a commit's diff runs | ||
`git show`. Each page is one document with its styles inline and two fixed | ||
scripts, allowed by hash in a `default-src 'none'` content security policy: | ||
| ||
- **Highlighting.** Code is sent as escaped text. Tree-sitter grammars for | ||
the common languages produce ranges, which travel in a `data-hl` | ||
attribute as `name:gap,len,…` in UTF-16 units; the script turns them into | ||
`Range`s for the CSS Custom Highlight API, and marks the line a `#L12` | ||
fragment names. Files over 1 MB, and languages without a grammar, are | ||
shown plain. | ||
- **Diffs.** A commit's patch is read into files and rows, each row a line | ||
with its old and new numbers, shown as a table. The ranges for a diff run | ||
over the rows' text as if joined by newlines, and the script cuts each | ||
range at row boundaries. Removed lines are highlighted as the old side of | ||
the hunks reads, the rest as the new side reads, so a comment or string | ||
that spans lines is still recognized. A removed line paired with an added | ||
one also gets the part between their common prefix and suffix marked, when | ||
that shared part is at least half the shorter line. A file with more than | ||
2000 rows is folded away. | ||
- **Navigation.** Speculation rules prerender a same-site link as the | ||
pointer settles on it, and cross-document view transitions keep the | ||
header still between pages. | ||
- **Caching.** Pages named by a commit hash are immutable; pages named by a | ||
branch are revalidated. HTML is compressed; git responses are not. | ||
- **Layout.** One stylesheet of custom properties, redefined for dark mode, | ||
with a single column under 640 px. Nothing is rendered lazily with | ||
`content-visibility`, so a full-page screenshot is the same every time. | ||
| ||
An account's page, at `/<handle>`, lists its public repositories from its | ||
own PDS rather than the index, so it is right without an appview, and shows | ||
the display name and description of its Bluesky profile record if it has | ||
one. Private repositories are in spaces and never listed there. | ||
| ||
A page asked for by a repository's old name redirects to the same page | ||
under its new name. The redirect is temporary, as the old name can be | ||
given to another repository. Clones over HTTP follow the old name without | ||
a redirect. | ||
| ||
README markdown is rendered with raw HTML shown as text, relative links | ||
pointed into the repository, and links with schemes other than http(s) | ||
and mailto dropped. Raw files are served sandboxed, typed only as text, | ||
an image, or bytes. A handle is shown for a DID only if it resolves back to | ||
that DID. | ||
| ||
## knotless-mcp | ||
| ||
An MCP server over stdio, acting as the logged-in account (or | ||
`KNOTLESS_IDENTIFIER` and `KNOTLESS_PASSWORD`). Its tools are the CLI's | ||
operations with structured results: reads answer from the cache mirror, | ||
writes go through the same code as `knotless commit`, `fork`, `pull` and | ||
`merge`, and errors come back as tool errors for the agent to read. | ||
| ||
## Integrity | ||
| ||
Every git object is named by its hash, and `index-pack` verifies every pack | ||
it reads. A PDS, or a `knotless-serve` in front of one, can withhold data | ||
but cannot substitute it. Ref targets come from the owner's signed atproto | ||
repo. `knotless` reads them with plain XRPC and does not check the commit | ||
signature itself; `com.atproto.sync.getRecord` returns the proof a client | ||
that wants to check it needs. | ||
| ||
A gateway can write anything to an owner's account that its app password | ||
allows; what it writes to a repository is still checked by every reader's | ||
`index-pack`. Paths in commits are refused if they are empty, contain `.`, | ||
`..` or `.git` segments, or contain NUL or newline characters. | ||
| ||
## Prior art | ||
| ||
- **pds.js `@pdsjs/git`** stored a repository as one record holding every | ||
ref plus a chain of git bundles. Any two pushes to the repository | ||
contended on that one record. knotless splits refs into records with | ||
their own compare-and-swap, and replaces bundles with self-contained | ||
packs that arrive inline with the listing. | ||
- **Tangled** keeps git on separate servers ("knots") and the social | ||
records in the PDS. knotless keeps git in the PDS, and adds a server | ||
(`knotless-serve`) as an optional cache that anyone can run and nobody | ||
has to trust, and as a gateway that only the owners who delegate to it | ||
trust. | ||
- **Anthropic's Scion** ("Redesigning source control at Anthropic", | ||
October 2026) replaced git internally once agents pushed merges past | ||
30,000 a day. Its lessons that carry over: clients are online, so fetch | ||
by immutable hash through tiers of caches (`knotless-serve`, the object | ||
API). Shared mutable state should be a compare-and-swap on one row (one | ||
record per ref). Refs are the scaling problem, so agents should work on | ||
their own branches. A working copy is optional (`knotless commit`). | ||
- **Cursor Origin** keeps git's wire protocol and builds the agent-facing | ||
forge around it. `knotless-serve` makes the same bet for reads: plain git | ||
clients, with no special client required. | ||
harness/changed.mjsnew+108−0
| @@ -0,0 +1,108 @@ | ||
// Which tests a change needs: the source files changed since tests last | ||
// passed through `just t-changed`, and the cargo test runs that cover them. | ||
// | ||
// node harness/changed.mjs list them | ||
// node harness/changed.mjs --run run them, against the harness | ||
// `just pds` left up if there is one, | ||
// and mark them passed | ||
// | ||
// Passing is a stamp file's time, target/tested.stamp; with none, every | ||
// file counts as changed. The map is coarse on purpose: a change to the | ||
// core crate runs everything, as everything is built on it. | ||
| ||
import { spawnSync } from 'node:child_process' | ||
import { existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from 'node:fs' | ||
import { dirname, join, relative, resolve } from 'node:path' | ||
import { fileURLToPath } from 'node:url' | ||
| ||
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') | ||
const stamp = join(root, 'target/tested.stamp') | ||
const since = existsSync(stamp) ? statSync(stamp).mtimeMs : 0 | ||
const skip = new Set(['target', 'node_modules', '.git', 'baseline']) | ||
| ||
const walk = (dir, out = []) => { | ||
for (const entry of readdirSync(dir, { withFileTypes: true })) { | ||
if (skip.has(entry.name)) continue | ||
const path = join(dir, entry.name) | ||
if (entry.isDirectory()) walk(path, out) | ||
else if (statSync(path).mtimeMs > since) out.push(relative(root, path)) | ||
} | ||
return out | ||
} | ||
const changed = ['crates', 'harness', 'Cargo.toml', 'Cargo.lock'] | ||
.map((p) => join(root, p)) | ||
.filter(existsSync) | ||
.flatMap((p) => (statSync(p).isDirectory() ? walk(p) : statSync(p).mtimeMs > since ? [relative(root, p)] : [])) | ||
| ||
const serve = (...tests) => [['-p', 'knotless-serve', '--lib'], ...tests.map((t) => ['-p', 'knotless-serve', '--test', t])] | ||
/// The test runs for a file, and whether the pages may look different. | ||
const runs = (file) => { | ||
const m = (re) => file.match(re) | ||
let x | ||
if (m(/^crates\/knotless\/(src|tests\/common)\//) || m(/^(Cargo\.(toml|lock)|harness\/pds\.mjs|harness\/package)/)) { | ||
return { runs: [['--workspace']], pages: true } | ||
} | ||
if ((x = m(/^crates\/(knotless-\w+)\/Cargo\.toml$/))) return { runs: [['-p', x[1]]], pages: x[1] === 'knotless-serve' } | ||
if (m(/^crates\/knotless\/Cargo\.toml$/)) return { runs: [['--workspace']], pages: true } | ||
if ((x = m(/^crates\/knotless\/tests\/(\w+)\.rs$/))) return { runs: [['-p', 'knotless', '--test', x[1]]] } | ||
if (m(/^crates\/knotless-mcp\//)) return { runs: [['-p', 'knotless-mcp']] } | ||
if ((x = m(/^crates\/knotless-serve\/tests\/(\w+)\.rs$/))) return { runs: [['-p', 'knotless-serve', '--test', x[1]]] } | ||
if ((x = m(/^crates\/knotless-serve\/src\/(\w+)\.rs$/))) { | ||
const pages = { runs: serve('web'), pages: true } | ||
return ( | ||
{ | ||
web: pages, | ||
highlight: pages, | ||
diff: pages, | ||
objects: pages, | ||
gateway: { runs: serve('gateway') }, | ||
api: { runs: serve('serve') }, | ||
smart: { runs: serve('serve') }, | ||
firehose: { runs: serve('jetstream', 'forge') }, | ||
jetstream: { runs: serve('jetstream', 'forge') }, | ||
index: { runs: serve('forge', 'web'), pages: true }, | ||
appview: { runs: serve('forge', 'web'), pages: true }, | ||
}[x[1]] ?? { runs: [['-p', 'knotless-serve'], ['-p', 'knotless-mcp']], pages: true } | ||
) | ||
} | ||
if (m(/^crates\/knotless-serve\//)) return { runs: [['-p', 'knotless-serve']], pages: true } | ||
if (m(/^harness\/shots\//)) return { runs: [], pages: true } | ||
return { runs: [] } | ||
} | ||
| ||
const planned = changed.map(runs) | ||
let all = planned.flatMap((p) => p.runs) | ||
// A whole package, or the workspace, covers the narrower runs within it. | ||
if (all.some((r) => r[0] === '--workspace')) all = [['--workspace']] | ||
const whole = new Set(all.filter((r) => r.length === 2).map((r) => r[1])) | ||
all = all.filter((r) => r.length === 2 || !whole.has(r[1])) | ||
const unique = [...new Map(all.map((r) => [r.join(' '), r])).values()] | ||
const pages = planned.some((p) => p.pages) | ||
| ||
if (changed.length === 0) { | ||
console.log('Nothing changed since tests last passed.') | ||
process.exit(0) | ||
} | ||
console.log(`Changed since tests last passed${since ? '' : ' (no record of a pass, so everything)'}:`) | ||
for (const f of changed.slice(0, 20)) console.log(` ${f}`) | ||
if (changed.length > 20) console.log(` … and ${changed.length - 20} more`) | ||
console.log(unique.length ? 'Tests:' : 'No tests cover these.') | ||
for (const r of unique) console.log(` cargo test ${r.join(' ')}`) | ||
if (pages) console.log('The pages may look different: `just shots` retakes those whose HTML changed.') | ||
| ||
if (process.argv.includes('--run')) { | ||
const started = Date.now() | ||
const env = { ...process.env, KNOTLESS_TEST_NET: join(root, 'target/test-net.json') } | ||
for (const r of unique) { | ||
console.log(`\n$ cargo test ${r.join(' ')}`) | ||
const res = spawnSync('rustup', ['run', 'stable', 'cargo', 'test', ...r], { cwd: root, env, stdio: 'inherit' }) | ||
if (res.status !== 0) process.exit(res.status ?? 1) | ||
} | ||
// Marked as of when the run began, so a file changed during it is not | ||
// taken for tested. | ||
mkdirSync(dirname(stamp), { recursive: true }) | ||
writeFileSync(stamp, '') | ||
const { utimesSync } = await import('node:fs') | ||
utimesSync(stamp, started / 1000, started / 1000) | ||
console.log('\nPassed; marked.') | ||
} | ||
harness/package-lock.jsonnew+9487−0
A large diff, not shown.
harness/package.jsonnew+12−0
| @@ -0,0 +1,12 @@ | ||
{ | ||
"name": "knotless-harness", | ||
"private": true, | ||
"type": "module", | ||
"description": "Runs the npm-published reference PDS and PLC for knotless tests", | ||
"dependencies": { | ||
"@atproto/dev-env": "0.6.26" | ||
}, | ||
"devDependencies": { | ||
"@playwright/test": "1.58.2" | ||
} | ||
} | ||
harness/pds.mjsnew+76−0
| @@ -0,0 +1,76 @@ | ||
// Runs the reference PDS (@atproto/pds via @atproto/dev-env) and a PLC | ||
// directory on free ports, creates test accounts, prints one JSON line | ||
// describing them, and shuts down when stdin closes. | ||
// | ||
// node harness/pds.mjs [--accounts alice,bob,carol] [--rate-limits] | ||
// | ||
// KNOTLESS_DEV_ENV names a built dev-env package to use instead of the npm | ||
// one, such as an atproto checkout's packages/dev-env on the spaces alpha. | ||
| ||
import { fileURLToPath, pathToFileURL } from 'node:url' | ||
import { dirname, resolve } from 'node:path' | ||
| ||
// Relative to the repository root. | ||
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') | ||
const devEnv = process.env.KNOTLESS_DEV_ENV | ||
const { TestNetworkNoAppView } = await import( | ||
devEnv | ||
? pathToFileURL(resolve(root, devEnv, 'dist/index.js')).href | ||
: '@atproto/dev-env' | ||
) | ||
| ||
const arg = (name) => { | ||
const i = process.argv.indexOf(name) | ||
return i === -1 ? undefined : process.argv[i + 1] | ||
} | ||
const names = (arg('--accounts') ?? 'alice,bob,carol').split(',') | ||
| ||
const net = await TestNetworkNoAppView.create({ | ||
pds: { | ||
hostname: 'localhost', | ||
rateLimitsEnabled: process.argv.includes('--rate-limits'), | ||
}, | ||
}) | ||
| ||
const accounts = [] | ||
for (const name of names) { | ||
const password = `${name}-password` | ||
const res = await fetch( | ||
`${net.pds.url}/xrpc/com.atproto.server.createAccount`, | ||
{ | ||
method: 'POST', | ||
headers: { 'content-type': 'application/json' }, | ||
body: JSON.stringify({ | ||
email: `${name}@test.com`, | ||
handle: `${name}.test`, | ||
password, | ||
}), | ||
}, | ||
) | ||
if (!res.ok) throw new Error(`createAccount ${name}: ${await res.text()}`) | ||
const { did, handle } = await res.json() | ||
accounts.push({ handle, did, password }) | ||
} | ||
| ||
const described = await fetch( | ||
`${net.pds.url}/xrpc/com.atproto.server.describeServer`, | ||
).then((res) => res.json()) | ||
| ||
console.log( | ||
JSON.stringify({ | ||
pds: net.pds.url, | ||
plc: net.plc.url, | ||
spaces: described.spacesAlpha === true, | ||
accounts, | ||
}), | ||
) | ||
| ||
const shutdown = async () => { | ||
await net.close() | ||
process.exit(0) | ||
} | ||
process.stdin.on('end', shutdown) | ||
process.stdin.on('close', shutdown) | ||
process.stdin.resume() | ||
process.on('SIGTERM', shutdown) | ||
process.on('SIGINT', shutdown) | ||
harness/shots/baseline/darwin/commit-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/commit-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
395dfe6b7d9b6dd3529765c95ecd6b390911cc2465155df44224590719577a33 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/commit-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/commit-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
395dfe6b7d9b6dd3529765c95ecd6b390911cc2465155df44224590719577a33 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/commit-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/commit-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
395dfe6b7d9b6dd3529765c95ecd6b390911cc2465155df44224590719577a33 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/commit-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/commit-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
395dfe6b7d9b6dd3529765c95ecd6b390911cc2465155df44224590719577a33 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/commits-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/commits-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
df500af42b1f11143328c2fd195ebb95ed00e9e333b4c5d501ba9a2baa24ba2b | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/commits-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/commits-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
df500af42b1f11143328c2fd195ebb95ed00e9e333b4c5d501ba9a2baa24ba2b | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/commits-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/commits-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
df500af42b1f11143328c2fd195ebb95ed00e9e333b4c5d501ba9a2baa24ba2b | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/commits-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/commits-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
df500af42b1f11143328c2fd195ebb95ed00e9e333b4c5d501ba9a2baa24ba2b | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/directory-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/directory-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
245bc4f87cc8dc053418d4d0ac1eea6f61e43323927f9c083d6de57a0f28f7d6 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/directory-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/directory-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
245bc4f87cc8dc053418d4d0ac1eea6f61e43323927f9c083d6de57a0f28f7d6 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/directory-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/directory-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
245bc4f87cc8dc053418d4d0ac1eea6f61e43323927f9c083d6de57a0f28f7d6 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/directory-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/directory-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
245bc4f87cc8dc053418d4d0ac1eea6f61e43323927f9c083d6de57a0f28f7d6 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/empty-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/empty-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
678ef4190c22bd0d9808e92c6b5558e991221dd1f34218a3d893c881ce98f92e | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/empty-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/empty-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
678ef4190c22bd0d9808e92c6b5558e991221dd1f34218a3d893c881ce98f92e | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/empty-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/empty-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
678ef4190c22bd0d9808e92c6b5558e991221dd1f34218a3d893c881ce98f92e | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/empty-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/empty-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
678ef4190c22bd0d9808e92c6b5558e991221dd1f34218a3d893c881ce98f92e | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/file-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/file-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
f65deb19f2417b3ff935c2c557764002393c26effd2a9d26014d138a5f65df47 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/file-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/file-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
f65deb19f2417b3ff935c2c557764002393c26effd2a9d26014d138a5f65df47 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/file-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/file-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
f65deb19f2417b3ff935c2c557764002393c26effd2a9d26014d138a5f65df47 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/file-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/file-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
f65deb19f2417b3ff935c2c557764002393c26effd2a9d26014d138a5f65df47 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/fork-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/fork-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
ff01a27651eaf9bfa3c74af1e01c2f54365b305abae358163d946203cbe9c563 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/fork-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/fork-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
ff01a27651eaf9bfa3c74af1e01c2f54365b305abae358163d946203cbe9c563 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/fork-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/fork-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
ff01a27651eaf9bfa3c74af1e01c2f54365b305abae358163d946203cbe9c563 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/fork-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/fork-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
ff01a27651eaf9bfa3c74af1e01c2f54365b305abae358163d946203cbe9c563 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/home-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/home-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7ebdd3b2b907ba4afa5e6ceff837c0ad9b15975f813b5b9299ab163807685c97 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/home-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/home-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7ebdd3b2b907ba4afa5e6ceff837c0ad9b15975f813b5b9299ab163807685c97 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/home-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/home-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7ebdd3b2b907ba4afa5e6ceff837c0ad9b15975f813b5b9299ab163807685c97 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/home-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/home-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7ebdd3b2b907ba4afa5e6ceff837c0ad9b15975f813b5b9299ab163807685c97 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/image-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/image-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
a556f5e05a48f6e28f48d198ace64a670b3eb76e3a86e17b1ed1e0ec845cb9e8 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/image-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/image-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
a556f5e05a48f6e28f48d198ace64a670b3eb76e3a86e17b1ed1e0ec845cb9e8 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/image-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/image-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
a556f5e05a48f6e28f48d198ace64a670b3eb76e3a86e17b1ed1e0ec845cb9e8 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/image-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/image-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
a556f5e05a48f6e28f48d198ace64a670b3eb76e3a86e17b1ed1e0ec845cb9e8 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/issue-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/issue-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
de3f0d2dca0027cf1b3e5f4d610a8098e22b5fac1835d6f9a8c383cd15aebcda | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/issue-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/issue-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
de3f0d2dca0027cf1b3e5f4d610a8098e22b5fac1835d6f9a8c383cd15aebcda | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/issue-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/issue-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
de3f0d2dca0027cf1b3e5f4d610a8098e22b5fac1835d6f9a8c383cd15aebcda | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/issue-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/issue-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
de3f0d2dca0027cf1b3e5f4d610a8098e22b5fac1835d6f9a8c383cd15aebcda | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/issues-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/issues-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
eea04c8136733b5f2e46410f9f19fe56d7a2a4baed7a4ba462c5ca3d7ab0ee6c | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/issues-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/issues-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
eea04c8136733b5f2e46410f9f19fe56d7a2a4baed7a4ba462c5ca3d7ab0ee6c | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/issues-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/issues-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
eea04c8136733b5f2e46410f9f19fe56d7a2a4baed7a4ba462c5ca3d7ab0ee6c | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/issues-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/issues-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
eea04c8136733b5f2e46410f9f19fe56d7a2a4baed7a4ba462c5ca3d7ab0ee6c | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/markdown-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/markdown-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
de18e32d46eff8532df160ab0d19a646aaf32c929cea1e2c996c15f75ef440ca | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/markdown-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/markdown-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
de18e32d46eff8532df160ab0d19a646aaf32c929cea1e2c996c15f75ef440ca | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/markdown-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/markdown-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
de18e32d46eff8532df160ab0d19a646aaf32c929cea1e2c996c15f75ef440ca | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/markdown-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/markdown-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
de18e32d46eff8532df160ab0d19a646aaf32c929cea1e2c996c15f75ef440ca | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/missing-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/missing-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
99cb2c771e1c620cf8f48907ab9ffd1e906b1771e91820c9a5e4dcb63fba4561 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/missing-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/missing-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
99cb2c771e1c620cf8f48907ab9ffd1e906b1771e91820c9a5e4dcb63fba4561 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/missing-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/missing-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
99cb2c771e1c620cf8f48907ab9ffd1e906b1771e91820c9a5e4dcb63fba4561 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/missing-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/missing-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
99cb2c771e1c620cf8f48907ab9ffd1e906b1771e91820c9a5e4dcb63fba4561 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/profile-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/profile-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
ff564a4b5f0667775a0efa4fe9df6a8887e29bcdb8f32621dc94ea535a9a47ce | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/profile-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/profile-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
ff564a4b5f0667775a0efa4fe9df6a8887e29bcdb8f32621dc94ea535a9a47ce | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/profile-fork-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/profile-fork-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7dc4d219eb757b1ad44cb42b0b8c8f86b7fae29f4f4ddb6f5accf1abef2e5f75 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/profile-fork-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/profile-fork-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7dc4d219eb757b1ad44cb42b0b8c8f86b7fae29f4f4ddb6f5accf1abef2e5f75 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/profile-fork-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/profile-fork-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7dc4d219eb757b1ad44cb42b0b8c8f86b7fae29f4f4ddb6f5accf1abef2e5f75 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/profile-fork-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/profile-fork-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7dc4d219eb757b1ad44cb42b0b8c8f86b7fae29f4f4ddb6f5accf1abef2e5f75 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/profile-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/profile-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
ff564a4b5f0667775a0efa4fe9df6a8887e29bcdb8f32621dc94ea535a9a47ce | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/profile-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/profile-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
ff564a4b5f0667775a0efa4fe9df6a8887e29bcdb8f32621dc94ea535a9a47ce | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pull-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pull-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
11cf70ccfbd7e11f7031f9244e37b1cd225d0ef465d60dfa006527f4d4d7065e | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pull-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pull-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
11cf70ccfbd7e11f7031f9244e37b1cd225d0ef465d60dfa006527f4d4d7065e | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pull-open-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pull-open-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
0ae0f190a6d78f5a4cde33afccbc3996eaea695e3affec567cb13a916204976b | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pull-open-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pull-open-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
0ae0f190a6d78f5a4cde33afccbc3996eaea695e3affec567cb13a916204976b | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pull-open-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pull-open-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
0ae0f190a6d78f5a4cde33afccbc3996eaea695e3affec567cb13a916204976b | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pull-open-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pull-open-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
0ae0f190a6d78f5a4cde33afccbc3996eaea695e3affec567cb13a916204976b | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pull-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pull-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
11cf70ccfbd7e11f7031f9244e37b1cd225d0ef465d60dfa006527f4d4d7065e | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pull-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pull-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
11cf70ccfbd7e11f7031f9244e37b1cd225d0ef465d60dfa006527f4d4d7065e | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pulls-closed-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pulls-closed-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7d4aa88c997ce4f8583bcb17f1d9c03986d6ab4b2e701edb34fcbf9a7d2a4b45 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pulls-closed-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pulls-closed-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7d4aa88c997ce4f8583bcb17f1d9c03986d6ab4b2e701edb34fcbf9a7d2a4b45 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pulls-closed-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pulls-closed-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7d4aa88c997ce4f8583bcb17f1d9c03986d6ab4b2e701edb34fcbf9a7d2a4b45 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pulls-closed-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pulls-closed-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
7d4aa88c997ce4f8583bcb17f1d9c03986d6ab4b2e701edb34fcbf9a7d2a4b45 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pulls-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pulls-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
6cdae0d8e63ec3c34c4f2fe200a96abac9ef12b0d295fbccb8a0a30b2b2ec7b7 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pulls-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pulls-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
6cdae0d8e63ec3c34c4f2fe200a96abac9ef12b0d295fbccb8a0a30b2b2ec7b7 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pulls-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pulls-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
6cdae0d8e63ec3c34c4f2fe200a96abac9ef12b0d295fbccb8a0a30b2b2ec7b7 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/pulls-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/pulls-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
6cdae0d8e63ec3c34c4f2fe200a96abac9ef12b0d295fbccb8a0a30b2b2ec7b7 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/repository-desktop-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/repository-desktop-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
549f070ed956084a55d56ad6a076d65eb6bf98c67374c205b00460aef47dbd55 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/repository-desktop-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/repository-desktop-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
549f070ed956084a55d56ad6a076d65eb6bf98c67374c205b00460aef47dbd55 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/repository-phone-dark.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/repository-phone-dark.sha256new+1−0
| @@ -0,0 +1 @@ | ||
549f070ed956084a55d56ad6a076d65eb6bf98c67374c205b00460aef47dbd55 | ||
| \ No newline at end of file | ||
harness/shots/baseline/darwin/repository-phone-light.pngnew+0−0
A binary file, not shown.
harness/shots/baseline/darwin/repository-phone-light.sha256new+1−0
| @@ -0,0 +1 @@ | ||
549f070ed956084a55d56ad6a076d65eb6bf98c67374c205b00460aef47dbd55 | ||
| \ No newline at end of file | ||
harness/shots/digest.mjsnew+10−0
| @@ -0,0 +1,10 @@ | ||
// What a page's screenshot follows from: its HTML, which carries the | ||
// styles and scripts too, less what differs between worlds: DIDs, and the | ||
// record keys in links to pulls and issues. | ||
| ||
import { createHash } from 'node:crypto' | ||
| ||
export const digest = (html) => | ||
createHash('sha256') | ||
.update(html.replace(/did:plc:[a-z2-7]{24}/g, 'did').replace(/\/[2-7a-z]{13}(?=["/?#])/g, '/rkey')) | ||
.digest('hex') | ||
harness/shots/fixture/count/lib.rsnew+30−0
| @@ -0,0 +1,30 @@ | ||
//! Counting, kept apart from reading so it can be tested. | ||
| ||
/// What a text holds. | ||
#[derive(Debug, Default, PartialEq)] | ||
pub struct Counts { | ||
pub lines: usize, | ||
pub words: usize, | ||
pub bytes: usize, | ||
} | ||
| ||
/// Counts `text` in one pass. | ||
pub fn count(text: &str) -> Counts { | ||
let mut counts = Counts { bytes: text.len(), ..Counts::default() }; | ||
for line in text.lines() { | ||
counts.lines += 1; | ||
counts.words += line.split_whitespace().count(); | ||
} | ||
counts | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn counts() { | ||
let c = count("one two\nthree\n"); | ||
assert_eq!(c, Counts { lines: 2, words: 3, bytes: 14 }); | ||
} | ||
} | ||
harness/shots/fixture/count/main.rsnew+8−0
| @@ -0,0 +1,8 @@ | ||
use tally::count; | ||
| ||
fn main() { | ||
let path = std::env::args().nth(1).expect("a file to count"); | ||
let text = std::fs::read_to_string(&path).unwrap(); | ||
let c = count(&text); | ||
println!("{:>8} {:>8} {:>8} {}", c.lines, c.words, c.bytes, path); | ||
} | ||
harness/shots/fixture/docs/README.mdnew+27−0
| @@ -0,0 +1,27 @@ | ||
# tally | ||
| ||
Counts lines, words and bytes, like `wc`, with output that lines up. | ||
| ||
 | ||
| ||
## Usage | ||
| ||
```sh | ||
cargo install --path . | ||
tally notes.txt | ||
echo "one two" | tally | ||
``` | ||
| ||
## Output | ||
| ||
| Column | Meaning | | ||
| --- | --- | | ||
| 1 | lines | | ||
| 2 | words, split on whitespace | | ||
| 3 | bytes | | ||
| ||
See [the design notes](docs/design.md) for why it reads the whole file. | ||
| ||
## License | ||
| ||
MIT | ||
harness/shots/fixture/docs/design.mdnew+4−0
| @@ -0,0 +1,4 @@ | ||
# Design | ||
| ||
tally reads the whole input before counting. Inputs are small, and one | ||
pass over a string is simpler than a streaming reader. | ||
harness/shots/fixture/fix/parse.rsnew+20−0
| @@ -0,0 +1,20 @@ | ||
//! Splitting text into the things tally counts. | ||
| ||
/// The words of `line`: runs of anything but whitespace. | ||
pub fn words(line: &str) -> usize { | ||
line.split_whitespace().count() | ||
} | ||
| ||
/// The lines of `text`. A last line without a newline still counts, and | ||
/// empty input has none. | ||
pub fn lines(text: &str) -> impl Iterator<Item = &str> { | ||
text.lines().filter(|_| !text.is_empty()) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
#[test] | ||
fn empty_input_has_no_lines() { | ||
assert_eq!(super::lines("").count(), 0); | ||
} | ||
} | ||
harness/shots/fixture/json/main.rsnew+23−0
| @@ -0,0 +1,23 @@ | ||
use std::io::Read; | ||
use tally::count; | ||
| ||
fn main() { | ||
let mut args: Vec<String> = std::env::args().skip(1).collect(); | ||
let json = args.iter().position(|a| a == "--json").map(|i| args.remove(i)).is_some(); | ||
let path = args.first(); | ||
let text = match path { | ||
Some(path) => std::fs::read_to_string(path).unwrap(), | ||
None => { | ||
let mut text = String::new(); | ||
std::io::stdin().read_to_string(&mut text).unwrap(); | ||
text | ||
} | ||
}; | ||
let c = count(&text); | ||
let name = path.map(String::as_str).unwrap_or("-"); | ||
if json { | ||
println!(r#"{{"lines":{},"words":{},"bytes":{},"name":"{name}"}}"#, c.lines, c.words, c.bytes); | ||
} else { | ||
println!("{:>8} {:>8} {:>8} {name}", c.lines, c.words, c.bytes); | ||
} | ||
} | ||
harness/shots/fixture/logo/logo.svgnew+1−0
| @@ -0,0 +1 @@ | ||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 120 120" width="120" height="120"><rect width="120" height="120" rx="28" fill="#1c1d20"/><g stroke="#fff" stroke-width="8" stroke-linecap="round"><path d="M36 34v52M52 34v52M68 34v52M84 34v52"/><path d="M28 72l64-24"/></g></svg> | ||
harness/shots/fixture/notes/README.mdnew+3−0
| @@ -0,0 +1,3 @@ | ||
# notes | ||
| ||
Things worth remembering. | ||
harness/shots/fixture/parse/lib.rsnew+37−0
| @@ -0,0 +1,37 @@ | ||
//! Counting, kept apart from reading so it can be tested. | ||
| ||
mod parse; | ||
| ||
/// What a text holds. | ||
#[derive(Debug, Default, PartialEq)] | ||
pub struct Counts { | ||
pub lines: usize, | ||
pub words: usize, | ||
pub bytes: usize, | ||
} | ||
| ||
/// Counts `text` in one pass. | ||
pub fn count(text: &str) -> Counts { | ||
let mut counts = Counts { bytes: text.len(), ..Counts::default() }; | ||
for line in parse::lines(text) { | ||
counts.lines += 1; | ||
counts.words += parse::words(line); | ||
} | ||
counts | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
| ||
#[test] | ||
fn counts() { | ||
let c = count("one two\nthree\n"); | ||
assert_eq!(c, Counts { lines: 2, words: 3, bytes: 14 }); | ||
} | ||
| ||
#[test] | ||
fn a_last_line_without_a_newline() { | ||
assert_eq!(count("a\nb").lines, 2); | ||
} | ||
} | ||
harness/shots/fixture/parse/parse.rsnew+11−0
| @@ -0,0 +1,11 @@ | ||
//! Splitting text into the things tally counts. | ||
| ||
/// The words of `line`: runs of anything but whitespace. | ||
pub fn words(line: &str) -> usize { | ||
line.split_whitespace().count() | ||
} | ||
| ||
/// The lines of `text`. A last line without a newline still counts. | ||
pub fn lines(text: &str) -> impl Iterator<Item = &str> { | ||
text.lines() | ||
} | ||
harness/shots/fixture/start/.gitignorenew+1−0
| @@ -0,0 +1 @@ | ||
/target | ||
harness/shots/fixture/start/Cargo.tomlnew+6−0
| @@ -0,0 +1,6 @@ | ||
[package] | ||
name = "tally" | ||
version = "0.1.0" | ||
edition = "2024" | ||
description = "Counts lines, words and bytes" | ||
license = "MIT" | ||
harness/shots/fixture/start/README.mdnew+3−0
| @@ -0,0 +1,3 @@ | ||
# tally | ||
| ||
Counts lines, words and bytes. | ||
harness/shots/fixture/start/main.rsnew+5−0
| @@ -0,0 +1,5 @@ | ||
fn main() { | ||
let path = std::env::args().nth(1).expect("a file to count"); | ||
let text = std::fs::read_to_string(&path).unwrap(); | ||
println!("{} {}", text.lines().count(), path); | ||
} | ||
harness/shots/fixture/stdin/main.rsnew+17−0
| @@ -0,0 +1,17 @@ | ||
use std::io::Read; | ||
use tally::count; | ||
| ||
fn main() { | ||
let path = std::env::args().nth(1); | ||
let text = match &path { | ||
Some(path) => std::fs::read_to_string(path).unwrap(), | ||
None => { | ||
let mut text = String::new(); | ||
std::io::stdin().read_to_string(&mut text).unwrap(); | ||
text | ||
} | ||
}; | ||
let c = count(&text); | ||
let name = path.as_deref().unwrap_or("-"); | ||
println!("{:>8} {:>8} {:>8} {name}", c.lines, c.words, c.bytes); | ||
} | ||
harness/shots/playwright.config.mjsnew+37−0
| @@ -0,0 +1,37 @@ | ||
// Screenshots of every page of the web interface, in light and dark, on a | ||
// desktop and a phone, compared pixel for pixel with the ones checked in. | ||
// | ||
// just shots take them again, replacing the baselines | ||
// just shots-check fail on any pixel that differs | ||
// | ||
// Baselines are per platform: fonts render differently on each. | ||
| ||
import { defineConfig, devices } from '@playwright/test' | ||
| ||
const desktop = { viewport: { width: 1280, height: 800 }, deviceScaleFactor: 2 } | ||
const phone = { | ||
...devices['iPhone 13'], | ||
browserName: 'chromium', | ||
defaultBrowserType: 'chromium', | ||
deviceScaleFactor: 2, | ||
} | ||
| ||
export default defineConfig({ | ||
testDir: '.', | ||
testMatch: 'views.spec.mjs', | ||
globalSetup: './setup.mjs', | ||
outputDir: '../../target/shots/results', | ||
snapshotPathTemplate: '{testDir}/baseline/{platform}/{arg}-{projectName}{ext}', | ||
fullyParallel: true, | ||
reporter: [['list']], | ||
expect: { | ||
toHaveScreenshot: { maxDiffPixels: 0, threshold: 0, animations: 'disabled', caret: 'hide', scale: 'device' }, | ||
}, | ||
use: { baseURL: 'http://127.0.0.1:2593', locale: 'en-US', timezoneId: 'UTC' }, | ||
projects: [ | ||
{ name: 'desktop-light', use: { ...desktop, colorScheme: 'light' } }, | ||
{ name: 'desktop-dark', use: { ...desktop, colorScheme: 'dark' } }, | ||
{ name: 'phone-light', use: { ...phone, colorScheme: 'light' } }, | ||
{ name: 'phone-dark', use: { ...phone, colorScheme: 'dark' } }, | ||
], | ||
}) | ||
harness/shots/setup.mjsnew+50−0
| @@ -0,0 +1,50 @@ | ||
// Before the screenshots: a world to shoot, the one `just shots-up` keeps | ||
// up if it is there, else a new one for this run, and knotless-serve from | ||
// this build in front of it. The pages are written to | ||
// target/shots/views.json, and those whose HTML is as it was when their | ||
// baselines were taken, in every project, to target/shots/unchanged.json: | ||
// views.spec.mjs skips them before a browser page is opened. Returns the | ||
// teardown. | ||
| ||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' | ||
import { join } from 'node:path' | ||
import { base, create, page, root, saved, serve, until } from './world.mjs' | ||
import { digest } from './digest.mjs' | ||
import config from './playwright.config.mjs' | ||
| ||
const here = join(root, 'harness/shots') | ||
const projects = config.projects.map((p) => p.name) | ||
| ||
const kept = async () => { | ||
if (!existsSync(saved)) return null | ||
const world = JSON.parse(readFileSync(saved, 'utf8')) | ||
const up = await fetch(`${world.net.pds}/xrpc/com.atproto.server.describeServer`).then((r) => r.ok, () => false) | ||
return up ? world : null | ||
} | ||
| ||
export default async function setup() { | ||
let world = await kept() | ||
let teardown | ||
if (world) { | ||
const server = await serve(world) | ||
teardown = async () => server.kill() | ||
// The index is on disk already; this waits only for the mirrors. | ||
await until('the pages', async () => (await page(world.views.pull)).includes('Merged as')) | ||
} else { | ||
world = await create() | ||
teardown = async () => world.stop() | ||
} | ||
mkdirSync(join(root, 'target/shots'), { recursive: true }) | ||
writeFileSync(join(root, 'target/shots/views.json'), JSON.stringify(world.views, null, 2)) | ||
const unchanged = [] | ||
if (!process.env.SHOTS_ALL) { | ||
for (const [name, path] of Object.entries(world.views)) { | ||
const sum = digest(await (await fetch(base + path)).text()) | ||
const at = (project, ext) => join(here, 'baseline', process.platform, `${name}-${project}.${ext}`) | ||
const same = (p) => existsSync(at(p, 'png')) && existsSync(at(p, 'sha256')) && readFileSync(at(p, 'sha256'), 'utf8') === sum | ||
if (projects.every(same)) unchanged.push(name) | ||
} | ||
} | ||
writeFileSync(join(root, 'target/shots/unchanged.json'), JSON.stringify(unchanged)) | ||
return teardown | ||
} | ||
harness/shots/views.spec.mjsnew+41−0
| @@ -0,0 +1,41 @@ | ||
// One screenshot per page, per project in playwright.config.mjs. The pages' | ||
// paths are known once setup.mjs has made them, so they are read in each test. | ||
// | ||
// A page's pixels follow from its HTML and the browser. Beside each | ||
// baseline is a digest of the HTML it was taken from, and setup.mjs lists | ||
// the pages whose HTML still matches in every project: those are skipped | ||
// as unchanged, before a browser page is opened. SHOTS_ALL=1 takes every | ||
// one regardless, as `just shots-check` does. | ||
| ||
import { test, expect } from '@playwright/test' | ||
import { existsSync, readFileSync, writeFileSync } from 'node:fs' | ||
import { dirname, resolve } from 'node:path' | ||
import { fileURLToPath } from 'node:url' | ||
import { digest } from './digest.mjs' | ||
| ||
const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..') | ||
const read = (file, otherwise) => { | ||
const path = resolve(root, 'target/shots', file) | ||
return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : otherwise | ||
} | ||
const unchanged = new Set(read('unchanged.json', [])) | ||
| ||
const names = [ | ||
'home', 'profile', 'profile-fork', 'repository', 'directory', 'file', 'markdown', 'image', 'commits', 'commit', | ||
'pulls', 'pulls-closed', 'pull', 'pull-open', 'issues', 'issue', 'fork', 'empty', 'missing', | ||
] | ||
| ||
for (const name of names) { | ||
test.describe(() => { | ||
test.skip(unchanged.has(name), 'unchanged') | ||
test(name, async ({ page, request }, info) => { | ||
const path = read('views.json', {})[name] | ||
const sum = digest(await (await request.get(path)).text()) | ||
await page.goto(path, { waitUntil: 'load' }) | ||
// A DID is new each run, as the PLC directory makes one per account: | ||
// it is covered, so the rest can match to the pixel. | ||
await expect(page).toHaveScreenshot(`${name}.png`, { fullPage: true, mask: [page.locator('.did')], maskColor: '#80808a' }) | ||
writeFileSync(info.snapshotPath(`${name}.sha256`), sum) | ||
}) | ||
}) | ||
} | ||
harness/shots/world.mjsnew+321−0
| @@ -0,0 +1,321 @@ | ||
// The world the screenshots show: a reference PDS holding the same | ||
// repositories, pulls and issues every time, so each page renders to the | ||
// same pixels. Commits carry fixed dates, so their hashes are fixed too, and | ||
// every record's createdAt is rewritten to a fixed time, in the order the | ||
// records were made. The index a knotless-serve built while it was seeded | ||
// is kept in `dir`, with its mirrors, for the next server to start from. | ||
// | ||
// node harness/shots/world.mjs seed one and keep it up, for | ||
// `just shots` runs to reuse, until | ||
// ctrl-c; written to target/shots/world.json | ||
| ||
import { spawn, execFileSync } from 'node:child_process' | ||
import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' | ||
import { tmpdir } from 'node:os' | ||
import { dirname, join, resolve } from 'node:path' | ||
import { createInterface } from 'node:readline' | ||
import { fileURLToPath } from 'node:url' | ||
| ||
const here = dirname(fileURLToPath(import.meta.url)) | ||
export const root = resolve(here, '../..') | ||
const fixture = join(here, 'fixture') | ||
const bin = join(root, 'target/debug') | ||
export const port = 2593 | ||
export const base = `http://127.0.0.1:${port}` | ||
export const saved = join(root, 'target/shots/world.json') | ||
| ||
const people = { | ||
alice: { name: 'Alice Moreau', email: 'alice@example.com' }, | ||
bob: { name: 'Bob Tanaka', email: 'bob@example.com' }, | ||
} | ||
| ||
export const until = async (what, check) => { | ||
for (let i = 0; i < 300; i++) { | ||
try { | ||
if (await check()) return | ||
} catch {} | ||
await new Promise((r) => setTimeout(r, 100)) | ||
} | ||
throw new Error(`timed out waiting for ${what}`) | ||
} | ||
| ||
export const page = async (path) => { | ||
const res = await fetch(base + path) | ||
return res.ok ? res.text() : '' | ||
} | ||
| ||
/// The environment `who` runs the CLI, git or the server in. | ||
const environment = (net, dir, who, when) => { | ||
const clean = Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith('KNOTLESS_'))) | ||
const person = people[who] ?? { name: who, email: `${who}@example.com` } | ||
return { | ||
...clean, | ||
PATH: `${bin}:${process.env.PATH}`, | ||
KNOTLESS_HANDLE_RESOLVER: net.pds, | ||
KNOTLESS_PLC_URL: net.plc, | ||
KNOTLESS_COMPACT: 'off', | ||
KNOTLESS_APPVIEW: base, | ||
KNOTLESS_CONFIG_DIR: join(dir, who, 'config'), | ||
KNOTLESS_CACHE_DIR: join(dir, who, 'cache'), | ||
GIT_CONFIG_GLOBAL: '/dev/null', | ||
GIT_CONFIG_NOSYSTEM: '1', | ||
GIT_AUTHOR_NAME: person.name, | ||
GIT_AUTHOR_EMAIL: person.email, | ||
GIT_COMMITTER_NAME: person.name, | ||
GIT_COMMITTER_EMAIL: person.email, | ||
GIT_AUTHOR_DATE: when ?? '2026-10-01T09:00:00Z', | ||
GIT_COMMITTER_DATE: when ?? '2026-10-01T09:00:00Z', | ||
} | ||
} | ||
| ||
/// knotless-serve, from this build, over the world's index and mirrors. | ||
export const serve = async (world) => { | ||
const child = spawn( | ||
join(bin, 'knotless-serve'), | ||
['--listen', `127.0.0.1:${port}`, '--index', join(world.dir, 'index.db'), '--firehose', world.net.pds.replace('http', 'ws')], | ||
{ env: environment(world.net, world.dir, 'server'), stdio: 'inherit' }, | ||
) | ||
await until('knotless-serve', async () => (await fetch(base)).ok) | ||
return child | ||
} | ||
| ||
/// A new world: its PDS, a server, and the seeded data. Returns them with | ||
/// the pages to shoot. | ||
export async function create() { | ||
const dir = mkdtempSync(join(tmpdir(), 'knotless-shots-')) | ||
const children = [] | ||
const stop = () => { | ||
for (const child of children.reverse()) child.kill() | ||
rmSync(dir, { recursive: true, force: true }) | ||
} | ||
try { | ||
const pds = spawn('node', ['harness/pds.mjs', '--accounts', 'alice,bob'], { | ||
cwd: root, | ||
stdio: ['pipe', 'pipe', 'inherit'], | ||
}) | ||
children.push(pds) | ||
const [line] = await createInterface({ input: pds.stdout })[Symbol.asyncIterator]().next().then((r) => [r.value]) | ||
const net = JSON.parse(line) | ||
const account = Object.fromEntries(net.accounts.map((a) => [a.handle.split('.')[0], a])) | ||
| ||
const work = dir | ||
const env = (who, when) => environment(net, dir, who, when) | ||
const run = (who, cmd, args, opts = {}) => | ||
execFileSync(cmd, args, { env: env(who, opts.when), cwd: opts.cwd ?? work, input: opts.input ?? '' }) | ||
.toString() | ||
.trim() | ||
const knotless = (who, ...args) => run(who, join(bin, 'knotless'), args) | ||
const git = (who, dir, when, ...args) => run(who, 'git', args, { cwd: dir, when }) | ||
/// Copies files from a fixture step into `dir` and commits them. | ||
const commit = (who, dir, when, message, files) => { | ||
for (const [from, to] of Object.entries(files)) { | ||
mkdirSync(dirname(join(dir, to)), { recursive: true }) | ||
cpSync(join(fixture, from), join(dir, to)) | ||
} | ||
git(who, dir, when, 'add', '-A') | ||
git(who, dir, when, 'commit', '-q', '-m', message) | ||
} | ||
| ||
children.push(await serve({ net, dir })) | ||
| ||
for (const who of ['alice', 'bob']) { | ||
run(who, join(bin, 'knotless'), ['login', account[who].handle, '--password-stdin'], { input: account[who].password }) | ||
} | ||
const tally = 'knotless://alice.test/tally' | ||
knotless('alice', 'create', tally, '--description', 'Counts lines, words and bytes, with output that lines up') | ||
const a = join(work, 'alice/tally') | ||
mkdirSync(a, { recursive: true }) | ||
git('alice', a, null, 'init', '-q', '-b', 'main') | ||
commit('alice', a, '2026-10-01T09:12:00Z', 'Start tally', { | ||
'start/Cargo.toml': 'Cargo.toml', | ||
'start/.gitignore': '.gitignore', | ||
'start/README.md': 'README.md', | ||
'start/main.rs': 'src/main.rs', | ||
}) | ||
commit('alice', a, '2026-10-02T14:40:00Z', 'Count words, lines and bytes', { | ||
'count/lib.rs': 'src/lib.rs', | ||
'count/main.rs': 'src/main.rs', | ||
}) | ||
commit( | ||
'alice', | ||
a, | ||
'2026-10-03T08:05:00Z', | ||
'Read from stdin when no file is given\n\nSo tally works at the end of a pipe, as wc does. The name shown\nfor stdin is "-".', | ||
{ 'stdin/main.rs': 'src/main.rs' }, | ||
) | ||
commit('alice', a, '2026-10-05T17:21:00Z', 'Add a logo', { 'logo/logo.svg': 'assets/logo.svg' }) | ||
commit('alice', a, '2026-10-06T10:02:00Z', 'Document usage and output', { | ||
'docs/README.md': 'README.md', | ||
'docs/design.md': 'docs/design.md', | ||
}) | ||
git('alice', a, null, 'mv', 'docs/design.md', 'docs/DESIGN.md') | ||
commit('alice', a, '2026-10-08T11:47:00Z', 'Split parsing into its own module', { | ||
'parse/parse.rs': 'src/parse.rs', | ||
'parse/lib.rs': 'src/lib.rs', | ||
}) | ||
const parse = git('alice', a, null, 'rev-parse', 'HEAD') | ||
git('alice', a, null, 'push', '-q', tally, 'main') | ||
| ||
const notes = 'knotless://alice.test/notes' | ||
knotless('alice', 'create', notes, '--description', 'Scratch notes') | ||
const n = join(work, 'alice/notes') | ||
mkdirSync(n, { recursive: true }) | ||
git('alice', n, null, 'init', '-q', '-b', 'main') | ||
commit('alice', n, '2026-09-20T19:30:00Z', 'Begin', { 'notes/README.md': 'README.md' }) | ||
git('alice', n, null, 'push', '-q', notes, 'main') | ||
knotless('alice', 'create', 'knotless://alice.test/scratch') | ||
| ||
const fork = knotless('bob', 'fork', tally) | ||
const b = join(work, 'bob/tally') | ||
git('bob', work, null, 'clone', '-q', fork, b) | ||
git('bob', b, null, 'checkout', '-q', '-b', 'empty-input') | ||
commit('bob', b, '2026-10-09T09:31:00Z', 'Count no lines in empty input', { 'fix/parse.rs': 'src/parse.rs' }) | ||
git('bob', b, null, 'push', '-q', 'origin', 'empty-input') | ||
git('bob', b, null, 'checkout', '-q', '-b', 'json', 'main') | ||
commit('bob', b, '2026-10-10T16:12:00Z', 'Print counts as JSON with --json', { 'json/main.rs': 'src/main.rs' }) | ||
git('bob', b, null, 'push', '-q', 'origin', 'json') | ||
| ||
const pull = knotless( | ||
'bob', 'pull', tally, 'main', fork, 'empty-input', 'Count no lines in empty input', | ||
'-m', '`tally` printed one line for an empty file. `lines` now yields nothing when the text is empty, and a test covers it.', | ||
) | ||
knotless('alice', 'comment', tally, pull, 'Could `lines` check for emptiness once, rather than in the filter?', '--changes') | ||
knotless('bob', 'comment', tally, pull, 'It only runs per line, so the cost is one comparison each. Happy to change it if you prefer.') | ||
knotless('alice', 'comment', tally, pull, 'Fair enough. Thanks!', '--approve') | ||
// Merging and closing read the item from the index, which follows the | ||
// PDS a moment behind. | ||
await until('the pull', async () => (await page('/alice.test/tally/pulls')).includes('Count no lines in empty input')) | ||
knotless('alice', 'merge', tally, pull) | ||
const json = knotless( | ||
'bob', 'pull', tally, 'main', fork, 'json', 'Print counts as JSON with --json', | ||
'-m', 'For scripts. The object has `lines`, `words`, `bytes` and `name`.', | ||
) | ||
const utf8 = knotless( | ||
'bob', 'issue', tally, 'Fails on files that are not UTF-8', | ||
'-m', 'Running `tally image.png` panics:\n\n```\nthread \'main\' panicked at src/main.rs:8:56:\nstream did not contain valid UTF-8\n```\n\nwc counts bytes for any file.', | ||
) | ||
knotless('alice', 'comment', tally, utf8, 'Confirmed. Reading bytes and counting words lossily would fix it.') | ||
const typo = knotless('bob', 'issue', tally, 'Typo in the README') | ||
await until('the issue', async () => (await page('/alice.test/tally/issues')).includes('Typo in the README')) | ||
knotless('alice', 'close', tally, typo) | ||
| ||
// Every record a fixed createdAt, keeping their order. | ||
const records = [] | ||
for (const who of ['alice', 'bob']) { | ||
const session = await fetch(`${net.pds}/xrpc/com.atproto.server.createSession`, { | ||
method: 'POST', | ||
headers: { 'content-type': 'application/json' }, | ||
body: JSON.stringify({ identifier: account[who].handle, password: account[who].password }), | ||
}).then((r) => r.json()) | ||
const did = account[who].did | ||
if (who === 'alice') { | ||
const res = await fetch(`${net.pds}/xrpc/com.atproto.repo.putRecord`, { | ||
method: 'POST', | ||
headers: { 'content-type': 'application/json', authorization: `Bearer ${session.accessJwt}` }, | ||
body: JSON.stringify({ | ||
repo: did, | ||
collection: 'app.bsky.actor.profile', | ||
rkey: 'self', | ||
record: { | ||
$type: 'app.bsky.actor.profile', | ||
displayName: people.alice.name, | ||
description: 'Small tools for text. Writes tally.', | ||
}, | ||
}), | ||
}) | ||
if (!res.ok) throw new Error(`profile: ${await res.text()}`) | ||
} | ||
const { collections } = await fetch(`${net.pds}/xrpc/com.atproto.repo.describeRepo?repo=${did}`).then((r) => r.json()) | ||
for (const collection of collections) { | ||
let cursor = '' | ||
do { | ||
const listed = await fetch( | ||
`${net.pds}/xrpc/com.atproto.repo.listRecords?repo=${did}&collection=${collection}&limit=100${cursor}`, | ||
).then((r) => r.json()) | ||
for (const r of listed.records) { | ||
if (typeof r.value.createdAt === 'string') records.push({ ...r, did, collection, jwt: session.accessJwt }) | ||
} | ||
cursor = listed.cursor ? `&cursor=${listed.cursor}` : '' | ||
} while (cursor) | ||
} | ||
} | ||
records.sort((x, y) => (x.value.createdAt < y.value.createdAt ? -1 : x.value.createdAt > y.value.createdAt ? 1 : 0)) | ||
const start = Date.parse('2026-10-09T10:00:00Z') | ||
const cids = new Map() | ||
// Strong references to a record whose CID changed are pointed at the new one. | ||
const repoint = (v) => { | ||
if (Array.isArray(v)) return v.map(repoint) | ||
if (v && typeof v === 'object') { | ||
const out = Object.fromEntries(Object.entries(v).map(([k, x]) => [k, repoint(x)])) | ||
if (typeof out.uri === 'string' && typeof out.cid === 'string' && cids.has(out.uri)) out.cid = cids.get(out.uri) | ||
return out | ||
} | ||
return v | ||
} | ||
for (const [i, r] of records.entries()) { | ||
const value = repoint({ ...r.value, createdAt: new Date(start + i * 37 * 60_000).toISOString() }) | ||
const res = await fetch(`${net.pds}/xrpc/com.atproto.repo.putRecord`, { | ||
method: 'POST', | ||
headers: { 'content-type': 'application/json', authorization: `Bearer ${r.jwt}` }, | ||
body: JSON.stringify({ repo: r.did, collection: r.collection, rkey: r.uri.split('/').pop(), record: value }), | ||
}) | ||
if (!res.ok) throw new Error(`putRecord ${r.uri}: ${await res.text()}`) | ||
cids.set(r.uri, (await res.json()).cid) | ||
} | ||
| ||
const item = (kind, uri) => { | ||
const [, , did, , rkey] = uri.split('/') | ||
return `/alice.test/tally/${kind}/${did}/${rkey}` | ||
} | ||
const views = { | ||
home: '/', | ||
profile: '/alice.test', | ||
'profile-fork': '/bob.test', | ||
repository: '/alice.test/tally', | ||
directory: '/alice.test/tally/tree/main/src', | ||
file: '/alice.test/tally/blob/main/src/lib.rs', | ||
markdown: '/alice.test/tally/blob/main/README.md', | ||
image: '/alice.test/tally/blob/main/assets/logo.svg', | ||
commits: '/alice.test/tally/commits/main', | ||
commit: `/alice.test/tally/commit/${parse}`, | ||
pulls: '/alice.test/tally/pulls', | ||
'pulls-closed': '/alice.test/tally/pulls?state=closed', | ||
pull: item('pull', pull), | ||
'pull-open': item('pull', json), | ||
issues: '/alice.test/tally/issues', | ||
issue: item('issue', utf8), | ||
fork: '/bob.test/tally', | ||
empty: '/alice.test/scratch', | ||
missing: '/alice.test/tally/blob/main/nowhere', | ||
} | ||
// The index has caught up when no page shows today's date, every | ||
// record now being dated before it, and the merge shows. | ||
const today = new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric', timeZone: 'UTC' }) | ||
await until('the index', async () => { | ||
const pages = await Promise.all(Object.values(views).map(page)) | ||
const all = pages.join('') | ||
return !all.includes(today) && all.includes('Merged as') && all.includes('Forked from') && all.includes('1 Closed') | ||
}) | ||
return { net, dir, views, pds, server: children[1], stop } | ||
} catch (e) { | ||
stop() | ||
throw e | ||
} | ||
} | ||
| ||
if (process.argv[1] === fileURLToPath(import.meta.url)) { | ||
const world = await create() | ||
world.server.kill() | ||
mkdirSync(dirname(saved), { recursive: true }) | ||
writeFileSync(saved, JSON.stringify({ net: world.net, dir: world.dir, views: world.views }, null, 2)) | ||
console.log(`seeded; kept up for \`just shots\` until ctrl-c (${saved})`) | ||
const end = () => { | ||
rmSync(saved, { force: true }) | ||
world.stop() | ||
process.exit(0) | ||
} | ||
process.on('SIGINT', end) | ||
process.on('SIGTERM', end) | ||
process.stdin.on('end', end) | ||
process.stdin.resume() | ||
} | ||
lexicons/dev/knotless/block.jsonnew+21−0
| @@ -0,0 +1,21 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.block", | ||
"description": "An account a repository's owner keeps out, as app.bsky.graph.block blocks one.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Kept in the owner's account, keyed by a TID; one naming a private repository is in their repo inside its space. Readers honor it only from the owner, and leave out what the subject writes about the owner's repositories, or about `repo` alone.", | ||
"key": "tid", | ||
"record": { | ||
"type": "object", | ||
"required": ["subject", "createdAt"], | ||
"properties": { | ||
"subject": { "type": "string", "format": "did", "description": "The account kept out." }, | ||
"repo": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The one repository it is kept out of; all the owner's when left out." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/claim.jsonnew+25−0
| @@ -0,0 +1,25 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.claim", | ||
"description": "The move of a ref off one generation: its key is the compare-and-swap, its body the reflog entry.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Keyed `<repo key>.<ref hash>.<generation moved from>`. Two writers that read the same generation both try to create the same key, and the PDS accepts only the first; the dev.knotless.ref update travelling with it in the same applyWrites is refused along with it. Never updated.", | ||
"key": "any", | ||
"record": { | ||
"type": "object", | ||
"required": ["repo", "name", "generation", "createdAt"], | ||
"properties": { | ||
"repo": { "type": "string", "format": "tid", "description": "The `key` of the dev.knotless.repo the ref belongs to." }, | ||
"name": { "type": "string", "maxLength": 1024, "description": "The full ref name, such as refs/heads/main." }, | ||
"generation": { "type": "integer", "minimum": 0, "description": "The generation the ref moved from." }, | ||
"from": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID the ref pointed at before; absent when the ref is new." }, | ||
"to": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID the ref points at after; absent when the ref is deleted." }, | ||
"pack": { "type": "string", "format": "record-key", "description": "The record key of the dev.knotless.pack written with the move, if any." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/comment.jsonnew+24−0
| @@ -0,0 +1,24 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.comment", | ||
"description": "A comment on a pull or issue. With a verdict it is a review.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Kept in the commenter's account; for a private repository, in their repo inside the repository's space.", | ||
"key": "tid", | ||
"record": { | ||
"type": "object", | ||
"required": ["repo", "subject", "body", "createdAt"], | ||
"properties": { | ||
"repo": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The repository the pull or issue is on." }, | ||
"subject": { "type": "string", "format": "at-uri", "description": "The dev.knotless.pull or dev.knotless.issue commented on." }, | ||
"body": { "type": "string", "maxLength": 20000, "description": "Markdown." }, | ||
"verdict": { "type": "string", "maxLength": 64, "knownValues": ["approve", "changes"], "description": "Present on a review: `approve`, or `changes` to ask for changes." }, | ||
"commit": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID of the head commit a review looked at." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/createCommit.jsonnew+59−0
| @@ -0,0 +1,59 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.createCommit", | ||
"description": "Commits file changes to a branch through a gateway its owner has delegated writes to, without a clone. Commits to one branch that arrive together are built into one chain and written as one ref update. Requires a service token from the caller's PDS, audience the gateway's DID, for this method; the caller must be the owner or a collaborator.", | ||
"defs": { | ||
"main": { | ||
"type": "procedure", | ||
"description": "Fails with 409 when a change touches a path that moved since `base`, or when the branch kept moving under the batch.", | ||
"input": { | ||
"encoding": "application/json", | ||
"schema": { | ||
"type": "object", | ||
"required": ["repo", "name", "branch", "message", "changes"], | ||
"properties": { | ||
"repo": { "type": "string", "format": "did", "description": "The repository's owner." }, | ||
"name": { "type": "string", "maxLength": 100, "description": "The repository's name: its dev.knotless.repo record key." }, | ||
"branch": { "type": "string", "maxLength": 255, "description": "Without refs/heads/. Created when absent." }, | ||
"message": { "type": "string", "maxLength": 20000 }, | ||
"author": { "type": "ref", "ref": "#author" }, | ||
"base": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID of the commit the changes were made against. Without it the changes apply to whatever the branch holds." }, | ||
"changes": { "type": "array", "items": { "type": "ref", "ref": "#change" } } | ||
} | ||
} | ||
}, | ||
"output": { | ||
"encoding": "application/json", | ||
"schema": { | ||
"type": "object", | ||
"required": ["sha", "changed", "batched"], | ||
"properties": { | ||
"sha": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID of the commit, or of the branch tip when nothing changed." }, | ||
"parent": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID the commit was built on; absent for a branch's first commit." }, | ||
"changed": { "type": "boolean", "description": "False when the changes left the tree as it was, and no commit was made." }, | ||
"batched": { "type": "integer", "minimum": 0, "description": "How many commits went out in the write that carried this one." } | ||
} | ||
} | ||
} | ||
}, | ||
"author": { | ||
"type": "object", | ||
"description": "Who the commit names as its author, when not the gateway's default.", | ||
"required": ["name", "email"], | ||
"properties": { | ||
"name": { "type": "string", "maxLength": 1000 }, | ||
"email": { "type": "string", "maxLength": 1000 } | ||
} | ||
}, | ||
"change": { | ||
"type": "object", | ||
"description": "One path written or deleted.", | ||
"required": ["path"], | ||
"properties": { | ||
"path": { "type": "string", "maxLength": 4096, "description": "Relative to the repository root." }, | ||
"data": { "type": "bytes", "description": "The file's new contents; absent to delete it." }, | ||
"executable": { "type": "boolean", "default": false } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/defs.jsonnew+16−0
| @@ -0,0 +1,16 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.defs", | ||
"description": "Definitions shared by the knotless lexicons.", | ||
"defs": { | ||
"repoRef": { | ||
"type": "object", | ||
"description": "A repository, as another record names it. The URI's name is the one the repository had when the record was written; a rename changes it, and a later repository can take it. The owner in the URI and `key` stay the repository's own, and are what a reader matches on.", | ||
"required": ["uri", "key"], | ||
"properties": { | ||
"uri": { "type": "string", "format": "at-uri", "description": "The dev.knotless.repo record, as it was named when this was written." }, | ||
"key": { "type": "string", "format": "tid", "description": "The repository's `key`." } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/gate.jsonnew+26−0
| @@ -0,0 +1,26 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.gate", | ||
"description": "What a repository's owner hides of what others wrote about it, as app.bsky.feed.threadgate hides replies.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Kept in the owner's account, keyed by the repository's `key`; for a private repository, in their repo inside its space. Readers honor it only from the owner. Accounts the owner blocks (dev.knotless.block) are left out as well, without a gate.", | ||
"key": "any", | ||
"record": { | ||
"type": "object", | ||
"required": ["repo", "createdAt"], | ||
"properties": { | ||
"repo": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The repository." }, | ||
"hidden": { | ||
"type": "array", | ||
"maxLength": 1000, | ||
"items": { "type": "string", "format": "at-uri" }, | ||
"description": "Pulls, issues and comments to leave out. Comments and state changes on a hidden pull or issue go with it." | ||
}, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/getForge.jsonnew+39−0
| @@ -0,0 +1,39 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.getForge", | ||
"description": "Every pull, issue, comment and state change anyone has written about a public repository, as an appview has indexed them from the firehose.", | ||
"defs": { | ||
"main": { | ||
"type": "query", | ||
"description": "Every pull, issue, comment and state change anyone has written about a public repository, as an appview has indexed them from the firehose. No authentication; responses are not personalized. Private repositories are never indexed: their readers list the members' records in the space.", | ||
"parameters": { | ||
"type": "params", | ||
"required": ["repo", "key"], | ||
"properties": { | ||
"repo": { "type": "string", "format": "at-uri", "description": "The repository's dev.knotless.repo record; only its owner is matched." }, | ||
"key": { "type": "string", "format": "tid", "description": "The repository's `key`. Records naming it are returned whatever name their URI has, with those naming a repository it was moved from with its owner's agreement." } | ||
} | ||
}, | ||
"output": { | ||
"encoding": "application/json", | ||
"schema": { | ||
"type": "object", | ||
"required": ["entries"], | ||
"properties": { | ||
"entries": { "type": "array", "items": { "type": "ref", "ref": "#entry" } } | ||
} | ||
} | ||
} | ||
}, | ||
"entry": { | ||
"type": "object", | ||
"description": "One record, as written.", | ||
"required": ["uri", "author", "value"], | ||
"properties": { | ||
"uri": { "type": "string", "format": "at-uri", "description": "The record's URI." }, | ||
"author": { "type": "string", "format": "did", "description": "The account the record is in." }, | ||
"value": { "type": "unknown", "description": "The record verbatim: a dev.knotless.pull, issue, comment or state." } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/issue.jsonnew+22−0
| @@ -0,0 +1,22 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.issue", | ||
"description": "An issue on a repository, kept in its author's account.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "For a private repository, written in the author's repo inside the repository's space.", | ||
"key": "tid", | ||
"record": { | ||
"type": "object", | ||
"required": ["repo", "title", "createdAt"], | ||
"properties": { | ||
"repo": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The target repository." }, | ||
"title": { "type": "string", "maxLength": 3000, "maxGraphemes": 300 }, | ||
"body": { "type": "string", "maxLength": 20000, "description": "Markdown." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/listReports.jsonnew+47−0
| @@ -0,0 +1,47 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.listReports", | ||
"defs": { | ||
"main": { | ||
"type": "query", | ||
"description": "The reports a moderation service was sent (com.atproto.moderation.createReport), newest first. For its operator: takes the admin password, as Basic auth for the user 'admin'.", | ||
"parameters": { | ||
"type": "params", | ||
"properties": { | ||
"limit": { "type": "integer", "minimum": 1, "maximum": 500, "default": 50 }, | ||
"cursor": { "type": "string", "description": "A report id: those before it." } | ||
} | ||
}, | ||
"output": { | ||
"encoding": "application/json", | ||
"schema": { | ||
"type": "object", | ||
"required": ["reports"], | ||
"properties": { | ||
"cursor": { "type": "string" }, | ||
"reports": { | ||
"type": "array", | ||
"items": { "type": "ref", "ref": "#report" } | ||
} | ||
} | ||
} | ||
} | ||
}, | ||
"report": { | ||
"type": "object", | ||
"description": "As com.atproto.moderation.createReport answered it.", | ||
"required": ["id", "reasonType", "subject", "reportedBy", "createdAt"], | ||
"properties": { | ||
"id": { "type": "integer" }, | ||
"reasonType": { "type": "ref", "ref": "com.atproto.moderation.defs#reasonType" }, | ||
"reason": { "type": "string", "maxGraphemes": 2000, "maxLength": 20000 }, | ||
"subject": { | ||
"type": "union", | ||
"refs": ["com.atproto.admin.defs#repoRef", "com.atproto.repo.strongRef"] | ||
}, | ||
"reportedBy": { "type": "string", "format": "did" }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/listRepos.jsonnew+42−0
| @@ -0,0 +1,42 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.listRepos", | ||
"description": "Public repositories an appview has indexed, newest first.", | ||
"defs": { | ||
"main": { | ||
"type": "query", | ||
"description": "Public repositories an appview has indexed, newest first. No authentication; responses are not personalized.", | ||
"parameters": { | ||
"type": "params", | ||
"properties": { | ||
"q": { "type": "string", "maxLength": 100, "description": "Only repositories whose name or description holds this text, ignoring case." }, | ||
"limit": { "type": "integer", "minimum": 1, "maximum": 500, "default": 50 }, | ||
"cursor": { "type": "string", "maxLength": 512 } | ||
} | ||
}, | ||
"output": { | ||
"encoding": "application/json", | ||
"schema": { | ||
"type": "object", | ||
"required": ["repos"], | ||
"properties": { | ||
"cursor": { "type": "string", "maxLength": 512 }, | ||
"repos": { "type": "array", "items": { "type": "ref", "ref": "#repoView" } } | ||
} | ||
} | ||
} | ||
}, | ||
"repoView": { | ||
"type": "object", | ||
"description": "A repository, from its dev.knotless.repo record.", | ||
"required": ["did", "name", "createdAt"], | ||
"properties": { | ||
"did": { "type": "string", "format": "did", "description": "The repository's owner." }, | ||
"name": { "type": "string", "maxLength": 100, "description": "The repository's name: its record key." }, | ||
"description": { "type": "string", "maxGraphemes": 300, "maxLength": 3000 }, | ||
"forkOf": { "type": "string", "format": "at-uri", "description": "The dev.knotless.repo record this repository was forked from." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/pack.jsonnew+27−0
| @@ -0,0 +1,27 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.pack", | ||
"description": "A self-contained git packfile: every object reachable from `tips` and not from `bases`.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Keyed `<repo key>.<tid>`, so a newest-first listing from the cursor `<repo key>/` walks one repository's packs in push order. Never updated; compaction writes one pack superseding many and deletes them in the same applyWrites.", | ||
"key": "any", | ||
"record": { | ||
"type": "object", | ||
"required": ["repo", "tips", "size", "objects", "createdAt"], | ||
"properties": { | ||
"repo": { "type": "string", "format": "tid", "description": "The `key` of the dev.knotless.repo this pack belongs to." }, | ||
"tips": { "type": "array", "items": { "type": "string", "minLength": 40, "maxLength": 64 }, "description": "Hex object IDs the pack holds everything reachable from, short of `bases`." }, | ||
"bases": { "type": "array", "items": { "type": "string", "minLength": 40, "maxLength": 64 }, "description": "Hex object IDs of objects the pack's history needs and does not hold." }, | ||
"size": { "type": "integer", "minimum": 12, "description": "Bytes in the pack." }, | ||
"objects": { "type": "integer", "minimum": 1, "description": "Objects in the pack." }, | ||
"data": { "type": "bytes", "maxLength": 262144, "description": "The pack itself, when small. Exactly one of data and parts is present. Inline so small pushes cost no blob uploads, which hosts rate-limit far more tightly than record writes." }, | ||
"parts": { "type": "array", "items": { "type": "blob", "accept": ["application/x-git-packed-objects"] }, "description": "The pack, split over blobs in order." }, | ||
"supersedes": { "type": "array", "items": { "type": "string", "format": "record-key" }, "description": "Record keys of the dev.knotless.pack records this one replaced." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/pull.jsonnew+33−0
| @@ -0,0 +1,33 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.pull", | ||
"description": "A request to merge a branch into a repository, kept in the requester's account.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "For a private repository, written in the requester's repo inside the repository's space.", | ||
"key": "tid", | ||
"record": { | ||
"type": "object", | ||
"required": ["repo", "base", "head", "title", "createdAt"], | ||
"properties": { | ||
"repo": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The target repository." }, | ||
"base": { "type": "string", "maxLength": 255, "description": "The branch to merge into, without refs/heads/." }, | ||
"head": { "type": "ref", "ref": "#head" }, | ||
"title": { "type": "string", "maxLength": 3000, "maxGraphemes": 300 }, | ||
"body": { "type": "string", "maxLength": 20000, "description": "Markdown." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
}, | ||
"head": { | ||
"type": "object", | ||
"description": "Where the changes to merge are.", | ||
"required": ["repo", "branch"], | ||
"properties": { | ||
"repo": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The repository holding the branch: the target, or a fork." }, | ||
"branch": { "type": "string", "maxLength": 255, "description": "Without refs/heads/." } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/pushRefs.jsonnew+53−0
| @@ -0,0 +1,53 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.pushRefs", | ||
"description": "Pushes to a repository through a gateway its owner has delegated writes to: the gateway checks the caller is the owner or a collaborator, then writes the pack and ref updates as the owner. Requires a service token from the caller's PDS, audience the gateway's DID, for this method.", | ||
"defs": { | ||
"main": { | ||
"type": "procedure", | ||
"description": "The body is one line of JSON, a #header, then a git packfile holding every object the updates need that the repository lacks (empty when every update is a deletion). Each update succeeds or fails on its own.", | ||
"input": { | ||
"encoding": "application/x-knotless-push" | ||
}, | ||
"output": { | ||
"encoding": "application/json", | ||
"schema": { | ||
"type": "object", | ||
"required": ["results"], | ||
"properties": { | ||
"results": { "type": "array", "items": { "type": "ref", "ref": "#result" } } | ||
} | ||
} | ||
} | ||
}, | ||
"header": { | ||
"type": "object", | ||
"description": "What precedes the pack in the body, on a line of its own.", | ||
"required": ["repo", "name", "updates"], | ||
"properties": { | ||
"repo": { "type": "string", "format": "did", "description": "The repository's owner." }, | ||
"name": { "type": "string", "maxLength": 100, "description": "The repository's name: its dev.knotless.repo record key." }, | ||
"updates": { "type": "array", "items": { "type": "ref", "ref": "#update" } } | ||
} | ||
}, | ||
"update": { | ||
"type": "object", | ||
"description": "One ref to move.", | ||
"required": ["dst"], | ||
"properties": { | ||
"dst": { "type": "string", "maxLength": 1024, "description": "The full ref name, such as refs/heads/main." }, | ||
"src": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID to point it at; absent to delete the ref." }, | ||
"force": { "type": "boolean", "default": false, "description": "Move it even when the move is not a fast-forward." } | ||
} | ||
}, | ||
"result": { | ||
"type": "object", | ||
"description": "How one update went.", | ||
"required": ["ref"], | ||
"properties": { | ||
"ref": { "type": "string", "maxLength": 1024, "description": "The update's `dst`." }, | ||
"error": { "type": "string", "maxLength": 3000, "description": "Why the update was refused; absent when it landed." } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/putDelegation.jsonnew+25−0
| @@ -0,0 +1,25 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.putDelegation", | ||
"description": "Delegates the caller's writes to a gateway by handing it an app password, replacing any it held. The gateway logs in with it and keeps the session, to write on the caller's behalf for collaborators and batched commits. Requires a service token from the caller's PDS, audience the gateway's DID, for this method.", | ||
"defs": { | ||
"main": { | ||
"type": "procedure", | ||
"description": "Fails with 403 when the password does not log in as the caller.", | ||
"input": { | ||
"encoding": "application/json", | ||
"schema": { | ||
"type": "object", | ||
"required": ["password"], | ||
"properties": { | ||
"password": { "type": "string", "maxLength": 256, "description": "An app password for the caller's account." } | ||
} | ||
} | ||
}, | ||
"output": { | ||
"encoding": "application/json", | ||
"schema": { "type": "object", "properties": {} } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/redirect.jsonnew+20−0
| @@ -0,0 +1,20 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.redirect", | ||
"description": "Left under a repository's old name when it is renamed, so clone URLs, remotes and links that use the old name still reach it.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Keyed by the old name. A rename repoints every redirect to the repository at its new name, so each is one hop. A repository created under the name replaces the redirect.", | ||
"key": "any", | ||
"record": { | ||
"type": "object", | ||
"required": ["to", "createdAt"], | ||
"properties": { | ||
"to": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The repository under the name it has now." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/ref.jsonnew+23−0
| @@ -0,0 +1,23 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.ref", | ||
"description": "One git ref of a repository.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Keyed `<repo key>.<ref hash>`, where the hash is the first 26 characters of the lowercase unpadded base32 SHA-256 of `name`. Written only in the same applyWrites as the dev.knotless.claim for its current generation.", | ||
"key": "any", | ||
"record": { | ||
"type": "object", | ||
"required": ["repo", "name", "generation", "updatedAt"], | ||
"properties": { | ||
"repo": { "type": "string", "format": "tid", "description": "The `key` of the dev.knotless.repo this ref belongs to." }, | ||
"name": { "type": "string", "maxLength": 1024, "description": "The full ref name, such as refs/heads/main." }, | ||
"target": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID the ref points at. Absent once the ref is deleted; the record stays so its generation never repeats." }, | ||
"generation": { "type": "integer", "minimum": 1, "description": "How many times the ref has moved." }, | ||
"updatedAt": { "type": "string", "format": "datetime", "description": "When the ref last moved." } | ||
} | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/repo.jsonnew+41−0
| @@ -0,0 +1,41 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.repo", | ||
"description": "A git repository hosted in this account: in its public repo, or, for a private repository, in a dev.knotless.space space keyed by the name.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Keyed by the repository's name, so a clone URL resolves with one getRecord. `key` prefixes the record keys of everything the repository holds; a rename writes a record under the new name with the same key, deletes the old one and leaves a dev.knotless.redirect under the old name.", | ||
"key": "any", | ||
"record": { | ||
"type": "object", | ||
"required": ["key", "name", "createdAt"], | ||
"properties": { | ||
"key": { "type": "string", "format": "tid", "description": "Minted when the repository is created; never reused. The `repo` of its dev.knotless.ref, dev.knotless.claim and dev.knotless.pack records." }, | ||
"name": { "type": "string", "maxLength": 100, "description": "Equal to the record key." }, | ||
"description": { "type": "string", "maxGraphemes": 300, "maxLength": 3000 }, | ||
"defaultBranch": { "type": "string", "maxLength": 255, "description": "The branch HEAD points at, without refs/heads/." }, | ||
"collaborators": { | ||
"type": "array", | ||
"maxLength": 1000, | ||
"items": { "type": "ref", "ref": "#collaborator" }, | ||
"description": "Accounts that may push. They push through a gateway the owner has delegated writes to, which checks this list." | ||
}, | ||
"forkOf": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The repository this one was forked from." }, | ||
"gateway": { "type": "string", "format": "did", "description": "A gateway the owner has delegated writes to. Collaborators push and commit through it; it batches concurrent commits to a branch into one write." }, | ||
"transfer": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "Where the owner has offered to move the repository: a record in another account, named before it exists. Once that account holds a dev.knotless.repo there whose `transferredFrom` names this one, the repository has moved, and readers follow." }, | ||
"transferredFrom": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The repository this one was moved from. Readers count records naming that one as naming this one, if its owner agreed: its `transfer`, or the redirect left in its place, leads here." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
}, | ||
"collaborator": { | ||
"type": "object", | ||
"description": "An account that may push to the repository.", | ||
"required": ["did"], | ||
"properties": { | ||
"did": { "type": "string", "format": "did" } | ||
} | ||
} | ||
} | ||
} | ||
lexicons/dev/knotless/state.jsonnew+24−0
| @@ -0,0 +1,24 @@ | ||
{ | ||
"lexicon": 1, | ||
"id": "dev.knotless.state", | ||
"description": "A pull or issue opened, closed or merged, or a pull pointed at another target branch.", | ||
"defs": { | ||
"main": { | ||
"type": "record", | ||
"description": "Kept in the account of whoever changed it. Readers take the latest from someone allowed to make it: the subject's author, the repository's owner or a collaborator; `merged` only from the latter two.", | ||
"key": "tid", | ||
"record": { | ||
"type": "object", | ||
"required": ["repo", "subject", "state", "createdAt"], | ||
"properties": { | ||
"repo": { "type": "ref", "ref": "dev.knotless.defs#repoRef", "description": "The repository the pull or issue is on." }, | ||
"subject": { "type": "string", "format": "at-uri", "description": "The dev.knotless.pull or dev.knotless.issue whose state changed." }, | ||
"state": { "type": "string", "maxLength": 64, "knownValues": ["open", "closed", "merged"], "description": "`merged` applies to pulls only." }, | ||
"commit": { "type": "string", "minLength": 40, "maxLength": 64, "description": "The hex object ID of the merge commit, for `merged`." }, | ||
"base": { "type": "string", "maxLength": 255, "description": "For a pull, the branch it targets from here on, without refs/heads/: set when the branch it targeted was renamed. `state` repeats the pull's state." }, | ||
"createdAt": { "type": "string", "format": "datetime" } | ||
} | ||
} | ||
} | ||
} | ||
} | ||