~ knotless Search/
↑↓ move↵ open⇧↵ all results

grain.social/grain
Switch repository
Loading…

photography-focused photo sharing platform

grain.social
1 collaborator
0
2

ci: The harness check keeps passing runs and fails network scans

Merged chadtmiller.com asked to merge into feature/rust from grain.social/grain:ci/harness-speed

Only what this pull adds. It is built on grain runs on a Rust server modelled on knotless, without hatk.

.knotless/workflows/e2e.yml+9−20
View file
@@ -55,32 +55,21 @@ cache:
harness/node_modules,
]
env:
NODE_VERSION: 24.22.1
PLAYWRIGHT_BROWSERS_PATH: /opt/browsers
# The spaces alpha the spaces flows run over, as harness.yml pins it:
# the two move together.
ATPROTO_COMMIT: 11c5d98cd50261c5e1201fedc5c6c05a7b8d1ccf
ALPHA_BUILD: "1"
steps:
- name: node
run: |
[ "$(/opt/node/bin/node --version 2>/dev/null)" = "v$NODE_VERSION" ] && exit 0
mkdir -p /opt/node && find /opt/node -mindepth 1 -delete
curl -fsSL "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" | tar -xJ -C /opt/node --strip-components 1
/opt/node/bin/node --version
- name: web packages
run: |
lock=$(sha256sum package-lock.json)
[ "$(cat node_modules/.lock 2>/dev/null)" = "$lock" ] && exit 0
PATH=/opt/node/bin:$PATH npm ci --no-audit --no-fund
echo "$lock" > node_modules/.lock
- name: harness
run: |
cd harness
lock=$(sha256sum package-lock.json)
[ "$(cat node_modules/.lock 2>/dev/null)" = "$lock" ] && exit 0
PATH=/opt/node/bin:$PATH npm ci --no-audit --no-fund
echo "$lock" > node_modules/.lock
# Node into /opt/node, and the web's and the harness's packages when
# their lockfiles change, each kept in the cache.
- uses: chadtmiller.com/steps/node@1
with: { version: 24.22.1 }
- uses: chadtmiller.com/steps/npm-ci@1
name: web packages
- uses: chadtmiller.com/steps/npm-ci@1
name: harness
with: { dir: harness }
- name: chromium
run: |
export PATH=/opt/node/bin:$PATH
.knotless/workflows/harness.yml+31−26
View file
@@ -3,7 +3,9 @@
# against @atproto/pds and a PLC directory from harness/pds.mjs, failing
# under harness/cov.json's lines or functions. Then that every XRPC answer
# grain-serve's tests received is as its lexicon says (harness/answers.mjs),
# and that the check would name a wrong one (harness/answers-test.sh).
# and that the check would name a wrong one (harness/answers-test.sh); and
# that no query they ran scans the network at prod's size
# (harness/plans.mjs).
#
# The test binaries named `spaces` run against the reference PDS on the
# permissioned spaces alpha, built from an atproto commit, as knotless's
@@ -12,7 +14,9 @@
#
# Node, the harness's packages, the alpha's build, cargo-llvm-cov and the
# build are kept between runs; each is made again only when what it is
# made from changes.
# made from changes. So are the test runs that passed, in /opt/testcache:
# a test binary the same byte for byte as one that passed, against the
# same harness, is not run again, and its profile counts (harness/cov.mjs).
#
# At cutover this becomes test.yml, as knotless's is, over the Node one,
# and rust.yml check.yml (see there).
@@ -34,28 +38,23 @@ cache:
/opt/tools,
/opt/atproto,
harness/node_modules,
/opt/testcache,
]
env:
NODE_VERSION: 24.22.1
LLVM_COV_VERSION: 0.9.1
# The spaces alpha the tests of spaces run against, as knotless's do.
ATPROTO_COMMIT: 11c5d98cd50261c5e1201fedc5c6c05a7b8d1ccf
# Bumped when the steps that build the alpha change, to build it again.
ALPHA_BUILD: "1"
steps:
- name: node
run: |
[ "$(/opt/node/bin/node --version 2>/dev/null)" = "v$NODE_VERSION" ] && exit 0
mkdir -p /opt/node && find /opt/node -mindepth 1 -delete
curl -fsSL "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" | tar -xJ -C /opt/node --strip-components 1
/opt/node/bin/node --version
- name: harness
run: |
cd harness
lock=$(sha256sum package-lock.json)
[ "$(cat node_modules/.lock 2>/dev/null)" = "$lock" ] && exit 0
PATH=/opt/node/bin:$PATH npm ci --no-audit --no-fund
echo "$lock" > node_modules/.lock
# Node into /opt/node, the harness's packages when their lockfile
# changes, and cargo-llvm-cov into /opt/tools, each kept in the cache.
- uses: chadtmiller.com/steps/node@1
with: { version: 24.22.1 }
- uses: chadtmiller.com/steps/npm-ci@1
name: harness
with: { dir: harness, omit: dev }
- uses: chadtmiller.com/steps/cargo-llvm-cov@1
with: { version: 0.9.1 }
- name: spaces alpha
run: |
[ "$(cat /opt/atproto/.built 2>/dev/null)" = "$ATPROTO_COMMIT $ALPHA_BUILD" ] && exit 0
@@ -80,20 +79,26 @@ steps:
cd "$(dirname "$(node -p "require.resolve('better-sqlite3/package.json', { paths: ['/opt/atproto/packages/pds'] })")")"
[ -f build/Release/better_sqlite3.node ] && exit 0
npm run install
- name: tools
run: |
rustup component add llvm-tools-preview
[ "$(cat /opt/tools/.llvm-cov 2>/dev/null)" = "$LLVM_COV_VERSION" ] && exit 0
mkdir -p /opt/tools
curl -fsSL "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v$LLVM_COV_VERSION/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz" | tar -xz -C /opt/tools
echo "$LLVM_COV_VERSION" > /opt/tools/.llvm-cov
- name: test
run: |
export PATH=/opt/node/bin:/opt/tools:$PATH GRAIN_DEV_ENV=/opt/atproto/packages/dev-env
COV_JOBS=$(( $(nproc) / 2 )) COV_HARNESSES=2 COV_TOOLCHAIN=$(rustup show active-toolchain | cut -d' ' -f1) \
export PATH=/opt/node/bin:$PATH GRAIN_DEV_ENV=/opt/atproto/packages/dev-env
# Half the host's cores, as two test runs go side by side, over two
# stock PDSes, as one PDS is one node process, and the alpha beside
# them, all at once: each binary against the stock PDS but those
# named `spaces`, which go against the alpha (harness/cov.mjs).
COV_STORE=/opt/testcache COV_JOBS=$(( $(nproc) / 2 )) COV_HARNESSES=2 COV_TOOLCHAIN=$(rustup show active-toolchain | cut -d' ' -f1) \
node harness/cov.mjs
- name: answers
run: |
# Every XRPC answer grain-serve's tests received, fresh or kept with
# a stored run, as its lexicon says.
export PATH=/opt/node/bin:$PATH
harness/answers-test.sh
node harness/answers.mjs target/llvm-cov-target/cov-report/answers
- name: plans
run: |
# No query the tests ran scans a table that grows with the network,
# or sorts many of its rows, as SQLite would plan it at prod's size,
# unless its SQL says so.
export PATH=/opt/node/bin:$PATH
node harness/plans.mjs target/llvm-cov-target/cov-report/plans
.knotless/workflows/rust.yml+2−4
View file
@@ -34,7 +34,5 @@ steps:
0) echo "named outside crates/grain-import, above"; exit 1 ;;
*) echo "git grep could not search the checkout"; exit 2 ;;
esac
- name: fmt
run: rustup component add rustfmt && cargo fmt --all --check
- name: clippy
run: rustup component add clippy && cargo clippy --workspace --all-targets --locked -- -D warnings
# cargo fmt --check, then clippy over the workspace, warnings denied.
- uses: chadtmiller.com/steps/rust-lint@1
.knotless/uses.lock+12−0
View file
@@ -1,4 +1,12 @@
# Written by `knotless uses update`. Commit it: runs read it.
chadtmiller.com/steps/cargo-llvm-cov@1:
version: 1.0.0
commit: 686cdbb38978730bb1ba5d1c999967e48497ce15
release: at://did:plc:bcgltzqazw5tb6k2g3ttenbj/dev.knotless.release/3mza7iq7tundv.tbnmqntp7ufhx7yd6i3rdqewys.sjjb7q6l3fsl3spvqsuzdoe73w
chadtmiller.com/steps/node@1:
version: 1.0.0
commit: 686cdbb38978730bb1ba5d1c999967e48497ce15
release: at://did:plc:bcgltzqazw5tb6k2g3ttenbj/dev.knotless.release/3mza7iq7tundv.qespbvwvof6xwjdaedpzq3koia.sjjb7q6l3fsl3spvqsuzdoe73w
chadtmiller.com/steps/npm-ci@1:
version: 1.0.0
commit: 686cdbb38978730bb1ba5d1c999967e48497ce15
@@ -7,6 +15,10 @@ chadtmiller.com/steps/publish-lexicons@1:
version: 1.0.0
commit: cc436177fab7461a33e84ec8876a2d66656a5c09
release: at://did:plc:bcgltzqazw5tb6k2g3ttenbj/dev.knotless.release/3mza7iq7tundv.ez5tgppxbpaxxrztjj3cnlyyhc.sjjb7q6l3fsl3spvqsuzdoe73w
chadtmiller.com/steps/rust-lint@1:
version: 1.0.0
commit: 686cdbb38978730bb1ba5d1c999967e48497ce15
release: at://did:plc:bcgltzqazw5tb6k2g3ttenbj/dev.knotless.release/3mza7iq7tundv.zvxvlxa6fe7s4pnwsnmgmm63aj.sjjb7q6l3fsl3spvqsuzdoe73w
chadtmiller.com/steps/wait-for-checks@2:
version: 2.0.0
commit: d5b057a2314833fdee9f600c7872db2916b0d0aa
crates/grain-import/src/tests.rs+23−5
View file
@@ -727,7 +727,12 @@ fn every_record_is_imported_as_it_was_indexed() {
assert_eq!(index.self_labels(uri).unwrap(), ["gore", "nudity"]);
let story = "at://did:plc:bob/social.grain.story/3l3qo2vuowo2a";
assert_eq!(index.self_labels(story).unwrap(), ["spam"]);
let generated = index.read(|db| dump(db, "SELECT h3, sort_at FROM gallery"));
let generated = index.read(|db| {
dump(
db,
"SELECT h3, sort_at FROM gallery -- scans: gallery, a test's own",
)
});
assert_eq!(generated, ["8a28a98a4c67fff|2025-01-01T00:00:00.000Z"]);
let photo = index
@@ -748,14 +753,24 @@ fn every_record_is_imported_as_it_was_indexed() {
let exif = exif.record;
assert_eq!((exif.iso, exif.f_number), (Some(400), Some(2_800_000)));
assert_eq!(exif.make.as_deref(), Some("RICOH"));
let item = index.read(|db| dump(db, "SELECT position FROM gallery_item ORDER BY uri"));
let item = index.read(|db| {
dump(
db,
"SELECT position FROM gallery_item ORDER BY uri -- scans: gallery_item, a test's own",
)
});
assert_eq!(item, ["2", "0"]);
let group = index.read(|db| dump(db, "SELECT group_did FROM group_submission"));
let group = index.read(|db| {
dump(
db,
"SELECT group_did FROM group_submission -- scans: group_submission, a test's own",
)
});
assert_eq!(group, ["did:plc:group"]);
let embed = index.read(|db| {
dump(
db,
"SELECT json_extract(embed, '$.external.title') FROM bsky_post",
"SELECT json_extract(embed, '$.external.title') FROM bsky_post -- scans: bsky_post, a test's own",
)
});
assert_eq!(embed, ["Fog"]);
@@ -816,7 +831,10 @@ fn strict_refuses_an_import_with_a_record_refused() {
.unwrap()
.query_row(sql, [], |r| r.get::<_, i64>(0))
};
assert_eq!(held("SELECT count(*) FROM gallery").unwrap(), 0);
assert_eq!(
held("SELECT count(*) FROM gallery -- scans: gallery, a test's own").unwrap(),
0
);
assert_eq!(held("SELECT count(*) FROM label").unwrap(), 0);
// What is left is empty, and taken again.
assert!(import(&old, &into, false, URL).unwrap().written);
crates/grain-serve/src/index/mod.rs+4−2
View file
@@ -19,6 +19,7 @@ pub mod graph;
pub mod moderation;
#[cfg(test)]
pub mod plan;
pub mod plans;
pub mod preferences;
pub mod push;
pub mod records;
@@ -233,7 +234,7 @@ impl Index {
/// That the index answers a read.
pub fn answers(&self) -> rusqlite::Result<()> {
let sql = "SELECT EXISTS (SELECT 1 FROM repo)";
let sql = "SELECT EXISTS (SELECT 1 FROM repo -- scans: repo, to its first row alone\n)";
self.read(|db| db.query_row(sql, [], |r| r.get::<_, bool>(0)).map(drop))
}
@@ -248,7 +249,8 @@ impl Index {
db.execute_batch("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;")?;
migrate(&mut db, steps)?;
let tracked = {
let mut stmt = db.prepare("SELECT did FROM repo")?;
let sql = "SELECT did FROM repo -- scans: repo, as the index opens, for whom it tracks";
let mut stmt = db.prepare(sql)?;
let dids = stmt.query_map([], |r| r.get(0))?;
dids.collect::<rusqlite::Result<HashSet<String>>>()?
};
crates/grain-serve/src/index/moderation.rs+4−2
View file
@@ -194,7 +194,8 @@ impl Index {
.collect::<rusqlite::Result<_>>()?;
let total = db.query_row(
"SELECT count(*) FROM report WHERE (?1 IS NULL OR status = ?1)
AND (?2 IS NULL OR label = ?2)",
AND (?2 IS NULL OR label = ?2)
-- scans: report, as only moderators ask, and reports are few beside records",
params![filter.status, filter.label],
|r| r.get(0),
)?;
@@ -252,7 +253,8 @@ impl Index {
let db = self.db.lock().unwrap();
let sql = format!(
"SELECT l.src, l.uri, l.val, l.neg, l.cts, l.exp FROM label l
WHERE {} AND {} ORDER BY l.id",
WHERE {} AND {} ORDER BY l.id
-- sorts: label, an account's own, by the range of its URIs",
of_account("l.uri", "?1"),
standing("l", "?2")
);
crates/grain-serve/src/index/plan.rs+1−44
View file
@@ -7,50 +7,7 @@ use std::collections::{HashMap, HashSet};
/// Each table said to hold a million rows, and each index's first column
/// to pick out a hundred, so SQLite plans as it would on prod's.
pub fn sized(db: &Connection) {
db.execute_batch("ANALYZE; DELETE FROM sqlite_stat1;")
.unwrap();
let names = |sql: &str, arg: &str| -> Vec<(String, bool)> {
let mut stmt = db.prepare(sql).unwrap();
let rows = stmt
.query_map([arg], |r| Ok((r.get(0)?, r.get(1)?)))
.unwrap();
rows.collect::<rusqlite::Result<_>>().unwrap()
};
let tables = names(
"SELECT name, 0 FROM sqlite_master WHERE type = 'table' AND name NOT LIKE ?1",
"sqlite_%",
);
for (table, _) in tables {
db.execute(
"INSERT INTO sqlite_stat1 VALUES (?1, NULL, '1000000')",
[&table],
)
.unwrap();
for (index, unique) in names("SELECT name, \"unique\" FROM pragma_index_list(?1)", &table) {
let columns: u32 = db
.query_row(
"SELECT count(*) FROM pragma_index_info(?1)",
[&index],
|r| r.get(0),
)
.unwrap();
let mut stat = "1000000".to_string();
for i in 0..columns {
let picked = if unique && i + 1 == columns {
1
} else {
(100 / 10u64.pow(i)).max(2)
};
stat += &format!(" {picked}");
}
db.execute(
"INSERT INTO sqlite_stat1 VALUES (?1, ?2, ?3)",
[&table, &index, &stat],
)
.unwrap();
}
}
db.execute_batch("ANALYZE sqlite_schema;").unwrap();
super::plans::sized(db, |_| super::plans::MANY).unwrap();
}
/// The tables `sql`'s plan reads whole, rather than by an index: what no
crates/grain-serve/src/index/plans.rsnew+595−0
View file
595 lines changed: not loaded until asked for.
crates/grain-serve/src/index/schema.rs+10−2
View file
@@ -360,12 +360,18 @@ CREATE TRIGGER repo_search_handle AFTER UPDATE OF handle ON repo BEGIN
(SELECT d.id FROM gallery g JOIN search_doc d ON d.uri = g.uri WHERE g.did = NEW.did);
END;
INSERT INTO search_doc (uri) SELECT uri FROM profile;
INSERT INTO search_doc (uri) SELECT uri FROM gallery;
INSERT INTO search_doc (uri)
-- scans: profile, as this version fills it once from what the index holds
SELECT uri FROM profile;
INSERT INTO search_doc (uri)
-- scans: gallery, as this version fills it once from what the index holds
SELECT uri FROM gallery;
INSERT INTO profile_search (rowid, handle, display_name, description)
-- scans: profile, as this version fills it once from what the index holds
SELECT d.id, r.handle, p.display_name, p.description
FROM profile p JOIN search_doc d ON d.uri = p.uri LEFT JOIN repo r ON r.did = p.did;
INSERT INTO gallery_search (rowid, handle, title, description, place)
-- scans: gallery, as this version fills it once from what the index holds
SELECT d.id, r.handle, g.title, g.description,
concat_ws(' ', json_extract(g.location, '$.name'), json_extract(g.address, '$.locality'),
json_extract(g.address, '$.region'), json_extract(g.address, '$.country'))
@@ -426,11 +432,13 @@ CREATE TRIGGER gallery_mention_delete AFTER DELETE ON gallery BEGIN
END;
INSERT OR IGNORE INTO mention (did, uri)
-- scans: comment, as this version fills it once from what the index holds
SELECT json_extract(c.facets, t.path || '.did'), c.uri
FROM comment c, json_tree(CASE WHEN json_valid(c.facets) THEN c.facets END) t
WHERE t.key = '$type' AND t.atom = 'app.bsky.richtext.facet#mention'
AND json_type(c.facets, t.path || '.did') = 'text';
INSERT OR IGNORE INTO mention (did, uri)
-- scans: gallery, as this version fills it once from what the index holds
SELECT json_extract(g.facets, t.path || '.did'), g.uri
FROM gallery g, json_tree(CASE WHEN json_valid(g.facets) THEN g.facets END) t
WHERE t.key = '$type' AND t.atom = 'app.bsky.richtext.facet#mention'
crates/grain-serve/src/index/sessions.rs+4−1
View file
@@ -194,7 +194,10 @@ mod tests {
.db
.lock()
.unwrap()
.execute("UPDATE web_session SET session = '{'", [])
.execute(
"UPDATE web_session SET session = '{' -- scans: web_session, a test's own",
[],
)
.unwrap();
assert_eq!(index.web_session("did:plc:a").unwrap(), None);
}
crates/grain-serve/src/index/stories.rs+8−2
View file
crates/grain-serve/src/index/tests.rs+34−13
View file
crates/grain-serve/src/read/actor.rs+3−1
View file
crates/grain-serve/src/read/camera.rs+2−2
View file
crates/grain-serve/src/read/foryou.rs+4−2
View file
crates/grain-serve/src/read/location.rs+2−1
View file
crates/grain-serve/src/read/sitemap.rs+7−3
View file
crates/grain-serve/src/spaces/group.rs+3−1
View file
crates/grain-serve/src/telemetry.rs+11−2
View file
crates/grain-serve/tests/classify.rs+7−2
View file
crates/grain-serve/tests/stories.rs+6−2
View file
harness/cov.mjs+414−106
View file
520 lines changed: not loaded until asked for.
harness/plans.mjsnew+56−0
View file
harness/quarantine.jsonnew+1−0
View file
AGENTS.md+26−2
View file
Justfile+5−0
View file