grain.social/grain
photography-focused photo sharing platform
grain.social
1 collaborator
- 1 collaborator
0
2
The import is its own crate, and grain's crates no longer name hatk
Only what this pull adds. It is built on grain runs on a Rust server modelled on knotless, without hatk.
.knotless/workflows/rust.yml+15−2
| @@ -1,7 +1,6 @@ | ||
# The Rust workspace: formatting and clippy with warnings denied, as `just | ||
# lint` runs them; harness.yml runs the tests. Kept apart from check.yml | ||
# and test.yml, which run the SvelteKit app and hatk's server in Node, | ||
# until the move off hatk folds them together. | ||
# and test.yml, which run the Node side of the repository. | ||
# | ||
# The build, the registry and the toolchain's components are kept between | ||
# runs, so a push builds only what it changed. | ||
| @@ -12,6 +11,20 @@ on: | ||
pull: { base: [refs/heads/main, refs/heads/feature/*] } | ||
cache: [target, /usr/local/cargo/registry, /usr/local/rustup] | ||
steps: | ||
# grain-import, the one-shot cutover from the old server's database, is | ||
# the only Rust that may name it: the rest describes grain on its own | ||
# terms. The pattern is bracketed so that this file does not match it. | ||
# git grep exits 1 when it finds nothing, and more when it cannot search. | ||
- name: the old server is named only in grain-import | ||
run: | | ||
git config --global --add safe.directory "$PWD" | ||
rc=0 | ||
git grep -in 'h[a]tk' -- crates harness .knotless/workflows/rust.yml ':(exclude)crates/grain-import' || rc=$? | ||
case $rc in | ||
1) exit 0 ;; | ||
0) echo "named outside crates/grain-import, above"; exit 1 ;; | ||
*) echo "git grep could not search the checkout"; exit 2 ;; | ||
esac | ||
- name: fmt | ||
run: rustup component add rustfmt && cargo fmt --all --check | ||
- name: clippy | ||
crates/grain/src/keys.rs+1−1
| @@ -21,7 +21,7 @@ pub fn now_secs() -> u64 { | ||
| ||
/// `micros` since the epoch as RFC 3339 in UTC to the millisecond, | ||
/// `2026-10-31T12:00:00.000Z`: as JavaScript's `toISOString` writes it, so | ||
/// such times sort as text beside the ones hatk wrote. | ||
/// that times grain writes and times clients write sort together as text. | ||
pub fn iso(micros: u64) -> String { | ||
let secs = micros / 1_000_000; | ||
let millis = (micros / 1_000) % 1_000; | ||
crates/grain/src/lexicon.rs+5−9
| @@ -2,17 +2,13 @@ | ||
//! schemas of the collections grain indexes, built in, and a validator of | ||
//! a record against one. | ||
//! | ||
//! The rules are hatk's (`@bigmoves/lexicon`'s `validateRecord`), so that | ||
//! what one refused the other refuses, with three differences that follow | ||
//! the atproto specs where it did not: a string's `maxLength` counts UTF-8 | ||
//! The rules follow the atproto specs: a string's `maxLength` counts UTF-8 | ||
//! bytes, not UTF-16 units; a `datetime` with an offset is valid whatever | ||
//! day it falls on in UTC; and a DID may hold colons past its method | ||
//! (`did:web:host:path`). | ||
//! | ||
//! As there: properties a schema does not name are allowed, a union's | ||
//! value is checked against the definition its `$type` names whether or | ||
//! not the union lists it (unless it is closed), and a string format this | ||
//! does not know passes. | ||
//! (`did:web:host:path`). Properties a schema does not name are allowed, a | ||
//! union's value is checked against the definition its `$type` names | ||
//! whether or not the union lists it (unless it is closed), and a string | ||
//! format this does not know passes. | ||
| ||
use serde_json::{Map, Value}; | ||
use std::collections::HashMap; | ||
crates/grain/src/session.rs+1−1
| @@ -301,7 +301,7 @@ mod tests { | ||
assert!(matches!(broken.auth(), Auth::Bearer(_))); | ||
assert!(broken.oauth.unwrap().key().is_err()); | ||
| ||
// Kept as camelCase, as knotless's and hatk's sessions are. | ||
// Kept as camelCase, as knotless's sessions are. | ||
let kept = serde_json::to_value(&session).unwrap(); | ||
assert_eq!(kept["accessJwt"], "old"); | ||
assert!(kept["oauth"]["expiresAt"].is_u64()); | ||
crates/grain-serve/src/index/import.rs → crates/grain-import/src/lib.rsrenamed+14−9
| @@ -1,5 +1,10 @@ | ||
//! `grain-serve import`: hatk's SQLite database made a new index, once, as | ||
//! grain moves off hatk. | ||
//! `grain-import`: hatk's SQLite database made a new grain-serve index. | ||
//! | ||
//! A one-shot cutover tool, run once against production's `grain.db` as | ||
//! grain moves off hatk, and deleted once that import has run. It is the | ||
//! only code that knows the old database's layout: grain-serve's index and | ||
//! its migrations hold grain's data alone, and this crate converts hatk's | ||
//! into it. | ||
//! | ||
//! The old database is opened read-only and read in one snapshot. The index | ||
//! it is read into must be new or empty, and it is written in one | ||
| @@ -62,11 +67,11 @@ | ||
//! of what the network cannot rebuild differs by a row, nothing is written | ||
//! and the import fails. | ||
| ||
use super::records::{self, Any}; | ||
use super::{migrate, record_uri, schema}; | ||
use anyhow::{Context, bail}; | ||
use base64::Engine; | ||
use grain::lexicon::{self, Lexicons}; | ||
use grain_serve::index::records::{self, Any}; | ||
use grain_serve::index::{migrate, record_uri, schema}; | ||
use rusqlite::types::{Value as Sql, ValueRef}; | ||
use rusqlite::{Connection, OpenFlags, ToSql}; | ||
use serde_json::{Map, Value, json}; | ||
| @@ -74,11 +79,11 @@ use std::collections::BTreeSet; | ||
use std::fmt; | ||
use std::path::{Path, PathBuf}; | ||
| ||
pub const USAGE: &str = "usage: grain-serve import <hatk's grain.db> [--into <index>] [--strict] | ||
pub const USAGE: &str = "usage: grain-import <hatk's grain.db> [--into <index>] [--strict] | ||
reads hatk's database into the index at --into, or GRAIN_INDEX, which | ||
must be new or empty; --strict fails on any record that does not fit"; | ||
| ||
/// What `grain-serve import` was asked to do. | ||
/// What `grain-import` was asked to do. | ||
#[derive(Debug, Clone, PartialEq)] | ||
pub struct Options { | ||
/// hatk's database. | ||
| @@ -89,7 +94,7 @@ pub struct Options { | ||
pub strict: bool, | ||
} | ||
| ||
/// The options `args`, those after `import`, give, the index being | ||
/// The options `args` give, the index being | ||
/// `GRAIN_INDEX` (by `var`) unless `--into` names one; or the usage. | ||
pub fn parse_args( | ||
args: &[String], | ||
| @@ -115,7 +120,7 @@ pub fn parse_args( | ||
Ok(Options { from, into, strict }) | ||
} | ||
| ||
/// `grain-serve import`, of `args` after `import`: the summary on stdout, | ||
/// `grain-import`, of its arguments `args`: the summary on stdout, | ||
/// and the exit code, 0 when the index was written. | ||
pub fn main(args: &[String], var: impl Fn(&str) -> Option<String>) -> i32 { | ||
let opts = match parse_args(args, var) { | ||
| @@ -131,7 +136,7 @@ pub fn main(args: &[String], var: impl Fn(&str) -> Option<String>) -> i32 { | ||
i32::from(!summary.written) | ||
} | ||
Err(e) => { | ||
eprintln!("grain-serve import: {e:#}"); | ||
eprintln!("grain-import: {e:#}"); | ||
1 | ||
} | ||
} | ||
crates/grain-import/src/main.rsnew+9−0
| @@ -0,0 +1,9 @@ | ||
//! `grain-import <hatk's grain.db> [--into <index>] [--strict]`: hatk's | ||
//! database read into a new index, at `GRAIN_INDEX` or `--into` | ||
//! ([`grain_import`]). | ||
| ||
fn main() { | ||
let args: Vec<String> = std::env::args().skip(1).collect(); | ||
let var = |name: &str| std::env::var(name).ok(); | ||
std::process::exit(grain_import::main(&args, var)); | ||
} | ||
crates/grain-serve/src/index/import/tests.rs → crates/grain-import/src/tests.rsrenamed+3−3
| @@ -1,6 +1,6 @@ | ||
use super::*; | ||
use crate::index::Index; | ||
use crate::index::records::{Gallery, GermDeclaration, Photo, PhotoExif}; | ||
use grain_serve::index::Index; | ||
use grain_serve::index::records::{Gallery, GermDeclaration, Photo, PhotoExif}; | ||
use std::collections::HashMap; | ||
| ||
const CID: &str = "bafyreia4w4bbvc5lqzkvexqlbwbagz5yfjfzzrqhfwqkhkk2xgekh4c6ai"; | ||
| @@ -466,7 +466,7 @@ fn every_row_of_what_the_network_cannot_rebuild_is_imported() { | ||
.map(|l| l.neg) | ||
.collect(); | ||
assert_eq!(negated, [false, true]); | ||
let label = index.label(&super::super::Label { | ||
let label = index.label(&grain_serve::index::Label { | ||
src: "did:web:grain.social".into(), | ||
uri: "did:plc:erin".into(), | ||
val: "spam".into(), | ||
crates/grain-import/tests/main.rsnew+43−0
| @@ -0,0 +1,43 @@ | ||
//! The `grain-import` binary: it reads hatk's database into the index | ||
//! `GRAIN_INDEX` names, and says how it is used when it is not. | ||
| ||
use std::process::Command; | ||
| ||
#[test] | ||
fn imports_hatks_database_into_its_index() { | ||
let dir = tempfile::tempdir().unwrap(); | ||
let old = dir.path().join("grain.db"); | ||
let db = rusqlite::Connection::open(&old).unwrap(); | ||
db.execute_batch( | ||
"CREATE TABLE _repos (did TEXT PRIMARY KEY, status TEXT NOT NULL DEFAULT 'pending', | ||
handle TEXT, backfilled_at TEXT, rev TEXT, retry_count INTEGER NOT NULL DEFAULT 0, | ||
retry_after INTEGER NOT NULL DEFAULT 0); | ||
INSERT INTO _repos (did, status) VALUES ('did:plc:alice', 'takendown');", | ||
) | ||
.unwrap(); | ||
drop(db); | ||
let index = dir.path().join("index.db"); | ||
let out = Command::new(env!("CARGO_BIN_EXE_grain-import")) | ||
.arg(&old) | ||
.env("GRAIN_INDEX", &index) | ||
.output() | ||
.unwrap(); | ||
let said = String::from_utf8_lossy(&out.stdout); | ||
assert!(out.status.success(), "{said}"); | ||
assert!( | ||
said.contains("takedown (_repos, status takendown)"), | ||
"{said}" | ||
); | ||
assert!(said.ends_with("written\n"), "{said}"); | ||
let held = rusqlite::Connection::open(&index).unwrap(); | ||
let did: String = held | ||
.query_row("SELECT did FROM takedown", [], |r| r.get(0)) | ||
.unwrap(); | ||
assert_eq!(did, "did:plc:alice"); | ||
| ||
let usage = Command::new(env!("CARGO_BIN_EXE_grain-import")) | ||
.output() | ||
.unwrap(); | ||
assert_eq!(usage.status.code(), Some(2)); | ||
assert!(String::from_utf8_lossy(&usage.stderr).starts_with("usage: grain-import")); | ||
} | ||
crates/grain-import/Cargo.tomlnew+18−0
| @@ -0,0 +1,18 @@ | ||
[package] | ||
name = "grain-import" | ||
description = "A one-shot cutover tool: hatk's grain.db read into a new grain-serve index" | ||
version.workspace = true | ||
edition.workspace = true | ||
license.workspace = true | ||
rust-version.workspace = true | ||
| ||
[dependencies] | ||
grain.workspace = true | ||
grain-serve.workspace = true | ||
anyhow.workspace = true | ||
base64.workspace = true | ||
rusqlite.workspace = true | ||
serde_json.workspace = true | ||
| ||
[dev-dependencies] | ||
tempfile.workspace = true | ||
crates/grain-serve/src/index/mod.rs+1−2
| @@ -14,7 +14,6 @@ | ||
//! leaves alone what the index has from a later one ([`Applied::Stale`]). | ||
| ||
pub mod backfill; | ||
pub mod import; | ||
#[cfg(test)] | ||
pub mod plan; | ||
pub mod records; | ||
| @@ -131,7 +130,7 @@ pub fn record_uri(uri: &str) -> Option<(&str, &str, &str)> { | ||
/// Brings `db`'s tables to the last of `steps`, running each it has not | ||
/// run, each in a transaction of its own. A database further on than | ||
/// `steps` is from a newer grain, and is refused rather than read. | ||
fn migrate(db: &mut Connection, steps: &[&str]) -> anyhow::Result<()> { | ||
pub fn migrate(db: &mut Connection, steps: &[&str]) -> anyhow::Result<()> { | ||
let at: i64 = db.query_row("PRAGMA user_version", [], |r| r.get(0))?; | ||
let at = at as usize; | ||
if at > steps.len() { | ||