~ knotless Search/
↑↓ move↵ open⇧↵ all results

grain.social/grain
Switch repository
Loading…

photography-focused photo sharing platform

grain.social
1 collaborator
0
2

grain-serve searches, suggests follows and lists cameras and places

Merged chadtmiller.com asked to merge into feature/rust from grain.social/grain:read/search

Only what this pull adds. It is built on grain runs on a Rust server modelled on knotless, without hatk.

crates/grain-serve/src/index/import.rs+12−2
View file
@@ -434,16 +434,26 @@ fn names(db: &Connection) -> rusqlite::Result<BTreeSet<String>> {
names.collect()
}
/// The tables of `db` that hold rows of their own: not a virtual table,
/// such as full-text search's, nor the ones SQLite keeps it in.
fn own(db: &Connection) -> rusqlite::Result<BTreeSet<String>> {
let sql = "SELECT name FROM pragma_table_list
WHERE schema = 'main' AND type = 'table' AND name NOT LIKE 'sqlite_%'";
let mut stmt = db.prepare(sql)?;
let names = stmt.query_map([], |r| r.get(0))?;
names.collect()
}
/// The index at `into`, made, or migrated if it is one already, so long as
/// it holds nothing; set for one large write.
fn empty_index(into: &Path) -> anyhow::Result<Connection> {
let mut db = Connection::open(into).with_context(|| format!("opening {}", into.display()))?;
let version: i64 = db.query_row("PRAGMA user_version", [], |r| r.get(0))?;
if let Some(table) = names(&db)?.first().filter(|_| version == 0) {
if let Some(table) = own(&db)?.first().filter(|_| version == 0) {
bail!("{} is not an index: it has {table}", into.display());
}
migrate(&mut db, schema::MIGRATIONS)?;
for table in names(&db)? {
for table in own(&db)? {
if db
.prepare(&format!("SELECT 1 FROM \"{table}\""))?
.exists([])?
crates/grain-serve/src/index/plan.rs+6−2
View file
@@ -54,7 +54,8 @@ pub fn sized(db: &Connection) {
}
/// The tables `sql`'s plan reads whole, rather than by an index: what no
/// query of a table that grows with the network may do.
/// query of a table that grows with the network may do. A full-text
/// search reads its table through the match, which is its index.
pub fn scans(db: &Connection, sql: &str) -> Vec<String> {
let words: Vec<&str> = sql
.split(|c: char| c.is_whitespace() || "(),".contains(c))
@@ -92,7 +93,10 @@ pub fn scans(db: &Connection, sql: &str) -> Vec<String> {
};
let name = rest.split_whitespace().next().unwrap_or_default();
let table = aliases.get(name).copied().unwrap_or(name);
tables.contains(table) && !rest.contains(" USING ")
// A virtual table read through a constraint, as full-text
// search reads one by MATCH, is read by its index.
let constrained = rest.contains(" VIRTUAL TABLE INDEX ") && !rest.ends_with(':');
tables.contains(table) && !rest.contains(" USING ") && !constrained
})
.collect()
}
crates/grain-serve/src/index/schema.rs+84−1
View file
@@ -29,6 +29,17 @@
//! is where each stream was read to, by source (`jetstream:<url>`, as
//! hatk's `_cursor` keys it).
//!
//! # Search
//!
//! `profile_search` and `gallery_search` are SQLite FTS5 tables of the
//! words a profile or gallery is found by: its author's handle, a
//! profile's name and description, a gallery's title, description and
//! place. Porter-stemmed and without diacritics, as hatk's `_fts_*`
//! tables were. Triggers on `profile`, `gallery` and `repo` keep them as
//! those tables are; `search_doc` gives each record the rowid it has in
//! them, since a record's own rowid may change on a `VACUUM`. They are
//! made from the records, so rebuilding these rebuilds them.
//!
//! `record_rev` is the revision each record was last written or deleted
//! at (`gone`), for backfill to leave alone what is newer than the repo it
//! read: a write the stream applied after the repo was exported, or a
@@ -60,7 +71,7 @@
/// Each step of the schema, run once, in order, by `PRAGMA user_version`.
/// A step is only ever appended: one that has run is never changed.
pub const MIGRATIONS: &[&str] = &[V1, V2, V3];
pub const MIGRATIONS: &[&str] = &[V1, V2, V3, V4];
const V1: &str = "
CREATE TABLE gallery (
@@ -248,6 +259,78 @@ CREATE INDEX follow_did_created ON follow (did, created_at);
CREATE INDEX favorite_subject_created ON favorite (subject, created_at);
";
/// Full-text search of profiles and galleries, kept by triggers, and
/// filled from what the index holds.
const V4: &str = "
CREATE TABLE search_doc (id INTEGER PRIMARY KEY, uri TEXT NOT NULL UNIQUE);
CREATE VIRTUAL TABLE profile_search USING fts5(
handle, display_name, description, tokenize = 'porter unicode61 remove_diacritics 2');
CREATE VIRTUAL TABLE gallery_search USING fts5(
handle, title, description, place, tokenize = 'porter unicode61 remove_diacritics 2');
CREATE TRIGGER profile_search_insert AFTER INSERT ON profile BEGIN
INSERT OR IGNORE INTO search_doc (uri) VALUES (NEW.uri);
INSERT INTO profile_search (rowid, handle, display_name, description)
SELECT d.id, (SELECT handle FROM repo WHERE did = NEW.did), NEW.display_name, NEW.description
FROM search_doc d WHERE d.uri = NEW.uri;
END;
CREATE TRIGGER profile_search_update AFTER UPDATE ON profile BEGIN
DELETE FROM profile_search WHERE rowid = (SELECT id FROM search_doc WHERE uri = OLD.uri);
INSERT INTO profile_search (rowid, handle, display_name, description)
SELECT d.id, (SELECT handle FROM repo WHERE did = NEW.did), NEW.display_name, NEW.description
FROM search_doc d WHERE d.uri = NEW.uri;
END;
CREATE TRIGGER profile_search_delete AFTER DELETE ON profile BEGIN
DELETE FROM profile_search WHERE rowid = (SELECT id FROM search_doc WHERE uri = OLD.uri);
DELETE FROM search_doc WHERE uri = OLD.uri;
END;
CREATE TRIGGER gallery_search_insert AFTER INSERT ON gallery BEGIN
INSERT OR IGNORE INTO search_doc (uri) VALUES (NEW.uri);
INSERT INTO gallery_search (rowid, handle, title, description, place)
SELECT d.id, (SELECT handle FROM repo WHERE did = NEW.did), NEW.title, NEW.description,
concat_ws(' ', json_extract(NEW.location, '$.name'), json_extract(NEW.address, '$.locality'),
json_extract(NEW.address, '$.region'), json_extract(NEW.address, '$.country'))
FROM search_doc d WHERE d.uri = NEW.uri;
END;
CREATE TRIGGER gallery_search_update AFTER UPDATE ON gallery BEGIN
DELETE FROM gallery_search WHERE rowid = (SELECT id FROM search_doc WHERE uri = OLD.uri);
INSERT INTO gallery_search (rowid, handle, title, description, place)
SELECT d.id, (SELECT handle FROM repo WHERE did = NEW.did), NEW.title, NEW.description,
concat_ws(' ', json_extract(NEW.location, '$.name'), json_extract(NEW.address, '$.locality'),
json_extract(NEW.address, '$.region'), json_extract(NEW.address, '$.country'))
FROM search_doc d WHERE d.uri = NEW.uri;
END;
CREATE TRIGGER gallery_search_delete AFTER DELETE ON gallery BEGIN
DELETE FROM gallery_search WHERE rowid = (SELECT id FROM search_doc WHERE uri = OLD.uri);
DELETE FROM search_doc WHERE uri = OLD.uri;
END;
CREATE TRIGGER repo_search_insert AFTER INSERT ON repo BEGIN
UPDATE profile_search SET handle = NEW.handle WHERE rowid IN
(SELECT d.id FROM profile p JOIN search_doc d ON d.uri = p.uri WHERE p.did = NEW.did);
UPDATE gallery_search SET handle = NEW.handle WHERE rowid IN
(SELECT d.id FROM gallery g JOIN search_doc d ON d.uri = g.uri WHERE g.did = NEW.did);
END;
CREATE TRIGGER repo_search_handle AFTER UPDATE OF handle ON repo BEGIN
UPDATE profile_search SET handle = NEW.handle WHERE rowid IN
(SELECT d.id FROM profile p JOIN search_doc d ON d.uri = p.uri WHERE p.did = NEW.did);
UPDATE gallery_search SET handle = NEW.handle WHERE rowid IN
(SELECT d.id FROM gallery g JOIN search_doc d ON d.uri = g.uri WHERE g.did = NEW.did);
END;
INSERT INTO search_doc (uri) SELECT uri FROM profile;
INSERT INTO search_doc (uri) SELECT uri FROM gallery;
INSERT INTO profile_search (rowid, handle, display_name, description)
SELECT d.id, r.handle, p.display_name, p.description
FROM profile p JOIN search_doc d ON d.uri = p.uri LEFT JOIN repo r ON r.did = p.did;
INSERT INTO gallery_search (rowid, handle, title, description, place)
SELECT d.id, r.handle, g.title, g.description,
concat_ws(' ', json_extract(g.location, '$.name'), json_extract(g.address, '$.locality'),
json_extract(g.address, '$.region'), json_extract(g.address, '$.country'))
FROM gallery g JOIN search_doc d ON d.uri = g.uri LEFT JOIN repo r ON r.did = g.did;
";
/// The label values that leave what carries them out of feeds: the
/// imperative ones, and those whose default is to hide.
pub const HIDDEN_LABELS: [&str; 9] = [
crates/grain-serve/src/index/tests.rs+0−1
View file
@@ -454,7 +454,6 @@ fn the_queries_still_to_port_read_by_index() {
// written, by `read::tests::the_reads_read_by_index`; these are the
// ones of what is still to be ported, as they will read.
let queries = [
"SELECT f.subject FROM bsky_follow f JOIN profile p ON p.did = f.subject WHERE f.did = ?1 LIMIT 10",
"SELECT f.did FROM follow f WHERE f.subject = ?1 ORDER BY f.did LIMIT 50",
"SELECT count(*) FROM group_submission s WHERE s.group_did = ?1
AND NOT EXISTS (SELECT 1 FROM group_item i WHERE i.did = s.group_did AND i.gallery = s.gallery)
crates/grain-serve/src/read/actor.rs+94−4
View file
@@ -1,9 +1,9 @@
//! `social.grain.actor.*`: an account's profile, and the galleries it
//! favorited.
//! `social.grain.actor.*`: an account's profile, the galleries it
//! favorited, and who it might follow.
use super::moderation::{shown, unhidden};
use super::moderation::{shown, unblocked, unhidden, unmuted};
use super::views::put;
use super::{Params, Reader, page};
use super::{Params, Reader, list, page};
use crate::cdn::AVATAR;
use crate::index::records::Json;
use crate::xrpc::Failure;
@@ -170,6 +170,95 @@ pub fn get_favorites(r: &Reader, p: &Params) -> Result<Value, Failure> {
Ok(answer)
}
/// SQL that holds where `did` is one `?1` might be shown to follow: shown,
/// not followed by `?1` yet, and neither blocking, blocked by nor muted
/// by them.
fn followable(did: &str) -> String {
format!(
"{} AND {did} NOT IN (SELECT subject FROM follow WHERE did = ?1) AND {} AND {}",
shown(did),
unblocked(did, "?1"),
unmuted(did, "?1")
)
}
/// The accounts `?1` follows on Bluesky that have a grain profile, `?2`
/// of them.
pub fn bsky_follows_sql() -> String {
format!(
"SELECT DISTINCT f.subject FROM bsky_follow f JOIN profile p ON p.did = f.subject
WHERE f.did = ?1 AND f.subject != ?1 AND {}
ORDER BY f.subject LIMIT ?2",
followable("f.subject")
)
}
/// The most followed accounts with a grain profile, other than `?2`'s,
/// `?3` of them.
pub fn popular_sql() -> String {
format!(
"SELECT s.subject FROM (SELECT subject, count(*) AS n FROM follow GROUP BY subject) s
WHERE EXISTS (SELECT 1 FROM profile p WHERE p.did = s.subject)
AND s.subject NOT IN (SELECT value FROM json_each(?2)) AND {}
ORDER BY s.n DESC, s.subject LIMIT ?3",
followable("s.subject")
)
}
/// How many follow each of `?1`.
pub const FOLLOWERS_SQL: &str = "SELECT subject, count(DISTINCT did) FROM follow
WHERE subject IN (SELECT value FROM json_each(?1)) GROUP BY subject";
/// `social.grain.actor.getSuggestedFollows`: who `actor` might follow on
/// grain, those they follow on Bluesky first, then the most followed.
pub fn get_suggested_follows(r: &Reader, p: &Params) -> Result<Value, Failure> {
let actor = r.actor(p)?;
let limit = p.limit(10, 20)?;
let mut dids: Vec<String> = r
.all::<(String,)>(&bsky_follows_sql(), params![actor, limit as i64])?
.into_iter()
.map(|(d,)| d)
.collect();
if dids.len() < limit {
let mut not = dids.clone();
not.push(actor.clone());
let more: Vec<(String,)> = r.all(
&popular_sql(),
params![actor, list(&not), (limit - dids.len()) as i64],
)?;
dids.extend(more.into_iter().map(|(d,)| d));
}
let dids: Vec<&str> = dids.iter().map(String::as_str).collect();
let actors = r.actors(&dids)?;
let followers: std::collections::HashMap<String, i64> = r
.all::<(String, i64)>(FOLLOWERS_SQL, [list(&dids)])?
.into_iter()
.collect();
let items: Vec<Value> = dids
.iter()
.map(|did| {
let actor = &actors[*did];
let mut item = json!({
"did": did,
"handle": actor.handle,
"followersCount": followers.get(*did).copied().unwrap_or(0),
});
if let Some(p) = &actor.profile {
put(&mut item, "displayName", p.record.display_name.clone());
put(&mut item, "description", p.record.description.clone());
let avatar = p.record.avatar.as_ref();
put(
&mut item,
"avatar",
avatar.and_then(|a| r.cdn.url(&p.did, &a.0, AVATAR)),
);
}
item
})
.collect();
Ok(json!({ "items": items }))
}
/// The SQL these queries run, for the plan check.
#[cfg(test)]
pub fn queries() -> Vec<String> {
@@ -177,5 +266,6 @@ pub fn queries() -> Vec<String> {
all.extend(COUNTS.iter().map(|s| s.to_string()));
all.push(GERM.into());
all.push(favorites_sql());
all.extend([bsky_follows_sql(), popular_sql(), FOLLOWERS_SQL.into()]);
all
}
crates/grain-serve/src/read/camera.rs+77−2
View file
@@ -4,9 +4,13 @@
//! find the same galleries.
use super::feed::{empty, filtered, listed_param};
use super::moderation::shown;
use super::{Params, Reader};
use crate::cdn::THUMBNAIL;
use crate::index::records::Photo;
use crate::xrpc::Failure;
use serde_json::Value;
use rusqlite::params;
use serde_json::{Value, json};
/// Every make and model the index holds, as EXIF writes them.
pub const CAMERAS_SQL: &str = "SELECT DISTINCT make || ' ' || model FROM photo_exif
@@ -36,6 +40,72 @@ pub fn feed(r: &Reader, p: &Params) -> Result<Value, Failure> {
filtered(r, p, FILTER, vec![listed_param(&raws)])
}
/// How many cameras `getCameras` lists.
const CAMERAS: usize = 30;
/// How many thumbnails a camera carries, enough for its tile.
const THUMBS: usize = 4;
/// How many photos each make and model the index holds, the most first.
pub const COUNTS_SQL: &str = "SELECT make, model, count(*) FROM photo_exif
WHERE make IS NOT NULL AND model IS NOT NULL GROUP BY make, model";
/// The latest photos taken with the make `?1` and model `?2`, of
/// accounts shown, `?3` of them.
pub fn latest_sql() -> String {
format!(
"SELECT p.* FROM photo_exif e JOIN photo p ON p.uri = e.photo
WHERE e.make = ?1 AND e.model = ?2 AND {}
ORDER BY p.created_at DESC, p.uri LIMIT ?3",
shown("p.did")
)
}
/// A camera: its name, its count, and the makes and models it is.
type Camera = (String, i64, Vec<(String, String)>);
/// `social.grain.gallery.getCameras`: the cameras the most photos were
/// taken with, by the name a card gives them, makes and models that read
/// as one counted as one, each with its latest photos.
pub fn get_cameras(r: &Reader, _: &Params) -> Result<Value, Failure> {
let rows: Vec<(String, String, i64)> = r.all(COUNTS_SQL, [])?;
let mut cameras: Vec<Camera> = Vec::new();
for (make, model, n) in rows {
let name = clean(&format!("{make} {model}"));
if name.is_empty() {
continue;
}
match cameras.iter_mut().find(|c| c.0 == name) {
Some(c) => {
c.1 += n;
c.2.push((make, model));
}
None => cameras.push((name, n, vec![(make, model)])),
}
}
cameras.sort_by(|a, b| {
b.1.cmp(&a.1)
.then_with(|| a.0.to_lowercase().cmp(&b.0.to_lowercase()))
.then_with(|| a.0.cmp(&b.0))
});
cameras.truncate(CAMERAS);
let mut answer = Vec::new();
for (camera, count, raws) in cameras {
let mut photos = Vec::new();
for (make, model) in &raws {
photos.extend(r.rows::<Photo>(&latest_sql(), params![make, model, THUMBS as i64])?);
}
photos.sort_by(|a, b| b.record.created_at.cmp(&a.record.created_at));
let thumbs: Vec<String> = photos
.iter()
.take(THUMBS)
.filter_map(|p| r.cdn.url(&p.did, &p.record.photo.0, THUMBNAIL))
.collect();
answer.push(json!({"camera": camera, "photoCount": count, "thumbs": thumbs}));
}
Ok(json!({ "cameras": answer }))
}
/// A step of a pattern [`RULES`] match.
#[derive(Clone, Copy)]
enum Step {
@@ -186,7 +256,12 @@ pub fn clean(raw: &str) -> String {
#[cfg(test)]
pub fn queries() -> Vec<String> {
vec![CAMERAS_SQL.into(), super::feed::filtered_sql(FILTER)]
vec![
CAMERAS_SQL.into(),
super::feed::filtered_sql(FILTER),
COUNTS_SQL.into(),
latest_sql(),
]
}
#[cfg(test)]
crates/grain-serve/src/read/location.rs+272−6
View file
@@ -1,13 +1,16 @@
//! The Location feed: the galleries at a place, named by the label a view
//! shows for it, by its H3 cell, or both, as hatk's
//! `server/feeds/location.ts` finds them.
//! Places: the Location feed, the galleries at a place, named by the label
//! a view shows for it, by its H3 cell, or both, as hatk's
//! `server/feeds/location.ts` finds them; and `getLocations`, the places
//! galleries were made at, as hatk's `getLocations.ts` counts them.
use super::feed::{empty, filtered};
use super::moderation::{shown, unhidden};
use super::place::{REGIONS, country_code};
use super::{Params, Reader};
use super::{Params, Reader, list};
use crate::cdn::THUMBNAIL;
use crate::xrpc::Failure;
use rusqlite::types::Value as Sql;
use serde_json::Value;
use serde_json::{Value, json};
/// Location: by `name`, the label a view shows for a place, where the
/// call gives one and it finds any; otherwise by `location`, an H3 cell,
@@ -219,11 +222,274 @@ fn resolution(index: u64) -> u64 {
(index >> 52) & 0xf
}
/// How many places `getLocations` lists in full.
const PLACES: usize = 30;
/// How many galleries' covers a place carries, enough for its tile.
const THUMBS: usize = 4;
/// The cell `cell` is in at a city's resolution: itself where it is a
/// city's or coarser; none where it is no cell.
fn city_of(cell: &str) -> Option<String> {
let index = u64::from_str_radix(cell, 16)
.ok()
.filter(|i| mode(*i) == 1)?;
if resolution(index) <= CITY {
return Some(cell.to_string());
}
let mut at = (index & !(0xf << 52)) | (CITY << 52);
for digit in CITY + 1..=15 {
at |= 0b111 << ((15 - digit) * 3);
}
Some(format!("{at:x}"))
}
/// Every gallery with a place, of an account shown and carrying no label
/// that hides, the latest first: where it is, and what its address says.
pub fn places_sql() -> String {
format!(
"SELECT g.uri, g.created_at, json_extract(g.location, '$.name'), g.h3,
json_extract(g.address, '$.locality'), json_extract(g.address, '$.region'),
json_extract(g.address, '$.country')
FROM gallery g INDEXED BY gallery_h3 WHERE g.h3 IS NOT NULL AND {} AND {}
ORDER BY g.created_at DESC, g.uri",
shown("g.did"),
unhidden("g.uri")
)
}
/// The first photo of each gallery of `?1`.
pub const COVERS_SQL: &str = "SELECT i.gallery, p.did, p.photo FROM gallery_item i
JOIN photo p ON p.uri = i.item
WHERE i.gallery IN (SELECT value FROM json_each(?1)) ORDER BY i.position, i.uri";
/// A gallery's place, as `places_sql` reads it.
type Spot = (
String,
String,
Option<String>,
String,
Option<String>,
Option<String>,
Option<String>,
);
/// `value`, trimmed, where something is left.
fn given(value: &Option<String>) -> Option<&str> {
value.as_deref().map(str::trim).filter(|v| !v.is_empty())
}
/// What a place is grouped by: its address, its name, or its city's
/// cell, whichever it has first.
fn key(spot: &Spot) -> Option<String> {
let (_, _, name, cell, locality, region, country) = spot;
let country = country.as_deref().and_then(country_code);
let (locality, region) = (given(locality), given(region));
if locality.is_some() || region.is_some() || country.is_some() {
let lower = |v: Option<&str>| v.map(str::to_lowercase).unwrap_or_default();
let country = country.unwrap_or_default();
return Some(format!("A:{country}|{}|{}", lower(region), lower(locality)));
}
if let Some(name) = given(name) {
return Some(format!("N:{}", name.to_lowercase()));
}
city_of(cell).map(|c| format!("H:{c}"))
}
/// What a place is called on its tile: its locality, region and
/// country's code; a country alone by its English name; or its own name.
fn label(spot: &Spot) -> Option<String> {
let (_, _, name, _, locality, region, country) = spot;
let country = country.as_deref().and_then(country_code);
let (locality, region) = (given(locality), given(region));
if locality.is_some() || region.is_some() {
let parts: Vec<&str> = [locality, region, country.as_deref()]
.into_iter()
.flatten()
.collect();
return Some(parts.join(", "));
}
if let Some(code) = country {
let named = REGIONS
.iter()
.find(|(c, _)| *c == code)
.map(|(_, n)| n.to_string());
return Some(named.unwrap_or(code));
}
given(name).map(str::to_string)
}
/// Counts of each of a place's names or cells, in the order first seen.
#[derive(Default, Clone)]
struct Tally(Vec<(String, usize)>);
impl Tally {
fn add(&mut self, value: &str, n: usize) {
match self.0.iter_mut().find(|(v, _)| v == value) {
Some((_, count)) => *count += n,
None => self.0.push((value.to_string(), n)),
}
}
/// Each value, the most counted first, those counted alike as seen.
fn ranked(&self) -> Vec<String> {
let mut all = self.0.clone();
all.sort_by_key(|a| std::cmp::Reverse(a.1));
all.into_iter().map(|(v, _)| v).collect()
}
}
/// A place: what its galleries call it and where they are, how many
/// they are, and the latest of them, with when.
#[derive(Default)]
struct Place {
names: Tally,
cells: Tally,
galleries: Vec<(String, String)>,
country: String,
region: String,
locality: String,
count: usize,
}
/// A place as listed: its name, its cell, how many galleries, all its
/// cells, and its latest galleries.
type Ranked = (String, String, usize, Vec<String>, Vec<String>);
/// `social.grain.gallery.getLocations`: the places galleries were made
/// at, the most galleries first; each of them only as a map plots it
/// where `pins` asks, and otherwise the first thirty, with their cells
/// and the covers of their latest galleries.
pub fn get_locations(r: &Reader, p: &Params) -> Result<Value, Failure> {
let pins = match p.get("pins") {
None | Some("false") => false,
Some("true") => true,
Some(_) => return Err(Failure::invalid("pins must be a boolean")),
};
let spots: Vec<Spot> = r.all(&places_sql(), [])?;
let mut keys: Vec<String> = Vec::new();
let mut places: std::collections::HashMap<String, Place> = Default::default();
for spot in &spots {
let (Some(key), Some(label)) = (key(spot), label(spot)) else {
continue;
};
let place = places.entry(key.clone()).or_insert_with(|| {
keys.push(key);
Place {
country: spot.6.as_deref().and_then(country_code).unwrap_or_default(),
region: given(&spot.5).unwrap_or_default().to_lowercase(),
locality: given(&spot.4).unwrap_or_default().to_lowercase(),
..Place::default()
}
});
place.count += 1;
if place.galleries.len() < THUMBS {
place.galleries.push((spot.0.clone(), spot.1.clone()));
}
place.names.add(&label, 1);
place.cells.add(&spot.3, 1);
}
// A place with no region, as one geocoder gives "Lisbon, PT" and
// another "Lisbon, Lisbon, PT", is folded into the one place of its
// locality and country that has one; where there are more, as
// Portland is in Oregon and in Maine, it stands alone.
for key in keys.clone() {
let Some(place) = places.get(&key) else {
continue;
};
if !place.region.is_empty() || place.locality.is_empty() {
continue;
}
let twins: Vec<&String> = keys
.iter()
.filter(|k| {
places.get(*k).is_some_and(|o| {
!o.region.is_empty()
&& o.locality == place.locality
&& o.country == place.country
})
})
.collect();
let [into] = twins[..] else { continue };
let into = into.clone();
let place = places.remove(&key).expect("present");
let twin = places.get_mut(&into).expect("present");
twin.count += place.count;
for (name, n) in &place.names.0 {
twin.names.add(name, *n);
}
for (cell, n) in &place.cells.0 {
twin.cells.add(cell, *n);
}
twin.galleries.extend(place.galleries);
twin.galleries.sort_by(|a, b| b.1.cmp(&a.1));
twin.galleries.truncate(THUMBS);
}
let mut ranked: Vec<Ranked> = keys
.iter()
.filter_map(|k| places.get(k))
.filter_map(|place| {
// The name most of its galleries give it, the first given
// where two are given alike.
let mut best: Option<&(String, usize)> = None;
for named in &place.names.0 {
if best.is_none_or(|b| named.1 > b.1) {
best = Some(named);
}
}
let cells = place.cells.ranked();
let (name, cell) = (best?.0.clone(), cells.first()?.clone());
let galleries = place.galleries.iter().map(|(u, _)| u.clone()).collect();
Some((name, cell, place.count, cells, galleries))
})
.collect();
ranked.sort_by(|a, b| {
b.2.cmp(&a.2)
.then_with(|| a.0.to_lowercase().cmp(&b.0.to_lowercase()))
.then_with(|| a.0.cmp(&b.0))
});
if pins {
let pins: Vec<Value> = ranked
.iter()
.map(|(name, cell, count, _, _)| json!({"name": name, "h3Index": cell, "galleryCount": count}))
.collect();
return Ok(json!({ "locations": pins }));
}
ranked.truncate(PLACES);
let uris: Vec<&str> = ranked
.iter()
.flat_map(|p| p.4.iter().map(String::as_str))
.collect();
let mut covers: std::collections::HashMap<String, String> = Default::default();
for (gallery, did, photo) in r.all::<(String, String, String)>(COVERS_SQL, [list(&uris)])? {
let blob: Value = serde_json::from_str(&photo).unwrap_or_default();
if let (false, Some(url)) = (
covers.contains_key(&gallery),
r.cdn.url(&did, &blob, THUMBNAIL),
) {
covers.insert(gallery, url);
}
}
let locations: Vec<Value> = ranked
.into_iter()
.map(|(name, cell, count, cells, galleries)| {
let thumbs: Vec<&String> = galleries.iter().filter_map(|g| covers.get(g)).collect();
json!({"name": name, "h3Index": cell, "galleryCount": count, "h3Cells": cells, "thumbs": thumbs})
})
.collect();
Ok(json!({ "locations": locations }))
}
#[cfg(test)]
pub fn queries() -> Vec<String> {
use super::feed::filtered_sql;
let (named, _) = by_name("The Space Needle, Seattle, Washington, US");
vec![filtered_sql(&named), filtered_sql(CELL_FILTER)]
vec![
filtered_sql(&named),
filtered_sql(CELL_FILTER),
places_sql(),
COVERS_SQL.into(),
]
}
#[cfg(test)]
crates/grain-serve/src/read/mod.rs+29−6
View file
@@ -22,6 +22,7 @@ pub mod graph;
pub mod location;
pub mod moderation;
pub mod place;
pub mod search;
pub mod story;
pub mod views;
@@ -71,10 +72,26 @@ pub const METHODS: &[(&str, Method)] = &[
("social.grain.story.getAuthors", story::get_authors),
("social.grain.feed.describeFeeds", feed::describe_feeds),
("social.grain.feed.getFeed", feed::get_feed),
("social.grain.actor.searchActors", search::search_actors),
(
"social.grain.actor.getSuggestedFollows",
actor::get_suggested_follows,
),
(
"social.grain.gallery.searchGalleries",
search::search_galleries,
),
("social.grain.gallery.getCameras", camera::get_cameras),
("social.grain.gallery.getLocations", location::get_locations),
];
/// `router` with each of [`METHODS`] at `/xrpc/<nsid>`.
/// `router` with each of [`METHODS`] at `/xrpc/<nsid>`, and typeahead,
/// which asks Bluesky's too ([`search::typeahead`]).
pub fn routes(router: Router<Arc<App>>) -> Router<Arc<App>> {
let router = router.route(
"/xrpc/social.grain.actor.searchActorsTypeahead",
get(search::typeahead),
);
METHODS.iter().fold(router, |router, &(nsid, method)| {
let path = format!("/xrpc/{nsid}");
router.route(
@@ -105,11 +122,7 @@ async fn answer(
why,
));
};
let params = Params(
Query::try_from_uri(&uri)
.map(|Query(q)| q)
.unwrap_or_default(),
);
let params = params(&uri);
let viewer = crate::viewer::viewer(&app, &headers, nsid).await?;
index.read(|db| {
let reader = Reader {
@@ -122,6 +135,15 @@ async fn answer(
})
}
/// The parameters `uri` gives, none where it gives none that parse.
fn params(uri: &Uri) -> Params {
Params(
Query::try_from_uri(uri)
.map(|Query(q)| q)
.unwrap_or_default(),
)
}
/// A query's parameters, as its URL names them.
#[derive(Debug, Default, Clone)]
pub struct Params(pub Vec<(String, String)>);
@@ -296,6 +318,7 @@ columns!(A 0);
columns!(A 0, B 1);
columns!(A 0, B 1, C 2);
columns!(A 0, B 1, C 2, D 3, E 4);
columns!(A 0, B 1, C 2, D 3, E 4, F 5, G 6);
#[cfg(test)]
mod tests;
crates/grain-serve/src/read/search.rsnew+846−0
View file
846 lines changed: not loaded until asked for.
crates/grain-serve/src/read/tests.rs+1−0
View file
@@ -13,6 +13,7 @@ fn the_reads_read_by_index() {
queries.extend(story::queries());
queries.extend(feed::queries());
queries.extend(views::queries());
queries.extend(search::queries());
index.read(|db| {
sized(db);
for sql in &queries {
crates/grain-serve/src/lib.rs+26−0
View file
@@ -40,6 +40,11 @@ pub struct Settings {
/// imgproxy, and the hex key and salt it signs paths with
/// ([`cdn::Cdn`]). None draws images from Bluesky's CDN.
pub cdn: [Option<String>; 3],
/// `GRAIN_BSKY_APPVIEW`: the Bluesky appview whose typeahead
/// `searchActorsTypeahead` fills out its own with, a URL, or `1` for
/// [`read::search::BSKY_APPVIEW`]. None, as when it is unset, empty
/// or `0`, asks none, as tests and dev run.
pub bsky_appview: Option<String>,
}
impl Settings {
@@ -55,6 +60,11 @@ impl Settings {
Some(url) => Some(url.into()),
},
cdn: ["GRAIN_CDN_URL", "GRAIN_CDN_KEY", "GRAIN_CDN_SALT"].map(set),
bsky_appview: match set("GRAIN_BSKY_APPVIEW").as_deref() {
None | Some("0") => None,
Some("1") => Some(read::search::BSKY_APPVIEW.into()),
Some(url) => Some(url.trim_end_matches('/').into()),
},
}
}
@@ -100,6 +110,7 @@ impl Settings {
index,
did,
cdn,
bsky_appview: self.bsky_appview.clone(),
})
}
}
@@ -114,6 +125,8 @@ pub struct App {
pub did: Option<String>,
/// Where views send clients for images.
pub cdn: cdn::Cdn,
/// The Bluesky appview typeahead asks too, if any.
pub bsky_appview: Option<String>,
}
impl App {
@@ -125,6 +138,7 @@ impl App {
index: None,
did: None,
cdn: cdn::Cdn::default(),
bsky_appview: None,
}
}
}
@@ -205,8 +219,20 @@ mod tests {
"GRAIN_INDEX" => Some("/data/index.db".into()),
"GRAIN_PUBLIC_URL" => Some("https://grain.social".into()),
"GRAIN_CDN_URL" => Some("https://cdn.grain.social".into()),
"GRAIN_BSKY_APPVIEW" => Some("1".into()),
_ => None,
});
assert_eq!(
set.bsky_appview.as_deref(),
Some(read::search::BSKY_APPVIEW)
);
let named = Settings::from_vars(|name| {
(name == "GRAIN_BSKY_APPVIEW").then(|| "http://127.0.0.1:2600/".to_string())
});
assert_eq!(named.bsky_appview.as_deref(), Some("http://127.0.0.1:2600"));
let off =
Settings::from_vars(|name| (name == "GRAIN_BSKY_APPVIEW").then(|| "0".to_string()));
assert_eq!(off.bsky_appview, None);
assert_eq!(set.cdn[0].as_deref(), Some("https://cdn.grain.social"));
assert!(
set.app(Ctx::new(Env::from_vars(|_| None))).is_err(),
crates/grain-serve/tests/index_records.rs+1−0
View file
crates/grain-serve/tests/jetstream.rs+6−2
View file
crates/grain-serve/tests/read.rs+447−0
View file
447 lines changed: not loaded until asked for.