~ knotless Search/
↑↓ move↵ open⇧↵ all results

grain.social/grain
Switch repository
Loading…

photography-focused photo sharing platform

grain.social
1 collaborator
0
2

server: Reports come in, and admins moderate with service tokens

Merged chadtmiller.com asked to merge into feature/rust from grain.social/grain:admin/moderation

Only what this pull adds. It is built on grain runs on a Rust server modelled on knotless, without hatk.

crates/grain/src/labels.rsnew+245−0
View file
@@ -0,0 +1,245 @@
//! The labels grain's labeler gives out, what a report asks for, and which
//! of them are about the safety of children.
//!
//! A report names its reason as atproto does
//! (`com.atproto.moderation.createReport`'s `reasonType`), and grain files
//! it under the label it asks for ([`label_for`]): resolving the report
//! applies that label. Child safety (`csae`) is set apart
//! ([`child_safety`]): what a report or label of it is about is for a
//! person to judge, and is never handed to a model.
use serde_json::{Value, json};
/// The label of child sexual abuse and exploitation.
pub const CSAE: &str = "csae";
/// A label grain's labeler gives out, as
/// `com.atproto.label.defs#labelValueDefinition` has one, in English.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Definition {
pub identifier: &'static str,
/// `inform`, `alert` or `none`.
pub severity: &'static str,
/// `content`, `media` or `none`.
pub blurs: &'static str,
/// `ignore`, `warn` or `hide`.
pub default_setting: &'static str,
pub name: &'static str,
pub description: &'static str,
}
/// Every label grain's labeler gives out, as its report sheet lists them.
pub const DEFINITIONS: &[Definition] = &[
Definition {
identifier: "copyright",
severity: "alert",
blurs: "content",
default_setting: "hide",
name: "Copyright Violation",
description: "Content that infringes on copyright or intellectual property.",
},
Definition {
identifier: CSAE,
severity: "alert",
blurs: "content",
default_setting: "hide",
name: "Child Safety",
description: "Sexual content involving a minor, or the exploitation, grooming or endangerment of a child.",
},
Definition {
identifier: "gore",
severity: "alert",
blurs: "media",
default_setting: "hide",
name: "Graphic Media",
description: "Graphic or violent imagery.",
},
Definition {
identifier: "harassment",
severity: "alert",
blurs: "none",
default_setting: "warn",
name: "Harassment",
description: "Targeted harassment, bullying, or hate speech.",
},
Definition {
identifier: "misleading",
severity: "inform",
blurs: "none",
default_setting: "warn",
name: "Misleading",
description: "Impersonation, misinformation, or deceptive content.",
},
Definition {
identifier: "nudity",
severity: "inform",
blurs: "media",
default_setting: "warn",
name: "Nudity",
description: "Content containing nudity.",
},
Definition {
identifier: "other",
severity: "none",
blurs: "none",
default_setting: "ignore",
name: "Other",
description: "Content that does not fit other report categories.",
},
Definition {
identifier: "sexual",
severity: "inform",
blurs: "media",
default_setting: "warn",
name: "Sexual Content",
description: "Unwanted or mislabeled sexual content.",
},
Definition {
identifier: "spam",
severity: "inform",
blurs: "content",
default_setting: "hide",
name: "Spam",
description: "Unwanted or repetitive promotional content.",
},
];
impl Definition {
/// As `com.atproto.label.defs#labelValueDefinition`.
pub fn json(&self) -> Value {
json!({
"identifier": self.identifier,
"severity": self.severity,
"blurs": self.blurs,
"defaultSetting": self.default_setting,
"locales": [{"lang": "en", "name": self.name, "description": self.description}],
})
}
}
/// atproto's reason types (`com.atproto.moderation.defs#reason…`).
const ATPROTO: &str = "com.atproto.moderation.defs#reason";
/// Ozone's finer ones (`tools.ozone.report.defs#reason…`).
const OZONE: &str = "tools.ozone.report.defs#reason";
/// grain's own, for the labels neither has a reason for.
const GRAIN: &str = "social.grain.moderation.defs#reason";
/// The label a report for `reason_type` asks for, if grain takes the
/// reason: each of atproto's and Ozone's, and grain's own
/// `social.grain.moderation.defs#reasonCopyright` and `#reasonNudity`.
pub fn label_for(reason_type: &str) -> Option<&'static str> {
if let Some(reason) = reason_type.strip_prefix(ATPROTO) {
return Some(match reason {
"Spam" => "spam",
"Misleading" => "misleading",
"Sexual" => "sexual",
"Rude" => "harassment",
"Violation" | "Other" | "Appeal" => "other",
_ => return None,
});
}
if let Some(reason) = reason_type.strip_prefix(OZONE) {
let label = match reason {
"MisleadingSpam" => "spam",
"ViolenceGraphicContent" => "gore",
r if r.starts_with("ChildSafety") => CSAE,
r if r.starts_with("Misleading") => "misleading",
r if r.starts_with("Sexual") => "sexual",
r if r.starts_with("Harassment") => "harassment",
r if ["Appeal", "Other", "Violence", "Rule", "SelfHarm"]
.iter()
.any(|kind| r.starts_with(kind)) =>
{
"other"
}
_ => return None,
};
return Some(label);
}
match reason_type.strip_prefix(GRAIN)? {
"Copyright" => Some("copyright"),
"Nudity" => Some("nudity"),
_ => None,
}
}
/// Whether a report or label of `label`, or made for the reason type
/// `reason`, is about the safety of children.
pub fn child_safety(label: &str, reason: Option<&str>) -> bool {
label == CSAE || reason.and_then(label_for) == Some(CSAE)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn each_reason_is_filed_under_a_label() {
let cases = [
("com.atproto.moderation.defs#reasonSpam", "spam"),
("com.atproto.moderation.defs#reasonMisleading", "misleading"),
("com.atproto.moderation.defs#reasonSexual", "sexual"),
("com.atproto.moderation.defs#reasonRude", "harassment"),
("com.atproto.moderation.defs#reasonViolation", "other"),
("com.atproto.moderation.defs#reasonOther", "other"),
("com.atproto.moderation.defs#reasonAppeal", "other"),
("tools.ozone.report.defs#reasonMisleadingSpam", "spam"),
(
"tools.ozone.report.defs#reasonViolenceGraphicContent",
"gore",
),
("tools.ozone.report.defs#reasonChildSafetyCSAM", CSAE),
("tools.ozone.report.defs#reasonChildSafetyGroom", CSAE),
("tools.ozone.report.defs#reasonMisleadingScam", "misleading"),
("tools.ozone.report.defs#reasonSexualNCII", "sexual"),
(
"tools.ozone.report.defs#reasonHarassmentDoxxing",
"harassment",
),
("tools.ozone.report.defs#reasonViolenceThreats", "other"),
("tools.ozone.report.defs#reasonRuleBanEvasion", "other"),
("tools.ozone.report.defs#reasonSelfHarmED", "other"),
("tools.ozone.report.defs#reasonAppeal", "other"),
("social.grain.moderation.defs#reasonCopyright", "copyright"),
("social.grain.moderation.defs#reasonNudity", "nudity"),
];
for (reason, label) in cases {
assert_eq!(label_for(reason), Some(label), "{reason}");
assert!(DEFINITIONS.iter().any(|d| d.identifier == label));
}
for unknown in [
"com.atproto.moderation.defs#reasonBoring",
"tools.ozone.report.defs#reasonBoring",
"social.grain.moderation.defs#reasonBoring",
"spam",
"",
] {
assert_eq!(label_for(unknown), None, "{unknown}");
}
}
#[test]
fn child_safety_is_known_by_its_label_or_its_reason() {
assert!(child_safety(CSAE, None));
assert!(child_safety(
"other",
Some("tools.ozone.report.defs#reasonChildSafetyOther")
));
assert!(!child_safety("spam", None));
assert!(!child_safety(
"spam",
Some("com.atproto.moderation.defs#reasonSpam")
));
}
#[test]
fn definitions_read_as_atprotos() {
let csae = DEFINITIONS.iter().find(|d| d.identifier == CSAE).unwrap();
let json = csae.json();
assert_eq!(json["defaultSetting"], "hide");
assert_eq!(json["locales"][0]["name"], "Child Safety");
let mut ids: Vec<_> = DEFINITIONS.iter().map(|d| d.identifier).collect();
ids.dedup();
assert_eq!(ids.len(), DEFINITIONS.len());
}
}
crates/grain/src/lib.rs+1−0
View file
@@ -13,6 +13,7 @@ pub mod guard;
pub mod identity;
pub mod jetstream;
pub mod keys;
pub mod labels;
pub mod lexicon;
pub mod oauth;
pub mod repo;
crates/grain-import/src/lib.rs+25−0
View file
@@ -32,6 +32,12 @@
//! `_reports`, `_mutes`, `_preferences`, `_push_tokens`, `_story_views`,
//! `_space_invites`, `_space_support`.
//!
//! hatk gave its labels out as `admin`, from its admin routes, and `self`,
//! from its label rules, which name no labeler: each is grain's, and given
//! out as grain-serve's DID, the `did:web` of `GRAIN_PUBLIC_URL`, or
//! [`LABELER`] where it is unset. A label hatk had from a labeler by its
//! DID keeps it.
//!
//! hatk's `_repos.status` said two things at once, kept apart here. An
//! admin's takedown, `takendown`, which only hatk's admin routes wrote,
//! is a row of `takedown`, at the time of the import as hatk kept none.
@@ -108,6 +114,9 @@ use std::collections::BTreeSet;
use std::fmt;
use std::path::{Path, PathBuf};
/// The DID grain gives out labels as, where `GRAIN_PUBLIC_URL` names none.
pub const LABELER: &str = "did:web:grain.social";
pub const USAGE: &str = "usage: grain-import <hatk's grain.db> [--into <index>] [--strict]
[--public-url <url>]
reads hatk's database into the index at --into, or GRAIN_INDEX, which
@@ -498,6 +507,10 @@ pub fn import(
strict: bool,
public_url: Option<&str>,
) -> anyhow::Result<Summary> {
let labeler = match public_url {
Some(url) => grain_serve::service_did(url)?,
None => LABELER.to_string(),
};
let src = Connection::open_with_flags(from, OpenFlags::SQLITE_OPEN_READ_ONLY)
.with_context(|| format!("opening {}", from.display()))?;
// One snapshot, for the counts to be of the rows read.
@@ -533,6 +546,18 @@ pub fn import(
};
let carried = copy_sessions(&src, &tx, &tables, server.as_deref(), public_url)?;
(summary.sessions, summary.sessions_left) = carried;
let mine = "UPDATE label SET src = ?1 WHERE src NOT LIKE 'did:%'";
tx.execute(mine, [&labeler])?;
// A report names who made and closed it by DID, as atproto's do: the
// old server's own reports (its classifiers', as `system:<name>`) and
// its admin's closings are the server's, the old reporter kept at the
// head of the reason.
let ours = "UPDATE report SET reported_by = ?1, reason = reported_by
|| CASE WHEN reason IS NULL THEN '' ELSE ': ' || reason END
WHERE reported_by NOT LIKE 'did:%'";
tx.execute(ours, [&labeler])?;
let closed = "UPDATE report SET resolved_by = ?1 WHERE resolved_by NOT LIKE 'did:%'";
tx.execute(closed, [&labeler])?;
let sql = format!(
"SELECT status, count(*) FROM _repos WHERE status NOT IN {STATUSES} GROUP BY status"
);
crates/grain-import/src/tests.rs+36−7
View file
@@ -121,7 +121,8 @@ INSERT INTO _labels VALUES
(9, 'self', 'at://did:plc:carol/social.grain.photo/3l3qo2vuowo2a', 'spam', NULL, '2025-01-05T00:00:00.000Z', NULL);
INSERT INTO _reports VALUES
(1, 'at://did:plc:bob/social.grain.gallery/3l3qo2vuowo2a', 'did:plc:bob', 'spam', 'ads', 'did:plc:alice', 'open', NULL, NULL, '2025-01-06T00:00:00.000Z'),
(4, 'did:plc:carol', 'did:plc:carol', 'rude', NULL, 'did:plc:alice', 'resolved', 'did:plc:admin', '2025-01-08T00:00:00.000Z', '2025-01-07T00:00:00.000Z');
(4, 'did:plc:carol', 'did:plc:carol', 'rude', NULL, 'did:plc:alice', 'resolved', 'did:plc:admin', '2025-01-08T00:00:00.000Z', '2025-01-07T00:00:00.000Z'),
(7, 'at://did:plc:bob/social.grain.photo/3l3qo2vuowo2a', 'did:plc:bob', 'nudity', 'nsfw: nudity 0.91', 'system:nsfw', 'dismissed', 'admin', '2025-01-09T00:00:00.000Z', '2025-01-08T00:00:00.000Z');
INSERT INTO _push_tokens VALUES
('did:plc:alice', 'apns-token', 'apns', '2025-01-01T00:00:00.000Z'),
('did:plc:alice', 'fcm-token', 'fcm', '2025-01-02T00:00:00.000Z'),
@@ -369,16 +370,19 @@ fn every_row_of_what_the_network_cannot_rebuild_is_imported() {
[
"3|did:web:grain.social|at://did:plc:alice/social.grain.gallery/3l3qo2vuowo2a|nudity|0|2025-01-02T00:00:00.000Z|null",
// An account's, on its bare DID, given and then negated.
"5|admin|did:plc:bob|spam|0|2025-01-03T00:00:00.000Z|null",
"6|admin|did:plc:bob|spam|1|2025-01-04T00:00:00.000Z|2026-01-01T00:00:00.000Z",
"9|self|at://did:plc:carol/social.grain.photo/3l3qo2vuowo2a|spam|0|2025-01-05T00:00:00.000Z|null",
// hatk's own, from its admin routes and its rules, grain's.
"5|did:web:grain.social|did:plc:bob|spam|0|2025-01-03T00:00:00.000Z|null",
"6|did:web:grain.social|did:plc:bob|spam|1|2025-01-04T00:00:00.000Z|2026-01-01T00:00:00.000Z",
"9|did:web:grain.social|at://did:plc:carol/social.grain.photo/3l3qo2vuowo2a|spam|0|2025-01-05T00:00:00.000Z|null",
]
);
assert_eq!(
rows("SELECT * FROM report ORDER BY id"),
[
"1|at://did:plc:bob/social.grain.gallery/3l3qo2vuowo2a|did:plc:bob|spam|ads|did:plc:alice|open|null|null|2025-01-06T00:00:00.000Z",
"4|did:plc:carol|did:plc:carol|rude|null|did:plc:alice|resolved|did:plc:admin|2025-01-08T00:00:00.000Z|2025-01-07T00:00:00.000Z",
"1|at://did:plc:bob/social.grain.gallery/3l3qo2vuowo2a|did:plc:bob|spam|ads|did:plc:alice|open|null|null|2025-01-06T00:00:00.000Z|null",
"4|did:plc:carol|did:plc:carol|rude|null|did:plc:alice|resolved|did:plc:admin|2025-01-08T00:00:00.000Z|2025-01-07T00:00:00.000Z|null",
// The old server's own, and its admin's closing, are the server's.
"7|at://did:plc:bob/social.grain.photo/3l3qo2vuowo2a|did:plc:bob|nudity|system:nsfw: nsfw: nudity 0.91|did:web:grain.social|dismissed|did:web:grain.social|2025-01-09T00:00:00.000Z|2025-01-08T00:00:00.000Z|null",
]
);
assert_eq!(
@@ -461,7 +465,7 @@ fn every_row_of_what_the_network_cannot_rebuild_is_imported() {
("takedown", 2, 2),
("stream_cursor", 1, 1),
("label", 4, 4),
("report", 2, 2),
("report", 3, 3),
("mute", 2, 2),
("preference", 1, 1),
("notification_seen", 2, 2),
@@ -940,6 +944,31 @@ fn it_reads_only_hatks_database() {
);
}
#[test]
fn hatks_own_labels_are_given_out_as_the_servers_did() {
let srcs = |public_url: Option<&str>| {
let dir = tempfile::tempdir().unwrap();
let (old, into) = (full(dir.path()), dir.path().join("index.db"));
import(&old, &into, false, public_url).unwrap();
let index = Connection::open(&into).unwrap();
let mut stmt = index
.prepare("SELECT src FROM label GROUP BY src ORDER BY min(id)")
.unwrap();
let srcs = stmt.query_map([], |r| r.get::<_, String>(0)).unwrap();
srcs.collect::<rusqlite::Result<Vec<_>>>().unwrap()
};
assert_eq!(
srcs(Some("http://127.0.0.1:2600")),
["did:web:grain.social", "did:web:127.0.0.1%3A2600"],
"a labeler's own DID is kept"
);
assert_eq!(srcs(None), [LABELER]);
let dir = tempfile::tempdir().unwrap();
let old = full(dir.path());
let why = import(&old, &dir.path().join("i.db"), false, Some("grain.social"));
assert!(why.unwrap_err().to_string().starts_with("grain.social"));
}
#[test]
fn arguments_name_the_database_and_the_index() {
let args = |a: &[&str]| a.iter().map(|s| s.to_string()).collect::<Vec<_>>();
crates/grain-serve/src/classify/mod.rs+19−7
View file
@@ -14,9 +14,10 @@
//! same state before (its fingerprint, [`Index::fingerprint`]), and
//! otherwise asks Clef ([`clef`]) and keeps the scores
//! ([`Index::classified`]). Nothing acts on a score: one at or past its
//! threshold files a report ([`Index::report_once`]), by
//! `system:<classifier>`, into the queue a person's report lands in, and a
//! person makes every decision.
//! threshold files a report ([`Index::file_report_once`]), by grain's
//! labeler with the classifier named in its reason, into the queue a
//! person's report lands in, and a person makes every decision. With no
//! labeler, none is filed.
//!
//! Nothing of an account taken down, or gone from its host, is scored,
//! and nothing of one that a report or a label ever said was about the
@@ -42,6 +43,7 @@ pub mod classifiers;
pub mod clef;
use crate::cdn::{Cdn, THUMBNAIL};
use crate::index::moderation::NewReport;
use crate::index::{Index, record_uri, records};
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
@@ -300,12 +302,22 @@ impl Classify {
let Some((signal, score)) = crossing else {
return Ok(Scored::Asked { filed: false });
};
// A report names its reporter by DID: the server's, as its labels.
let Some(by) = self.labeler.as_deref() else {
return Ok(Scored::Asked { filed: false });
};
let reason = format!("{}: {signal} {score:.2}", c.name);
let by = format!("system:{}", c.name);
let label = (c.label)(signal);
let filed = self
.index
.report_once((subject, did), label, &reason, &by, &now)?;
let filed = self.index.file_report_once(&NewReport {
subject,
subject_did: did,
label,
reason_type: None,
reason: Some(&reason),
reported_by: by,
created_at: &now,
})?;
let filed = filed.is_some();
if filed {
tracing::info!(classifier = c.name, %subject, label, "classify: filed a report");
}
crates/grain-serve/src/index/classify.rs+11−37
View file
@@ -111,27 +111,6 @@ impl Index {
.optional()
}
/// Files a report on `subject`, of the account `did`, asking for
/// `label`, by `by` at `now`, unless one asking for it is open; whether
/// it filed one. A second report would only split the review.
pub fn report_once(
&self,
(subject, did): (&str, &str),
label: &str,
reason: &str,
by: &str,
now: &str,
) -> rusqlite::Result<bool> {
let db = self.db.lock().unwrap();
let filed = db.execute(
"INSERT INTO report (subject_uri, subject_did, label, reason, reported_by, created_at)
SELECT ?1, ?2, ?3, ?4, ?5, ?6 WHERE NOT EXISTS (SELECT 1 FROM report
WHERE subject_uri = ?1 AND label = ?3 AND status = 'open')",
params![subject, did, label, reason, by, now],
)?;
Ok(filed > 0)
}
/// Gives out `src`'s `val` on `uri` at `now`, unless any label of
/// `val` was ever given it, or negated: a moderator who took one off
/// is not overruled. Whether it gave one.
@@ -147,23 +126,18 @@ impl Index {
}
/// Whether what the account `did` writes may be shown to a model: it
/// is shown here, and nothing of it was ever reported or labeled as
/// being about the safety of children (`csae`), by grain, a reporter
/// or its author, negated since or not. What a report or label of that
/// is about is for a person to judge, and is never handed to a model.
/// is shown here, and nothing of it is about the safety of children
/// ([`Index::child_safety`]: a report by its label or its reason type,
/// in any state, or a `csae` label, negated since or not, or its
/// author's own). What a report or label of that is about is for a
/// person to judge, and is never handed to a model.
pub fn may_classify(&self, did: &str) -> rusqlite::Result<bool> {
let db = self.db.lock().unwrap();
let ours = "(uri = ?1 OR (uri > 'at://' || ?1 || '/' AND uri < 'at://' || ?1 || '0'))";
let sql = format!(
"SELECT {shown}
AND NOT EXISTS (SELECT 1 FROM label WHERE val = 'csae' AND {ours})
AND NOT EXISTS (SELECT 1 FROM self_label WHERE val = 'csae' AND {ours})
AND NOT EXISTS (SELECT 1 FROM report WHERE label = 'csae' AND (subject_did = ?1
OR subject_uri = ?1 OR (subject_uri > 'at://' || ?1 || '/'
AND subject_uri < 'at://' || ?1 || '0')))",
shown = shown("?1"),
);
db.query_row(&sql, [did], |r| r.get(0))
let shown: bool = {
let db = self.db.lock().unwrap();
let sql = format!("SELECT {}", shown("?1"));
db.query_row(&sql, [did], |r| r.get(0))?
};
Ok(shown && !self.child_safety(did)?)
}
/// The handle of `did`, if the index tracks it, as it keeps it.
crates/grain-serve/src/index/mod.rs+2−8
View file
@@ -15,6 +15,7 @@
pub mod backfill;
pub mod classify;
pub mod moderation;
#[cfg(test)]
pub mod plan;
pub mod preferences;
@@ -501,14 +502,7 @@ impl Index {
/// Gives out `label`; its sequence number.
pub fn label(&self, label: &Label) -> rusqlite::Result<i64> {
let db = self.db.lock().unwrap();
db.execute(
"INSERT INTO label (src, uri, val, neg, cts, exp) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![
label.src, label.uri, label.val, label.neg, label.cts, label.exp
],
)?;
Ok(db.last_insert_rowid())
moderation::give(&self.db.lock().unwrap(), label)
}
/// The labels given out on `uri`, in the order they were.
crates/grain-serve/src/index/moderation.rsnew+416−0
View file
416 lines changed: not loaded until asked for.
crates/grain-serve/src/index/records.rs+77−0
View file
@@ -108,6 +108,13 @@ macro_rules! records {
/// Each table's columns past the envelope.
pub const COLUMNS: &[(&str, &[&str])] = &[ $( ($table, $name::COLUMNS), )* ];
/// Each table's columns past the envelope, as the fields they hold.
pub const FIELDS: &[(&str, &[Field])] = &[ $( ($table, &[ $( Field {
column: stringify!($field),
name: $json,
json: holds_json(stringify!($ty)),
}, )* ]), )* ];
};
}
@@ -248,6 +255,57 @@ records! {
}
}
/// A column of a record's table, and the field of the record it holds.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Field {
pub column: &'static str,
/// The field's name in the record.
pub name: &'static str,
/// Whether the column holds the field as JSON text ([`Json`]).
pub json: bool,
}
/// Whether a field of the type written `ty` is kept as [`Json`].
const fn holds_json(ty: &str) -> bool {
let (ty, word) = (ty.as_bytes(), b"Json");
let mut at = 0;
while at + word.len() <= ty.len() {
let mut same = 0;
while same < word.len() && ty[at + same] == word[same] {
same += 1;
}
if same == word.len() {
return true;
}
at += 1;
}
false
}
/// The record of `collection` a result row holds, as the index keeps it:
/// its fields from the columns of its table ([`FIELDS`]), each the row
/// names, and its `$type`. A field the record does not have is left out.
pub fn value(collection: &str, row: &rusqlite::Row) -> rusqlite::Result<Value> {
let mut record = serde_json::Map::new();
record.insert("$type".into(), collection.into());
let fields = table_of(collection).and_then(|t| FIELDS.iter().find(|(table, _)| *table == t));
for field in fields.map(|(_, fields)| *fields).unwrap_or_default() {
// Every column is text or an integer: a field kept as JSON is
// read as JSON, as it was checked against its lexicon going in.
let value = match row.get_ref(field.column)? {
ValueRef::Null => continue,
ValueRef::Integer(n) => Value::from(n),
text => {
let text = String::from_utf8_lossy(text.as_bytes().unwrap_or_default());
let parsed = field.json.then(|| serde_json::from_str(&text).ok());
parsed.flatten().unwrap_or(Value::String(text.into_owned()))
}
};
record.insert(field.name.into(), value);
}
Ok(Value::Object(record))
}
/// The table of `collection`, if the index keeps it.
pub fn table_of(collection: &str) -> Option<&'static str> {
COLLECTIONS
@@ -266,3 +324,22 @@ pub fn self_labels(record: &Value) -> Vec<&str> {
.map(|values| values.iter().filter_map(|v| v["val"].as_str()).collect())
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_field_is_kept_as_json_when_its_type_names_it() {
// The same function [`FIELDS`] is made with, run where a test sees it.
let kept = std::hint::black_box(holds_json);
assert!(kept("Json"));
assert!(kept("Option<Json>"));
assert!(!kept("String"));
assert!(!kept("Jso"));
assert!(!kept(""));
let gallery = FIELDS.iter().find(|(t, _)| *t == "gallery").unwrap().1;
let facets = gallery.iter().find(|f| f.name == "facets").unwrap();
assert!(facets.json);
}
}
crates/grain-serve/src/index/schema.rs+16−3
View file
@@ -62,8 +62,14 @@
//! - `label`: the labels grain gives out, as `com.atproto.label.defs#label`
//! has them; one on an account names it by its bare DID, not
//! `at://<did>`, as Bluesky's do;
//! - `report`, `mute`, `preference`, `push_token`, `story_view`,
//! `space_invite`, `space_support`;
//! - `report`, what someone reported: its subject (an account's bare DID
//! or a record's AT-URI) and that subject's account (`subject_did`), the
//! label it asks for, the `reason_type` it was made with
//! (`com.atproto.moderation.defs#reasonType`, none for one made before
//! grain took them by it) and its `reason`, who made it, and whether it is
//! `open`, `resolved` or `dismissed`, by whom and when;
//! - `mute`, `preference`, `push_token`, `story_view`, `space_invite`,
//! `space_support`;
//! - `notification_seen`, when each account last saw its notifications
//! (`updateSeen`'s `seenAt`, in UTC as [`grain::keys::iso`] writes it),
//! and `notification_pref`, how each kind of notification reaches it
@@ -107,7 +113,7 @@
/// Each step of the schema, run once, in order, by `PRAGMA user_version`.
/// A step is only ever appended: one that has run is never changed.
pub const MIGRATIONS: &[&str] = &[V1, V2, V3, V4, V5, V6, V7, V8, V9, V10];
pub const MIGRATIONS: &[&str] = &[V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11];
const V1: &str = "
CREATE TABLE gallery (
@@ -504,3 +510,10 @@ CREATE TABLE classification (
) WITHOUT ROWID;
CREATE INDEX classification_did ON classification (subject_did);
";
/// Reports by the reason atproto names them with, and found by the account
/// they are about.
const V11: &str = "
ALTER TABLE report ADD COLUMN reason_type TEXT;
CREATE INDEX report_subject_did ON report (subject_did, id);
";
crates/grain-serve/src/lib.rs+21−5
View file
@@ -2,10 +2,10 @@
//! `social.grain.sync.indexRecords` ([`sync`]), the queries that read
//! the index alone ([`read`]), what an account has seen of its
//! notifications ([`notification`]) and the devices they are pushed to
//! ([`push`]), and the web's writes ([`write`]), draws the cards links
//! unfurl into ([`og`]), and logs the web in ([`login`]). Off the path of
//! any request, its classifiers score what arrives for moderators to
//! review ([`classify`]).
//! ([`push`]), moderation ([`moderation`]), and the web's writes
//! ([`write`]), draws the cards links unfurl into ([`og`]), and logs the
//! web in ([`login`]). Off the path of any request, its classifiers score
//! what arrives for moderators to review ([`classify`]).
pub mod backfill;
pub mod cdn;
@@ -13,6 +13,7 @@ pub mod classify;
pub mod index;
pub mod jetstream;
pub mod login;
pub mod moderation;
pub mod notification;
pub mod og;
pub mod preferences;
@@ -60,6 +61,9 @@ pub struct Settings {
/// `GRAIN_SIGN_UP_PDS`: where someone making an account on the web
/// makes it. None is [`login::SIGN_UP_PDS`].
pub sign_up_pds: Option<String>,
/// `GRAIN_ADMINS`: the DIDs of the accounts that moderate, separated
/// by commas or spaces ([`moderation`]).
pub admins: Vec<String>,
/// `GRAIN_CARRY_COOKIES_UNTIL`: the last moment, an RFC 3339 time, the
/// web carries the cookies browsers held before grain-serve
/// ([`login::carryover`]). None carries none.
@@ -94,6 +98,12 @@ impl Settings {
Some(url) => Some(url.trim_end_matches('/').into()),
},
sign_up_pds: set("GRAIN_SIGN_UP_PDS"),
admins: set("GRAIN_ADMINS")
.unwrap_or_default()
.split([',', ' '])
.filter(|did| !did.is_empty())
.map(str::to_string)
.collect(),
carry_cookies_until: set("GRAIN_CARRY_COOKIES_UNTIL"),
push: push::Config::from_vars(&get),
clef: classify::clef::Config::from_vars(&get),
@@ -192,6 +202,7 @@ impl Settings {
cdn,
bsky_appview: self.bsky_appview.clone(),
web,
admins: self.admins.clone(),
cards: og::Cards::default(),
})
}
@@ -211,6 +222,8 @@ pub struct App {
pub bsky_appview: Option<String>,
/// The web's logins, for a server with a public URL and an index.
pub web: Option<Arc<login::Web>>,
/// The accounts that moderate, by DID ([`moderation`]).
pub admins: Vec<String>,
/// The OpenGraph cards drawn lately.
pub cards: og::Cards,
}
@@ -226,6 +239,7 @@ impl App {
cdn: cdn::Cdn::default(),
bsky_appview: None,
web: None,
admins: Vec::new(),
cards: og::Cards::default(),
}
}
@@ -246,7 +260,7 @@ pub fn service_did(public_url: &str) -> anyhow::Result<String> {
/// The routes the server answers.
pub fn router(app: Arc<App>) -> Router {
og::routes(read::routes(Router::new()))
moderation::routes(og::routes(read::routes(Router::new())))
.route("/xrpc/_health", get(health))
.route(
"/xrpc/social.grain.sync.indexRecords",
@@ -354,8 +368,10 @@ mod tests {
"GRAIN_CDN_URL" => Some("https://cdn.grain.social".into()),
"GRAIN_BSKY_APPVIEW" => Some("1".into()),
"GRAIN_SIGN_UP_PDS" => Some("https://pds.test".into()),
"GRAIN_ADMINS" => Some("did:plc:alice, did:web:bob.test".into()),
_ => None,
});
assert_eq!(set.admins, ["did:plc:alice", "did:web:bob.test"]);
assert_eq!(
set.bsky_appview.as_deref(),
Some(read::search::BSKY_APPVIEW)
crates/grain-serve/src/moderation.rsnew+585−0
View file
585 lines changed: not loaded until asked for.
crates/grain-serve/tests/web/mod.rs+2−0
View file
crates/grain-serve/tests/classify.rs+46−24
View file
crates/grain-serve/tests/index_records.rs+1−0
View file
crates/grain-serve/tests/moderation.rsnew+697−0
View file
697 lines changed: not loaded until asked for.
crates/grain-serve/tests/notifications.rs+1−0
View file
crates/grain-serve/tests/push.rs+1−0
View file
crates/grain-serve/tests/read.rs+3−0
View file
crates/grain-serve/Cargo.toml+1−0
View file
lexicons/com/atproto/admin/defs.jsonnew+47−0
View file
lexicons/com/atproto/admin/getSubjectStatus.jsonnew+43−0
View file
lexicons/com/atproto/admin/updateSubjectStatus.jsonnew+56−0
View file
lexicons/com/atproto/label/queryLabels.jsonnew+47−0
View file
lexicons/com/atproto/moderation/createReport.jsonnew+78−0
View file
lexicons/com/atproto/moderation/defs.json+56−8
View file
lexicons/social/grain/admin/defs.json+5−1
View file
lexicons/social/grain/admin/getReviewSubject.json+13−3
View file
lexicons/social/grain/moderation/defs.jsonnew+14−0
View file
lexicons/README.md+6−0
View file
Cargo.lock+1−0
View file
hatk.generated.client.ts+2−2
View file
hatk.generated.ts+32−8
View file