import type { Moderation } from "@workspace/types"; import { childSafetyReport, CSAE, HUMAN, withheldFromModels, withhold } from "./child-safety.ts"; import { applyLabel, listLabelDefinitions, moderatorConfigured, openReports, resolveReport, takeDown } from "./grain.ts"; import { update, type Item } from "./store.ts"; // Acting on a report through grain-serve's moderation methods, as the // maintainer asked in the item's conversation. Every action is on the item's // own subject, never on one the agent names: text it read cannot point it at // someone else. // // Dismiss and label happen when asked. A takedown waits for the maintainer to // confirm it in the dashboard, because it is the action a prompt injection in // a report would most want to steer toward. // // The agent never acts on anything about the safety of a child: it may not // give out or take back the csae label, nor act at all on a subject whose // account is child safety (child-safety.ts). Those are for a person, with the // dashboard's buttons or the grain command line. /** Who asks for an action: the agent, through the moderate tool, or a person, through the dashboard. */ export type Actor = "model" | "person"; export const configured = moderatorConfigured; /** The labels the agent may give out: grain's, but for those only an admin's own action gives, and csae, which is a person's alone. */ export async function labelNames(): Promise { const { definitions } = await listLabelDefinitions(); return definitions.map((d) => d.identifier).filter((id) => !id.startsWith("!") && id !== CSAE); } /** Why the agent may not take `m` on the item: HUMAN, when it concerns the safety of a child; null when it may. */ export async function modelRefusal(item: Item, m: Pick): Promise { if (m.label === CSAE) return HUMAN; if (await withheldFromModels(item)) return HUMAN; return null; } /** Whether an item is a report with a subject grain-serve can act on. */ export function actionable(item: Item): string | null { if (item.source === "bluesky") return "this is a Bluesky post, not a report; there is nothing on grain to act on"; if (!item.subject_uri || !item.subject_did) return "this report has no subject recorded"; if (item.moderation_done) return "this report has already been acted on"; return null; } /** * Act on the item's subject, then close every open report on it, the * classifier's and any a person filed, and the item is done. * * - takedown takes the subject's account down, and resolves the reports; * - label gives out the label: a report that asked for it is resolved, which * gives it out, and the rest are dismissed; * - dismiss dismisses them. * * `by` the agent, it is refused (HUMAN) when it concerns the safety of a * child, by the label, the item, its account, or any open report on the * subject. */ export async function act(item: Item, m: Moderation, by: Actor): Promise { if (by === "model") { const refusal = await modelRefusal(item, m); if (refusal) throw new Error(refusal); } const subject = item.subject_uri; const open = (await openReports()).filter((r) => r.subject === subject); if (by === "model" && open.some(childSafetyReport)) { withhold(item); throw new Error(HUMAN); } if (m.action === "takedown") await takeDown(item.subject_did); if (m.action === "label") { if (!m.label) throw new Error("a label action names its label"); if (!open.some((r) => r.label === m.label)) await applyLabel(subject, m.label); } for (const r of open) { const upheld = m.action === "takedown" || (m.action === "label" && r.label === m.label); await resolveReport(r.id, upheld ? "resolve" : "dismiss"); } update(item.id, { status: "done", moderation_pending: "", moderation_done: JSON.stringify({ ...m, at: new Date().toISOString() }), }); } export function parse(json: string): Moderation | null { return json ? (JSON.parse(json) as Moderation) : null; }