~ knotless Search/
↑↓ move↵ open⇧↵ all results

grain.social/grain-support-agent

grain's support agent: triages mentions and reports, investigates bugs, proposes fixes

1 collaborator
0
0
main
88 lines · 3.9 KB Raw History
import type { Moderation } from "@workspace/types";
import { childSafetyReport, CSAE, HUMAN, withheldFromModels, withhold } from "./child-safety.ts";
import { applyLabel, listLabelDefinitions, moderatorConfigured, openReports, resolveReport, takeDown } from "./grain.ts";
import { update, type Item } from "./store.ts";

// Acting on a report through grain-serve's moderation methods, as the
// maintainer asked in the item's conversation. Every action is on the item's
// own subject, never on one the agent names: text it read cannot point it at
// someone else.
//
// Dismiss and label happen when asked. A takedown waits for the maintainer to
// confirm it in the dashboard, because it is the action a prompt injection in
// a report would most want to steer toward.
//
// The agent never acts on anything about the safety of a child: it may not
// give out or take back the csae label, nor act at all on a subject whose
// account is child safety (child-safety.ts). Those are for a person, with the
// dashboard's buttons or the grain command line.

/** Who asks for an action: the agent, through the moderate tool, or a person, through the dashboard. */
export type Actor = "model" | "person";

export const configured = moderatorConfigured;

/** The labels the agent may give out: grain's, but for those only an admin's own action gives, and csae, which is a person's alone. */
export async function labelNames(): Promise<string[]> {
  const { definitions } = await listLabelDefinitions();
  return definitions.map((d) => d.identifier).filter((id) => !id.startsWith("!") && id !== CSAE);
}

/** Why the agent may not take `m` on the item: HUMAN, when it concerns the safety of a child; null when it may. */
export async function modelRefusal(item: Item, m: Pick<Moderation, "label">): Promise<string | null> {
  if (m.label === CSAE) return HUMAN;
  if (await withheldFromModels(item)) return HUMAN;
  return null;
}

/** Whether an item is a report with a subject grain-serve can act on. */
export function actionable(item: Item): string | null {
  if (item.source === "bluesky") return "this is a Bluesky post, not a report; there is nothing on grain to act on";
  if (!item.subject_uri || !item.subject_did) return "this report has no subject recorded";
  if (item.moderation_done) return "this report has already been acted on";
  return null;
}

/**
 * Act on the item's subject, then close every open report on it, the
 * classifier's and any a person filed, and the item is done.
 *
 * - takedown takes the subject's account down, and resolves the reports;
 * - label gives out the label: a report that asked for it is resolved, which
 *   gives it out, and the rest are dismissed;
 * - dismiss dismisses them.
 *
 * `by` the agent, it is refused (HUMAN) when it concerns the safety of a
 * child, by the label, the item, its account, or any open report on the
 * subject.
 */
export async function act(item: Item, m: Moderation, by: Actor): Promise<void> {
  if (by === "model") {
    const refusal = await modelRefusal(item, m);
    if (refusal) throw new Error(refusal);
  }
  const subject = item.subject_uri;
  const open = (await openReports()).filter((r) => r.subject === subject);
  if (by === "model" && open.some(childSafetyReport)) {
    withhold(item);
    throw new Error(HUMAN);
  }
  if (m.action === "takedown") await takeDown(item.subject_did);
  if (m.action === "label") {
    if (!m.label) throw new Error("a label action names its label");
    if (!open.some((r) => r.label === m.label)) await applyLabel(subject, m.label);
  }
  for (const r of open) {
    const upheld = m.action === "takedown" || (m.action === "label" && r.label === m.label);
    await resolveReport(r.id, upheld ? "resolve" : "dismiss");
  }
  update(item.id, {
    status: "done",
    moderation_pending: "",
    moderation_done: JSON.stringify({ ...m, at: new Date().toISOString() }),
  });
}

export function parse(json: string): Moderation | null {
  return json ? (JSON.parse(json) as Moderation) : null;
}