//! A decision model as the classifier: any that speaks the decision API //! TypeSafe's Jev does, as Cloudflare's Clef does too. Such a model reads //! an issue and answers typed questions about it, each with a probability, //! and writes no text of its own, so what an issue says can move //! probabilities but never step outside the labels asked about. //! //! The questions come from the repository's labels. A scope whose labels //! are all exclusive, as `kind/bug` and `kind/feature` can be, is one //! `choice` among them, with a way to say none fits; any other scope asks //! of each label whether it fits (`noul`). An answer short of the //! threshold sets nothing, so the triager says nothing of a scope rather //! than guess, and every probability goes into the reason it gives. use anyhow::{Context, Result, bail}; use knotless::records::Label; use knotless::triage::{Choice, Pending}; use serde_json::{Map, Value, json}; /// The option of a `choice` that says none of its labels fits. const NONE: &str = "none_of_these"; /// Where and how decisions are asked for. #[derive(Clone, PartialEq)] pub struct Decider { /// The endpoint taking `{model, state, questions}`, as /// `https://api.typesafe.ai/v1/systemone`, or a model's on Workers AI. pub endpoint: String, /// Sent as a bearer token. pub token: String, /// The model asked, as `jev-latest` or `clef`. pub model: String, /// The least probability that sets a label. pub threshold: f64, } /// What one question asks: which of a scope's labels fits, or whether one /// label does. enum Question<'a> { Choice { scope: &'a str, options: Vec<(String, &'a Label)>, }, Fits(&'a Label), } impl Question<'_> { fn schema(&self) -> Value { match self { Question::Choice { scope, options } => { let mut criteria: Map = (options.iter()) .map(|(key, label)| (key.clone(), meaning(label).into())) .collect(); criteria.insert(NONE.into(), "None of these fits".into()); let instructions = format!("Which {scope} is this issue?"); json!({"type": "choice", "instructions": instructions, "criteria": criteria}) } Question::Fits(label) => { let instructions = format!("Does the label {} fit this issue?", meaning(label)); json!({"type": "noul", "instructions": instructions}) } } } } /// A label's name, and what it means when its definition says. fn meaning(label: &Label) -> String { match &label.description { Some(about) => format!("{}: {about}", label.name), None => label.name.clone(), } } /// `name` as a question's or option's key, lowercase letters, digits and `_`, made /// unlike every key in `taken`. fn key(name: &str, taken: &[String]) -> String { let mut key: String = (name.chars()) .map(|c| match c.is_ascii_alphanumeric() { true => c.to_ascii_lowercase(), false => '_', }) .collect(); while taken.contains(&key) { key.push('_'); } key } /// The questions `labels` make, by key, in the order the repository /// defines them. fn ask(labels: &[Label]) -> Vec<(String, Question<'_>)> { let mut scopes: Vec<&str> = Vec::new(); for scope in labels.iter().filter_map(Label::scope) { if !scopes.contains(&scope) { scopes.push(scope); } } let mut asked: Vec<(String, Question)> = Vec::new(); for scope in scopes { let of: Vec<&Label> = labels.iter().filter(|l| l.scope() == Some(scope)).collect(); let taken: Vec = asked.iter().map(|(k, _)| k.clone()).collect(); if of.iter().all(|l| l.exclusive) { let mut options: Vec<(String, &Label)> = Vec::new(); for label in of { let leaf = &label.name[scope.len() + 1..]; let mut keys: Vec = options.iter().map(|(k, _)| k.clone()).collect(); keys.push(NONE.into()); options.push((key(leaf, &keys), label)); } asked.push((key(scope, &taken), Question::Choice { scope, options })); } else { for label in of { let taken: Vec = asked.iter().map(|(k, _)| k.clone()).collect(); asked.push((key(&label.name, &taken), Question::Fits(label))); } } } asked } /// What the model chooses for `issue`: of each question, the label it gives at /// least the threshold's probability. pub async fn classify( http: &reqwest::Client, decider: &Decider, issue: &Pending, ) -> Result { let asked = ask(&issue.labels); let questions: Map = (asked.iter()) .map(|(key, q)| (key.clone(), q.schema())) .collect(); let state = json!({ "what": "An issue opened on a git repository", "title": issue.title, "body": issue.body, }); let body = json!({"model": decider.model, "state": state, "questions": questions}); let res = http.post(&decider.endpoint).bearer_auth(&decider.token); let res = res.json(&body); let res = res.send().await?; let status = res.status(); let text = res.text().await?; if !status.is_success() { bail!( "{}: {status}: {}", decider.endpoint, text.chars().take(300).collect::() ); } let body: Value = serde_json::from_str(&text)?; // Workers AI wraps the answer in `result`. let body = body.get("result").unwrap_or(&body); let answers = body .get("answers") .context("the model answered no questions")?; let mut chosen = Vec::new(); let mut said = Vec::new(); for (key, question) in &asked { let answer = &answers[key]; let (label, p) = match question { Question::Choice { options, .. } => { let picked = options.iter().find(|(k, _)| answer["choice"] == k.as_str()); (picked.map(|(_, l)| *l), answer["confidence"].as_f64()) } Question::Fits(label) => (Some(*label), answer["noul"].as_f64()), }; let p = p.unwrap_or(0.0); let named = label.map_or_else(|| format!("{key}: none"), |l| l.name.clone()); said.push(format!("{named} {p:.2}")); if let Some(label) = label.filter(|_| p >= decider.threshold) { chosen.push(label.name.clone()); } } Ok(Choice { labels: chosen, // The version that answered, when the answer says. model: Some(body["model"].as_str().unwrap_or(&decider.model).to_string()), reason: Some(said.join("; ")), }) } #[cfg(test)] mod tests { use super::*; fn label(name: &str, exclusive: bool) -> Label { let mut label = Label::new(name); label.exclusive = exclusive; label } #[test] fn questions_from_labels() { let mut bug = label("kind/bug", true); bug.description = Some("Something is broken".into()); let labels = [ bug, label("kind/bug-fix", true), label("kind/bug_fix", true), label("kind/none-of-these", true), label("area/web", false), label("area/cli", true), label("kind_bug/x", false), ]; let asked = ask(&labels); let keys: Vec<&str> = asked.iter().map(|(k, _)| k.as_str()).collect(); assert_eq!(keys, ["kind", "area_web", "area_cli", "kind_bug_x"]); let kind = asked[0].1.schema(); assert_eq!(kind["type"], "choice"); assert_eq!(kind["instructions"], "Which kind is this issue?"); let criteria = kind["criteria"].as_object().unwrap(); let options: Vec<&str> = criteria.keys().map(String::as_str).collect(); assert_eq!( options, [ "bug", "bug_fix", "bug_fix_", "none_of_these", "none_of_these_" ] ); assert_eq!(criteria["bug"], "kind/bug: Something is broken"); assert_eq!(criteria["none_of_these"], "None of these fits"); let web = asked[1].1.schema(); assert_eq!(web["type"], "noul"); assert_eq!( web["instructions"], "Does the label area/web fit this issue?" ); assert_eq!(key("kind", &["kind".into(), "kind_".into()]), "kind__"); } }