johnandersen777.bsky.social/knotless

Switch repository

Loading…

//! A decision model as the classifier: any that speaks the decision API
//! TypeSafe's Jev does, as Cloudflare's Clef does too. Such a model reads
//! an issue and answers typed questions about it, each with a probability,
//! and writes no text of its own, so what an issue says can move
//! probabilities but never step outside the labels asked about.
//!
//! The questions come from the repository's labels. A scope whose labels
//! are all exclusive, as `kind/bug` and `kind/feature` can be, is one
//! `choice` among them, with a way to say none fits; any other scope asks
//! of each label whether it fits (`noul`). An answer short of the
//! threshold sets nothing, so the triager says nothing of a scope rather
//! than guess, and every probability goes into the reason it gives.

use anyhow::{Context, Result, bail};
use knotless::records::Label;
use knotless::triage::{Choice, Pending};
use serde_json::{Map, Value, json};

/// The option of a `choice` that says none of its labels fits.
const NONE: &str = "none_of_these";

/// Where and how decisions are asked for.
#[derive(Clone, PartialEq)]
pub struct Decider {
    /// The endpoint taking `{model, state, questions}`, as
    /// `https://api.typesafe.ai/v1/systemone`, or a model's on Workers AI.
    pub endpoint: String,
    /// Sent as a bearer token.
    pub token: String,
    /// The model asked, as `jev-latest` or `clef`.
    pub model: String,
    /// The least probability that sets a label.
    pub threshold: f64,
}

/// What one question asks: which of a scope's labels fits, or whether one
/// label does.
enum Question<'a> {
    Choice {
        scope: &'a str,
        options: Vec<(String, &'a Label)>,
    },
    Fits(&'a Label),
}

impl Question<'_> {
    fn schema(&self) -> Value {
        match self {
            Question::Choice { scope, options } => {
                let mut criteria: Map<String, Value> = (options.iter())
                    .map(|(key, label)| (key.clone(), meaning(label).into()))
                    .collect();
                criteria.insert(NONE.into(), "None of these fits".into());
                let instructions = format!("Which {scope} is this issue?");
                json!({"type": "choice", "instructions": instructions, "criteria": criteria})
            }
            Question::Fits(label) => {
                let instructions = format!("Does the label {} fit this issue?", meaning(label));
                json!({"type": "noul", "instructions": instructions})
            }
        }
    }
}

/// A label's name, and what it means when its definition says.
fn meaning(label: &Label) -> String {
    match &label.description {
        Some(about) => format!("{}: {about}", label.name),
        None => label.name.clone(),
    }
}

/// `name` as a question's or option's key, lowercase letters, digits and `_`, made
/// unlike every key in `taken`.
fn key(name: &str, taken: &[String]) -> String {
    let mut key: String = (name.chars())
        .map(|c| match c.is_ascii_alphanumeric() {
            true => c.to_ascii_lowercase(),
            false => '_',
        })
        .collect();
    while taken.contains(&key) {
        key.push('_');
    }
    key
}

/// The questions `labels` make, by key, in the order the repository
/// defines them.
fn ask(labels: &[Label]) -> Vec<(String, Question<'_>)> {
    let mut scopes: Vec<&str> = Vec::new();
    for scope in labels.iter().filter_map(Label::scope) {
        if !scopes.contains(&scope) {
            scopes.push(scope);
        }
    }
    let mut asked: Vec<(String, Question)> = Vec::new();
    for scope in scopes {
        let of: Vec<&Label> = labels.iter().filter(|l| l.scope() == Some(scope)).collect();
        let taken: Vec<String> = asked.iter().map(|(k, _)| k.clone()).collect();
        if of.iter().all(|l| l.exclusive) {
            let mut options: Vec<(String, &Label)> = Vec::new();
            for label in of {
                let leaf = &label.name[scope.len() + 1..];
                let mut keys: Vec<String> = options.iter().map(|(k, _)| k.clone()).collect();
                keys.push(NONE.into());
                options.push((key(leaf, &keys), label));
            }
            asked.push((key(scope, &taken), Question::Choice { scope, options }));
        } else {
            for label in of {
                let taken: Vec<String> = asked.iter().map(|(k, _)| k.clone()).collect();
                asked.push((key(&label.name, &taken), Question::Fits(label)));
            }
        }
    }
    asked
}

/// What the model chooses for `issue`: of each question, the label it gives at
/// least the threshold's probability.
pub async fn classify(
    http: &reqwest::Client,
    decider: &Decider,
    issue: &Pending,
) -> Result<Choice> {
    let asked = ask(&issue.labels);
    let questions: Map<String, Value> = (asked.iter())
        .map(|(key, q)| (key.clone(), q.schema()))
        .collect();
    let state = json!({
        "what": "An issue opened on a git repository",
        "title": issue.title,
        "body": issue.body,
    });
    let body = json!({"model": decider.model, "state": state, "questions": questions});
    let res = http.post(&decider.endpoint).bearer_auth(&decider.token);
    let res = res.json(&body);
    let res = res.send().await?;
    let status = res.status();
    let text = res.text().await?;
    if !status.is_success() {
        bail!(
            "{}: {status}: {}",
            decider.endpoint,
            text.chars().take(300).collect::<String>()
        );
    }
    let body: Value = serde_json::from_str(&text)?;
    // Workers AI wraps the answer in `result`.
    let body = body.get("result").unwrap_or(&body);
    let answers = body
        .get("answers")
        .context("the model answered no questions")?;
    let mut chosen = Vec::new();
    let mut said = Vec::new();
    for (key, question) in &asked {
        let answer = &answers[key];
        let (label, p) = match question {
            Question::Choice { options, .. } => {
                let picked = options.iter().find(|(k, _)| answer["choice"] == k.as_str());
                (picked.map(|(_, l)| *l), answer["confidence"].as_f64())
            }
            Question::Fits(label) => (Some(*label), answer["noul"].as_f64()),
        };
        let p = p.unwrap_or(0.0);
        let named = label.map_or_else(|| format!("{key}: none"), |l| l.name.clone());
        said.push(format!("{named} {p:.2}"));
        if let Some(label) = label.filter(|_| p >= decider.threshold) {
            chosen.push(label.name.clone());
        }
    }
    Ok(Choice {
        labels: chosen,
        // The version that answered, when the answer says.
        model: Some(body["model"].as_str().unwrap_or(&decider.model).to_string()),
        reason: Some(said.join("; ")),
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    fn label(name: &str, exclusive: bool) -> Label {
        let mut label = Label::new(name);
        label.exclusive = exclusive;
        label
    }

    #[test]
    fn questions_from_labels() {
        let mut bug = label("kind/bug", true);
        bug.description = Some("Something is broken".into());
        let labels = [
            bug,
            label("kind/bug-fix", true),
            label("kind/bug_fix", true),
            label("kind/none-of-these", true),
            label("area/web", false),
            label("area/cli", true),
            label("kind_bug/x", false),
        ];
        let asked = ask(&labels);
        let keys: Vec<&str> = asked.iter().map(|(k, _)| k.as_str()).collect();
        assert_eq!(keys, ["kind", "area_web", "area_cli", "kind_bug_x"]);
        let kind = asked[0].1.schema();
        assert_eq!(kind["type"], "choice");
        assert_eq!(kind["instructions"], "Which kind is this issue?");
        let criteria = kind["criteria"].as_object().unwrap();
        let options: Vec<&str> = criteria.keys().map(String::as_str).collect();
        assert_eq!(
            options,
            [
                "bug",
                "bug_fix",
                "bug_fix_",
                "none_of_these",
                "none_of_these_"
            ]
        );
        assert_eq!(criteria["bug"], "kind/bug: Something is broken");
        assert_eq!(criteria["none_of_these"], "None of these fits");
        let web = asked[1].1.schema();
        assert_eq!(web["type"], "noul");
        assert_eq!(
            web["instructions"],
            "Does the label area/web fit this issue?"
        );
        assert_eq!(key("kind", &["kind".into(), "kind_".into()]), "kind__");
    }
}