johnandersen777.bsky.social/knotless
git that lives in your Atmosphere account
Forked from chadtmiller.com/knotless
0
0
//! What a repository is written in, as GitHub's linguist says it, simply:
//! the bytes of each language's files in the default branch's tree, each
//! file known by its extension or its name. What linguist leaves out by
//! default is left out here: what is vendored or generated, what is
//! documentation, and prose and data (Markdown, JSON, YAML), which the
//! table below never names. A binary is never named either.
//!
//! Sizes are read from each object's header, never its contents, and the
//! answer is kept by tree: a commit's is counted once.
//!
//! What a public repository is mostly written in, which lists of them
//! show, is kept in the index ([`follow`]), so a list reads no repository.
//! The shared steps its lock pins are read from the same tree as it is
//! counted ([`crate::shared`]).
use crate::App;
use crate::objects::{self, Objects};
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::{Arc, LazyLock, Mutex};
/// Each language's name, its color in linguist, and the extensions (with
/// their dot) and file names that are written in it.
const KINDS: &[(&str, &str, &[&str])] = &[
("Rust", "#dea584", &[".rs"]),
("TypeScript", "#3178c6", &[".ts", ".mts", ".cts"]),
("TSX", "#3178c6", &[".tsx"]),
("JavaScript", "#f1e05a", &[".js", ".mjs", ".cjs", ".jsx"]),
("Svelte", "#ff3e00", &[".svelte"]),
("Vue", "#41b883", &[".vue"]),
("Astro", "#ff5a03", &[".astro"]),
("CSS", "#663399", &[".css"]),
("SCSS", "#c6538c", &[".scss"]),
("Sass", "#a53b70", &[".sass"]),
("Less", "#1d365d", &[".less"]),
("HTML", "#e34c26", &[".html", ".htm"]),
("MDX", "#fcb32c", &[".mdx"]),
("Shell", "#89e051", &[".sh", ".bash", ".zsh"]),
("fish", "#4aae47", &[".fish"]),
("PowerShell", "#012456", &[".ps1", ".psm1"]),
("Batchfile", "#c1f12e", &[".bat", ".cmd"]),
("Python", "#3572A5", &[".py", ".pyi"]),
("Go", "#00ADD8", &[".go"]),
("Java", "#b07219", &[".java"]),
("Kotlin", "#A97BFF", &[".kt", ".kts"]),
("Scala", "#c22d40", &[".scala"]),
("Groovy", "#4298b8", &[".groovy", ".gradle"]),
("Clojure", "#db5855", &[".clj", ".cljs", ".cljc"]),
("Swift", "#F05138", &[".swift"]),
("Objective-C", "#438eff", &[".m"]),
("Objective-C++", "#6866fb", &[".mm"]),
("C", "#555555", &[".c", ".h"]),
(
"C++",
"#f34b7d",
&[".cpp", ".cc", ".cxx", ".hpp", ".hh", ".hxx"],
),
("C#", "#178600", &[".cs"]),
("F#", "#b845fc", &[".fs", ".fsx"]),
("Zig", "#ec915c", &[".zig"]),
("Nim", "#ffc200", &[".nim"]),
("Assembly", "#6E4C13", &[".asm", ".s"]),
("WebAssembly", "#04133b", &[".wat"]),
("Ruby", "#701516", &[".rb", "Rakefile", "Gemfile"]),
("PHP", "#4F5D95", &[".php"]),
("Perl", "#0298c3", &[".pl", ".pm"]),
("Lua", "#000080", &[".lua"]),
("R", "#198CE7", &[".r"]),
("Julia", "#a270ba", &[".jl"]),
("Dart", "#00B4AB", &[".dart"]),
("Elixir", "#6e4a7e", &[".ex", ".exs"]),
("Erlang", "#B83998", &[".erl", ".hrl"]),
("Gleam", "#ffaff3", &[".gleam"]),
("Elm", "#60B5CC", &[".elm"]),
("Haskell", "#5e5086", &[".hs"]),
("OCaml", "#ef7a08", &[".ml", ".mli"]),
("Emacs Lisp", "#c065db", &[".el"]),
("Vim Script", "#199f4b", &[".vim"]),
("Solidity", "#AA6746", &[".sol"]),
("HCL", "#844FBA", &[".tf", ".hcl"]),
("Nix", "#7e7eff", &[".nix"]),
(
"Starlark",
"#76d275",
&[".bzl", ".star", "BUILD", "BUILD.bazel", "WORKSPACE"],
),
("TeX", "#3D6117", &[".tex"]),
("CoffeeScript", "#244776", &[".coffee"]),
("Dockerfile", "#384d54", &[".dockerfile", "Dockerfile"]),
(
"Makefile",
"#427819",
&[".mk", "Makefile", "GNUmakefile", "makefile"],
),
// `.justfile` is known by its extension, as a name with nothing before.
(
"Just",
"#384d54",
&[".just", ".justfile", "justfile", "Justfile"],
),
];
/// What languages under one percent are shown as, together, and its color.
pub const OTHER: (&str, &str) = ("Other", "#ededed");
/// Directories whose files are someone else's, anywhere in a path.
const VENDORED: &[&str] = &[
"node_modules",
"vendor",
"third_party",
"bower_components",
"dist",
];
/// Directories of documentation and examples, at the top of a tree.
const DOCUMENTATION: &[&str] = &["docs", "doc", "documentation", "examples", "demo", "demos"];
/// Files a build wrote, by how their names end.
const GENERATED: &[&str] = &[".min.js", ".min.css", "-min.js", ".bundle.js"];
/// Each kind of file by its whole name, and by its extension, lowercased.
type Known = (HashMap<&'static str, usize>, HashMap<String, usize>);
static KNOWN: LazyLock<Known> = LazyLock::new(|| {
let (mut names, mut extensions) = (HashMap::new(), HashMap::new());
for (i, (_, _, patterns)) in KINDS.iter().enumerate() {
for p in *patterns {
match p.strip_prefix('.') {
Some(ext) => extensions.insert(ext.to_string(), i),
None => names.insert(*p, i),
};
}
}
(names, extensions)
});
/// Prose and data, which are never counted, and their colors in linguist:
/// a code search finds them all the same.
const UNCOUNTED: &[(&str, &str)] = &[
("Markdown", "#083fa1"),
("JSON", "#292929"),
("YAML", "#cb171e"),
("TOML", "#9c4221"),
];
/// The color linguist gives the language it names `name`, as Zoekt names
/// languages too, if it is one known here.
pub fn color(name: &str) -> Option<&'static str> {
let counted = KINDS.iter().find(|(n, ..)| *n == name).map(|(_, c, _)| *c);
let uncounted = || UNCOUNTED.iter().find(|(n, _)| *n == name).map(|(_, c)| *c);
counted.or_else(uncounted)
}
/// The language `path` is written in, by its index in `KINDS`, if it is
/// one that is counted.
pub fn kind(path: &str) -> Option<usize> {
let mut dirs = path.split('/').collect::<Vec<_>>();
let name = dirs.pop()?;
if dirs.iter().any(|d| VENDORED.contains(d))
|| dirs
.first()
.is_some_and(|d| DOCUMENTATION.contains(&d.to_ascii_lowercase().as_str()))
{
return None;
}
let lower = name.to_ascii_lowercase();
if GENERATED.iter().any(|g| lower.ends_with(g)) {
return None;
}
let (names, extensions) = &*KNOWN;
if let Some(&i) = names.get(name) {
return Some(i);
}
let (_, ext) = lower.rsplit_once('.')?;
extensions.get(ext).copied()
}
/// One language a repository is written in.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct Language {
/// Its name, or `Other` for the languages each under one percent.
pub name: String,
/// Its color, as linguist has it.
pub color: String,
pub bytes: u64,
/// Its share of all the bytes counted, in percent, to a tenth.
pub percent: f64,
}
/// The languages of `files`, each a kind and its size: most bytes first,
/// those each under one percent together last, as `Other`, when there is
/// more than one of them.
pub fn breakdown(files: impl IntoIterator<Item = (usize, u64)>) -> Vec<Language> {
let mut bytes = vec![0u64; KINDS.len()];
for (kind, size) in files {
bytes[kind] += size;
}
let total: u64 = bytes.iter().sum();
let percent = |n: u64| (n as f64 * 1000.0 / total as f64).round() / 10.0;
let mut found: Vec<(usize, u64)> = bytes
.into_iter()
.enumerate()
.filter(|(_, n)| *n > 0)
.collect();
found.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0)));
let small = found.iter().filter(|(_, n)| n * 100 < total).count();
let shown = if small > 1 {
found.len() - small
} else {
found.len()
};
let mut out: Vec<Language> = found[..shown]
.iter()
.map(|&(kind, n)| {
let (name, color, _) = KINDS[kind];
Language {
name: name.into(),
color: color.into(),
bytes: n,
percent: percent(n),
}
})
.collect();
if shown < found.len() {
let rest = found[shown..].iter().map(|(_, n)| n).sum();
out.push(Language {
name: OTHER.0.into(),
color: OTHER.1.into(),
bytes: rest,
percent: percent(rest),
});
}
out
}
/// Each tree's languages, as they were counted; forgotten all at once past
/// a number of them.
static COUNTED: LazyLock<Mutex<HashMap<String, Vec<Language>>>> = LazyLock::new(Mutex::default);
const KEPT: usize = 1024;
/// The languages of the tree `rev` names in `objects`.
pub fn of(objects: &Objects, rev: &str) -> Result<Vec<Language>> {
let tree = objects.tree_id(rev)?;
if let Some(known) =
knotless::telemetry::looked("languages", COUNTED.lock().unwrap().get(&tree))
{
return Ok(known.clone());
}
let found = breakdown(objects.files(rev, kind)?);
let mut counted = COUNTED.lock().unwrap();
if counted.len() >= KEPT {
counted.clear();
}
counted.insert(tree, found.clone());
Ok(found)
}
/// The language most of what `objects`' HEAD names is written in: none
/// when no file of it is counted. `Other` is never one.
pub fn main_of(objects: &Objects) -> Option<Language> {
let found = of(objects, "HEAD").unwrap_or_default();
found.into_iter().find(|l| l.name != OTHER.0)
}
/// Keeps what each public repository is mostly written in, in the index,
/// as [`crate::index::Index::recount`] asks: for one a list showed that
/// was never counted, and again for one that was as its refs or record
/// change. One at a time, each from its mirror synced, so what a list
/// shows is never waited on and a push costs one count.
pub async fn follow(app: Arc<App>) {
let Some(mut asked) = app.index.as_ref().and_then(|i| i.counting()) else {
return;
};
while let Some((did, key)) = asked.recv().await {
recount(&app, &did, &key).await;
}
}
/// Counts `did`'s repository with `key` again, and keeps the answer; and
/// what shared steps its lock pins, read from the same tree
/// ([`crate::shared::read_uses`]).
async fn recount(app: &App, did: &str, key: &str) -> Option<()> {
let index = app.index.as_ref()?;
index.answered(did, key);
let name = index.name(did, key).ok().flatten()?;
let git = app.mirror(did, &name).await.ok()?;
// Read, and let go of, before what is awaited: a reader belongs to
// one thread.
let (value, lock) = {
let objects = objects::open(&git.dir).ok()?;
let value = serde_json::to_string(&main_of(&objects)).ok()?;
(value, objects.blob("HEAD", knotless::uses::LOCK).ok())
};
index.set_language(did, key, &value).ok()?;
crate::shared::read_uses(app, index, did, key, lock.as_deref()).await;
Some(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn colors() {
assert_eq!(color("Rust"), Some("#dea584"));
assert_eq!(color("Markdown"), Some("#083fa1"));
assert_eq!(color("Klingon"), None);
}
fn name(path: &str) -> Option<&'static str> {
kind(path).map(|i| KINDS[i].0)
}
#[test]
fn files_are_known_by_extension_and_name() {
for (path, want) in [
("src/main.rs", Some("Rust")),
("web/src/App.svelte", Some("Svelte")),
("a/B.TSX", Some("TSX")),
("lib.d.ts", Some("TypeScript")),
("Dockerfile", Some("Dockerfile")),
("deploy/Dockerfile", Some("Dockerfile")),
("Justfile", Some("Just")),
(".justfile", Some("Just")),
("x.sh", Some("Shell")),
("x.h", Some("C")),
("Makefile", Some("Makefile")),
// Prose, data and binaries are never counted.
("README.md", None),
("package.json", None),
("ci.yml", None),
("Cargo.lock", None),
("logo.png", None),
("LICENSE", None),
(".gitignore", None),
("trailing.", None),
// Nor what is vendored, generated or documentation.
("node_modules/x/index.js", None),
("a/vendor/b.go", None),
("web/dist/app.js", None),
("app.min.js", None),
("docs/example.py", None),
("Docs/example.py", None),
("src/docs/page.py", Some("Python")),
("", None),
] {
assert_eq!(name(path), want, "{path}");
}
}
#[test]
fn the_small_are_other() {
let rust = kind("a.rs").unwrap();
let ts = kind("a.ts").unwrap();
let (sh, css) = (kind("a.sh").unwrap(), kind("a.css").unwrap());
let shown = |files: Vec<(usize, u64)>| {
breakdown(files)
.into_iter()
.map(|l| (l.name, l.bytes, l.percent))
.collect::<Vec<_>>()
};
assert!(shown(vec![]).is_empty());
assert_eq!(
shown(vec![(rust, 600), (ts, 200), (rust, 194), (sh, 3), (css, 3)]),
[
("Rust".into(), 794, 79.4),
("TypeScript".into(), 200, 20.0),
("Other".into(), 6, 0.6)
]
);
// One alone is shown as itself.
assert_eq!(
shown(vec![(rust, 997), (sh, 3)]),
[("Rust".into(), 997, 99.7), ("Shell".into(), 3, 0.3)]
);
// As many bytes, by the table's order.
assert_eq!(shown(vec![(ts, 1), (rust, 1)])[0].0, "Rust");
let rust = &breakdown([(rust, 1)])[0];
assert_eq!((rust.color.as_str(), rust.percent), ("#dea584", 100.0));
}
#[test]
fn counted_once_a_tree() {
let dir = tempfile::tempdir().unwrap();
let d = dir.path();
let run = |args: &[&str]| {
let out = std::process::Command::new("git")
.args(args)
.current_dir(d)
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_AUTHOR_NAME", "A")
.env("GIT_AUTHOR_EMAIL", "a@x")
.env("GIT_COMMITTER_NAME", "A")
.env("GIT_COMMITTER_EMAIL", "a@x")
.output()
.unwrap();
assert!(out.status.success(), "{out:?}");
};
run(&["init", "-q", "-b", "main"]);
std::fs::create_dir_all(d.join("src/node_modules")).unwrap();
std::fs::write(d.join("src/a.rs"), "fn a() {}\n").unwrap();
std::fs::write(d.join("src/node_modules/x.js"), "x\n").unwrap();
std::fs::write(d.join("run.sh"), "x\n").unwrap();
std::fs::write(d.join("README.md"), "# A\n").unwrap();
#[cfg(unix)]
std::os::unix::fs::symlink("src/a.rs", d.join("link.rs")).unwrap();
run(&["add", "-A"]);
run(&["commit", "-q", "-m", "one"]);
let o = crate::objects::open(&d.join(".git")).unwrap();
// Fill what is kept, so that this tree's is the first after.
{
let mut counted = COUNTED.lock().unwrap();
for i in counted.len()..KEPT {
counted.insert(format!("filler {i}"), Vec::new());
}
}
let found = of(&o, "main").unwrap();
let shown: Vec<(&str, u64)> = found.iter().map(|l| (l.name.as_str(), l.bytes)).collect();
assert_eq!(shown, [("Rust", 10), ("Shell", 2)]);
assert!(!COUNTED.lock().unwrap().contains_key("filler 0"));
// Kept: a tree's answer is the one counted, whatever it now says.
let tree = o.tree_id("main").unwrap();
COUNTED.lock().unwrap().insert(tree, Vec::new());
assert_eq!(of(&o, "HEAD").unwrap(), []);
assert!(of(&o, "nope").is_err());
}
}