~ knotless Search/
↑↓ move↵ open⇧↵ all results

chadtmiller.com/knotless
Switch repository
Loading…

git that lives in your Atmosphere account

knotless.dev
1 collaborator

knotless 0.13.0 Latest 2 packages, 1 image

1
10

A workflow step can use a step another repository shares

Merged chadtmiller.com asked to merge into main from chadtmiller.com/knotless:run/uses
.changeset/shared-steps.mdnew+7−0
View file
@@ -0,0 +1,7 @@
---
default: minor
---
# A workflow step can use a step another repository shares
A step with `uses: <owner>/<repo>/<step>@<version>` runs the steps of `.knotless/steps/<step>.yml` in that repository, at the commit `.knotless/uses.lock` pins it to, with the inputs `with` gives, only the secrets the step passes it, and its own repository checked out beside the checkout as `KNOTLESS_STEP_DIR`. A step file can use shared steps of its own. Release a step with `knotless publish <url> @step/<step> <version> .knotless/steps/<step>.yml --commit <sha>`, and pin the ones a checkout uses with `knotless uses update`. Each step is now given its secrets as it starts, rather than the container holding them all.
crates/knotless/src/cli.rs+23−0
View file
@@ -344,6 +344,10 @@ const USAGE: &str = "usage: knotless <command>
--oci adds an image's, from an OCI
image layout, for knotless-serve's /v2/
releases <url> [<package>] the versions published, highest first
uses update [-C <dir>] [<uses>...] pin the shared steps the workflows use
in .knotless/uses.lock: each to the
highest release its @ takes (those
named, if any; the rest as pinned)
download <url> <package> [<version>] [--file <name>]... [-o <dir>]
a version's files (the latest's), each
checked, into a directory (here)
@@ -1126,6 +1130,25 @@ async fn command(
}
}
"theme" => return theme(ctx, &mut args, stdin, out).await,
"uses" => {
match args.next("update")? {
"update" => {}
other => bail!("{other}: knotless uses update"),
}
let (mut dir, mut only) = (PathBuf::from("."), Vec::new());
while let Some(arg) = args.rest.next() {
match arg.as_str() {
"-C" => dir = args.next("<dir>")?.into(),
flag if flag.starts_with('-') => bail!("unexpected {flag}"),
uses => only.push(uses.to_string()),
}
}
for p in crate::uses::update(ctx, &dir, &only).await? {
let was = p.was.as_deref().unwrap_or("-");
let (now, commit) = (&p.locked.version, &p.locked.commit[..12]);
writeln!(out, "{}\t{was}\t{now}\t{commit}", p.uses)?;
}
}
"owners" => {
let text = args.next("<text>")?;
let limit = match args.rest.next().map(String::as_str) {
crates/knotless/src/lib.rs+1−0
View file
@@ -68,5 +68,6 @@ pub mod triage;
pub mod update;
pub mod url;
pub mod usage;
pub mod uses;
pub mod workflow;
pub mod xrpc;
crates/knotless/src/uses.rsnew+667−0
View file
667 lines changed: not loaded until asked for.
crates/knotless/src/workflow.rs+87−3
View file
@@ -173,19 +173,63 @@ pub struct Pull {
pub paths: Option<Vec<String>>,
}
#[derive(Deserialize, Debug, Clone, PartialEq)]
#[derive(Deserialize, Debug, Clone, Default, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct Step {
#[serde(default)]
pub name: String,
/// A script for `sh -c`.
/// A script for `sh -c`. A step has this or [`Step::uses`].
#[serde(default)]
pub run: String,
/// A step another repository shares, `<owner>/<repo>/<name>@<version>`
/// (see [`crate::uses`]), which the runner puts in its place.
#[serde(default)]
pub uses: Option<String>,
/// The inputs a shared step is given, by name.
#[serde(default)]
pub with: BTreeMap<String, String>,
/// On a shared step, the workflow's secrets it is given. On the steps
/// it is put in place of, the only ones they see; on any other step,
/// `None`, and it sees every secret the run is granted.
#[serde(default)]
pub secrets: Option<Vec<String>>,
/// Variables of this step's own, as a shared step's inputs; never in
/// a workflow file.
#[serde(skip)]
pub env: BTreeMap<String, String>,
/// Which of the run's shared steps' repositories it comes from, checked
/// out beside the checkout for it as `KNOTLESS_STEP_DIR`; never in a
/// workflow file.
#[serde(skip)]
pub from: Option<usize>,
/// How long it may take, in minutes, if not the runner's limit: within
/// it, or past it only in a run its owner asked for.
#[serde(default)]
pub timeout: Option<u64>,
}
impl Step {
/// Whether it sees the secret `name`, of those the run is granted.
pub fn sees(&self, name: &str) -> bool {
self.secrets
.as_ref()
.is_none_or(|only| only.iter().any(|n| n == name))
}
/// Why it is not a step: it has both `run` and `uses`, or neither, or
/// what only a shared step has without `uses`.
pub fn invalid(&self) -> Option<&'static str> {
match (self.run.is_empty(), self.uses.is_some()) {
(false, true) => Some("has both `run` and `uses`"),
(true, false) => Some("has neither `run` nor `uses`"),
(false, false) if !self.with.is_empty() || self.secrets.is_some() => {
Some("has `with` or `secrets`, which only a step with `uses` takes")
}
_ => None,
}
}
}
impl Workflow {
/// Whether a push to `name` runs this workflow.
pub fn runs_on_push(&self, name: &str) -> bool {
@@ -308,8 +352,27 @@ pub fn parse(file: &str, text: &[u8]) -> Result<Workflow> {
}
}
for (i, step) in workflow.steps.iter_mut().enumerate() {
if let Some(why) = step.invalid() {
bail!("{file}: step {} {why}", i + 1);
}
if let Some(passed) = &step.secrets {
let unasked = passed.iter().find(|n| !workflow.secrets.contains(n));
if let Some(name) = unasked {
bail!(
"{file}: step {} passes the secret {name}, which the workflow's `secrets` does not name",
i + 1
);
}
}
let used = match &step.uses {
Some(uses) => match crate::uses::Use::parse(uses) {
Ok(used) => Some(used.step),
Err(err) => bail!("{file}: step {}: {err:#}", i + 1),
},
None => None,
};
if step.name.is_empty() {
step.name = format!("step {}", i + 1);
step.name = used.unwrap_or_else(|| format!("step {}", i + 1));
}
}
Ok(workflow)
@@ -409,6 +472,27 @@ mod tests {
assert_eq!(w.superseded, Superseded::Cancel);
}
#[test]
fn a_step_runs_or_uses() {
let text = "secrets: [KEY]\nsteps:\n - uses: a.test/steps/wait@1\n with: {workflows: test}\n secrets: [KEY]\n - {name: own, uses: a.test/steps/wait@1}\n";
let w = parse("d.yml", text.as_bytes()).unwrap();
assert_eq!(
(w.steps[0].name.as_str(), w.steps[1].name.as_str()),
("wait", "own")
);
assert_eq!(w.steps[0].with["workflows"], "test");
let wrong = |text: &str| parse("d.yml", text.as_bytes()).unwrap_err().to_string();
let both = wrong("steps:\n - {run: x, uses: a/b/c@1}\n");
assert!(both.contains("step 1 has both `run` and `uses`"), "{both}");
assert!(wrong("steps:\n - {name: x}\n").contains("neither `run` nor `uses`"));
let with = wrong("steps:\n - {run: x, with: {a: b}}\n");
assert!(with.contains("only a step with `uses`"), "{with}");
let secret = wrong("steps:\n - {uses: a/b/c@1, secrets: [KEY]}\n");
assert!(secret.contains("does not name"), "{secret}");
let shape = wrong("steps:\n - {uses: a/b@1}\n");
assert!(shape.contains("d.yml: step 1: `uses: a/b@1`"), "{shape}");
}
#[test]
fn superseded() {
let w = parse("d.yml", b"superseded: finish\nsteps: []\n").unwrap();
crates/knotless/tests/cli.rs+151−0
View file
@@ -1330,3 +1330,154 @@ fn releases() {
);
}
}
#[test]
fn shared_steps_are_pinned() {
let mut s = Sandbox::new();
let handle = s.account.handle.clone();
s.set("KNOTLESS_IDENTIFIER", &handle);
let url = s.url("steps");
let work = s.work("steps", 1);
let dir = work.join(".knotless/steps");
std::fs::create_dir_all(&dir).unwrap();
let greet = format!("steps:\n - uses: {handle}/steps/inner@1\n - run: echo hi\n");
std::fs::write(dir.join("greet.yml"), &greet).unwrap();
std::fs::write(dir.join("inner.yml"), "steps: [{run: echo inner}]\n").unwrap();
s.git(&work, &["add", "."]);
s.git(&work, &["commit", "-q", "-m", "steps"]);
let first = s.git(&work, &["rev-parse", "HEAD"]);
std::fs::write(
dir.join("greet.yml"),
format!("{greet} - run: echo again\n"),
)
.unwrap();
std::fs::write(dir.join("bad.yml"), "steps: [{run: echo bad}]\n").unwrap();
s.git(&work, &["add", "."]);
s.git(&work, &["commit", "-q", "-m", "more"]);
let second = s.git(&work, &["rev-parse", "HEAD"]);
s.git(&work, &["push", "-q", &url, "main"]);
let home = s.path("");
let file = |name: &str| home.join(name).to_string_lossy().into_owned();
let step_file = |name: &str| dir.join(name).to_string_lossy().into_owned();
let publish = |package: &str, version: &str, path: &str, commit: Option<&str>| {
let mut args = vec!["publish", &url, package, version, path];
args.extend(commit.map(|c| ["--commit", c]).into_iter().flatten());
let (code, out) = s.cli(&args, b"");
assert_eq!(code, 0, "{package} {version}: {out}");
};
// Each release holds the step file, as it was at its commit.
s.git(&work, &["checkout", "-q", &first]);
publish(
"@step/greet",
"1.0.0",
&step_file("greet.yml"),
Some(&first),
);
publish(
"@step/inner",
"1.0.0",
&step_file("inner.yml"),
Some(&first),
);
s.git(&work, &["checkout", "-q", "main"]);
for version in ["1.1.0", "2.0.0", "1.2.0-rc.1"] {
publish(
"@step/greet",
version,
&step_file("greet.yml"),
Some(&second),
);
}
std::fs::write(s.path("bad.yml"), "steps: [{run: echo worse}]\n").unwrap();
publish("@step/bad", "1.0.0", &file("bad.yml"), Some(&second));
publish("@step/nocommit", "1.0.0", &step_file("inner.yml"), None);
publish(
"@step/nofile",
"1.0.0",
&step_file("inner.yml"),
Some(&second),
);
std::fs::write(s.path("gone.yml"), "steps: [{run: x}]\n").unwrap();
publish("@step/gone", "1.0.0", &file("gone.yml"), Some(&first));
let project = |name: &str, step: &str| {
let wf = s.path(name).join(".knotless/workflows");
std::fs::create_dir_all(&wf).unwrap();
let text = format!("steps:\n - uses: {handle}/steps/{step}\n - run: 'true'\n");
std::fs::write(wf.join("ci.yml"), text).unwrap();
std::fs::write(wf.join("notes.txt"), "not a workflow").unwrap();
s.path(name).to_string_lossy().into_owned()
};
let proj = project("proj", "greet@1");
// A step two workflows use is pinned once.
let again = format!("steps:\n - uses: {handle}/steps/greet@1\n");
std::fs::write(s.path("proj/.knotless/workflows/deploy.yml"), again).unwrap();
let (code, out) = s.cli(&["uses", "update", "-C", &proj], b"");
assert_eq!(code, 0, "{out}");
let (g, i) = (
format!("{handle}/steps/greet@1"),
format!("{handle}/steps/inner@1"),
);
assert_eq!(
out,
format!(
"{g}\t-\t1.1.0\t{}\n{i}\t-\t1.0.0\t{}",
&second[..12],
&first[..12]
)
);
let lock = std::fs::read_to_string(s.path("proj/.knotless/uses.lock")).unwrap();
assert!(
lock.starts_with("# Written by `knotless uses update`"),
"{lock}"
);
assert!(lock.contains(&format!("commit: {second}")), "{lock}");
let release = format!("release: at://{}/dev.knotless.release/", s.account.did);
assert!(lock.contains(&release), "{lock}");
// Named, only those move; the rest stay as pinned.
s.git(&work, &["checkout", "-q", "main"]);
publish(
"@step/greet",
"1.3.0",
&step_file("greet.yml"),
Some(&second),
);
let (code, out) = s.cli(&["uses", "update", "-C", &proj, &i], b"");
assert_eq!(code, 0, "{out}");
assert!(out.contains(&format!("{g}\t1.1.0\t1.1.0\t")), "{out}");
let (code, out) = s.cli(&["uses", "update", "-C", &proj], b"");
assert_eq!(code, 0, "{out}");
assert!(out.contains(&format!("{g}\t1.1.0\t1.3.0\t")), "{out}");
// What cannot be pinned says why.
for (name, step, said) in [
("p3", "greet@3", "has released no @step/greet that @3 takes"),
(
"pbad",
"bad@1",
"@step/bad 1.0.0's bad.yml is not .knotless/steps/bad.yml at",
),
("pnc", "nocommit@1", "@step/nocommit 1.0.0 names no commit"),
("pnf", "nofile@1", "@step/nofile 1.0.0 holds no nofile.yml"),
(
"pgone",
"gone@1",
&format!(".knotless/steps/gone.yml at {first} of"),
),
] {
let dir = project(name, step);
let (ok, _, err) = s.knotless(&["uses", "update", "-C", &dir], b"");
assert!(!ok && err.contains(said), "{step}: {err}");
}
let (ok, _, err) = s.knotless(
&["uses", "update", "-C", &s.path("none").to_string_lossy()],
b"",
);
assert!(!ok && err.contains("has no .knotless/workflows"), "{err}");
let (ok, _, err) = s.knotless(&["uses", "pin"], b"");
assert!(!ok && err.contains("pin: knotless uses update"), "{err}");
let (ok, _, err) = s.knotless(&["uses", "update", "--all"], b"");
assert!(!ok && err.contains("unexpected --all"), "{err}");
}
crates/knotless-run/src/executor.rs+147−17
View file
@@ -5,7 +5,7 @@
use anyhow::{Context, Result, bail};
use rustix::process::{Pid, Signal};
use std::collections::HashSet;
use std::collections::{BTreeSet, HashSet};
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Mutex;
@@ -117,6 +117,9 @@ pub struct Setup<'a> {
/// Where steps leave what the run made to be looked at, named to them
/// as `KNOTLESS_RUN_OUTPUTS`.
pub outputs: &'a Path,
/// The checkouts of the shared steps' repositories, if any: mounted
/// read-only at [`STEPS`] under docker.
pub steps: Option<&'a Path>,
/// Whose the run is, as its container is labeled under docker.
pub owner: Owner<'a>,
/// Whether it gives way to everything else on the host for CPU, as a
@@ -144,6 +147,10 @@ pub struct Owner<'a> {
/// The variable naming [`Setup::outputs`] to the steps.
pub const OUTPUTS: &str = "KNOTLESS_RUN_OUTPUTS";
/// Where the shared steps' repositories are, in the container: each at
/// `<STEPS>/<n>`, named to its steps as `KNOTLESS_STEP_DIR`.
pub const STEPS: &str = "/steps";
/// An overlay of a cache, mounted at `path` in a run's container: a
/// volume of docker's `local` driver over a base, `lower`, read only, with
/// the run's own `upper` layer taking what it writes.
@@ -178,6 +185,15 @@ impl Executor {
}
}
/// Where the steps find the `i`th shared step's repository, checked
/// out at `at` on the host: under docker, where it is mounted.
pub fn step_dir(&self, i: usize, at: &Path) -> String {
match self {
Executor::Shell => at.to_string_lossy().into_owned(),
Executor::Docker { .. } => format!("{STEPS}/{i}"),
}
}
/// Starts what a run's steps run in, as `setup` says: under docker, a
/// container whose start, an image pulled among it, is written to
/// `log` and may take `timeout`.
@@ -205,6 +221,7 @@ impl Executor {
image,
caches,
outputs,
steps: _,
owner: _,
behind: _,
vouched: _,
@@ -314,8 +331,11 @@ async fn container(
.arg("-v")
.arg(format!("{}:/work", setup.dir.display()))
.arg("-v")
.arg(format!("{}:/outputs", setup.outputs.display()))
.args(["-w", "/work"]);
.arg(format!("{}:/outputs", setup.outputs.display()));
if let Some(steps) = setup.steps {
cmd.arg("-v").arg(format!("{}:{STEPS}:ro", steps.display()));
}
cmd.args(["-w", "/work"]);
if setup.behind {
cmd.args(["--cpu-shares", BEHIND]);
}
@@ -541,6 +561,11 @@ async fn remove_volumes(docker: &Path, names: &[String]) {
.await;
}
/// Lists the processes in a container by ID, but for itself, from `/proc`,
/// which every image has where `ps` may not be.
const RUNNING: &str =
r#"for p in /proc/[0-9]*; do p=${p#/proc/}; [ "$p" = "$$" ] || echo "$p"; done"#;
/// The docker client, seeing `vars` and the variables in [`PASSED`].
fn client(docker: &Path, vars: &[(String, String)]) -> Command {
let mut cmd = Command::new(docker);
@@ -562,24 +587,77 @@ fn passed(cmd: &mut Command, vars: &[(String, String)]) {
impl Session {
/// The process that runs `script` as a step: `sh -e`, so a command
/// that fails ends the step, failed, rather than only its last one
/// counting.
pub fn command(&self, script: &str) -> Command {
/// counting. It sees the run's variables and `own`, this step's alone,
/// such as the secrets it is given, which the container never holds.
pub fn command(&self, script: &str, own: &[(String, String)]) -> Command {
match self {
Session::Shell { dir, vars } => {
let mut cmd = Command::new("sh");
cmd.arg("-ec").arg(script).current_dir(dir);
passed(&mut cmd, vars);
cmd.envs(own.iter().map(|(k, v)| (k, v)));
cmd
}
Session::Docker { docker, id, .. } => {
// The container has the run's variables already.
let mut cmd = client(docker, &[]);
cmd.args(["exec", id, "sh", "-ec", script]);
// The container has the run's variables already. Named
// only: the values reach the step from the client's
// environment, never its arguments.
let mut cmd = client(docker, own);
cmd.arg("exec");
for (k, _) in own {
cmd.arg("-e").arg(k);
}
cmd.args([id, "sh", "-ec", script]);
cmd
}
}
}
/// The processes running in the container, by ID, for
/// [`Session::end_left`] to tell which a step left: none under the
/// shell executor, whose steps share the runner's host.
pub async fn running(&self) -> Option<BTreeSet<u32>> {
let Session::Docker { docker, id, .. } = self else {
return None;
};
let mut list = client(docker, &[]);
list.args(["exec", id, "sh", "-c", RUNNING]);
let ids = words(&mut list).await;
Some(ids.iter().filter_map(|p| p.parse().ok()).collect())
}
/// Ends what is running in the container that was not in `before`, as
/// [`Session::running`] had it before `step`, and says so in `log`: the
/// processes a step given secrets left, which would hold them for the
/// steps after it to read. Nothing, with no `before`.
pub async fn end_left(
&self,
before: Option<&BTreeSet<u32>>,
step: &str,
mut log: &std::fs::File,
) -> Result<()> {
let (Some(before), Session::Docker { docker, id, .. }) = (before, self) else {
return Ok(());
};
let now = self.running().await.unwrap_or_default();
let left: Vec<String> = now.difference(before).map(u32::to_string).collect();
if left.is_empty() {
return Ok(());
}
let mut kill = client(docker, &[]);
kill.args(["exec", id, "kill", "-9"]).args(&left);
let _ = kill
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.await;
use std::io::Write;
let n = left.len();
let said = format!("==> ended {n} of what {step} left running, as it was given secrets\n");
log.write_all(said.as_bytes())?;
Ok(())
}
/// Ends the run's container, and with it anything a step left
/// running, one killed for its time among them; then removes its
/// overlay volumes, which unmounts them, for its layers to be kept or
@@ -663,24 +741,34 @@ mod tests {
fn executors() {
assert_eq!(Executor::parse("shell", "x"), Some(Executor::Shell));
assert_eq!(Executor::parse("vm", "x"), None);
let at = Path::new("/tmp/run/steps/2");
assert_eq!(Executor::Shell.step_dir(2, at), "/tmp/run/steps/2");
let docker = Executor::parse("docker", "x").unwrap();
assert_eq!(docker.step_dir(2, at), format!("{STEPS}/2"));
let vars = [("SECRET".to_string(), "v".to_string())];
let shell = Session::Shell {
dir: "/w".into(),
vars: vars.to_vec(),
};
let cmd = shell.command("make");
let own = [("KEY".to_string(), "k".to_string())];
let cmd = shell.command("make", &own);
assert_eq!(args(&cmd), ["sh", "-ec", "make"]);
let env: Vec<_> = cmd.as_std().get_envs().collect();
assert!(env.contains(&("SECRET".as_ref(), Some("v".as_ref()))));
assert!(env.contains(&("KEY".as_ref(), Some("k".as_ref()))));
let docker = Session::Docker {
docker: "docker".into(),
id: "c1".into(),
volumes: Vec::new(),
};
assert_eq!(
args(&docker.command("make")).join(" "),
args(&docker.command("make", &[])).join(" "),
"docker exec c1 sh -ec make"
);
let cmd = docker.command("make", &own);
assert_eq!(args(&cmd).join(" "), "docker exec -e KEY c1 sh -ec make");
let env: Vec<_> = cmd.as_std().get_envs().collect();
assert!(env.contains(&("KEY".as_ref(), Some("k".as_ref()))));
}
/// A stand-in docker client, in `dir`, that writes what it is asked
@@ -694,7 +782,7 @@ mod tests {
let path = dir.join("docker");
let calls = dir.join("calls");
let script = format!(
"#!/bin/sh\necho \"$*\" >> {calls}\ncase \"$*\" in\n 'network inspect have') exit 0;;\n 'network inspect raced') test -e {raced}; exit $?;;\n 'network inspect '*) exit 1;;\n 'network create '*' raced') touch {raced}; exit 1;;\n 'network create '*' gone') echo 'no network' >&2; exit 1;;\n 'network create '*) exit 0;;\n *' bad tail '*) echo 'no such image' >&2; exit 1;;\n *' slow tail '*) sleep 5;;\n run\\ *) echo c1;;\n *' knotless-overlay-bad') echo 'no overlay' >&2; exit 1;;\n exec\\ *) shift 2; exec \"$@\";;\n 'volume ls -q --filter label='*) echo o1; echo o2;;\n 'volume ls -q --filter name='*) echo old;;\n 'volume ls --filter '*) echo r1; echo r1; echo r0;;\n 'ps -aq') echo c7; echo c8; echo c9; echo c10;;\n inspect\\ *) echo 'c7|did:left|r7|/x;'; echo 'c8|did:left|r8|/w/work/a/src;'; echo 'c9|||/w/work/b/src;/w/work/b/outputs;'; echo 'c10|did:other||/w/workshop;';;\nesac\n",
"#!/bin/sh\necho \"$*\" >> {calls}\ncase \"$*\" in\n 'network inspect have') exit 0;;\n 'network inspect raced') test -e {raced}; exit $?;;\n 'network inspect '*) exit 1;;\n 'network create '*' raced') touch {raced}; exit 1;;\n 'network create '*' gone') echo 'no network' >&2; exit 1;;\n 'network create '*) exit 0;;\n *' bad tail '*) echo 'no such image' >&2; exit 1;;\n *' slow tail '*) sleep 5;;\n run\\ *) echo c1;;\n *' knotless-overlay-bad') echo 'no overlay' >&2; exit 1;;\n *' for p in /proc/'*) echo 1; echo 7; echo 9; echo x;;\n 'exec c1 kill -9 '*) ;;\n exec\\ *) shift 2; exec \"$@\";;\n 'volume ls -q --filter label='*) echo o1; echo o2;;\n 'volume ls -q --filter name='*) echo old;;\n 'volume ls --filter '*) echo r1; echo r1; echo r0;;\n 'ps -aq') echo c7; echo c8; echo c9; echo c10;;\n inspect\\ *) echo 'c7|did:left|r7|/x;'; echo 'c8|did:left|r8|/w/work/a/src;'; echo 'c9|||/w/work/b/src;/w/work/b/outputs;'; echo 'c10|did:other||/w/workshop;';;\nesac\n",
calls = calls.display(),
raced = dir.join("raced").display()
);
@@ -714,6 +802,45 @@ mod tests {
path
}
#[tokio::test]
async fn what_a_step_leaves_running_is_ended() {
let dir = tempfile::tempdir().unwrap();
let docker = fake_docker(dir.path());
let log_path = dir.path().join("log");
let log = std::fs::File::create(&log_path).unwrap();
let session = Session::Docker {
docker,
id: "c1".into(),
volumes: Vec::new(),
};
let running = session.running().await.unwrap();
assert_eq!(running, BTreeSet::from([1, 7, 9]));
let before = BTreeSet::from([1, 7]);
session
.end_left(Some(&before), "deploy", &log)
.await
.unwrap();
session
.end_left(Some(&running), "quiet", &log)
.await
.unwrap();
session.end_left(None, "unwatched", &log).await.unwrap();
let calls = std::fs::read_to_string(dir.path().join("calls")).unwrap();
assert!(calls.contains("exec c1 kill -9 9\n"), "{calls}");
assert_eq!(calls.matches("kill").count(), 1, "{calls}");
let said = std::fs::read_to_string(&log_path).unwrap();
assert_eq!(
said,
"==> ended 1 of what deploy left running, as it was given secrets\n"
);
let shell = Session::Shell {
dir: "/w".into(),
vars: Vec::new(),
};
assert_eq!(shell.running().await, None);
shell.end_left(Some(&before), "s", &log).await.unwrap();
}
fn overlay_of(volume: &str, path: &str) -> Cache {
Cache {
volume: volume.into(),
@@ -741,6 +868,7 @@ mod tests {
image,
caches,
outputs: Path::new("/w/outputs"),
steps: None,
owner: Owner {
runner: "did:r",
run: "k1",
@@ -751,6 +879,7 @@ mod tests {
// A pull's, behind the rest for CPU.
let behind = Setup {
behind: true,
steps: Some(Path::new("/w/steps")),
..setup(Some("rust:1"), &vars, &caches)
};
let session = executor
@@ -758,7 +887,7 @@ mod tests {
.await
.unwrap();
let ran = run(
session.command("echo ran"),
session.command("echo ran", &[]),
&log,
time,
std::future::pending(),
@@ -797,7 +926,7 @@ mod tests {
volume create --driver local --label dev.knotless.overlay=r1 --opt type=overlay \
--opt device=overlay --opt o=lowerdir=/c/bases/t/0,upperdir=/c/runs/r1/y/upper,\
workdir=/c/runs/r1/y/work,userxattr y\n\
run -d --init --label dev.knotless.runner=did:r --label dev.knotless.run=k1 -v /w/src:/work -v /w/outputs:/outputs -w /work \
run -d --init --label dev.knotless.runner=did:r --label dev.knotless.run=k1 -v /w/src:/work -v /w/outputs:/outputs -v /w/steps:/steps:ro -w /work \
--cpu-shares 256 --mount type=volume,src=x,dst=/work/target \
--mount type=volume,src=y,dst=/root \
-e SECRET -e KNOTLESS_RUN_OUTPUTS rust:1 tail -f /dev/null\n\
@@ -914,6 +1043,7 @@ mod tests {
image: None,
caches: &caches,
outputs: Path::new("/w/outputs"),
steps: None,
owner: Owner {
runner: "did:r",
run: "k1",
@@ -1024,7 +1154,7 @@ mod tests {
dir: dir.path().into(),
vars: vars.to_vec(),
};
let cmd = shell.command("echo hi $WHO; echo err >&2; exit 3");
let cmd = shell.command("echo hi $WHO; echo err >&2; exit 3", &[]);
let never = std::future::pending::<()>;
let exit = run(cmd, &log, Duration::from_secs(10), never())
.await
@@ -1032,21 +1162,21 @@ mod tests {
assert_eq!(exit.code, Some(3));
assert!(!exit.success());
// A child of the shell is killed with it.
let cmd = shell.command("sleep 30 & sleep 30");
let cmd = shell.command("sleep 30 & sleep 30", &[]);
let exit = run(cmd, &log, Duration::from_millis(200), never())
.await
.unwrap();
assert_eq!((exit.code, exit.cancelled), (None, false));
assert!(exit.elapsed < Duration::from_secs(5));
// Cancelled, likewise.
let cmd = shell.command("sleep 30 & sleep 30");
let cmd = shell.command("sleep 30 & sleep 30", &[]);
let soon = tokio::time::sleep(Duration::from_millis(200));
let exit = run(cmd, &log, Duration::from_secs(10), soon).await.unwrap();
assert_eq!((exit.code, exit.cancelled), (None, true));
assert!(exit.elapsed < Duration::from_secs(5));
let text = std::fs::read_to_string(dir.path().join("log")).unwrap();
assert_eq!(text, "hi step\nerr\n");
let mut missing = shell.command("true");
let mut missing = shell.command("true", &[]);
missing.current_dir(dir.path().join("gone"));
assert!(
run(missing, &log, Duration::from_secs(1), never())
crates/knotless-run/src/lib.rs+115−40
View file
@@ -801,6 +801,9 @@ struct Pull {
/// Whether it runs after the runs of the merge queue it is not in,
/// and the pushes'.
later: bool,
/// The accounts whose shared steps its lock adds that the repository
/// does not vouch for, which make it untrusted.
strangers: Vec<String>,
/// The other runners the repository names, by DID: a pre-flight one
/// of them is running, or passed, at the same head and base is not
/// run here too.
@@ -1513,6 +1516,13 @@ impl Runner {
Ok(merged) => changes(git, &d.base_sha, merged).await,
Err(_) => None,
};
let strangers = match &merged {
Ok(merged) => {
self.strangers(git, &target.record, owner, &d.base_sha, merged)
.await
}
Err(_) => Vec::new(),
};
let subject = Subject {
ref_name: d.branch,
commit: d.sha,
@@ -1524,8 +1534,9 @@ impl Runner {
base_commit: d.base_sha,
main_generation: d.main_generation,
merged,
trusted: d.trusted,
trusted: d.trusted && strangers.is_empty(),
later: d.later,
strangers,
others: others.clone(),
}),
tree: None,
@@ -2134,6 +2145,28 @@ impl Runner {
Ok(check.status)
}
/// The accounts the lock at `tip` names shared steps in that the lock
/// at `base` does not, and the repository does not vouch for: a pull
/// or batch bringing one is code no one has vouched for, as an agent's
/// is, until it lands. One whose name does not resolve is among them.
pub(crate) async fn strangers(
&self,
git: &Git,
record: &RepoRecord,
owner: &str,
base: &str,
tip: &str,
) -> Vec<String> {
let mut strangers = Vec::new();
for who in knotless::uses::added(git, base, tip).await {
let resolved = knotless::identity::resolve(&self.ctx, &who).await;
if !resolved.is_ok_and(|id| vouched(record, owner, &id.did)) {
strangers.push(who);
}
}
strangers
}
/// The other runners `record` names, by DID.
fn others(&self, record: &RepoRecord) -> Vec<String> {
let me = &self.reporter.identity.did;
@@ -2703,7 +2736,11 @@ impl Runner {
cancelled,
said,
} = started;
let (vars, secrets, ungranted) = self.vars(watch, subject, wf).await?;
// Each step that uses a shared step is put in place of by its
// steps, as the repository's lock pins them, before anything runs.
let expanded = knotless::uses::expanded(&self.ctx, git, subject.checkout(), wf).await?;
let wf = &expanded.workflow;
let (vars, granted, secrets, ungranted) = self.vars(watch, subject, wf).await?;
let dir = match &self.opts.work {
Some(work) => {
std::fs::create_dir_all(work)?;
@@ -2712,33 +2749,16 @@ impl Runner {
None => tempfile::tempdir()?,
};
let src = dir.path().join("src");
let src_text = src.to_string_lossy();
// Its objects hard links to the mirror's, not borrowed from it, so
// git works in a container that sees only the checkout.
git.run(
&[
"clone",
"-q",
"--no-checkout",
&git.dir.to_string_lossy(),
&src_text,
],
None,
)
.await?;
Git::new(src.join(".git"))
.run(
&[
"--work-tree",
&src_text,
"checkout",
"-q",
"--detach",
subject.checkout(),
],
None,
)
.await?;
checkout(git, &src, subject.checkout()).await?;
// Each shared step's repository, at the commit the lock pins, beside
// the checkout, for its steps to find what it keeps beside them.
let shared = dir.path().join("steps");
let mut step_dirs = Vec::new();
for (i, source) in expanded.sources.iter().enumerate() {
let at = shared.join(i.to_string());
checkout(&source.git, &at, &source.commit).await?;
step_dirs.push(self.opts.executor.step_dir(i, &at));
}
let log_path = dir.path().join("log");
let mut log = std::fs::File::create(&log_path)?;
log.write_all(said.as_bytes())?;
@@ -2747,6 +2767,15 @@ impl Runner {
writeln!(log, "==> asked for by name{}", given.collect::<String>())?;
}
log.write_all(not_granted(&ungranted, watch, subject).as_bytes())?;
let pull = subject.pull.as_ref().map(|p| &p.strangers);
let strangers = pull.or(subject.batch.as_ref().map(|b| &b.strangers));
if let Some(who) = strangers.filter(|s| !s.is_empty()) {
let who = who.join(", ");
let said = format!(
"==> runs as code no one vouches for, as its lock adds shared steps from {who}\n"
);
log.write_all(said.as_bytes())?;
}
// What it writes goes to the appview as it is written, ending
// once all of it is, before it is filed.
let rkey = run_rkey(watch, subject, &wf.name);
@@ -2772,6 +2801,7 @@ impl Runner {
image: wf.image.as_deref(),
caches: &caches,
outputs: &outputs,
steps: (!step_dirs.is_empty()).then_some(shared.as_path()),
owner: executor::Owner {
runner: &self.reporter.identity.did,
run: &run_rkey(watch, subject, &wf.name),
@@ -2788,6 +2818,8 @@ impl Runner {
let steps = Steps {
log,
progress: &progress,
granted: &granted,
step_dirs: &step_dirs,
};
let ran = (self.run_steps(&session, wf, check, steps, cancelled, asked)).await;
session.stop().await;
@@ -2996,10 +3028,16 @@ impl Runner {
mut cancelled: Cancelled,
asked: bool,
) -> Result<()> {
let Steps { mut log, progress } = steps;
let Steps {
mut log,
progress,
granted,
step_dirs,
} = steps;
let mut failed = false;
let begun = std::time::Instant::now();
let whole = self.opts.run_timeout.filter(|_| !asked);
let shared = wf.steps.iter().any(|s| s.from.is_some());
for step in &wf.steps {
let left = whole.map(|run| run.saturating_sub(begun.elapsed()));
let (limit, last) = within(self.limit(step, asked), left);
@@ -3010,7 +3048,20 @@ impl Runner {
writeln!(log, "==> {}", step.name)?;
progress.send_modify(|p| p.going = Some(step.name.clone()));
telemetry::step(&check.commit, &wf.name, &step.name, None, None, None);
let cmd = session.command(&step.run);
let own = step.env.iter().map(|(k, v)| (k.clone(), v.clone()));
let seen = granted.iter().filter(|(name, _)| step.sees(name)).cloned();
let from = step
.from
.map(|i| ("KNOTLESS_STEP_DIR".into(), step_dirs[i].clone()));
let given = seen.clone().next().is_some();
let own: Vars = own.chain(seen).chain(from).collect();
// Beside shared steps, what a step given secrets leaves running
// is ended once it ends, so none holds them for the next.
let before = match shared && given {
true => session.running().await,
false => None,
};
let cmd = session.command(&step.run, &own);
// The run's task holds the sender to its end, so this ends only
// when it is set.
let cancel = cancelled.wait();
@@ -3024,6 +3075,7 @@ impl Runner {
let said = killed(limit, last.then_some(whole).flatten());
writeln!(log, "==> {said}")?;
}
session.end_left(before.as_ref(), &step.name, &log).await?;
failed = !exit.success();
let ended: &str = match (exit.cancelled, failed) {
(true, _) => status::CANCELLED,
@@ -3062,21 +3114,22 @@ impl Runner {
}
/// What a step sees beyond the runner's tools: the workflow's own
/// variables, those a run asked for by name was given, the run's, and
/// the secrets both the workflow asks for and the runner grants this
/// repository, in its secrets file or sealed to it by the owner, never
/// to a pull's run; those secrets' values, to keep out of the log; and
/// variables, those a run asked for by name was given, and the run's;
/// apart from them, the secrets both the workflow asks for and the
/// runner grants this repository, in its secrets file or sealed to it
/// by the owner, never to a pull's run, each given only to the steps
/// that may see it; those secrets' values, to keep out of the log; and
/// the names of those it asks for and is not given.
async fn vars(
&self,
watch: &Watch,
subject: &Subject,
wf: &Workflow,
) -> Result<(Vars, Vec<String>, Vec<String>)> {
) -> Result<(Vars, Vars, Vec<String>, Vec<String>)> {
let mut env = wf.env.clone();
env.extend(given(subject, wf).cloned());
let mut vars: Vars = env.into_iter().collect();
let (mut secrets, mut given) = (Vec::new(), Vec::new());
let (mut secrets, mut given, mut granted_vars) = (Vec::new(), Vec::new(), Vars::new());
let pull = subject.pull.as_ref().map(|p| p.uri.as_str());
for (k, v) in [
("KNOTLESS_RUN", Some("1")),
@@ -3101,7 +3154,7 @@ impl Runner {
for name in &wf.secrets {
if let Some(value) = granted.remove(name).and_then(|s| s.to(&subject.ref_name)) {
secrets.push(value.clone());
vars.push((name.clone(), value));
granted_vars.push((name.clone(), value));
given.push(name);
}
}
@@ -3112,12 +3165,12 @@ impl Runner {
if sealed {
for (name, value) in self.sealed(watch, subject, &missing).await? {
secrets.push(value.clone());
vars.push((name.clone(), value));
granted_vars.push((name.clone(), value));
given.push(name);
}
}
let ungranted = wf.secrets.iter().filter(|n| !given.contains(n));
Ok((vars, secrets, ungranted.cloned().collect()))
Ok((vars, granted_vars, secrets, ungranted.cloned().collect()))
}
/// Whether `wf` is this runner's to run: it names no runners, or this
@@ -3702,6 +3755,12 @@ impl Drop for Tracked<'_> {
struct Steps<'a> {
log: std::fs::File,
progress: &'a tokio::sync::watch::Sender<live::Progress>,
/// The secrets the run is granted, by name, for each step to be given
/// those it may see.
granted: &'a Vars,
/// Where each shared step's repository is checked out, as its steps
/// see it.
step_dirs: &'a [String],
}
struct Started<'a> {
@@ -3820,6 +3879,20 @@ fn kept_apart(subject: &Subject, wf: &Workflow) -> bool {
(pull || batch) && !wf.secrets.is_empty()
}
/// Checks out `commit` of the mirror `git` at `at`, its objects hard links
/// to the mirror's rather than borrowed from it, so git works in a
/// container that sees only the checkout.
async fn checkout(git: &Git, at: &Path, commit: &str) -> Result<()> {
let text = at.to_string_lossy();
let mirror = git.dir.to_string_lossy();
git.run(&["clone", "-q", "--no-checkout", &mirror, &text], None)
.await?;
let cloned = Git::new(at.join(".git"));
let args = ["--work-tree", &text, "checkout", "-q", "--detach", commit];
cloned.run(&args, None).await?;
Ok(())
}
/// Whether `subject` is code its repository's owner vouches for: a push,
/// or a pull or batch every writer of which the owner vouches for.
fn vouched_for(subject: &Subject) -> bool {
@@ -3950,6 +4023,7 @@ mod tests {
base_commit: "b".into(),
main_generation: Some(1),
merged: Ok("m".into()),
strangers: Vec::new(),
trusted,
later: false,
others: Vec::new(),
@@ -4000,6 +4074,7 @@ mod tests {
base_commit: "b".into(),
main_generation: Some(1),
merged,
strangers: Vec::new(),
trusted,
later: false,
others: Vec::new(),
crates/knotless-run/src/queue.rs+11−0
View file
@@ -82,6 +82,9 @@ pub(crate) struct Batch {
/// Whether it was built on a batch still being checked: its runs go
/// as a pull's do.
pub ahead: bool,
/// The accounts whose shared steps its lock adds that the repository
/// does not vouch for, which make it untrusted.
pub strangers: Vec<String>,
runs: Arc<Mutex<BTreeMap<String, Option<String>>>>,
}
@@ -93,6 +96,7 @@ impl Batch {
pulls,
trusted,
ahead: false,
strangers: Vec::new(),
runs: Arc::default(),
}
}
@@ -654,6 +658,8 @@ impl Runner {
let branch = item.head.as_ref().expect("a pull has a head");
let trusted =
vouched(record, owner, &item.author) && vouched(record, owner, branch.repo.did());
let strangers = self.strangers(on.git, record, owner, tip, merge).await;
let trusted = trusted && strangers.is_empty();
// Where the pull ran, so a pass over the pulls finds it there, and
// does not cancel it as moved.
let id = on.watch.id();
@@ -675,6 +681,7 @@ impl Runner {
merged,
trusted,
later: false,
strangers,
others: self.others(record),
}),
tree: None,
@@ -702,6 +709,9 @@ impl Runner {
let head = s.pull.head.as_ref().map_or("", |h| h.repo.did());
vouched(record, owner, &s.pull.author) && vouched(record, owner, head)
});
let strangers = self.strangers(on.git, record, owner, &stack.base, stack.tip());
let strangers = strangers.await;
let trusted = trusted && strangers.is_empty();
let pulls = stack.stacked.iter().map(|s| Batched {
pull: s.pull.uri.clone(),
commit: s.commit.clone(),
@@ -710,6 +720,7 @@ impl Runner {
let refname = format!("refs/heads/{}", on.branch);
let mut batch = Batch::new(&refname, &stack.base, pulls.collect(), trusted);
batch.ahead = ahead;
batch.strangers = strangers;
let changed = changes(on.git, &stack.base, stack.tip()).await;
let subject = Subject {
ref_name: refname,
crates/knotless-run/tests/run.rs+225−0
View file
@@ -2006,6 +2006,43 @@ fn a_pull_runs_its_head() {
collaborators(serde_json::json!([]));
contrib.git(&clone, &["checkout", "-q", "late"]);
// A collaborator's pull whose lock adds shared steps from an account the
// repository does not vouch for runs as code no one vouches for, and
// says so; the owner's own add nothing.
collaborators(serde_json::json!([{"did": contrib.account.did}]));
contrib.git(&clone, &["checkout", "-q", "-b", "locked", "late"]);
let (mine, theirs) = (&owner.account.handle, &runner.account.handle);
let commit = "a".repeat(40);
let lock = format!(
"{mine}/steps/a@1: {{version: 1.0.0, commit: {commit}}}\n{theirs}/steps/b@1: {{version: 1.0.0, commit: {commit}}}\n"
);
std::fs::write(clone.join(".knotless/uses.lock"), lock).unwrap();
contrib.git(&clone, &["add", "."]);
contrib.git(&clone, &["commit", "-q", "-m", "locked"]);
contrib.git(&clone, &["push", "-q", "origin", "locked"]);
let locked = pull("locked");
wait_until(|| {
js.send(
&contrib.account.did,
"dev.knotless.pull",
"p",
Some(record.clone()),
);
check(&locked)["finishedAt"].is_string()
});
let ran = check(&locked);
assert_eq!(ran["status"], "success", "{ran}");
let log = blob(
&runner.account.did,
ran["log"]["ref"]["$link"].as_str().unwrap(),
);
let said = format!(
"==> runs as code no one vouches for, as its lock adds shared steps from {theirs}\n"
);
assert!(log.contains(&said), "{log}");
collaborators(serde_json::json!([]));
contrib.git(&clone, &["checkout", "-q", "late"]);
// A pull's run of a head and base it is no longer at is cancelled, as
// when its base moves on, which runs anyone else's pull no further.
let go = runner.path("go");
@@ -4593,3 +4630,191 @@ fn a_run_shows_as_it_runs() {
fresh.lock().unwrap()
);
}
/// A step another repository shares: it takes inputs, needs a command, may
/// be given one secret, and runs a script kept beside it.
const GREET: &str = r#"
description: Says who it greets.
inputs:
who: {required: true}
how: {default: hello}
needs: [sh]
secrets: [TOKEN]
steps:
- name: say
run: |
echo "$INPUT_HOW $INPUT_WHO token=${TOKEN:-none} other=${OTHER:-none}"
- name: script
run: sh "$KNOTLESS_STEP_DIR/greet.sh"
"#;
#[test]
fn a_workflow_uses_a_shared_step() {
let owner = Sandbox::new();
let mut runner = Sandbox::new();
let handle = runner.account.handle.clone();
runner.set("KNOTLESS_IDENTIFIER", &handle);
let h = owner.account.handle.clone();
// The shared steps, in a repository of their own, at the commit the
// lock pins.
let steps = owner.work("steps", 1);
let files = steps.join(".knotless/steps");
std::fs::create_dir_all(&files).unwrap();
let write = |name: &str, text: &str| std::fs::write(files.join(name), text).unwrap();
write("greet.yml", GREET);
std::fs::write(steps.join("greet.sh"), "echo a script of its own\n").unwrap();
// One that uses another, passing on what it is given; one that uses
// itself; and a chain past the deepest.
let outer = format!(
"secrets: [TOKEN]\nsteps:\n - {{uses: {h}/steps/greet@1, with: {{who: nested}}, secrets: [TOKEN]}}\n"
);
write("outer.yml", &outer);
write(
"loop.yml",
&format!("steps: [{{uses: {h}/steps/loop@1}}]\n"),
);
for d in 1..=5 {
let next = format!("steps: [{{uses: {h}/steps/d{}@1}}]\n", d + 1);
write(&format!("d{d}.yml"), &next);
}
owner.git(&steps, &["add", "."]);
owner.git(&steps, &["commit", "-q", "-m", "steps"]);
let pinned = owner.git(&steps, &["rev-parse", "HEAD"]);
owner.git(&steps, &["remote", "add", "origin", &owner.url("steps")]);
owner.git(&steps, &["push", "-q", "origin", "main"]);
// What it says later never runs where the lock pins the earlier.
let later = "steps: [{run: exit 9}]\n";
owner.commit_file(&steps, ".knotless/steps/greet.yml", later);
owner.git(&steps, &["push", "-q", "origin", "main"]);
let uses = format!("{h}/steps/greet@1");
let work = owner.work("proj", 1);
let dir = work.join(".knotless/workflows");
std::fs::create_dir_all(&dir).unwrap();
let on = "on: {push: {refs: [refs/heads/main]}}\nsecrets: [TOKEN, OTHER]\n";
let shared = format!(
"{on}steps:\n - run: echo \"own token=${{TOKEN:-none}} other=${{OTHER:-none}}\"\n - uses: {uses}\n with: {{who: world}}\n secrets: [TOKEN]\n - {{name: quiet, uses: {uses}, with: {{who: you, how: hi}}}}\n"
);
std::fs::write(dir.join("shared.yml"), shared).unwrap();
let using =
|step: &str, more: &str| format!("{on}steps:\n - {{uses: {h}/steps/{step}{more}}}\n");
let workflows = [
("unpinned", using("other@1", "")),
("newer", using("greet@2", ", with: {who: x}")),
("gone", using("gone@1", "")),
("nested", using("outer@1", ", secrets: [TOKEN]")),
("looped", using("loop@1", "")),
("deep", using("d1@1", "")),
];
for (name, text) in &workflows {
std::fs::write(dir.join(format!("{name}.yml")), text).unwrap();
}
let mut lock = String::new();
let pin = |lock: &mut String, step: &str, version: &str| {
let line = format!("{h}/steps/{step}: {{version: {version}, commit: {pinned}}}\n");
lock.push_str(&line);
};
pin(&mut lock, "greet@1", "1.0.0");
pin(&mut lock, "greet@2", "1.0.0");
pin(&mut lock, "gone@1", "1.1.0");
for step in ["outer", "loop", "d1", "d2", "d3", "d4", "d5"] {
pin(&mut lock, &format!("{step}@1"), "1.0.0");
}
std::fs::write(work.join(".knotless/uses.lock"), lock).unwrap();
owner.git(&work, &["add", "."]);
owner.git(&work, &["commit", "-q", "-m", "shared steps"]);
owner.git(&work, &["remote", "add", "origin", &owner.url("proj")]);
owner.git(&work, &["push", "-q", "origin", "main"]);
let key = owner.record("dev.knotless.repo", "proj")["key"]
.as_str()
.unwrap()
.to_string();
let ref_key = ref_rkey(&key, "refs/heads/main");
let rt = tokio::runtime::Runtime::new().unwrap();
let js = jetstream(&rt);
let opts = runner_opts(&runner, &owner, &js);
rt.spawn(knotless_run::run(opts, common::ctx(&runner)));
wait_until(|| runner.path("state.json").exists());
owner.commit_file(&work, "f", "again");
owner.git(&work, &["push", "-q", "origin", "main"]);
js.ring(&owner.account.did, &ref_key);
let all = [
"shared", "unpinned", "newer", "gone", "nested", "looped", "deep",
];
wait_until(|| {
all.iter()
.all(|wf| check_of(&runner, &owner, &key, wf)["finishedAt"].is_string())
});
let ran = check_of(&runner, &owner, &key, "shared");
assert_eq!(ran["status"], "success", "{ran}");
let names: Vec<&str> = ran["steps"]
.as_array()
.unwrap()
.iter()
.map(|s| s["name"].as_str().unwrap())
.collect();
assert_eq!(
names,
[
"step 1",
"greet / needs",
"greet / say",
"greet / script",
"quiet / needs",
"quiet / say",
"quiet / script",
]
);
let log_of = |check: &serde_json::Value| {
blob(
&runner.account.did,
check["log"]["ref"]["$link"].as_str().unwrap(),
)
};
let log = log_of(&ran);
// Its own step sees every secret granted; a shared one only those it
// is given, and none it is not.
assert!(log.contains("own token=*** other=***"), "{log}");
assert!(log.contains("hello world token=*** other=none"), "{log}");
assert!(log.contains("hi you token=none other=none"), "{log}");
assert!(log.contains("a script of its own"), "{log}");
// A shared step's own shared steps, given what it passes on.
let nested = check_of(&runner, &owner, &key, "nested");
assert_eq!(nested["status"], "success", "{nested}");
assert_eq!(nested["steps"][1]["name"], "outer / greet / say");
let log = log_of(&nested);
assert!(log.contains("hello nested token=*** other=none"), "{log}");
let said = |wf: &str| {
let check = check_of(&runner, &owner, &key, wf);
assert_eq!(check["status"], "error", "{check}");
check["error"].as_str().unwrap().to_string()
};
for (wf, says) in [
("unpinned", "does not pin".to_string()),
("newer", "which @2 does not take".to_string()),
("gone", format!(".knotless/steps/gone.yml at {pinned} of")),
("looped", "loop@1 uses itself, through".to_string()),
("deep", "d5@1 is past 4 shared steps deep".to_string()),
] {
let error = said(wf);
assert!(error.contains(&says), "{wf}: {error}");
}
// Without a lock, nothing shared runs.
owner.git(&work, &["rm", "-q", ".knotless/uses.lock"]);
owner.git(&work, &["commit", "-q", "-m", "no lock"]);
owner.git(&work, &["push", "-q", "origin", "main"]);
let unlocked = owner.git(&work, &["rev-parse", "HEAD"]);
js.ring(&owner.account.did, &ref_key);
wait_until(|| {
let check = check_of(&runner, &owner, &key, "shared");
check["commit"] == unlocked.as_str() && check["finishedAt"].is_string()
});
let error = said("shared");
assert!(error.contains("there is no .knotless/uses.lock"), "{error}");
}
docs/design/README.md+7−5
View file
@@ -133,10 +133,12 @@ what each one forces.
moderation and web pages do, and how fresh they are.
7. [The runner](runner.md): which workflows run, when, in what order, and
with which caches and secrets.
8. [Merge queue](merge-queue.md): how queued pulls are checked and landed
8. [Shared steps](reusable-workflows.md): how a workflow uses steps
another repository publishes, pinned so they cannot change under it.
9. [Merge queue](merge-queue.md): how queued pulls are checked and landed
together in batches.
9. [Releases](packages.md): how a repository publishes versions of a
package, as records that never change.
10. [Integrity and trust](security.md): what each party can and cannot do.
11. [Prior art](prior-art.md): what came before, and what knotless takes
10. [Releases](packages.md): how a repository publishes versions of a
package, as records that never change.
11. [Integrity and trust](security.md): what each party can and cannot do.
12. [Prior art](prior-art.md): what came before, and what knotless takes
from it.
docs/design/reusable-workflows.mdnew+238−0
View file
docs/design/runner.md+17−1
View file
docs/design/security.md+11−0
View file
docs/reference/cli.md+21−0
View file
docs/reference/workflows.md+109−2
View file
docs-site/astro.config.mjs+1−0
View file