~ knotless Search/
↑↓ move↵ open⇧↵ all results

chadtmiller.com/knotless

git that lives in your Atmosphere account

knotless.dev
1 collaborator

knotless 0.13.0 Latest 2 packages, 1 image

1
10

knotless-run keeps a key for secrets, and publishes it

Merged chadtmiller.com asked to merge into main from chadtmiller.com/knotless:run/runner-key
.changeset/runner-key.mdnew+7−0
@@ -0,0 +1,7 @@
---
default: minor
---
# knotless-run keeps a key for secrets
`knotless-run` now makes a P-256 key on its first start, kept in `--secrets-key <file>` (`secrets-key.json` beside its state file by default), and publishes its public half as its `dev.knotless.runnerKey` record. Owners will seal the secrets they give a runner to it, so only the runner can read them.
crates/knotless/src/lib.rs+1−0
@@ -53,6 +53,7 @@ pub mod release;
pub mod remote;
pub mod revert;
pub mod reviews;
pub mod sealed;
pub mod session;
pub mod social;
pub mod space;
crates/knotless/src/sealed.rsnew+224−0
@@ -0,0 +1,224 @@
//! Secrets an owner gives a runner, sealed so only the runner reads them.
//!
//! A runner makes a P-256 key and publishes its public half as its
//! `dev.knotless.runnerKey` record. An owner seals each secret to it with
//! HPKE (RFC 9180) in base mode: DHKEM(P-256, HKDF-SHA256), HKDF-SHA256 and
//! AES-128-GCM. Whatever holds the sealed value, the appview or anyone who
//! reads it, holds only ciphertext. The value is bound to where it is for,
//! as the AEAD's associated data: the owner, the repository's key, the
//! runner, the secret's name and the refs it is limited to. The runner
//! rebuilds that from where it found the value, so one copied to another
//! repository, name or runner does not open. Base mode does not say who
//! sealed it: anyone can seal a value to a runner, and only the owner's
//! word for where it is for keeps it there.
use crate::crypto::{P256Key, PublicKey};
use anyhow::{Context, Result, anyhow, bail, ensure};
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use hpke::aead::AesGcm128;
use hpke::kdf::HkdfSha256;
use hpke::kem::DhP256HkdfSha256;
use hpke::{Deserializable, Kem, OpModeR, OpModeS, Serializable};
use serde::{Deserialize, Serialize};
use std::path::Path;
/// The record a runner publishes its key as, under [`KEY_RKEY`].
pub const KEY_NSID: &str = "dev.knotless.runnerKey";
pub const KEY_RKEY: &str = "self";
/// What each sealing is for, so a value sealed for anything else of the
/// same key does not open as a secret.
const INFO: &[u8] = b"dev.knotless.secret v1";
/// The bytes of the key a sealing encapsulates, uncompressed P-256.
const ENCAPPED: usize = 65;
/// Where a sealed secret is for.
pub struct Bound<'a> {
/// The repository owner's DID.
pub owner: &'a str,
/// The repository's key, its record key: a rename keeps it, and a new
/// repository under the old name has another.
pub repo: &'a str,
/// The runner's DID.
pub runner: &'a str,
pub name: &'a str,
/// The refs it is limited to, none for every push.
pub refs: &'a [String],
}
impl Bound<'_> {
/// Each field, length first, so no two differ only in where one ends.
fn aad(&self) -> Vec<u8> {
let fields = [self.owner, self.repo, self.runner, self.name];
let refs = self.refs.iter().map(String::as_str);
let mut aad = Vec::new();
for field in fields.into_iter().chain(refs) {
aad.extend((field.len() as u32).to_be_bytes());
aad.extend(field.as_bytes());
}
aad
}
}
/// `value` sealed to the runner key `to` for `bound`: the encapsulated key,
/// then the ciphertext.
pub fn seal(to: &PublicKey, bound: &Bound, value: &[u8]) -> Result<Vec<u8>> {
let PublicKey::P256(key) = to else {
bail!("a runner's key is P-256");
};
let point = key.to_encoded_point(false);
let to = <DhP256HkdfSha256 as Kem>::PublicKey::from_bytes(point.as_bytes())
.expect("a P-256 key's point is one");
let mut rng = rand_core::OsRng;
let aad = bound.aad();
let (encapped, sealed) = hpke::single_shot_seal::<AesGcm128, HkdfSha256, DhP256HkdfSha256, _>(
&OpModeS::Base,
&to,
INFO,
value,
&aad,
&mut rng,
)
.expect("sealing to a P-256 key cannot fail");
Ok([&encapped.to_bytes()[..], &sealed].concat())
}
/// A runner's keys as it keeps them: each secret scalar, base64, the
/// newest first.
#[derive(Serialize, Deserialize)]
struct Kept {
keys: Vec<String>,
}
/// A runner's keys, the newest first: it publishes that one, and opens what
/// was sealed to any of them.
pub struct RunnerKeys(Vec<P256Key>);
impl RunnerKeys {
/// Those kept at `path`, made and kept there if there are none.
pub fn open_or_make(path: &Path) -> Result<RunnerKeys> {
let kept = match crate::store::read::<Kept>(path) {
Some(kept) => kept,
None => {
let kept = Kept {
keys: vec![STANDARD.encode(P256Key::generate().to_bytes())],
};
crate::store::write(path, &kept, true)?;
kept
}
};
let keys = kept
.keys
.iter()
.map(|k| P256Key::from_bytes(&STANDARD.decode(k)?));
let keys = keys.collect::<Result<Vec<_>>>();
let keys =
keys.with_context(|| format!("reading the runner's keys, {}", path.display()))?;
ensure!(!keys.is_empty(), "{} holds no key", path.display());
Ok(RunnerKeys(keys))
}
/// The newest key's public half, which it publishes.
pub fn public(&self) -> PublicKey {
self.0[0].public()
}
/// What was sealed for `bound` to the key whose `did:key` is `to`.
pub fn open(&self, to: &str, bound: &Bound, sealed: &[u8]) -> Result<Vec<u8>> {
let key = self.0.iter().find(|k| k.public().to_did_key() == to);
let key =
key.with_context(|| format!("sealed to {to}, a key this runner does not hold"))?;
ensure!(sealed.len() > ENCAPPED, "too short to be sealed");
let (encapped, sealed) = sealed.split_at(ENCAPPED);
let encapped = <DhP256HkdfSha256 as Kem>::EncappedKey::from_bytes(encapped);
let encapped = encapped.map_err(|_| anyhow!("not sealed to a P-256 key"))?;
let mine = <DhP256HkdfSha256 as Kem>::PrivateKey::from_bytes(&key.to_bytes())
.expect("a P-256 secret is one");
let opened = hpke::single_shot_open::<AesGcm128, HkdfSha256, DhP256HkdfSha256>(
&OpModeR::Base,
&mine,
&encapped,
INFO,
sealed,
&bound.aad(),
);
opened.map_err(|_| {
anyhow!("it does not open: sealed for another repository, name, runner or refs")
})
}
}
#[cfg(test)]
mod tests {
use super::*;
fn bound<'a>(name: &'a str, refs: &'a [String]) -> Bound<'a> {
Bound {
owner: "did:plc:owner",
repo: "3kabc",
runner: "did:plc:runner",
name,
refs,
}
}
#[test]
fn opens_only_where_it_was_sealed_for() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("key.json");
let keys = RunnerKeys::open_or_make(&path).unwrap();
// Kept, and the same key on reading it again.
let again = RunnerKeys::open_or_make(&path).unwrap();
assert_eq!(keys.public(), again.public());
let to = keys.public();
let did = to.to_did_key();
let main = vec!["refs/heads/main".to_string()];
let sealed = seal(&to, &bound("TOKEN", &main), b"s3cret").unwrap();
let opened = keys.open(&did, &bound("TOKEN", &main), &sealed).unwrap();
assert_eq!(opened, b"s3cret");
// Each field it was bound to, changed, keeps it shut.
let other = |b: Bound| keys.open(&did, &b, &sealed).unwrap_err().to_string();
assert!(other(bound("OTHER", &main)).contains("does not open"));
assert!(other(bound("TOKEN", &[])).contains("does not open"));
let moved = Bound {
repo: "3kxyz",
..bound("TOKEN", &main)
};
assert!(other(moved).contains("does not open"));
// Nor does another key's, a short one, or one to a key not held.
let theirs = P256Key::generate().public();
let sealed = seal(&theirs, &bound("TOKEN", &[]), b"x").unwrap();
let err = keys.open(&theirs.to_did_key(), &bound("TOKEN", &[]), &sealed);
assert!(err.unwrap_err().to_string().contains("does not hold"));
let err = keys.open(&did, &bound("TOKEN", &[]), b"short");
assert!(err.unwrap_err().to_string().contains("too short"));
let mut garbled = vec![9u8; ENCAPPED];
garbled.push(0);
let err = keys.open(&did, &bound("TOKEN", &[]), &garbled);
assert!(
err.unwrap_err()
.to_string()
.contains("not sealed to a P-256")
);
}
#[test]
fn a_runner_key_is_p256() {
let k256 = PublicKey::K256(
*k256::ecdsa::SigningKey::random(&mut rand_core::OsRng).verifying_key(),
);
let err = seal(&k256, &bound("T", &[]), b"x").unwrap_err();
assert!(err.to_string().contains("P-256"));
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("key.json");
std::fs::write(&path, r#"{"keys": []}"#).unwrap();
assert!(RunnerKeys::open_or_make(&path).is_err());
std::fs::write(&path, r#"{"keys": ["!"]}"#).unwrap();
let err = RunnerKeys::open_or_make(&path).err().unwrap();
assert!(format!("{err:#}").contains("reading the runner's keys"));
}
}
crates/knotless/Cargo.toml+1−0
@@ -27,6 +27,7 @@ bs58.workspace = true
blake3.workspace = true
ciborium.workspace = true
hmac.workspace = true
hpke.workspace = true
tokio.workspace = true
tokio-tungstenite.workspace = true
tracing.workspace = true
crates/knotless-run/src/check.rs+31−0
@@ -4,10 +4,12 @@
//! says anything of it.
use anyhow::Result;
use knotless::crypto::PublicKey;
use knotless::env::Ctx;
use knotless::identity::{self, Identity};
use knotless::records::{BlobRef, MinutesRecord, nsid};
use knotless::remote::{self, Remote};
use knotless::sealed;
use knotless::session;
use knotless::space;
use knotless::xrpc::{Auth, Xrpc, XrpcError};
@@ -170,6 +172,35 @@ impl Reporter {
self.write("com.atproto.repo.putRecord", &body).await
}
/// Publishes `key` as the key owners seal its secrets to, unless its
/// record already holds it.
pub async fn put_key(&self, key: &PublicKey) -> Result<()> {
let key = key.to_did_key();
let params = [
("repo", self.identity.did.as_str()),
("collection", sealed::KEY_NSID),
("rkey", sealed::KEY_RKEY),
];
let held =
self.xrpc
.query::<serde_json::Value>("com.atproto.repo.getRecord", &params, None);
let held = held.await.ok().map(|r| r["value"]["key"].clone());
if held.is_some_and(|k| k == key.as_str()) {
return Ok(());
}
let body = json!({
"repo": self.identity.did,
"collection": sealed::KEY_NSID,
"rkey": sealed::KEY_RKEY,
"record": {
"$type": sealed::KEY_NSID,
"key": key,
"createdAt": knotless::records::now(),
},
});
self.write("com.atproto.repo.putRecord", &body).await
}
/// Calls the procedure `nsid` with `body`, as [`Reporter::retried`].
async fn write(&self, nsid: &str, body: &serde_json::Value) -> Result<()> {
self.retried(|auth| async move {
crates/knotless-run/src/lib.rs+12−1
@@ -100,6 +100,7 @@ use knotless::records::{
RerunRecord, RerunVar, now, nsid,
};
use knotless::remote::{Remote, Snapshot, Stored};
use knotless::sealed;
use knotless::session;
use knotless::space;
use knotless::store;
@@ -115,7 +116,7 @@ use tokio::sync::{Notify, Semaphore};
use turns::{Class, Rank, Turns, Whose};
use workflow::{Superseded, Workflow};
const USAGE: &str = "usage: knotless-run [--jetstream <wss://host>] [--executor docker|shell] [--image <image>] [--secrets <file>] [--state <file>] [--work <dir>] [--step-timeout <secs>] [--run-timeout <secs>] [--cpus <n>] [--memory <GiB>] [--pids <n>] [--network <name>] [--runtime <name>] [--trusted-runtime <name>] [--trust <did>] [--resolv <file>] [--jobs <n>] [--pull-jobs <n>] [--owner-jobs <n>] [--allowance <minutes>] [--private-allowance <minutes>] [--cache-max <GiB>] [--caches <dir>] [--paused <file>] [--keep-checks <days>] [--sweep <secs>] [--stop-wait <secs>] [--poll <secs>] [--pulls] [--pulls-only <handle>/<name>] [--unmetered <did>] [--queue [--batch <n>] [--depth 1|2] [--notice <secs>]] [--discover] [<handle>/<name>...]";
const USAGE: &str = "usage: knotless-run [--jetstream <wss://host>] [--executor docker|shell] [--image <image>] [--secrets <file>] [--secrets-key <file>] [--state <file>] [--work <dir>] [--step-timeout <secs>] [--run-timeout <secs>] [--cpus <n>] [--memory <GiB>] [--pids <n>] [--network <name>] [--runtime <name>] [--trusted-runtime <name>] [--trust <did>] [--resolv <file>] [--jobs <n>] [--pull-jobs <n>] [--owner-jobs <n>] [--allowance <minutes>] [--private-allowance <minutes>] [--cache-max <GiB>] [--caches <dir>] [--paused <file>] [--keep-checks <days>] [--sweep <secs>] [--stop-wait <secs>] [--poll <secs>] [--pulls] [--pulls-only <handle>/<name>] [--unmetered <did>] [--queue [--batch <n>] [--depth 1|2] [--notice <secs>]] [--discover] [<handle>/<name>...]";
#[derive(Debug, Clone, PartialEq)]
pub struct Options {
@@ -135,6 +136,10 @@ pub struct Options {
/// "value"}}`, or `{"value": "...", "refs": ["refs/heads/main"]}` for
/// one granted only to pushes to those refs.
pub secrets: Option<PathBuf>,
/// Where it keeps the key owners seal secrets to, which it publishes as
/// its `dev.knotless.runnerKey` record. Defaults to `secrets-key.json`
/// beside the state file.
pub secrets_key: Option<PathBuf>,
/// The generation each ref was last run at. Defaults to one in the cache
/// directory.
pub state: Option<PathBuf>,
@@ -228,6 +233,7 @@ impl Options {
discover: false,
executor: Executor::Shell,
secrets: None,
secrets_key: None,
state: None,
work: None,
step_timeout: Duration::from_secs(30 * 60),
@@ -360,6 +366,7 @@ pub fn parse_args(args: &[String]) -> Result<Options, String> {
"--executor" => executor = value()?,
"--image" => image = value()?,
"--secrets" => opts.secrets = Some(value()?.into()),
"--secrets-key" => opts.secrets_key = Some(value()?.into()),
"--state" => opts.state = Some(value()?.into()),
"--work" => opts.work = Some(value()?.into()),
"--caches" => opts.caches = Some(value()?.into()),
@@ -1073,6 +1080,10 @@ impl Runner {
.clone()
.unwrap_or_else(|| ctx.env.cache_dir.join("run").join("state.json"));
let state: State = store::read(&state_path).unwrap_or_default();
let key_path = opts.secrets_key.clone();
let key_path = key_path.unwrap_or_else(|| state_path.with_file_name("secrets-key.json"));
let keys = sealed::RunnerKeys::open_or_make(&key_path)?;
reporter.put_key(&keys.public()).await?;
// The runs whose layers may be kept: those a push may yet take.
let kept: Vec<String> = state.passed.values().flatten().map(Passed::rkey).collect();
let root = opts
crates/knotless-run/tests/run.rs+13−0
@@ -154,6 +154,7 @@ fn runner_opts(runner: &Sandbox, owner: &Sandbox, js: &Jetstream) -> knotless_ru
discover: false,
executor: Executor::Shell,
secrets: Some(secrets),
secrets_key: None,
state: Some(runner.path("state.json")),
work: Some(runner.path("work")),
step_timeout: Duration::from_secs(60),
@@ -237,6 +238,12 @@ fn a_push_runs_its_workflows() {
// Watching starts where the refs are: nothing runs for what was there.
wait_until(|| runner.path("state.json").exists());
// It made a key, kept beside its state, and published its public half
// for owners to seal secrets to.
let keys = knotless::sealed::RunnerKeys::open_or_make(&runner.path("secrets-key.json"));
let sealing = keys.unwrap().public().to_did_key();
let published = runner.record(knotless::sealed::KEY_NSID, knotless::sealed::KEY_RKEY);
assert_eq!(published["key"], sealing.as_str(), "{published}");
js.ring(&owner.account.did, &ref_key);
std::thread::sleep(Duration::from_millis(500));
assert!(check_of(&runner, &owner, &key, "ci").is_null());
@@ -739,6 +746,7 @@ fn what_goes_wrong_is_reported() {
discover: false,
executor: Executor::Shell,
secrets: Some(runner.path("secrets.json")),
secrets_key: None,
state: Some(runner.path("state.json")),
work: None,
step_timeout: Duration::from_secs(1),
@@ -946,6 +954,7 @@ fn a_deploy_is_never_cut_off() {
discover: false,
executor: Executor::Shell,
secrets: None,
secrets_key: None,
state: Some(runner.path("state.json")),
work: Some(runner.path("work")),
step_timeout: Duration::from_secs(60),
@@ -1086,6 +1095,7 @@ fn a_stop_drops_what_has_not_started() {
discover: false,
executor: Executor::Shell,
secrets: None,
secrets_key: None,
state: Some(runner.path("state.json")),
work: Some(runner.path("work")),
step_timeout: Duration::from_secs(60),
@@ -1354,6 +1364,7 @@ fn a_failing_pds_wedges_nothing() {
discover: false,
executor: Executor::Shell,
secrets: None,
secrets_key: None,
state: Some(runner.path("state.json")),
work: Some(runner.path("work")),
step_timeout: Duration::from_secs(60),
@@ -2081,6 +2092,7 @@ fn a_merged_pull_passes_its_push() {
discover: false,
executor: Executor::Shell,
secrets: None,
secrets_key: None,
state: Some(runner.path("state.json")),
work: Some(runner.path("work")),
step_timeout: Duration::from_secs(60),
@@ -3577,6 +3589,7 @@ fn a_workflow_asked_for_by_name() {
discover: false,
executor: Executor::Shell,
secrets: Some(secrets),
secrets_key: None,
state: Some(runner.path("state.json")),
work: Some(runner.path("work")),
// Shorter than a step of `costly`, which only a run asked for may
docs/design/runner.md+27−0
@@ -762,6 +762,33 @@ Values under 4 bytes are left alone, since masking them would mask
ordinary output. Masking catches a secret only as written. A step can
still leak one transformed, encoded, or a bit at a time.
### The runner's key
Owners will give a runner secrets without the operator's hands: each
sealed to the runner, so whatever holds it on the way holds only
ciphertext. For that the runner keeps a P-256 key, made on its first
start, in the file `--secrets-key` names (`secrets-key.json` beside the
state file by default, readable by its owner alone), and publishes its
public half as its `dev.knotless.runnerKey` record, as a `did:key`. It
writes the record again only when the key differs.
A value is sealed with HPKE (RFC 9180) in base mode: DHKEM(P-256,
HKDF-SHA256), HKDF-SHA256 and AES-128-GCM, as GitHub seals a repository's
Actions secrets to a key of its own. What it is for is bound to it as the
AEAD's associated data: the owner's DID, the repository's key (its record
key, so a rename keeps its secrets and a repository made later under the
old name gets none), the runner's DID, the secret's name and the refs it
is limited to. The runner rebuilds those from where it found the value,
so one copied to another repository, name, runner or refs does not open:
nobody can carry an owner's sealed token into a repository of their own
to print it. Base mode does not say who sealed a value, so whoever holds
the sealed values could swap one in, though never read one.
The file holds a list of keys, the newest first: the runner publishes the
newest and opens what was sealed to any of them, each value naming the
key it was sealed to, so a key can be replaced while values sealed to the
old one are set again.
## Check records
Checks are `dev.knotless.check` records in the runner's account, written
docs/reference/cli.md+1−0
@@ -403,6 +403,7 @@ add` make it one, and invite it, which `--discover` reads.
| `--cache-max <GiB>` | 16 | a whole number of GiB: the size at which a cache's new base is not used, to start afresh, and the most kept layers may take in all |
| `--caches <dir>` | `<cache dir>/run/caches` | where caches' bases and runs' layers are kept; a path the docker daemon sees under the same name |
| `--secrets <file>` | | the [secrets file](workflows.md#secrets-file) |
| `--secrets-key <file>` | `secrets-key.json` beside the state file | where the runner keeps the key owners seal secrets to, made if missing, and published as its `dev.knotless.runnerKey` record (see [The runner's key](../design/runner.md#the-runners-key)) |
| `--state <file>` | `<cache dir>/run/state.json` | the state file |
| `--work <dir>` | the system's temporary directory | where checkouts go while their steps run; made if missing |
| `--jetstream <wss://host>` | `wss://jetstream2.us-east.bsky.network` | where events come from |
lexicons/dev/knotless/runnerKey.jsonnew+20−0
@@ -0,0 +1,20 @@
{
"lexicon": 1,
"id": "dev.knotless.runnerKey",
"description": "The key a knotless runner opens the secrets owners give it with.",
"defs": {
"main": {
"type": "record",
"description": "Kept in the runner's account. An owner seals each secret it gives the runner to this key, with HPKE (RFC 9180) in base mode: DHKEM(P-256, HKDF-SHA256), HKDF-SHA256 and AES-128-GCM, so only the runner, which keeps the private half, can open it. The runner writes it again when its key changes; a value sealed to an older key names that key.",
"key": "literal:self",
"record": {
"type": "object",
"required": ["key", "createdAt"],
"properties": {
"key": { "type": "string", "format": "did", "description": "The public key, as a P-256 did:key." },
"createdAt": { "type": "string", "format": "datetime", "description": "When the runner published this key." }
}
}
}
}
}
Cargo.lock+174−1
Cargo.toml+2−0