//! Skills: an id is a handle, and the body it loads is the trail to the state the
//! id names.
//!
//! Two roots form one namespace. `builtin` ships with the checkout the daemon was
//! started in; `managed` lives in the data directory and outlives a checkout move.
//! An id defined in both roots is refused rather than resolved: with a silent
//! winner, which body the model reads would depend on root order, and this runtime
//! refuses half-read state everywhere else too.
//!
//! The title rule is the entire schema. The first non-empty line is `# <id>` and
//! the directory name must equal it, so a body has no fields to keep in sync with
//! a header. A description stored beside the body would only become a second,
//! staler definition of the same thing.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use anyhow::{bail, ensure, Context, Result};
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
pub const SKILL_FILE: &str = "SKILL.md";
/// Bounded so a stray file inside a skill directory cannot become an unbounded
/// read on the model's behalf.
const MAX_SKILL_BYTES: usize = 262_144;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Origin {
Builtin,
Managed,
}
impl Origin {
pub fn as_str(self) -> &'static str {
match self {
Self::Builtin => "builtin",
Self::Managed => "managed",
}
}
}
/// The roots skills are read from. Their order is not precedence.
#[derive(Clone, Debug)]
pub struct SkillRoots {
pub builtin: PathBuf,
pub managed: Option<PathBuf>,
}
impl SkillRoots {
/// What a workspace implies when nothing else is configured. A missing
/// directory means "no skills here", never an error: most checkouts are not
/// the runtime's own.
pub fn for_workspace(workspace: &Path) -> Self {
Self {
builtin: workspace.join("skills"),
managed: None,
}
}
pub fn with_managed(mut self, managed: impl Into<PathBuf>) -> Self {
self.managed = Some(managed.into());
self
}
pub fn builtin_root(&self) -> &Path {
&self.builtin
}
pub fn managed_root(&self) -> Option<&Path> {
self.managed.as_deref()
}
fn roots(&self) -> impl Iterator<Item = (Origin, &Path)> {
std::iter::once((Origin::Builtin, self.builtin.as_path()))
.chain(self.managed.as_deref().map(|path| (Origin::Managed, path)))
}
/// Every id that is offered, plus every id that was refused and why.
pub fn catalog(&self) -> Catalog {
let mut catalog = Catalog::default();
let mut defined: BTreeMap<String, (Origin, PathBuf)> = BTreeMap::new();
for (origin, root) in self.roots() {
for found in scan(root, origin) {
let entry = match found {
Ok(entry) => entry,
Err(refusal) => {
catalog.refused.push(refusal);
continue;
}
};
match defined.remove(&entry.id) {
None => {
defined.insert(entry.id.clone(), (entry.origin, entry.path.clone()));
catalog.skills.push(entry);
}
Some((first_origin, first_path)) => {
catalog.skills.retain(|skill| skill.id != entry.id);
catalog.refused.push(Refusal {
id: entry.id,
reason: format!(
"defined twice: {} at {}, and {} at {}",
first_origin.as_str(),
first_path.display(),
entry.origin.as_str(),
entry.path.display()
),
});
}
}
}
}
catalog.skills.sort_by(|a, b| a.id.cmp(&b.id));
catalog.refused.sort_by(|a, b| a.id.cmp(&b.id));
catalog
}
/// Read one body by id. A body that changed since it was indexed is refused
/// rather than returned: the model would otherwise be reading bytes the host
/// never vouched for.
pub fn load(&self, id: &str) -> Result<SkillBody> {
let catalog = self.catalog();
let entry = match catalog.find(id) {
Some(entry) => entry.clone(),
None => match catalog.refusal(id) {
Some(refusal) => bail!("skill {id:?} is not offered: {}", refusal.reason),
None => bail!(
"unknown skill {id:?}; available: {}",
catalog.ids().join(", ")
),
},
};
let bytes = std::fs::read(&entry.path)
.with_context(|| format!("cannot read {}", entry.path.display()))?;
let digest = digest_hex(&bytes);
ensure!(
digest == entry.digest,
"skill {id:?} changed on disk since it was indexed: indexed {}, found {digest}",
entry.digest
);
let text = String::from_utf8(bytes)
.with_context(|| format!("{} is not UTF-8", entry.path.display()))?;
Ok(SkillBody {
id: entry.id,
origin: entry.origin,
path: entry.path,
digest: entry.digest,
text,
})
}
}
#[derive(Clone, Debug)]
pub struct SkillEntry {
pub id: String,
pub origin: Origin,
pub path: PathBuf,
pub digest: String,
pub bytes: usize,
}
impl SkillEntry {
pub fn report(&self) -> Value {
json!({
"id": self.id,
"origin": self.origin.as_str(),
"path": self.path.display().to_string(),
"digest": self.digest,
"bytes": self.bytes,
})
}
}
#[derive(Clone, Debug)]
pub struct Refusal {
pub id: String,
pub reason: String,
}
impl Refusal {
pub fn report(&self) -> Value {
json!({"id": self.id, "reason": self.reason})
}
}
#[derive(Clone, Debug, Default)]
pub struct Catalog {
pub skills: Vec<SkillEntry>,
pub refused: Vec<Refusal>,
}
impl Catalog {
/// Content identity of the whole offered set.
///
/// This is the same shape as a behavior fingerprint and for the same reason: a
/// worker generation can then be described by the instruction set it was
/// spawned with, and a body that changed later cannot silently claim to be the
/// one that generation ran under.
pub fn revision(&self) -> String {
let mut hasher = Sha256::new();
for skill in &self.skills {
hasher.update((skill.id.len() as u64).to_be_bytes());
hasher.update(skill.id.as_bytes());
hasher.update((skill.digest.len() as u64).to_be_bytes());
hasher.update(skill.digest.as_bytes());
}
format!("{:x}", hasher.finalize())
}
pub fn ids(&self) -> Vec<&str> {
self.skills.iter().map(|skill| skill.id.as_str()).collect()
}
pub fn find(&self, id: &str) -> Option<&SkillEntry> {
self.skills.iter().find(|skill| skill.id == id)
}
pub fn refusal(&self, id: &str) -> Option<&Refusal> {
self.refused.iter().find(|refusal| refusal.id == id)
}
pub fn report(&self) -> Value {
json!({
"index": self.skills.iter().map(SkillEntry::report).collect::<Vec<_>>(),
"refused": self.refused.iter().map(Refusal::report).collect::<Vec<_>>(),
})
}
}
#[derive(Clone, Debug)]
pub struct SkillBody {
pub id: String,
pub origin: Origin,
pub path: PathBuf,
pub digest: String,
pub text: String,
}
impl SkillBody {
pub fn report(&self) -> Value {
json!({
"id": self.id,
"origin": self.origin.as_str(),
"path": self.path.display().to_string(),
"digest": self.digest,
"bytes": self.text.len(),
"text": self.text,
})
}
}
/// What the host hands a worker at spawn.
///
/// The worker does not read the roots: it is given the bodies the host already
/// validated, so a cell cannot reach an id the host refused, and the set a
/// generation runs under is fixed when that generation starts rather than
/// changing underneath a turn. `refused` rides along so the refusal is visible
/// where the ids are, instead of only where the files are.
pub fn payload(roots: &SkillRoots) -> Result<Value> {
let catalog = roots.catalog();
let mut skills = Vec::with_capacity(catalog.skills.len());
let mut total = 0usize;
for entry in &catalog.skills {
let body = roots
.load(&entry.id)
.with_context(|| format!("cannot read skill {:?}", entry.id))?;
total = total.saturating_add(body.text.len());
ensure!(
total <= MAX_PAYLOAD_BYTES,
"skill bodies exceed the {MAX_PAYLOAD_BYTES}-byte spawn payload"
);
skills.push(json!({
"id": body.id,
"origin": body.origin.as_str(),
"path": body.path.display().to_string(),
"digest": body.digest,
"text": body.text,
}));
}
Ok(json!({
"revision": catalog.revision(),
// The roots are part of the answer, not decoration: without them an id that
// is simply absent from an older set has no explanation, and the only place
// left to look for one is the filesystem a reader cannot check from here.
"roots": {
"builtin": roots.builtin_root().display().to_string(),
"managed": roots.managed_root().map(|path| path.display().to_string()),
},
"skills": skills,
"refused": catalog.refused.iter().map(Refusal::report).collect::<Vec<_>>(),
}))
}
/// Write that payload where a worker will read it before it connects.
pub fn materialize(roots: &SkillRoots, directory: &Path) -> Result<PathBuf> {
let path = directory.join("skills.json");
let encoded = serde_json::to_vec(&payload(roots)?).context("cannot encode skill payload")?;
std::fs::write(&path, encoded).with_context(|| format!("cannot write {}", path.display()))?;
Ok(path)
}
/// Bounded so a runaway skills directory cannot turn every spawn into a stall.
const MAX_PAYLOAD_BYTES: usize = 2_097_152;
/// A directory is a skill because it holds a `SKILL.md`; a directory without one
/// is simply not a skill. Once that file exists the rules are enforced, because
/// its presence is a claim that this is one.
fn scan(root: &Path, origin: Origin) -> Vec<Result<SkillEntry, Refusal>> {
let Ok(entries) = std::fs::read_dir(root) else {
return Vec::new();
};
let mut candidates: Vec<PathBuf> = entries
.filter_map(|entry| entry.ok().map(|entry| entry.path()))
.collect();
candidates.sort();
candidates
.into_iter()
.filter_map(|directory| {
let name = directory.file_name()?.to_str()?.to_string();
if name.starts_with('.') {
return None;
}
let file = directory.join(SKILL_FILE);
file.is_file()
.then(|| read_skill(&directory, &name, &file, origin))
})
.collect()
}
fn read_skill(
directory: &Path,
name: &str,
file: &Path,
origin: Origin,
) -> Result<SkillEntry, Refusal> {
let refuse = |reason: String| {
Err(Refusal {
id: name.to_string(),
reason,
})
};
if !is_id(name) {
return refuse(format!(
"skill id {name:?} must be lower-case letters, digits, and single dashes"
));
}
if directory.is_symlink() || file.is_symlink() {
return refuse("skill directories and files cannot be symlinks".into());
}
let bytes = match std::fs::read(file) {
Ok(bytes) => bytes,
Err(error) => return refuse(format!("cannot read {SKILL_FILE}: {error}")),
};
if bytes.len() > MAX_SKILL_BYTES {
return refuse(format!(
"{SKILL_FILE} is {} bytes, over the {MAX_SKILL_BYTES}-byte limit",
bytes.len()
));
}
let text = match std::str::from_utf8(&bytes) {
Ok(text) => text,
Err(error) => return refuse(format!("{SKILL_FILE} is not UTF-8: {error}")),
};
match title_of(text) {
None => return refuse(format!("{SKILL_FILE} must open with `# {name}`")),
Some(title) if title != name => {
return refuse(format!(
"directory {name:?} does not match the title it declares: {title:?}"
));
}
Some(_) => {}
}
Ok(SkillEntry {
id: name.to_string(),
origin,
path: file.to_path_buf(),
digest: digest_hex(&bytes),
bytes: bytes.len(),
})
}
/// The first non-empty line must be the title. `## heading` is not a title, so a
/// body cannot declare its id with a subsection.
fn title_of(text: &str) -> Option<&str> {
text.lines()
.find(|line| !line.trim().is_empty())
.and_then(|line| line.strip_prefix("# "))
.map(str::trim)
}
fn is_id(name: &str) -> bool {
!name.is_empty()
&& !name.starts_with('-')
&& !name.ends_with('-')
&& !name.contains("--")
&& name
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
}
fn digest_hex(bytes: &[u8]) -> String {
format!("{:x}", Sha256::digest(bytes))
}
#[cfg(test)]
mod tests {
use super::*;
fn write_skill(root: &Path, id: &str, text: &str) {
let directory = root.join(id);
std::fs::create_dir_all(&directory).unwrap();
std::fs::write(directory.join(SKILL_FILE), text).unwrap();
}
fn roots(builtin: &Path, managed: &Path) -> SkillRoots {
SkillRoots {
builtin: builtin.to_path_buf(),
managed: Some(managed.to_path_buf()),
}
}
#[test]
fn both_roots_form_one_namespace_and_a_duplicate_swaps_for_a_refusal() {
let builtin = tempfile::tempdir().unwrap();
let managed = tempfile::tempdir().unwrap();
write_skill(builtin.path(), "workspace", "# workspace\n\nbuiltin body\n");
write_skill(managed.path(), "remote", "# remote\n\nmanaged body\n");
let catalog = roots(builtin.path(), managed.path()).catalog();
assert_eq!(catalog.ids(), vec!["remote", "workspace"]);
assert!(catalog.refused.is_empty());
write_skill(managed.path(), "workspace", "# workspace\n\nmanaged body\n");
let catalog = roots(builtin.path(), managed.path()).catalog();
assert_eq!(catalog.ids(), vec!["remote"]);
assert_eq!(catalog.refused.len(), 1);
assert!(catalog.refused[0].reason.contains("defined twice"));
}
#[test]
fn a_title_that_disagrees_with_its_directory_is_refused_not_renamed() {
let builtin = tempfile::tempdir().unwrap();
let managed = tempfile::tempdir().unwrap();
write_skill(builtin.path(), "workspace", "# workspaces\n\nbody\n");
write_skill(builtin.path(), "Peer", "# Peer\n\nbody\n");
write_skill(builtin.path(), "cli", "## cli\n\nbody\n");
let roots = roots(builtin.path(), managed.path());
let catalog = roots.catalog();
assert!(catalog.skills.is_empty());
assert_eq!(catalog.refused.len(), 3);
let error = roots.load("cli").unwrap_err().to_string();
assert!(error.contains("not offered"), "{error}");
}
#[test]
fn loading_always_describes_the_bytes_that_are_on_disk_now() {
let builtin = tempfile::tempdir().unwrap();
let managed = tempfile::tempdir().unwrap();
let body = "# workspace\n\nthe trail, not a signature list\n";
write_skill(builtin.path(), "workspace", body);
let roots = roots(builtin.path(), managed.path());
let loaded = roots.load("workspace").unwrap();
assert_eq!(loaded.text, body);
assert_eq!(loaded.origin, Origin::Builtin);
assert_eq!(loaded.digest, digest_hex(body.as_bytes()));
// An agent that edits its own skill gets the new body on the next load, and
// the digest it is told describes exactly those bytes - never the previous
// body's hash attached to the new text.
let edited = "# workspace\n\nedited\n";
std::fs::write(builtin.path().join("workspace").join(SKILL_FILE), edited).unwrap();
let reloaded = roots.load("workspace").unwrap();
assert_eq!(reloaded.text, edited);
assert_eq!(reloaded.digest, digest_hex(edited.as_bytes()));
assert_eq!(
roots.catalog().find("workspace").unwrap().digest,
reloaded.digest
);
}
#[test]
fn the_revision_covers_the_offered_set_and_ignores_a_changed_body() {
let builtin = tempfile::tempdir().unwrap();
let managed = tempfile::tempdir().unwrap();
write_skill(builtin.path(), "workspace", "# workspace\n\nfirst\n");
write_skill(managed.path(), "peer", "# peer\n\nsecond\n");
let roots = roots(builtin.path(), managed.path());
let before = roots.catalog().revision();
// Adding a refused directory must not move the identity of what is offered.
std::fs::create_dir_all(builtin.path().join("Bad")).unwrap();
std::fs::write(builtin.path().join("Bad").join(SKILL_FILE), "# Bad\n").unwrap();
assert_eq!(roots.catalog().revision(), before);
// A changed body must.
std::fs::write(
builtin.path().join("workspace").join(SKILL_FILE),
"# workspace\n\nedited\n",
)
.unwrap();
assert_ne!(roots.catalog().revision(), before);
}
#[test]
fn the_spawn_payload_carries_bodies_and_refusals_and_is_written_for_the_worker() {
let builtin = tempfile::tempdir().unwrap();
let managed = tempfile::tempdir().unwrap();
write_skill(builtin.path(), "workspace", "# workspace\n\nthe trail\n");
write_skill(managed.path(), "Broken", "# Broken\n");
let roots = roots(builtin.path(), managed.path());
let encoded = payload(&roots).unwrap();
assert_eq!(encoded["skills"].as_array().unwrap().len(), 1);
assert_eq!(encoded["skills"][0]["text"], "# workspace\n\nthe trail\n");
assert_eq!(encoded["refused"][0]["id"], "Broken");
assert_eq!(encoded["revision"], roots.catalog().revision());
let out = tempfile::tempdir().unwrap();
let written = materialize(&roots, out.path()).unwrap();
let read_back: serde_json::Value =
serde_json::from_slice(&std::fs::read(&written).unwrap()).unwrap();
assert_eq!(read_back, encoded);
}
#[test]
fn a_missing_root_is_no_skills_rather_than_a_failure() {
let missing = tempfile::tempdir().unwrap().path().join("absent");
let roots = SkillRoots {
builtin: missing,
managed: None,
};
assert!(roots.catalog().skills.is_empty());
assert!(roots.catalog().refused.is_empty());
}
}