use std::collections::BTreeSet;
use std::path::Path;
use std::thread::JoinHandle;
use anyhow::Context as _;
use crate::cas::Digest;
use crate::git_export;
use crate::manifest::GenerationManifest;
use crate::out::out;
use crate::plan::{self, GlobalInputs, RepoOutcome};
use super::RepoPlan;
pub(crate) fn write_www(build_dir: &Path, relative: &str, bytes: &[u8]) -> anyhow::Result<()> {
let path = build_dir.join("www").join(relative);
crate::cas::atomic_write(&path, bytes).with_context(|| format!("write {}", path.display()))
}
pub(crate) const COMPLETION_FILE: &str = ".complete.json";
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
pub(crate) struct FileInventory {
pub(crate) path: String,
pub(crate) bytes: u64,
pub(crate) sha256: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub(crate) struct Completion {
pub(crate) version: u32,
pub(crate) generation_id: String,
pub(crate) files: Vec<FileInventory>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Publication {
Published,
Reused,
}
impl Publication {
pub(crate) const fn verb(self) -> &'static str {
match self {
Self::Published => "generated",
Self::Reused => "reused",
}
}
}
/// dropping this joins the worker. declare it after the lock guards so it drops first.
/// it joins rather than cancels, since the worker has no cancellation point.
pub(crate) struct WorkerJoin<T: Send + 'static> {
handle: Option<JoinHandle<anyhow::Result<T>>>,
}
impl<T: Send + 'static> WorkerJoin<T> {
pub(crate) fn spawn(work: impl FnOnce() -> anyhow::Result<T> + Send + 'static) -> Self {
Self {
handle: Some(std::thread::spawn(work)),
}
}
pub(crate) fn join(mut self) -> anyhow::Result<T> {
let handle = self.handle.take().expect("build child already joined");
handle
.join()
.map_err(|_| anyhow::anyhow!("build child panicked"))?
}
}
impl<T: Send + 'static> Drop for WorkerJoin<T> {
fn drop(&mut self) {
if let Some(handle) = self.handle.take() {
match handle.join() {
Ok(Ok(_)) => {}
Ok(Err(error)) => {
out!("build child failed while the build was already failing: {error:#}")
}
Err(_) => out!("build child panicked on the error path"),
}
}
}
}
pub(crate) struct PrivateGeneration {
directory: tempfile::TempDir,
}
impl PrivateGeneration {
pub(crate) fn new(generations_root: &Path) -> anyhow::Result<Self> {
let directory = tempfile::Builder::new()
.prefix(".building-")
.tempdir_in(generations_root)
.context("create private generation directory")?;
Ok(Self { directory })
}
pub(crate) fn path(&self) -> &Path {
self.directory.path()
}
pub(crate) fn publish(
self,
destination: &Path,
expected: &GenerationManifest,
) -> anyhow::Result<Publication> {
if let Some(output_root) = destination.parent().and_then(Path::parent)
&& let Some(active) = crate::publish::current_generation(output_root)
{
let active = output_root.join("generations").join(active);
share_unchanged(self.path(), &active).context("share unchanged files")?;
}
prepare_completion(self.path(), expected)?;
crate::cas::seal_read_only(self.path()).context("seal generation files")?;
if destination.exists() {
let span = crate::profile::Span::start("pub/validate_existing");
let existing = validate_published_generation(destination, expected);
span.end();
match existing {
Ok(()) => return Ok(Publication::Reused),
Err(error) => {
tracing::warn!(
"generation {} exists but is incomplete or corrupt ({error:#}); replacing it",
expected.generation_id
);
std::fs::remove_dir_all(destination).with_context(|| {
format!("remove corrupt generation {}", destination.display())
})?;
}
}
}
std::fs::rename(self.path(), destination).with_context(|| {
format!(
"publish generation {} to {}",
expected.generation_id,
destination.display()
)
})?;
Ok(Publication::Published)
}
}
/// a file byte-identical to the active generation's copy becomes a link to it, so a write's
/// generation only takes room for what it changed. both are read-only once published.
fn share_unchanged(root: &Path, active: &Path) -> anyhow::Result<()> {
let mut dirs = vec![root.to_path_buf()];
while let Some(dir) = dirs.pop() {
for entry in std::fs::read_dir(&dir).with_context(|| format!("list {}", dir.display()))? {
let entry = entry?;
let kind = entry.file_type()?;
if kind.is_dir() {
dirs.push(entry.path());
continue;
}
let path = entry.path();
let relative = path.strip_prefix(root).expect("below the root");
let theirs = active.join(relative);
let meta = entry.metadata()?;
let len = meta.len();
// renaming a link over another link to the same file does nothing, and leaves it
let linked = std::fs::symlink_metadata(&theirs).is_ok_and(|theirs| {
use std::os::unix::fs::MetadataExt as _;
(theirs.dev(), theirs.ino()) == (meta.dev(), meta.ino())
});
let same = !linked
&& kind.is_file()
&& std::fs::symlink_metadata(&theirs)
.is_ok_and(|meta| meta.is_file() && meta.len() == len)
&& std::fs::read(&theirs)? == std::fs::read(&path)?;
if same {
let link = dir.join(format!(
".share-{}",
relative.file_name().unwrap_or_default().display()
));
std::fs::hard_link(&theirs, &link)
.with_context(|| format!("link {}", theirs.display()))?;
std::fs::rename(&link, &path)
.with_context(|| format!("share {}", path.display()))?;
}
}
}
Ok(())
}
pub(crate) fn prepare_completion(root: &Path, expected: &GenerationManifest) -> anyhow::Result<()> {
validate_payload(root, expected)?;
let completion = Completion {
version: 1,
generation_id: expected.generation_id.clone(),
files: payload_inventory(root)?,
};
let bytes = serde_json::to_vec_pretty(&completion)?;
crate::cas::atomic_write(&root.join(COMPLETION_FILE), &bytes)?;
Ok(())
}
pub(crate) fn validate_published_generation(
root: &Path,
expected: &GenerationManifest,
) -> anyhow::Result<()> {
validate_payload(root, expected)?;
let bytes =
std::fs::read(root.join(COMPLETION_FILE)).context("read generation completion marker")?;
let completion: Completion =
serde_json::from_slice(&bytes).context("parse generation completion marker")?;
anyhow::ensure!(
completion.version == 1,
"unsupported completion marker version"
);
anyhow::ensure!(
completion.generation_id == expected.generation_id,
"completion marker generation ID mismatch"
);
for file in &completion.files {
let path = root.join(&file.path);
let metadata = std::fs::symlink_metadata(&path)
.with_context(|| format!("generation file {} is missing", path.display()))?;
anyhow::ensure!(
metadata.is_file() && metadata.len() == file.bytes,
"generation file {} length mismatch",
path.display()
);
}
Ok(())
}
pub(crate) fn validate_payload(root: &Path, expected: &GenerationManifest) -> anyhow::Result<()> {
let manifest = GenerationManifest::load(root)?;
anyhow::ensure!(
manifest.generation_id == expected.generation_id,
"manifest generation ID mismatch"
);
anyhow::ensure!(
manifest.compute_id() == manifest.generation_id,
"manifest generation ID is not its semantic digest"
);
for repo in &manifest.repos {
let export = root.join("www/_git").join(plan::repo_token(&repo.repo_did));
anyhow::ensure!(
export.is_dir(),
"missing git export at {}",
export.display()
);
if let Some(expected_hash) = &repo.git_export_hash {
let actual = std::fs::read_to_string(export.join("export.fingerprint"))
.with_context(|| format!("read export fingerprint for {}", repo.repo_did))?;
anyhow::ensure!(
actual.trim() == expected_hash,
"git export fingerprint mismatch for {}: actual={:?} expected={:?}",
repo.repo_did,
actual.trim(),
expected_hash
);
}
}
anyhow::ensure!(
root.join("nginx/knot-static.conf").is_file(),
"missing nginx generation configuration"
);
anyhow::ensure!(
root.join("nginx/maps").is_dir(),
"missing nginx maps directory"
);
Ok(())
}
pub(crate) fn payload_inventory(root: &Path) -> anyhow::Result<Vec<FileInventory>> {
let span = crate::profile::Span::start("pub/inventory");
fn visit(root: &Path, directory: &Path, files: &mut Vec<FileInventory>) -> anyhow::Result<()> {
for entry in std::fs::read_dir(directory)
.with_context(|| format!("read generation directory {}", directory.display()))?
{
let entry = entry?;
let path = entry.path();
let metadata = std::fs::symlink_metadata(&path)?;
if metadata.is_symlink() {
continue;
} else if metadata.is_dir() {
visit(root, &path, files)?;
} else if metadata.is_file() {
let relative = path
.strip_prefix(root)
.expect("inventory path is below generation root");
if relative == Path::new(COMPLETION_FILE) {
continue;
}
let sha256 = match crate::cas::written_digest(&path) {
Some(hex) => hex,
None => Digest::of_file(&path)?.hex(),
};
files.push(FileInventory {
path: relative
.to_str()
.context("generation path is not UTF-8")?
.to_string(),
bytes: metadata.len(),
sha256,
});
} else {
anyhow::bail!("unsupported generation entry at {}", path.display());
}
}
Ok(())
}
let mut files = Vec::new();
visit(root, root, &mut files)?;
files.sort_by(|left, right| left.path.cmp(&right.path));
span.end();
Ok(files)
}
/// a deleted repo must not pass as unchanged. the hosted set has to match the generation's exactly,
/// and a generation another binary built is built again once.
/// this runs before the route sections are read.
pub(crate) fn unchanged_shortcut(
snapshots: &[crate::snapshot::RepoSnapshot],
outcomes: &[RepoOutcome],
carried_dids: &BTreeSet<String>,
hosted: &BTreeSet<String>,
active: Option<&plan::ActiveGeneration>,
) -> bool {
let Some(active) = active else {
return false;
};
if active.manifest.built_by != crate::identity::built_by() {
return false;
}
if !outcomes
.iter()
.all(|o| matches!(o, RepoOutcome::Unchanged { .. }))
{
return false;
}
let active_dids: BTreeSet<&str> = active
.manifest
.repos
.iter()
.map(|repo| repo.repo_did.as_str())
.collect();
let hosted_matches_active = hosted.len() == active_dids.len()
&& hosted.iter().all(|did| active_dids.contains(did.as_str()));
let merged_covers_active = snapshots.len() + carried_dids.len() == active_dids.len()
&& snapshots
.iter()
.all(|s| active_dids.contains(s.repo_did.as_str()))
&& carried_dids
.iter()
.all(|did| active_dids.contains(did.as_str()));
hosted_matches_active && merged_covers_active
}
pub(crate) fn build_plan(
snapshot: &crate::snapshot::RepoSnapshot,
outcome: RepoOutcome,
global: &GlobalInputs,
active: Option<&plan::ActiveGeneration>,
) -> RepoPlan {
let entry = active.and_then(|active| {
active
.manifest
.repos
.iter()
.find(|repo| repo.repo_did == snapshot.repo_did.as_str())
});
RepoPlan {
snapshot: snapshot.clone(),
identity: plan::repo_identity(snapshot, global),
outcome,
prior_release: entry.and_then(|e| {
let id = e.release_id.as_ref()?;
let hash = e.git_export_hash.as_ref()?;
Some((id.clone(), hash.clone()))
}),
prior_export: entry.and_then(|e| {
Some(git_export::ExportPrior {
fingerprint: e.git_export_hash.clone()?,
refs: e.public_refs.iter().chain(&e.cob_refs).cloned().collect(),
bases: e.fetch_bases.clone(),
})
}),
}
}
/// only ever mount a sealed release export, never the mutable source repo or an evictable cache entry.
pub(crate) fn mount_release_export(release_dir: &Path, destination: &Path) -> anyhow::Result<()> {
let release = crate::release::PublishedRelease::open(release_dir)?;
release.validate()?;
let source = std::fs::canonicalize(release_dir.join("export"))?;
anyhow::ensure!(source.is_dir(), "release export is not a directory");
if let Some(parent) = destination.parent() {
std::fs::create_dir_all(parent)?;
}
std::os::unix::fs::symlink(&source, destination)?;
Ok(())
}
pub(crate) fn compute_dir_size(path: &Path) -> usize {
let Ok(entries) = std::fs::read_dir(path) else {
return 0;
};
let mut total = 0;
for entry in entries.flatten() {
let Ok(file_type) = entry.file_type() else {
continue;
};
if file_type.is_dir() {
total += compute_dir_size(&entry.path());
} else if (file_type.is_file() || file_type.is_symlink())
&& let Ok(meta) = entry.metadata()
{
total += meta.len() as usize;
}
}
total
}
#[cfg(test)]
mod tests {
use std::os::unix::fs::MetadataExt as _;
#[test]
fn a_file_already_linked_to_the_active_ones_is_left_as_it_is() {
let dir = tempfile::tempdir().unwrap();
let (root, active) = (dir.path().join("new"), dir.path().join("active"));
crate::cas::atomic_write(&active.join("www/og/card.png"), b"card").unwrap();
crate::cas::atomic_write(&active.join("www/same"), b"same").unwrap();
crate::cas::atomic_write(&root.join("www/same"), b"same").unwrap();
std::fs::create_dir_all(root.join("www/og")).unwrap();
std::fs::hard_link(active.join("www/og/card.png"), root.join("www/og/card.png")).unwrap();
super::share_unchanged(&root, &active).unwrap();
let og: Vec<_> = std::fs::read_dir(root.join("www/og"))
.unwrap()
.map(|entry| entry.unwrap().file_name())
.collect();
assert_eq!(og, ["card.png"]);
let ino = |path: std::path::PathBuf| std::fs::metadata(path).unwrap().ino();
assert_eq!(ino(root.join("www/same")), ino(active.join("www/same")));
}
}