#[cfg(any(test, feature = "bench"))]
pub(crate) mod bench_fixture;
use self::fetch::MOVED_WANTS;
use self::full::Full;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::Path;
use anyhow::Context as _;
use gix::hash::ObjectId;
use sha2::{Digest as _, Sha256};
use self::partitions::{Added, Partitions};
use crate::config::ValidatedConfig;
use crate::snapshot::RepoSnapshot;
use crate::store::RepoStore;
pub mod atomic;
#[cfg(feature = "bench")]
mod benches;
pub mod cmd;
pub mod commit_graph;
pub mod consolidate;
pub mod depth_one;
pub mod depth_one_map;
pub mod fetch;
pub mod full;
pub mod hasher;
pub mod midx_rev;
pub mod packfile;
pub mod partitions;
pub(crate) mod reach;
pub mod refs;
pub mod replies;
pub mod reply_pack;
pub mod spare;
#[cfg(test)]
mod tests;
pub mod wire;
pub(crate) use atomic::{hardlink_tree, write_unlinked, write_unlinked_making};
pub(crate) use cmd::{git_command, run_cmd, stdin_bytes, true_history};
pub use fetch::{DEPTH_ONE_BRANCHES, SHALLOW_BRANCHES, have_replies};
pub(crate) use refs::{
assert_export_refs, head_tip, init_bare_export, is_reserved_ref, reset_export_dir,
set_export_head, source_path, update_refs,
};
pub(crate) use wire::pkt_line;
pub(crate) const EXPORT_CACHE_KIND: &str = "export";
const CONTROL_FILES: &[&str] = &["HEAD", "export.fingerprint"];
pub(crate) struct ExportTrace {
enabled: bool,
}
impl ExportTrace {
pub(crate) fn new() -> Self {
Self {
enabled: std::env::var_os("KNOT_PLAN_TRACE").is_some(),
}
}
fn time<R>(
&mut self,
label: &'static str,
work: impl FnOnce() -> anyhow::Result<R>,
) -> anyhow::Result<()> {
if !self.enabled {
work().map(|_| ())
} else {
let started = std::time::Instant::now();
work()?;
eprintln!(
"[trace] {label}: {:.1}ms",
started.elapsed().as_secs_f64() * 1000.0
);
Ok(())
}
}
pub(crate) fn note(&mut self, label: impl std::fmt::Display) {
if self.enabled {
eprintln!("[trace] {label}");
}
}
}
/// bump it when the hashed fingerprint fields change, or old cache entries keep matching.
const EXPORT_IDENTITY_VERSION: u32 = 12;
pub const FETCH_BASES: usize = 8;
pub fn export_fingerprint(
snapshot: &RepoSnapshot,
default_branch: &str,
prior: Option<&ExportPrior>,
) -> String {
let mut hasher = Sha256::new();
crate::plan::hash_fields::u32(&mut hasher, EXPORT_IDENTITY_VERSION);
crate::plan::hash_fields::text(&mut hasher, b"snapshot", &snapshot.snapshot_identity());
crate::plan::hash_fields::text(&mut hasher, b"default_branch", default_branch);
for base in fetch_candidates(prior) {
crate::plan::hash_fields::text(&mut hasher, b"fetch_base", &base.join(" "));
}
crate::cas::Digest::from_raw(hasher.finalize().into()).hex()
}
#[derive(Debug, Clone)]
pub struct ExportPrior {
pub fingerprint: String,
/// raw targets, which keep an annotated tag as the tag object.
pub refs: BTreeMap<String, String>,
pub bases: Vec<Vec<String>>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FetchReplies {
pub bases: Vec<Vec<String>>,
pub shallow_branches: Vec<String>,
pub shallow_head: Option<String>,
pub moved_wants: Vec<String>,
}
fn fetch_candidates(prior: Option<&ExportPrior>) -> Vec<Vec<String>> {
let Some(prior) = prior else {
return Vec::new();
};
let heads = prior
.refs
.iter()
.filter(|(name, _)| name.starts_with("refs/heads/"))
.map(|(_, raw)| raw.clone())
.collect();
std::iter::once(heads)
.chain(prior.bases.iter().cloned())
.collect()
}
fn exported_refs(snapshot: &RepoSnapshot) -> impl Iterator<Item = (&str, &str)> {
let public = snapshot
.public_refs
.iter()
.map(|r| (r.name.as_str(), r.raw.as_str()));
let cobs = snapshot
.cob_refs
.iter()
.map(|(name, raw)| (name.as_str(), raw.as_str()));
public.chain(cobs)
}
/// a cob ref checks every exported ref because either partition can reach its old target.
fn refs_fast_forwarded(
prior: &ExportPrior,
snapshot: &RepoSnapshot,
config: &ValidatedConfig,
) -> anyhow::Result<bool> {
let layout = config.layout().context("open layout for ff check")?;
let source = layout
.open(&snapshot.repo_did)
.context("open source repo")?;
let oid = |hex: &str| ObjectId::from_hex(hex.as_bytes()).with_context(|| format!("oid {hex}"));
let public = snapshot
.public_refs
.iter()
.map(|r| Ok((oid(&r.raw)?, oid(r.peeled.as_deref().unwrap_or(&r.raw))?)))
.collect::<anyhow::Result<Vec<_>>>()?;
// cobs go last so the walk starts from them.
let mut exported = public.clone();
for (_, raw) in &snapshot.cob_refs {
exported.push((oid(raw)?, oid(raw)?));
}
let current: HashMap<&str, &str> = exported_refs(snapshot).collect();
let graph = source.git().commit_graph().ok();
for (name, prior_raw) in &prior.refs {
if current.get(name.as_str()) == Some(&prior_raw.as_str()) {
continue;
}
let remaining = if crate::snapshot::is_cob_ref(name) {
&exported
} else {
&public
};
if !reached(source.git(), graph.as_ref(), oid(prior_raw)?, remaining)? {
return Ok(false);
}
}
Ok(true)
}
/// pruning by generation can only answer false, which packs anew.
fn reached(
git: &gix::Repository,
graph: Option<&gix::commitgraph::Graph>,
old: ObjectId,
tips: &[(ObjectId, ObjectId)],
) -> anyhow::Result<bool> {
if tips
.iter()
.any(|&(raw, peeled)| raw == old || peeled == old)
{
return Ok(true);
}
let kind = git.try_find_header(old).context("read the old target")?;
if kind.is_none_or(|header| header.kind() != gix::object::Kind::Commit) {
return Ok(false);
}
let graph = graph.map(|graph| {
let floor = graph
.commit_by_id(old)
.map_or(u32::MAX, |old| old.generation());
(graph, floor)
});
let mut seen = HashSet::new();
let mut walk: Vec<ObjectId> = tips.iter().map(|&(_, peeled)| peeled).collect();
while let Some(id) = walk.pop() {
if id == old {
return Ok(true);
}
if !seen.insert(id) {
continue;
}
let in_graph = graph
.and_then(|(graph, floor)| graph.commit_by_id(id).map(|commit| (graph, floor, commit)));
let parents: Vec<ObjectId> = match in_graph {
Some((_, floor, commit)) if commit.generation() <= floor => continue,
Some((graph, _, commit)) => commit
.iter_parents()
.map(|parent| Ok(graph.commit_at(parent?).id().to_owned()))
.collect::<anyhow::Result<_>>()?,
None => match git.find_object(id).context("read a tip's history")? {
object if object.kind == gix::object::Kind::Commit => object
.into_commit()
.parent_ids()
.map(|parent| parent.detach())
.collect(),
_ => continue,
},
};
walk.extend(parents.into_iter().rev());
}
Ok(false)
}
pub fn materialize_export(
config: &ValidatedConfig,
snapshot: &RepoSnapshot,
export_dir: &Path,
fingerprint: &str,
store: Option<&RepoStore>,
prior: Option<&ExportPrior>,
) -> anyhow::Result<()> {
if let Some(r) = snapshot
.public_refs
.iter()
.find(|r| is_reserved_ref(&r.name))
{
anyhow::bail!("snapshot public_refs contains reserved ref: {}", r.name);
}
let mut exporter = ExportTrace::new();
if export_dir.exists() {
std::fs::remove_dir_all(export_dir)
.with_context(|| format!("clean existing export dir {}", export_dir.display()))?;
}
std::fs::create_dir_all(export_dir)
.with_context(|| format!("create export dir {}", export_dir.display()))?;
exporter.time("export: git init --bare", || {
init_bare_export(export_dir, snapshot.object_format)
})?;
let cached = store.and_then(|store| store.read_directory(EXPORT_CACHE_KIND, fingerprint));
// the prior partitions stay exact because a fast-forward only adds to what refs reach.
let mut seeded_from_cache = false;
let mut added = None;
let prior_entry = prior.and_then(|prior| {
store.and_then(|store| store.read_directory(EXPORT_CACHE_KIND, &prior.fingerprint))
});
if let Some(entry) = &cached {
exporter.time("export: hardlink cached entry", || {
hardlink_tree(entry, export_dir).with_context(|| {
format!(
"install cached export entry {} into {}",
entry.display(),
export_dir.display()
)
})
})?;
seeded_from_cache = true;
} else if let Some(prior) = prior
&& let Some(prior_entry) = &prior_entry
&& let Some(partitions) = prior_partitions(prior_entry)
&& refs_fast_forwarded(prior, snapshot, config)?
&& exporter
.time("export: hardlink prior entry", || {
hardlink_tree(prior_entry, export_dir)
})
.is_ok()
{
exporter.note("export: seeded from prior entry");
exporter.time("export: pack what the moved refs add", || {
match add_reached(config, snapshot, export_dir, partitions) {
Ok(segments) => added = Some(segments),
Err(error) => {
ExportTrace::new().note(format_args!("export: packed anew: {error:#}"));
reset_export_dir(export_dir, snapshot.object_format)?;
}
}
Ok(())
})?;
}
let mut fresh = None;
if !seeded_from_cache && added.is_none() {
exporter.time("export: pack every exported object (cache miss)", || {
fresh = Some(pack_exported(config, snapshot, export_dir, store)?);
Ok(())
})?;
}
exporter.time("export: set HEAD", || {
set_export_head(config, snapshot, export_dir)
})?;
if !seeded_from_cache {
// HEAD has to be set first because the smart advertisement carries its symref.
let prior_smart = prior_entry.as_ref().map(|entry| entry.join("smart"));
let candidates = fetch_candidates(prior);
let _graph = LentGraph::lend(&source_path(config, snapshot)?, export_dir);
let mut smart = |full: Full| {
exporter.time("export: smart artifacts", || {
full::write_smart_artifacts(
snapshot,
export_dir,
prior_smart.as_deref(),
prior.map(|prior| &prior.refs),
full,
&candidates,
)
})
};
match (&fresh, &added, &prior_smart) {
(Some(fresh), _, _) => smart(Full::Frame(fresh))?,
(None, Some(added), Some(smart_dir)) => {
let tips = [
partitions::object_ids(snapshot.public_refs.iter().map(|r| r.raw.as_str()))?,
partitions::object_ids(exported_refs(snapshot).map(|(_, raw)| raw))?,
];
let full = Full::Grow {
prior: smart_dir,
added,
tips: &tips,
};
if let Err(error) = smart(full) {
ExportTrace::new().note(format_args!("export: packed anew: {error:#}"));
reset_export_dir(export_dir, snapshot.object_format)?;
let fresh = pack_exported(config, snapshot, export_dir, store)?;
set_export_head(config, snapshot, export_dir)?;
smart(Full::Frame(&fresh))?;
}
}
_ => unreachable!("an export not from the cache is packed fresh or grown"),
}
}
if added.is_some()
&& let Some(tails) = crate::consolidate::tails(&export_dir.join("smart"))
{
let framed = [0, 1].map(|at| tails[0].packs()[at].max(tails[1].packs()[at]));
exporter.time("export: merge the packs past the replies", || {
Partitions::merge_tails(export_dir, framed, snapshot.object_format)
})?;
}
let fp_path = export_dir.join("export.fingerprint");
write_unlinked(&fp_path, format!("{fingerprint}\n").as_bytes())
.with_context(|| format!("write export.fingerprint at {}", fp_path.display()))?;
exporter.time("export: assert refs", || {
assert_export_refs(snapshot, export_dir)
})?;
let alternates = export_dir.join("objects/info/alternates");
if alternates.exists() {
anyhow::bail!(
"objects/info/alternates exists in export at {}",
alternates.display()
);
}
if !seeded_from_cache && let Some(store) = store {
exporter.time("export: publish cache entry", || {
store
.publish_directory(EXPORT_CACHE_KIND, fingerprint, export_dir, CONTROL_FILES)
.context("publish export cache entry")
})?;
}
Ok(())
}
/// git cuts replies faster with the graph, without it every `--not` walk goes through all history.
/// it's dropped before publish because the graph names the source's hidden commits too.
struct LentGraph(Vec<std::path::PathBuf>);
impl LentGraph {
fn lend(source: &Path, export_dir: &Path) -> Self {
let mut lent = Self(Vec::new());
let Ok(repo) = knot_git::Repo::open(source) else {
return lent;
};
let from = repo.objects_dir().join("info");
let to = export_dir.join("objects/info");
let linked = (|| -> std::io::Result<()> {
if from.join("commit-graph").is_file() {
std::fs::hard_link(from.join("commit-graph"), to.join("commit-graph"))?;
lent.0.push(to.join("commit-graph"));
}
let Ok(layers) = std::fs::read_dir(from.join("commit-graphs")) else {
return Ok(());
};
std::fs::create_dir_all(to.join("commit-graphs"))?;
lent.0.push(to.join("commit-graphs"));
for layer in layers {
let layer = layer?;
std::fs::hard_link(
layer.path(),
to.join("commit-graphs").join(layer.file_name()),
)?;
}
Ok(())
})();
if let Err(error) = linked {
ExportTrace::new().note(format_args!("export: no commit-graph lent: {error:#}"));
}
lent
}
}
impl Drop for LentGraph {
fn drop(&mut self) {
for path in &self.0 {
let _ = std::fs::remove_file(path).or_else(|_| std::fs::remove_dir_all(path));
}
}
}
pub fn fetch_replies(export_dir: &Path) -> anyhow::Result<FetchReplies> {
let smart = export_dir.join("smart");
let bases = match std::fs::read_to_string(smart.join("fetch-bases")) {
Ok(text) => text
.lines()
.map(|line| line.split(' ').map(str::to_string).collect())
.collect(),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
Err(error) => return Err(error).context("read smart/fetch-bases"),
};
let shallow_branches = match std::fs::read_to_string(smart.join(SHALLOW_BRANCHES)) {
Ok(text) => text.lines().map(str::to_string).collect(),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
Err(error) => return Err(error).context("read smart/shallow-branches"),
};
let moved_wants = match std::fs::read_to_string(smart.join(MOVED_WANTS)) {
Ok(text) => text.split_whitespace().map(str::to_string).collect(),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
Err(error) => return Err(error).context("read smart/moved-wants"),
};
Ok(FetchReplies {
bases,
shallow_branches,
moved_wants,
shallow_head: smart
.join("shallow-head")
.is_file()
.then(|| head_tip(export_dir))
.flatten(),
})
}
fn prior_partitions(prior_entry: &Path) -> Option<Partitions> {
Partitions::read(prior_entry).unwrap_or_else(|error| {
ExportTrace::new().note(format_args!("export: prior partitions: {error:#}"));
None
})
}
fn add_reached(
config: &ValidatedConfig,
snapshot: &RepoSnapshot,
export_dir: &Path,
partitions: Partitions,
) -> anyhow::Result<Added> {
let source = source_path(config, snapshot)?;
let public = partitions::object_ids(snapshot.public_refs.iter().map(|r| r.raw.as_str()))?;
let exported = partitions::object_ids(exported_refs(snapshot).map(|(_, raw)| raw))?;
let added = partitions.grow(
&source,
export_dir,
&public,
&exported,
snapshot.object_format,
)?;
update_refs(export_dir, snapshot)?;
Ok(added)
}
fn pack_exported(
config: &ValidatedConfig,
snapshot: &RepoSnapshot,
export_dir: &Path,
store: Option<&RepoStore>,
) -> anyhow::Result<partitions::Fresh> {
let source = source_path(config, snapshot)?;
let public = partitions::object_ids(snapshot.public_refs.iter().map(|r| r.raw.as_str()))?;
let cob = partitions::object_ids(snapshot.cob_refs.iter().map(|(_, raw)| raw.as_str()))?;
let format = snapshot.object_format;
let fresh = Partitions::pack(&source, export_dir, &public, &cob, format, store)?;
update_refs(export_dir, snapshot)?;
Ok(fresh)
}
#[cfg(test)]
pub(crate) use {
cmd::{REPO_LOCATING_ENV, without_repo_env},
fetch::kept_pack,
full::FULL_REPLIES,
full::{CloneFile, Growth, SAME_PACK, full_prefixes, grow_reply},
refs::export_head_state,
replies::REPLY_KEYS,
wire::ensure_advertised,
wire::{v0_advertisement, v2_advertisement, v2_command, v2_ls_refs},
};