use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use crate::datastore::Durability;
use crate::generate::Tracer;
use anyhow::Context as _;
use super::{Action, git, run_steps};
pub(crate) enum Before {
Previous(Previous),
Fresh(Fresh),
}
pub(crate) struct Previous {
pub(crate) tips: [String; 2],
pub(crate) last: Option<(crate::bitmap::BitmapIndex, [Reached; 2])>,
}
#[derive(Debug)]
pub(crate) enum Fresh {
Moved,
Mismatch,
Failed(String),
}
pub(crate) enum Bitmap {
Built(Duration),
Cold(Duration),
Fresh(Fresh, Duration),
}
impl std::fmt::Display for Fresh {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Moved => f.write_str("tips moved"),
Self::Mismatch => f.write_str("guard mismatch"),
Self::Failed(error) => write!(f, "knot2 failed: {error}"),
}
}
}
impl std::fmt::Display for Bitmap {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Built(took) => {
write!(
f,
"knot2 incremental, guard ok, {:.1} s",
took.as_secs_f64()
)
}
Self::Cold(took) => {
write!(f, "knot2 cold, guard ok, {:.1} s", took.as_secs_f64())
}
Self::Fresh(why, took) => write!(f, "git ({why}), {:.1} s", took.as_secs_f64()),
}
}
}
impl Before {
pub(crate) fn read(repo_path: &Path) -> Self {
match Previous::read(repo_path) {
Ok(previous) => Self::Previous(previous),
Err(error) => Self::Fresh(Fresh::Failed(format!("{error:#}"))),
}
}
}
impl Previous {
fn read(repo_path: &Path) -> anyhow::Result<Self> {
let repo = knot_git::Repo::open(repo_path).context("open repo for its bitmap")?;
let tips = crate::snapshot::mark_tips(&repo)?;
let last = || -> anyhow::Result<_> {
crate::midx_rev::complete(&repo.objects_dir().join("pack"), repo.object_format())?;
let Some(index) = crate::bitmap::open_bitmap_index(&repo).context("open the bitmap")?
else {
return Ok(None);
};
let public = reached(repo_path, &tips[0], true)?;
let exported = if tips[1] == tips[0] {
public.clone()
} else {
reached(repo_path, &tips[1], true)?
};
let reached = [public, exported];
Ok(Some((index, reached)))
};
// any read error means no last bitmap, and the write goes cold.
let last = last().unwrap_or_else(|error| {
eprintln!("maintenance: {error:#}; writing the bitmap cold");
None
});
Ok(Self { tips, last })
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) struct Reached {
count: u64,
sum: u128,
xor: [u8; 32],
}
impl Reached {
fn add(&mut self, id: &[u8]) {
self.count += 1;
let high = u128::from_be_bytes(id[..16].try_into().unwrap());
self.sum = self.sum.wrapping_add(high);
self.xor
.iter_mut()
.zip(id)
.for_each(|(xor, byte)| *xor ^= byte);
}
}
pub(crate) fn marked(
repo_path: &Path,
tips: &str,
kind: gix::hash::Kind,
) -> anyhow::Result<Reached> {
let tips: Vec<gix::ObjectId> = (tips.lines())
.map(|tip| gix::ObjectId::from_hex(tip.as_bytes()).context("a tip"))
.collect::<anyhow::Result<_>>()?;
let mut reached = Reached::default();
match kind {
gix::hash::Kind::Sha1 => (crate::partitions::reach::<20>(repo_path, &tips, 0)?)
.iter()
.for_each(|id| reached.add(id)),
gix::hash::Kind::Sha256 => (crate::partitions::reach::<32>(repo_path, &tips, 0)?)
.iter()
.for_each(|id| reached.add(id)),
other => anyhow::bail!("no mark for {other:?} ids"),
}
Ok(reached)
}
pub(crate) fn reached(repo_path: &Path, tips: &str, bitmap: bool) -> anyhow::Result<Reached> {
use std::io::{BufRead as _, Read as _, Seek as _};
let errors = tempfile::tempfile().context("create stderr file")?;
let mut walk = crate::git_export::git_command(repo_path);
let args = crate::partitions::MARK_WALK.iter();
walk.args(args.filter(|arg| bitmap || **arg != "--use-bitmap-index"))
.stdin(crate::git_export::stdin_bytes(tips.as_bytes())?)
.stdout(std::process::Stdio::piped())
.stderr(errors.try_clone()?);
let mut child = walk.spawn().context("spawn git rev-list")?;
let mut stdout = std::io::BufReader::with_capacity(
1 << 20,
child.stdout.take().context("git rev-list's stdout")?,
);
let (mut reached, mut line) = (Reached::default(), Vec::new());
while stdout
.read_until(b'\n', &mut line)
.context("read git rev-list")?
> 0
{
let hex = line.strip_suffix(b"\n").unwrap_or(&line);
let oid = gix::ObjectId::from_hex(hex).context("an id git rev-list listed")?;
reached.add(oid.as_bytes());
line.clear();
}
let status = child.wait().context("wait for git rev-list")?;
if !status.success() {
let (mut errors, mut text) = (errors, String::new());
errors.rewind()?;
errors.read_to_string(&mut text)?;
anyhow::bail!("git rev-list --objects failed ({status}): {}", text.trim());
}
Ok(reached)
}
pub(crate) fn distinct_tip_lists(tips: &[String; 2]) -> impl Iterator<Item = (usize, &String)> {
tips.iter()
.enumerate()
.filter(|(n, list)| !tips[..*n].contains(list))
}
pub(crate) fn run_unlocked(
repo_path: &Path,
before: Option<Before>,
) -> anyhow::Result<Option<Bitmap>> {
(before.map(|before| write_bitmap(repo_path, before, write_knot_bitmap))).transpose()
}
pub(crate) fn write_knot_bitmap(
repo: &knot_git::Repo,
previous: Option<&crate::bitmap::BitmapIndex>,
) -> anyhow::Result<bool> {
Ok(crate::bitmap::write_midx_bitmap(repo, previous)?)
}
pub(crate) fn write_bitmap(
repo_path: &Path,
before: Before,
write: impl FnOnce(&knot_git::Repo, Option<&crate::bitmap::BitmapIndex>) -> anyhow::Result<bool>,
) -> anyhow::Result<Bitmap> {
let started = Instant::now();
let why = match before {
Before::Previous(previous) => match build_on(repo_path, &previous, write) {
Ok(()) if previous.last.is_some() => return Ok(Bitmap::Built(started.elapsed())),
Ok(()) => return Ok(Bitmap::Cold(started.elapsed())),
Err(why) => why,
},
Before::Fresh(why) => why,
};
Tracer::new().time("bitmap: git writer", || write_fresh(repo_path))?;
Ok(Bitmap::Fresh(why, started.elapsed()))
}
/// built in a scratch dir and moved in only after the guard passes.
/// no export marks through it before that.
pub(crate) fn build_on(
repo_path: &Path,
previous: &Previous,
write: impl FnOnce(&knot_git::Repo, Option<&crate::bitmap::BitmapIndex>) -> anyhow::Result<bool>,
) -> Result<(), Fresh> {
let failed = |error: anyhow::Error| Fresh::Failed(format!("{error:#}"));
let trace = Tracer::new();
let live = knot_git::Repo::open(repo_path)
.context("open repo for its packs")
.map_err(failed)?;
let packs = live.objects_dir().join("pack");
let scratch = scratch(&live.objects_dir()).map_err(failed)?;
// the index comes from git, since it reads pack order over knot_git's `.rev`.
trace
.time("bitmap: scratch index", || git(scratch.path(), MIDX, b""))
.map_err(failed)?;
let repo = knot_git::Repo::open(scratch.path())
.context("open the scratch repo")
.map_err(failed)?;
let last = previous.last.as_ref().map(|(index, _)| index);
std::thread::scope(|scope| {
// the walk runs beside the write, and the scope joins it even on an early return.
let trace = &trace;
let mut walks = distinct_tip_lists(&previous.tips)
.filter(|_| last.is_none())
.map(|(_, tips)| {
scope.spawn(move || trace.time("bitmap: walk", || reached(repo_path, tips, false)))
})
.collect::<Vec<_>>()
.into_iter();
if !trace
.time("bitmap: knot2 write", || write(&repo, last))
.map_err(failed)?
{
return Err(Fresh::Failed("knot_git wrote no bitmap".to_string()));
}
// the write may index commits pushed after the tips were read. recheck them.
if crate::snapshot::mark_tips(&live).map_err(failed)? != previous.tips {
return Err(Fresh::Moved);
}
let kind = repo.object_format().kind();
for (n, tips) in distinct_tip_lists(&previous.tips) {
let before = match &previous.last {
Some((_, reached)) => reached[n].clone(),
None => {
let walk = walks.next().expect("a walk per tip list");
walk.join().expect("git's walk panicked").map_err(failed)?
}
};
let after = trace.time("bitmap: mark", || marked(scratch.path(), tips, kind));
if after.map_err(failed)? != before {
return Err(Fresh::Mismatch);
}
}
trace
.time("bitmap: publish", || {
publish(&scratch.path().join("objects/pack"), &packs)
})
.map_err(failed)
})
}
pub(crate) const SCRATCH: &str = ".ks-bitmap-";
pub(crate) fn scratch(objects: &Path) -> anyhow::Result<tempfile::TempDir> {
let objects = objects
.canonicalize()
.context("resolve the repo's objects")?;
let git_dir = objects.parent().context("a repo's objects in no git dir")?;
let listed =
std::fs::read_dir(git_dir).with_context(|| format!("list {}", git_dir.display()))?;
for entry in listed {
let path = entry?.path();
if path
.file_name()
.unwrap_or_default()
.to_string_lossy()
.starts_with(SCRATCH)
{
std::fs::remove_dir_all(&path).with_context(|| format!("remove {}", path.display()))?;
}
}
let dir = tempfile::Builder::new()
.prefix(SCRATCH)
.tempdir_in(git_dir)
.context("create a scratch git dir")?;
let (pack, info) = (
dir.path().join("objects/pack"),
dir.path().join("objects/info"),
);
std::fs::create_dir_all(&pack).context("create its pack dir")?;
std::fs::create_dir_all(&info).context("create its info dir")?;
let alternates = format!("{}\n", objects.display());
crate::datastore::write::publish(
&info.join("alternates"),
alternates.as_bytes(),
Durability::Rename,
)
.context("point it at the repo's objects")?;
for name in ["commit-graph", "commit-graphs"] {
let graph = objects.join("info").join(name);
if graph.exists() {
std::os::unix::fs::symlink(&graph, info.join(name))
.with_context(|| format!("link {name}"))?;
}
}
// HEAD is copied, not linked, since git won't take one that links outside refs/.
let head = std::fs::read(git_dir.join("HEAD")).context("read HEAD")?;
crate::datastore::write::publish(&dir.path().join("HEAD"), &head, Durability::Rename)
.context("copy HEAD")?;
// these link to the live repo, and scratch never writes through them.
for name in ["config", "refs", "packed-refs"] {
if git_dir.join(name).exists() {
std::os::unix::fs::symlink(git_dir.join(name), dir.path().join(name))
.with_context(|| format!("link {name}"))?;
}
}
for entry in std::fs::read_dir(objects.join("pack")).context("list the repo's packs")? {
let path = entry?.path();
let name = path
.file_name()
.unwrap_or_default()
.to_string_lossy()
.into_owned();
let packed = [".pack", ".idx", ".rev"]
.iter()
.any(|ext| name.ends_with(ext));
if name.starts_with("pack-") && packed {
std::fs::hard_link(&path, pack.join(&name)).with_context(|| format!("link {name}"))?;
}
}
// git walks up from the cwd to the live repo if it won't take this dir.
let found = git(dir.path(), &["rev-parse", "--absolute-git-dir"], b"")?;
let found = PathBuf::from(String::from_utf8_lossy(&found).trim());
anyhow::ensure!(
found.canonicalize()? == dir.path().canonicalize()?,
"git takes {} for the scratch git dir",
found.display()
);
Ok(dir)
}
/// the index moves last, since readers find the bitmap by the index's checksum.
/// a crash between leaves files no index names, which the next publish drops.
pub(crate) fn publish(scratch: &Path, live: &Path) -> anyhow::Result<()> {
let mut kept = vec![MIDX_FILE.to_string()];
for entry in
std::fs::read_dir(scratch).with_context(|| format!("list {}", scratch.display()))?
{
let name = entry?.file_name().to_string_lossy().into_owned();
if name.starts_with("multi-pack-index-") {
std::fs::rename(scratch.join(&name), live.join(&name))
.with_context(|| format!("move {name} in"))?;
kept.push(name);
}
}
std::fs::rename(scratch.join(MIDX_FILE), live.join(MIDX_FILE))
.context("move the multi-pack index in")?;
drop_bitmaps_but(live, &kept)
}
pub(super) const MIDX_FILE: &str = "multi-pack-index";
const MIDX: &[&str] = &[
"-c",
"core.packedGitLimit=256m",
"-c",
"core.packedGitWindowSize=32m",
"multi-pack-index",
"write",
];
/// drop them first, since git reuses any bitmap it finds for its packs.
/// git also won't rewrite a midx that names a pack that's gone.
pub(crate) fn write_fresh(repo_path: &Path) -> anyhow::Result<()> {
drop_bitmaps(repo_path)?;
let steps = Action::Repack.unlocked_args();
let Err(error) = run_steps(repo_path, steps) else {
return Ok(());
};
// the empty tree can be in no pack, and a bitmap needs every object its commits reach.
// storing it and packing what's loose only adds a pack, as a push does.
eprintln!("maintenance: {error:#}; packing the empty tree");
pack_empty_tree(repo_path)?;
run_steps(repo_path, steps)
}
pub(crate) fn pack_empty_tree(repo_path: &Path) -> anyhow::Result<()> {
let empty = git(repo_path, &["hash-object", "-t", "tree", "/dev/null"], b"")?;
let pack = knot_git::Repo::open(repo_path)
.context("open repo for its packs")?
.objects_dir()
.join("pack/pack");
let pack = pack.to_str().context("a repo path that isn't utf-8")?;
git(repo_path, &["pack-objects", "-q", pack], &empty)?;
Ok(())
}
/// drop them all, since knot_git reads a bitmap left by `git gc`.
pub(crate) fn drop_bitmaps(repo_path: &Path) -> anyhow::Result<()> {
let packs = knot_git::Repo::open(repo_path)
.context("open repo for its bitmaps")?
.objects_dir()
.join("pack");
drop_bitmaps_but(&packs, &[])
}
pub(crate) fn drop_bitmaps_but(packs: &Path, kept: &[String]) -> anyhow::Result<()> {
for entry in std::fs::read_dir(packs).with_context(|| format!("list {}", packs.display()))? {
let path = entry?.path();
let name = path.file_name().unwrap_or_default().to_string_lossy();
let bitmap = name.starts_with("pack-") && name.ends_with(".bitmap");
let index = name.starts_with(MIDX_FILE) && !name.ends_with(".lock");
if (bitmap || index) && !kept.iter().any(|kept| *kept == name) {
std::fs::remove_file(&path).with_context(|| format!("remove {}", path.display()))?;
}
}
Ok(())
}