use super::bitmap::MIDX_FILE;
use super::bitmap::*;
use super::*;
use crate::midx_rev::tests::{git as git_in, push};
use std::time::Duration;
#[test]
fn equal_tip_lists_are_walked_once() {
let tips = |a: &str, b: &str| [a.to_string(), b.to_string()];
let places = |tips: &[String; 2]| distinct_tip_lists(tips).map(|(n, _)| n).collect::<Vec<_>>();
assert_eq!(places(&tips("a\nb\n", "a\nb\n")), [0]);
assert_eq!(places(&tips("a\n", "a\nb\n")), [0, 1]);
}
fn wait_released(root: &Path, key: &str, mode: LockMode) {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
if crate::repolock::try_acquire(root, key, mode)
.unwrap()
.is_some()
{
break;
}
assert!(std::time::Instant::now() < deadline, "lock not released");
std::thread::sleep(std::time::Duration::from_millis(5));
}
}
#[test]
fn parses_known_actions_and_rejects_others() {
assert_eq!(Action::parse("repack").unwrap(), Action::Repack);
assert_eq!(Action::parse("gc").unwrap(), Action::Gc);
assert_eq!(Action::parse("prune").unwrap(), Action::Prune);
assert!(Action::parse("rm -rf").is_err());
}
#[test]
fn prune_expires_reflogs_before_pruning() {
let args = Action::Prune.git_args();
assert_eq!(args[0][0], "reflog");
assert_eq!(args[1][0], "prune");
assert!(args[0].contains(&"--all"));
}
#[test]
fn only_a_repack_writes_indexes_after_the_lock() {
assert_eq!(Action::Gc.unlocked_args().len(), 0);
assert_eq!(Action::Prune.unlocked_args().len(), 0);
let locked = Action::Repack.git_args().concat();
assert!(locked.contains(&"-d"));
assert!(
!locked
.iter()
.any(|arg| arg.contains("bitmap") || arg.contains("midx"))
);
}
#[test]
fn work_runs_under_the_exclusive_lock_and_lock_is_released_after() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().to_path_buf();
let did = "did:plc:maintenance-test";
let key = crate::plan::repo_key(did);
let mut observed = None;
let path = maintain_locked(&root, did, dir.path(), Action::Repack, |_, _| {
observed = crate::repolock::try_acquire(&root, &key, LockMode::Lifetime)
.unwrap()
.is_none()
.then_some(());
Ok(())
})
.unwrap();
assert_eq!(observed, Some(()), "work must run under the exclusive lock");
assert_eq!(path, crate::repolock::lock_path(&root, &key));
wait_released(&root, &key, LockMode::Lifetime);
}
#[test]
fn the_scratch_repo_reads_the_live_commit_graph() {
for split in [false, true] {
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo");
std::fs::create_dir(&repo).unwrap();
git_in(&repo, &["init", "--bare", "-q", "-b", "main"], "");
push(&repo, "refs/heads/main", "a");
let mut args = vec!["commit-graph", "write", "--reachable"];
if split {
args.push("--split");
}
git_in(&repo, &args, "");
let tip =
gix::ObjectId::from_hex(git_in(&repo, &["rev-parse", "main"], "").as_bytes()).unwrap();
let scratch = scratch(&repo.join("objects")).unwrap();
let opened = knot_git::Repo::open(scratch.path()).unwrap();
let graph = opened
.git()
.commit_graph()
.expect("the scratch writer's commit-graph");
assert!(graph.lookup(tip).is_some(), "split={split}");
}
}
#[test]
fn a_prune_leaves_a_commit_graph_git_accepts() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().to_path_buf();
let repo = root.join("repo");
let git = |cwd: &Path, args: &[&str]| {
let output = crate::test_git()
.args(args)
.current_dir(cwd)
.env("GIT_AUTHOR_NAME", "t")
.env("GIT_AUTHOR_EMAIL", "t@t")
.env("GIT_COMMITTER_NAME", "t")
.env("GIT_COMMITTER_EMAIL", "t@t")
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_CONFIG_SYSTEM", "/dev/null")
.output()
.expect("run git");
assert!(
output.status.success(),
"git {args:?}: {}",
String::from_utf8_lossy(&output.stderr)
);
String::from_utf8(output.stdout).unwrap().trim().to_string()
};
git(&root, &["init", "-q", "-b", "main", "repo"]);
std::fs::write(repo.join("a.txt"), "a").unwrap();
git(&repo, &["add", "-A"]);
git(&repo, &["commit", "-q", "-m", "kept"]);
git(&repo, &["branch", "keep"]);
std::fs::write(repo.join("b.txt"), "b").unwrap();
git(&repo, &["add", "-A"]);
git(&repo, &["commit", "-q", "-m", "pruned"]);
crate::commit_graph::refresh(&repo);
git(&repo, &["symbolic-ref", "HEAD", "refs/heads/keep"]);
git(&repo, &["update-ref", "-d", "refs/heads/main"]);
maintain_locked(
&root,
"did:plc:maintenance-prune",
&repo,
Action::Prune,
run_git,
)
.unwrap();
git(&repo, &["commit-graph", "verify"]);
}
#[test]
fn a_repack_keeps_the_big_pack_and_rolls_up_the_rest() {
use std::os::unix::fs::MetadataExt as _;
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo");
let git = |args: &[&str]| {
let output = crate::test_git()
.args(args)
.current_dir(dir.path().join("repo"))
.env("GIT_AUTHOR_NAME", "t")
.env("GIT_AUTHOR_EMAIL", "t@t")
.env("GIT_COMMITTER_NAME", "t")
.env("GIT_COMMITTER_EMAIL", "t@t")
.output()
.expect("run git");
assert!(output.status.success(), "git {args:?}");
String::from_utf8(output.stdout).unwrap()
};
std::fs::create_dir(&repo).unwrap();
git(&["init", "-q", "-b", "main"]);
let commit = |name: &str, len: usize| {
let text: String = (0..len).map(|n| format!("{name} {n}\n")).collect();
std::fs::write(repo.join(name), text).unwrap();
git(&["add", "-A"]);
git(&["commit", "-q", "-m", name]);
};
// the first pack needs many objects because geometric repacks weigh packs by object count
for n in 0..100 {
std::fs::write(repo.join(format!("{n}.txt")), format!("{n}\n")).unwrap();
}
commit("big.txt", 100);
git(&["repack", "-adq"]);
let packs = || -> Vec<(String, u64)> {
let mut packs: Vec<_> = std::fs::read_dir(repo.join(".git/objects/pack"))
.unwrap()
.map(|entry| entry.unwrap())
.filter(|entry| entry.path().extension().is_some_and(|ext| ext == "pack"))
.map(|entry| {
(
entry.file_name().into_string().unwrap(),
entry.metadata().unwrap().ino(),
)
})
.collect();
packs.sort();
packs
};
let big = packs();
for name in ["a.txt", "b.txt", "c.txt"] {
commit(name, 100);
git(&["repack", "-dq"]);
}
commit("d.txt", 10);
let root = dir.path().join("out");
let did = "did:plc:maintenance-bitmap";
let before = packs();
let held = hold_maintenance_lock(&root, did);
assert!(!maintain(&root, did, &repo, Action::Repack, run_unlocked).unwrap());
assert_eq!(packs(), before);
drop(held);
run_git(&repo, Action::Repack).unwrap();
let after = packs();
assert_eq!(after.len(), 2, "{after:?}");
assert!(after.contains(&big[0]), "the big pack kept: {after:?}");
assert!(git(&["count-objects", "-v"]).starts_with("count: 0\n"));
assert_eq!(bitmaps(&repo), 0, "the bitmap isn't written under the lock");
assert!(maintain(&root, did, &repo, Action::Repack, run_unlocked).unwrap());
assert_eq!(
bitmaps(&repo),
1,
"an export's walk reads the multi-pack bitmap"
);
git(&["fsck", "--strict"]);
}
fn bitmaps(repo: &Path) -> usize {
std::fs::read_dir(repo.join(".git/objects/pack"))
.unwrap()
.filter(|entry| {
let name = entry.as_ref().unwrap().file_name();
let name = name.to_string_lossy();
name.starts_with("multi-pack-index-") && name.ends_with(".bitmap")
})
.count()
}
fn hold_maintenance_lock(root: &Path, did: &str) -> std::fs::File {
let dir = crate::repolock::repo_dir(root, &crate::plan::repo_key(did));
std::fs::create_dir_all(&dir).unwrap();
let file = std::fs::File::create(dir.join(MAINTENANCE_LOCK)).unwrap();
file.lock().unwrap();
file
}
#[test]
fn a_multi_pack_index_naming_a_gone_pack_is_written_afresh() {
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo");
let git = |args: &[&str]| {
let output = crate::test_git()
.args(args)
.current_dir(&repo)
.env("GIT_AUTHOR_NAME", "t")
.env("GIT_AUTHOR_EMAIL", "t@t")
.env("GIT_COMMITTER_NAME", "t")
.env("GIT_COMMITTER_EMAIL", "t@t")
.output()
.expect("run git");
assert!(output.status.success(), "git {args:?}");
};
std::fs::create_dir(&repo).unwrap();
git(&["init", "-q", "-b", "main"]);
for name in ["a.txt", "b.txt"] {
std::fs::write(repo.join(name), name).unwrap();
git(&["add", "-A"]);
git(&["commit", "-q", "-m", name]);
git(&["repack", "-dq"]);
}
git(&["multi-pack-index", "write", "--bitmap"]);
let packs = repo.join(".git/objects/pack");
let named: Vec<PathBuf> = std::fs::read_dir(&packs)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension().is_some_and(|ext| ext == "pack"))
.collect();
git(&["repack", "-aq"]);
std::fs::remove_file(&named[0]).unwrap();
std::fs::remove_file(named[0].with_extension("idx")).unwrap();
write_fresh(&repo).unwrap();
git(&["multi-pack-index", "verify"]);
assert_eq!(bitmaps(&repo), 1);
assert!(!packs.join("multi-pack-index.lock").exists());
}
#[test]
fn a_bitmap_covers_a_commit_whose_empty_tree_was_never_stored() {
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo");
let git = |args: &[&str]| {
let output = crate::test_git()
.args(args)
.current_dir(&repo)
.env("GIT_AUTHOR_NAME", "t")
.env("GIT_AUTHOR_EMAIL", "t@t")
.env("GIT_COMMITTER_NAME", "t")
.env("GIT_COMMITTER_EMAIL", "t@t")
.output()
.expect("run git");
assert!(output.status.success(), "git {args:?}");
String::from_utf8(output.stdout).unwrap().trim().to_string()
};
std::fs::create_dir(&repo).unwrap();
git(&["init", "-q"]);
let empty = git(&["hash-object", "-t", "tree", "/dev/null"]);
let commit = git(&["commit-tree", &empty, "-m", "empty"]);
git(&["update-ref", "refs/heads/main", &commit]);
run_git(&repo, Action::Repack).unwrap();
assert!(
run_steps(&repo, Action::Repack.unlocked_args()).is_err(),
"no pack holds the empty tree"
);
write_fresh(&repo).unwrap();
assert_eq!(bitmaps(&repo), 1);
git(&["multi-pack-index", "verify"]);
let listed = git(&["rev-list", "--objects", "--use-bitmap-index", "main"]);
assert_eq!(listed.lines().count(), 2, "{listed}");
}
/// no other maintenance may run alongside the bitmap write, because it reads packs.
#[test]
fn what_runs_unlocked_runs_outside_the_repo_lock_but_inside_the_maintenance_one() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("out");
let repo = dir.path().join("repo");
let init = crate::test_git()
.args(["init", "-q", "--bare"])
.arg(&repo)
.output()
.unwrap();
assert!(init.status.success());
let did = "did:plc:maintenance-unlocked";
let key = crate::plan::repo_key(did);
let mut ran = false;
maintain(&root, did, &repo, Action::Repack, |_, _| {
// a git forked by another test holds the lock fd until it execs. wait for the release
wait_released(&root, &key, LockMode::Exclusive);
assert!(!maintain(&root, did, &repo, Action::Gc, |_, _| Ok(None)).unwrap());
ran = true;
Ok(None)
})
.unwrap();
assert!(ran);
}
#[test]
fn lock_is_released_when_work_fails() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().to_path_buf();
let did = "did:plc:maintenance-fail";
let key = crate::plan::repo_key(did);
let error = maintain_locked(&root, did, dir.path(), Action::Gc, |_, _| {
anyhow::bail!("simulated git failure")
})
.expect_err("work failure must propagate");
assert!(format!("{error:#}").contains("simulated git failure"));
wait_released(&root, &key, LockMode::Exclusive);
}
fn bare(repo: &Path) {
std::fs::create_dir_all(repo).unwrap();
git_in(repo, &["init", "-q", "--bare", "-b", "main"], "");
}
fn reach(repo: &Path, bitmap: bool) -> Vec<std::collections::BTreeSet<String>> {
let opened = knot_git::Repo::open(repo).unwrap();
crate::snapshot::mark_tips(&opened)
.unwrap()
.iter()
.map(|tips| {
let mut args = vec!["rev-list", "--objects", "--no-object-names", "--stdin"];
if bitmap {
args.push("--use-bitmap-index");
}
git_in(repo, &args, tips)
.lines()
.map(str::to_string)
.collect()
})
.collect()
}
#[test]
fn a_repack_builds_its_bitmap_on_the_last_one() {
let dir = tempfile::tempdir().unwrap();
let (root, repo) = (dir.path().join("out"), dir.path().join("repo.git"));
bare(&repo);
for n in 0..30 {
push(&repo, "refs/heads/main", &format!("{n}"));
}
push(&repo, "refs/cobs/ticket", "cob");
push(&repo, "refs/hidden/secret", "secret");
git_in(&repo, &["repack", "-adq", "--no-write-bitmap-index"], "");
push(&repo, "refs/heads/main", "small");
let did = "did:plc:maintenance-incremental";
let mut nights = Vec::new();
for name in ["pushed", "after"] {
let ran = maintain(&root, did, &repo, Action::Repack, |path, before| {
let bitmap = run_unlocked(path, before)?;
nights.push(bitmap.as_ref().unwrap().to_string());
Ok(bitmap)
});
assert!(ran.unwrap());
assert_eq!(reach(&repo, true), reach(&repo, false), "{nights:?}");
push(&repo, "refs/heads/main", name);
}
assert!(nights[0].starts_with("knot2 cold, guard ok"), "{nights:?}");
assert!(
nights[1].starts_with("knot2 incremental, guard ok"),
"{nights:?}"
);
}
#[test]
fn a_bitmap_that_lies_is_replaced_by_gits() {
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo.git");
bare(&repo);
for name in ["a", "b"] {
push(&repo, "refs/heads/main", name);
}
push(&repo, "refs/hidden/secret", "secret");
git_in(&repo, &["multi-pack-index", "write", "--bitmap"], "");
push(&repo, "refs/heads/main", "c");
let before = Before::read(&repo);
assert!(matches!(before, Before::Previous(_)));
let bitmap = write_bitmap(&repo, before, |opened, previous| {
let wrote = write_knot_bitmap(opened, previous)?;
let pack_dir = opened.objects_dir().join("pack");
let written = std::fs::read_dir(&pack_dir)?
.map(|entry| entry.unwrap().path())
.find(|path| path.extension().is_some_and(|ext| ext == "bitmap"))
.unwrap();
crate::test_bitmap::claim_everything(&written);
Ok(wrote)
});
let bitmap = bitmap.unwrap().to_string();
assert!(bitmap.starts_with("git (guard mismatch)"), "{bitmap}");
assert_eq!(reach(&repo, true), reach(&repo, false));
}
/// the index git wrote keeps its own pack order, with the oldest pack first.
/// knot2 has to build its bitmap on an index that lacks that order.
#[test]
fn knot2s_bitmap_is_built_on_an_index_without_gits_pack_order() {
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo.git");
bare(&repo);
for name in ["a", "b", "c"] {
push(&repo, "refs/heads/main", name);
}
push(&repo, "refs/hidden/secret", "secret");
// the last pack by name gets the oldest mtime. git's order differs from the name order
let mut packs: Vec<PathBuf> = std::fs::read_dir(repo.join("objects/pack"))
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension().is_some_and(|ext| ext == "pack"))
.collect();
packs.sort();
let oldest = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000_000);
std::fs::File::open(packs.last().unwrap())
.unwrap()
.set_modified(oldest)
.unwrap();
git_in(&repo, &["multi-pack-index", "write", "--bitmap"], "");
let before = Before::read(&repo);
assert!(matches!(before, Before::Previous(_)));
let bitmap = write_bitmap(&repo, before, write_knot_bitmap)
.unwrap()
.to_string();
assert!(
bitmap.starts_with("knot2 incremental, guard ok"),
"{bitmap}"
);
assert_eq!(reach(&repo, true), reach(&repo, false));
}
/// a push after reading the tips adds commits knot2's walk misses, and git writes the bitmap.
#[test]
fn tips_that_move_before_the_bitmap_is_built_get_gits() {
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo.git");
bare(&repo);
for name in ["a", "b"] {
push(&repo, "refs/heads/main", name);
}
git_in(&repo, &["multi-pack-index", "write", "--bitmap"], "");
let before = Before::read(&repo);
assert!(matches!(before, Before::Previous(_)));
push(&repo, "refs/heads/main", "c");
let bitmap = write_bitmap(&repo, before, write_knot_bitmap)
.unwrap()
.to_string();
assert!(bitmap.starts_with("git (tips moved)"), "{bitmap}");
assert_eq!(reach(&repo, true), reach(&repo, false));
}
#[test]
fn a_bitmap_gc_left_on_a_pack_goes() {
let dir = tempfile::tempdir().unwrap();
let (root, repo) = (dir.path().join("out"), dir.path().join("repo.git"));
bare(&repo);
for name in ["a", "b"] {
push(&repo, "refs/heads/main", name);
}
git_in(&repo, &["gc", "--quiet"], "");
push(&repo, "refs/heads/main", "c");
let did = "did:plc:maintenance-gc-bitmap";
assert!(maintain(&root, did, &repo, Action::Repack, run_unlocked).unwrap());
let packs = std::fs::read_dir(repo.join("objects/pack")).unwrap();
let left: Vec<_> = (packs.map(|entry| entry.unwrap().file_name()))
.filter(|name| name.to_string_lossy().starts_with("pack-"))
.filter(|name| name.to_string_lossy().ends_with(".bitmap"))
.collect();
assert!(left.is_empty(), "{left:?}");
assert_eq!(reach(&repo, true), reach(&repo, false));
}
#[test]
fn a_bitmap_is_moved_in_only_once_its_guard_passes() {
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo.git");
bare(&repo);
for name in ["a", "b", "c"] {
push(&repo, "refs/heads/main", name);
}
push(&repo, "refs/hidden/secret", "secret");
git_in(&repo, &["multi-pack-index", "write", "--bitmap"], "");
let crashed = repo.join(format!("{SCRATCH}crashed"));
std::fs::create_dir(&crashed).unwrap();
let live = || {
let mut names: Vec<String> = std::fs::read_dir(repo.join("objects/pack"))
.unwrap()
.map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())
.filter(|name| name.starts_with(MIDX_FILE) || name.ends_with(".bitmap"))
.collect();
names.sort();
names
};
let before = Before::read(&repo);
assert!(matches!(before, Before::Previous(_)));
let last = live();
let bitmap = write_bitmap(&repo, before, |opened, previous| {
let wrote = write_knot_bitmap(opened, previous)?;
assert_eq!(live(), last, "knot_git's bitmap is read before its guard");
Ok(wrote)
});
assert!(
bitmap
.unwrap()
.to_string()
.starts_with("knot2 incremental, guard ok")
);
assert_ne!(live(), last);
assert!(!crashed.exists());
assert_eq!(reach(&repo, true), reach(&repo, false));
}
#[test]
fn a_cold_bitmap_that_lies_is_replaced_by_gits() {
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("repo.git");
bare(&repo);
for name in ["a", "b"] {
push(&repo, "refs/heads/main", name);
}
push(&repo, "refs/hidden/secret", "secret");
let before = Before::read(&repo);
assert!(matches!(
before,
Before::Previous(Previous { last: None, .. })
));
let bitmap = write_bitmap(&repo, before, |opened, previous| {
let wrote = write_knot_bitmap(opened, previous)?;
let pack_dir = opened.objects_dir().join("pack");
let written = std::fs::read_dir(&pack_dir)?
.map(|entry| entry.unwrap().path())
.find(|path| path.extension().is_some_and(|ext| ext == "bitmap"))
.unwrap();
crate::test_bitmap::claim_everything(&written);
Ok(wrote)
});
let bitmap = bitmap.unwrap().to_string();
assert!(bitmap.starts_with("git (guard mismatch)"), "{bitmap}");
assert_eq!(reach(&repo, true), reach(&repo, false));
}