use super::*;
#[test]
fn the_tail_starts_where_the_reply_was_last_framed_whole() -> anyhow::Result<()> {
let root = tempfile::tempdir()?;
let store = RepoStore::open(&root.path().join("repo-store"));
let source = init_test_source(root.path());
commit(&source, "big.txt", &noise(3, 64 * 1024), "big");
let (first, prior) = export_against(root.path(), &source, "export-1", None, &store);
commit(&source, "new.txt", &noise(4, 4096), "new");
let (second, _) = export_against(root.path(), &source, "export-2", Some(&prior), &store);
let format = knot_types::ObjectFormat::SHA1;
let body = |export: &Path, at: usize| {
let file = std::fs::File::open(export.join("smart").join(super::FULL_REPLIES[at]))?;
anyhow::Ok(crate::reply_pack::Framed::of(&file, format)?.body())
};
let tails = crate::consolidate::tails(&first.join("smart")).expect("tails");
let starts = tails.clone().map(|tail| tail.start());
assert_eq!(starts, [body(&first, 0)?, body(&first, 1)?]);
assert_eq!(
crate::consolidate::tails(&second.join("smart")),
Some(tails)
);
assert!(body(&second, 0)? > starts[0], "the push grew the reply");
Ok(())
}
#[test]
fn a_tail_past_its_bound_is_consolidated() -> anyhow::Result<()> {
let root = tempfile::tempdir()?;
let source = init_test_source(root.path());
let big = noise(5, 300 * 1024);
commit(&source, "a/b/big.txt", &big, "big");
let store = RepoStore::open(&root.path().join("repo-store"));
let export = |name: &str, prior: Option<&ExportPrior>| {
export_against(root.path(), &source, name, prior, &store)
};
let mut prior = export("export-0", None);
let replies = |export: &Path| -> anyhow::Result<[Vec<u8>; 2]> {
let read = |name: &str| std::fs::read(export.join("smart").join(name));
Ok([read(super::FULL_REPLIES[0])?, read(super::FULL_REPLIES[1])?])
};
let tails = |export: &Path| crate::consolidate::tails(&export.join("smart")).unwrap();
let mut steps = vec![(replies(&prior.0)?, tails(&prior.0))];
for step in 1..=6 {
commit(
&source,
"a/b/big.txt",
&format!("{big}{step}\n"),
&format!("step {step}"),
);
let next = export(&format!("export-{step}"), Some(&prior.1));
let hashed = replies(&next.0)?;
let tip = git(&source, &["rev-parse", "main"]);
for reply in &hashed {
assert_whole_pack(
root.path(),
&source,
reply,
std::slice::from_ref(&tip),
"sha1",
);
}
let [nak, packfile] = super::full_prefixes();
assert_eq!(
hashed[0][nak.len()..],
hashed[1][packfile.len()..],
"step {step}"
);
steps.push((hashed, tails(&next.0)));
prior = next;
}
let start = |step: usize| steps[step].1[0].start();
let len = |step: usize| steps[step].0[0].len();
for step in [2, 4, 6] {
assert_eq!(steps[step - 1].1, steps[step - 2].1, "step {}", step - 1);
assert!(
start(step) > start(step - 1) && len(step) < len(step - 1),
"push {step} consolidates"
);
let consolidated = start(step) - start(step - 1);
assert!(consolidated < big.len() as u64 / 4, "{consolidated} bytes");
assert_eq!(steps[step].1[0].regions().len(), step / 2, "step {step}");
assert_eq!(steps[step].1[0], steps[step].1[1]);
}
Ok(())
}
#[test]
fn the_replies_of_a_repo_with_cobs_consolidate_each_their_tail() -> anyhow::Result<()> {
for format in ["sha1", "sha256"] {
let object_format = knot_types::ObjectFormat::from_capability(format).unwrap();
let root = tempfile::tempdir()?;
let store = RepoStore::open(&root.path().join("repo-store"));
let source = root.path().join("plc").join("sm").join("arttest");
std::fs::create_dir_all(&source)?;
let object_format_arg = format!("--object-format={format}");
git(&source, &["init", "-q", "-b", "main", &object_format_arg]);
let big = noise(7, 300 * 1024);
commit(&source, "big.txt", &big, "big");
let cob = orphan_commit(&source, "cob");
let config = test_config(root.path());
let export = |name: &str, prior: Option<&ExportPrior>| {
let mut snap = snapshot(&source, heads(&source));
snap.object_format = object_format;
snap.cob_refs = vec![("refs/cobs/org.tangled.track.ticket/1".into(), cob.clone())];
let fingerprint = super::export_fingerprint(&snap, "main", prior);
let dir = root.path().join(name);
super::materialize_export(&config, &snap, &dir, &fingerprint, Some(&store), prior)
.unwrap();
let refs = exported_refs(&snap)
.map(|(name, raw)| (name.to_string(), raw.to_string()))
.collect();
let bases = super::fetch_replies(&dir).unwrap().bases;
let tails = crate::consolidate::tails(&dir.join("smart")).expect("tails");
let prior = ExportPrior {
fingerprint,
refs,
bases,
};
(dir, prior, tails)
};
let (_, mut prior, cold) = export("export-0", None);
let mut tails = vec![cold];
for step in 1..=2 {
commit(
&source,
"big.txt",
&format!("{big}{step}\n"),
&format!("step {step}"),
);
let (dir, next, grown) = export(&format!("export-{step}"), Some(&prior));
let main = git(&source, &["rev-parse", "main"]);
for (name, tips) in super::FULL_REPLIES
.iter()
.zip([vec![main.clone()], vec![main, cob.clone()]])
{
let reply = std::fs::read(dir.join("smart").join(name))?;
assert_whole_pack(root.path(), &source, &reply, &tips, format);
}
assert!(!dir.join("smart").join(super::SAME_PACK).exists());
tails.push(grown);
prior = next;
}
assert_eq!(tails[1], tails[0], "{format}");
for at in [0, 1] {
assert!(
tails[2][at].start() > tails[1][at].start(),
"{format} reply {at}"
);
}
assert_ne!(tails[2][0].regions(), tails[2][1].regions());
}
Ok(())
}
#[test]
fn a_big_reply_grows_into_its_spare_slots() -> anyhow::Result<()> {
for format in ["sha1", "sha256"] {
for with_cob in [false, true] {
let object_format = knot_types::ObjectFormat::from_capability(format).unwrap();
let root = tempfile::tempdir()?;
let store = RepoStore::open(&root.path().join("repo-store"));
let source = root.path().join("plc").join("sm").join("arttest");
std::fs::create_dir_all(&source)?;
let object_format_arg = format!("--object-format={format}");
git(&source, &["init", "-q", "-b", "main", &object_format_arg]);
commit(&source, "big.txt", noise(1, 8 << 20), "big");
let cob = with_cob.then(|| orphan_commit(&source, "cob"));
let (root, store, cob) = (root.path(), &store, cob.as_deref());
let export = |name: &str, prior: Option<&ExportPrior>| {
export_spared(root, &source, name, prior, store, object_format, cob)
};
let (first, prior, framed) = export("export-0", None);
let [Some(v0), Some(v2)] = framed.clone() else {
panic!("{format}: no spare slots");
};
assert_eq!((v0.first, v0.room), (0, 1024), "{format}");
assert_eq!(v0.nonce, v2.nonce, "{format}");
assert_eq!(v0 == v2, !with_cob, "{format}");
commit(&source, "small.txt", "small\n", "small");
let (second, prior, grown) = export("export-1", Some(&prior));
for at in [0, 1] {
let (before, after) = (framed[at].clone().unwrap(), grown[at].clone().unwrap());
assert_eq!(after.first, before.first + 3, "{format} {at}");
assert_eq!(after.nonce, before.nonce, "{format} {at}");
assert_eq!(after.since, before.since + 3, "{format} {at}");
let name = super::FULL_REPLIES[at];
assert_eq!(
reply_count(&second.join("smart").join(name)),
reply_count(&first.join("smart").join(name)),
"{format} {at}"
);
}
commit(&source, "small.txt", "smaller\n", "smaller");
let (_, _, again) = export("export-2", Some(&prior));
for at in [0, 1] {
let after = again[at].clone().unwrap();
assert_eq!((after.nonce, after.first), (v0.nonce, 6), "{format} {at}");
}
}
}
Ok(())
}
#[test]
fn a_push_past_its_spare_slots_reserves_anew() -> anyhow::Result<()> {
let format = knot_types::ObjectFormat::SHA1;
let root = tempfile::tempdir()?;
let store = RepoStore::open(&root.path().join("repo-store"));
let source = init_test_source(root.path());
commit(&source, "big.txt", noise(1, 8 << 20), "big");
let export = |name: &str, prior: Option<&ExportPrior>| {
export_spared(root.path(), &source, name, prior, &store, format, None)
};
let (_, prior, [Some(framed), _]) = export("export-0", None) else {
panic!("no spare slots");
};
commit(&source, "small.txt", "small\n", "small");
let (_, prior, [Some(taken), _]) = export("export-1", Some(&prior)) else {
panic!("no spare slots");
};
assert_eq!((taken.nonce, taken.first), (framed.nonce, 3));
let mut filler = taken.nonce.to_vec();
filler.extend((taken.room - 1).to_be_bytes());
commit(&source, "filler.bin", &filler, "a filler");
let (_, prior, [Some(planted), _]) = export("export-2", Some(&prior)) else {
panic!("no spare slots");
};
assert_ne!(planted.nonce, taken.nonce);
assert_eq!((planted.first, planted.since), (0, 0));
assert_eq!(planted.room, crate::spare::room(0, 3 + 3));
for n in 0..1100 {
std::fs::write(source.join(format!("many-{n:04}.txt")), format!("{n}\n"))?;
}
commit(&source, "many-last.txt", "last\n", "many");
let (_, _, [Some(past), _]) = export("export-3", Some(&prior)) else {
panic!("no spare slots");
};
assert_ne!(past.nonce, planted.nonce);
assert_eq!(past.first, 0);
assert_eq!(past.room, crate::spare::room(0, 1103));
Ok(())
}
#[test]
fn a_big_reply_consolidates_its_tail_before_its_spare_slots() -> anyhow::Result<()> {
for format in ["sha1", "sha256"] {
for with_cob in [false, true] {
let object_format = knot_types::ObjectFormat::from_capability(format).unwrap();
let root = tempfile::tempdir()?;
let store = RepoStore::open(&root.path().join("repo-store"));
let source = root.path().join("plc").join("sm").join("arttest");
std::fs::create_dir_all(&source)?;
let object_format_arg = format!("--object-format={format}");
git(&source, &["init", "-q", "-b", "main", &object_format_arg]);
commit(&source, "big.txt", noise(1, 8 << 20), "big");
let cob = with_cob.then(|| orphan_commit(&source, "cob"));
let (root, store, cob) = (root.path(), &store, cob.as_deref());
let export = |name: &str, prior: Option<&ExportPrior>| {
export_spared(root, &source, name, prior, store, object_format, cob)
};
let tails = |dir: &Path| crate::consolidate::tails(&dir.join("smart")).unwrap();
let (first, mut prior, framed) = export("export-0", None);
let mut before = tails(&first);
let mut spares = framed;
let steps: [&dyn Fn(); 5] = [
&|| commit(&source, "tail.txt", noise(2, 700 << 10), "tail"),
&|| commit(&source, "small.txt", "small\n", "small"),
&|| {
for n in 0..1100 {
let path = source.join(format!("many-{n:04}.txt"));
std::fs::write(path, format!("{n}\n")).unwrap();
}
commit(&source, "many-last.txt", "last\n", "many");
},
&|| commit(&source, "tail.txt", noise(3, 700 << 10), "tail again"),
&|| commit(&source, "tail.txt", noise(4, 700 << 10), "and again"),
];
for (step, push) in steps.iter().enumerate() {
push();
let (dir, next, grown) = export(&format!("export-{}", step + 1), Some(&prior));
let after = tails(&dir);
for at in [0, 1] {
let (was, is) = (spares[at].clone().unwrap(), grown[at].clone().unwrap());
let consolidated = after[at].start() != before[at].start();
assert_eq!(consolidated, step != 1 && step != 2, "{format} {step} {at}");
assert_eq!(is.nonce == was.nonce, step != 2, "{format} {step} {at}");
if consolidated {
assert_eq!(after[at].start(), is.at, "{format} {step} {at}");
assert_eq!(is.tail_state, Some(is.state), "{format} {step} {at}");
}
assert!(is.tail_state.is_some(), "{format} {step} {at}");
}
(prior, before, spares) = (next, after, grown);
}
}
}
Ok(())
}
#[test]
fn a_tail_that_cant_be_consolidated_is_grown() -> anyhow::Result<()> {
let root = tempfile::tempdir()?;
let store = RepoStore::open(&root.path().join("repo-store"));
let source = init_test_source(root.path());
let big = noise(6, 300 * 1024);
commit(&source, "big.txt", &big, "big");
let (_, mut prior) = export_against(root.path(), &source, "export-0", None, &store);
let mut tails = Vec::new();
for step in 1..=4 {
commit(
&source,
"big.txt",
&format!("{big}{step}\n"),
&format!("step {step}"),
);
let name = format!("export-{step}");
let (export, next) = export_against(root.path(), &source, &name, Some(&prior), &store);
let tip = git(&source, &["rev-parse", "main"]);
for name in super::FULL_REPLIES {
let reply = std::fs::read(export.join("smart").join(name))?;
assert_whole_pack(
root.path(),
&source,
&reply,
std::slice::from_ref(&tip),
"sha1",
);
}
let entry = store.entry_path(super::EXPORT_CACHE_KIND, &next.fingerprint);
tails.push(crate::consolidate::tails(&entry.join("smart")).expect("tails"));
if step == 2 {
// the same length keeps the store taking the entry
let region = tails[1][0].regions().first();
let path = entry
.join("smart")
.join(region.expect("the second push consolidates"));
let mut region = std::fs::read(&path)?;
for record in region.as_chunks_mut::<28>().0 {
record[20] = record[20].wrapping_add(1);
}
std::fs::remove_file(&path)?;
std::fs::write(&path, region)?;
}
prior = next;
}
assert_eq!(tails[3], tails[2], "the fourth push grew its tail");
Ok(())
}
#[test]
fn a_prior_reply_that_cant_grow_is_made_anew() -> anyhow::Result<()> {
use std::os::unix::fs::PermissionsExt as _;
let root = tempfile::tempdir()?;
let store = RepoStore::open(&root.path().join("repo-store"));
let source = init_test_source(root.path());
let (_, prior) = export_against(root.path(), &source, "export-1", None, &store);
let entry = store
.read_directory(super::EXPORT_CACHE_KIND, &prior.fingerprint)
.unwrap();
let smart = entry.join("smart");
let reply = smart.join("fetch-done");
// the same length keeps the store taking the entry
let zeroed = vec![0; std::fs::metadata(&reply)?.len() as usize];
std::fs::set_permissions(&smart, std::fs::Permissions::from_mode(0o755))?;
std::fs::remove_file(&reply)?;
std::fs::write(&reply, zeroed)?;
std::fs::set_permissions(&smart, std::fs::Permissions::from_mode(0o555))?;
commit(&source, "new.txt", &noise(4, 4096), "new");
let (export, _) = export_against(root.path(), &source, "export-2", Some(&prior), &store);
let clone = clone_over_http(root.path(), &export, "clone")?;
assert!(clone.join("new.txt").exists());
git(&clone, &["fsck", "--strict"]);
Ok(())
}
#[test]
fn a_cob_write_at_a_time_keeps_the_export_to_log_many_packs() -> anyhow::Result<()> {
let root = tempfile::tempdir()?;
let store = RepoStore::open(&root.path().join("repo-store"));
let source = root.path().join("plc").join("sm").join("arttest");
std::fs::create_dir_all(&source)?;
git(&source, &["init", "-q", "-b", "main"]);
commit(&source, "README.md", "# cobs\n", "initial");
let config = test_config(root.path());
let mut cob = orphan_commit(&source, "ticket");
let mut prior: Option<ExportPrior> = None;
for write in 1..=40 {
let blob = git_stdin(
&source,
&["hash-object", "-w", "--stdin"],
&format!("comment {write}\n"),
);
let tree = git_stdin(
&source,
&["mktree"],
&format!("100644 blob {blob}\tcomment\n"),
);
cob = git(
&source,
&[
"-c",
"user.name=t",
"-c",
"user.email=t@t",
"commit-tree",
&tree,
"-p",
&cob,
"-m",
&format!("comment {write}"),
],
);
let mut snap = snapshot(&source, heads(&source));
snap.cob_refs = vec![("refs/cobs/org.tangled.track.ticket/1".into(), cob.clone())];
let fingerprint = super::export_fingerprint(&snap, "main", prior.as_ref());
let dir = root.path().join(format!("export-{write}"));
super::materialize_export(
&config,
&snap,
&dir,
&fingerprint,
Some(&store),
prior.as_ref(),
)?;
let partitions = crate::partitions::Partitions::read(&dir)?.expect("partitions");
let packs = partitions.public().len() + partitions.cob().len();
assert!(
packs <= 2 + 2 * (write as f64).log2().ceil() as usize,
"{packs} packs after {write} writes"
);
if write % 8 == 0 {
let clone = root.path().join(format!("clone-{write}"));
git(
root.path(),
&[
"-c",
"transfer.fsckObjects=true",
"clone",
"-q",
"--mirror",
dir.to_str().unwrap(),
clone.to_str().unwrap(),
],
);
assert_eq!(
git(
&clone,
&["rev-parse", "refs/cobs/org.tangled.track.ticket/1"]
),
cob
);
git(&clone, &["fsck", "--strict"]);
for (name, tips) in super::FULL_REPLIES.iter().zip([
vec![git(&source, &["rev-parse", "main"])],
vec![git(&source, &["rev-parse", "main"]), cob.clone()],
]) {
let reply = std::fs::read(dir.join("smart").join(name))?;
assert_whole_pack(root.path(), &source, &reply, &tips, "sha1");
}
}
let refs = exported_refs(&snap)
.map(|(name, raw)| (name.to_string(), raw.to_string()))
.collect();
let bases = super::fetch_replies(&dir)?.bases;
prior = Some(ExportPrior {
fingerprint,
refs,
bases,
});
}
Ok(())
}