a repo's export keeps a log of its writes' packs, not one per write
Signed-off-by: dawn <90008@klbr.net>
38 unmodified lines39404142434445464748495051331 unmodified linesExpand all38 unmodified lines pub fn count(&self) -> u32 { self.count }
/// how many public and cob packs the reply holds whole pub fn packs(&self) -> [usize; 2] { [self.public, self.cob] }
#[cfg(test)] pub fn regions(&self) -> &[String] { &self.regions }331 unmodified linesExpand all361 unmodified linesExpand all362363364365366367368369370371372373374375376377151 unmodified linesExpand all361 unmodified linesExpand all } _ => unreachable!("an export not from the cache is packed fresh or grown"), } } if added.is_some() && let Some(tails) = crate::consolidate::tails(&export_dir.join("smart")) { let framed = [0, 1].map(|at| tails[0].packs()[at].max(tails[1].packs()[at])); exporter.time("export: merge the packs past the replies", || { Partitions::merge_tails(export_dir, framed, snapshot.object_format) })?; } let fp_path = export_dir.join("export.fingerprint"); write_unlinked(&fp_path, format!("{fingerprint}\n").as_bytes()) .with_context(|| format!("write export.fingerprint at {}", fp_path.display()))?;
151 unmodified linesExpand all184 unmodified linesExpand all1851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322335 unmodified lines239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277261 unmodified linesExpand all184 unmodified linesExpand all v2, }) }
/// the packs past what the replies frame stay geometric, each over twice the next, so an export /// holds O(log writes) of them. a merged pack is built from its packs' own indexes, which keeps /// every object in it once; the old files stay with the entries that link them. pub fn merge_tails( export_dir: &Path, framed: [usize; 2], format: knot_types::ObjectFormat, ) -> anyhow::Result<()> { let Some(mut partitions) = Self::read(export_dir)? else { return Ok(()); }; let mut merged = Vec::new(); merged.extend(merge_tail( export_dir, &mut partitions.public, framed[0], format, )?); merged.extend(merge_tail( export_dir, &mut partitions.cob, framed[1], format, )?); if merged.is_empty() { return Ok(()); } partitions.write(export_dir)?; let dir = export_dir.join("objects/pack"); for pack in merged { for entry in std::fs::read_dir(&dir).context("list the export's packs")? { let path = entry?.path(); if path.file_stem().is_some_and(|stem| *stem == *pack.0) { std::fs::remove_file(&path) .with_context(|| format!("remove {}", path.display()))?; } } } Ok(()) }
fn write(&self, export_dir: &Path) -> anyhow::Result<()> { let lines: String = (self .public .iter()5 unmodified lines .with_context(|| format!("write {}", path.display())) }}
fn merge_tail( export_dir: &Path, packs: &mut Vec<PartitionPack>, framed: usize, format: knot_types::ObjectFormat,) -> anyhow::Result<Vec<PartitionPack>> { let size = |pack: &PartitionPack| { std::fs::metadata(pack.path(export_dir, "pack")).map(|meta| meta.len()) }; let mut merged = Vec::new(); while packs.len() >= framed + 2 { let newer = &packs[packs.len() - 1]; let older = &packs[packs.len() - 2]; if size(older)? > 2 * size(newer)? { break; } let mut ids = HashSet::new(); for pack in [older, newer] { let idx = pack.path(export_dir, "idx"); let index = gix::odb::pack::index::File::at(&idx, format.kind()) .with_context(|| format!("open {}", idx.display()))?; ids.extend(index.iter().map(|entry| entry.oid)); } let list: String = ids.iter().map(|oid| format!("{oid}\n")).collect(); let pack = write_pack(export_dir, export_dir, &[], list.as_bytes())?; merged.extend(packs.drain(packs.len() - 2..)); packs.push(pack); } Ok(merged)}
pub(crate) fn object_ids<'a>( raws: impl IntoIterator<Item = &'a str>,) -> anyhow::Result<Vec<ObjectId>> { raws.into_iter()261 unmodified linesExpand all372 unmodified linesExpand all373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474372 unmodified linesExpand all assert!(clone.join("new.txt").exists()); git(&clone, &["fsck", "--strict"]); Ok(())}
#[test]fn a_cob_write_at_a_time_keeps_the_export_to_log_many_packs() -> anyhow::Result<()> { let root = tempfile::tempdir()?; let store = RepoStore::open(&root.path().join("repo-store")); let source = root.path().join("plc").join("sm").join("arttest"); std::fs::create_dir_all(&source)?; git(&source, &["init", "-q", "-b", "main"]); commit(&source, "README.md", "# cobs\n", "initial"); let config = test_config(root.path()); let mut cob = orphan_commit(&source, "ticket"); let mut prior: Option<ExportPrior> = None; for write in 1..=40 { let blob = git_stdin( &source, &["hash-object", "-w", "--stdin"], &format!("comment {write}\n"), ); let tree = git_stdin( &source, &["mktree"], &format!("100644 blob {blob}\tcomment\n"), ); cob = git( &source, &[ "-c", "user.name=t", "-c", "user.email=t@t", "commit-tree", &tree, "-p", &cob, "-m", &format!("comment {write}"), ], ); let mut snap = snapshot(&source, heads(&source)); snap.cob_refs = vec![("refs/cobs/org.tangled.track.ticket/1".into(), cob.clone())]; let fingerprint = super::export_fingerprint(&snap, "main", prior.as_ref()); let dir = root.path().join(format!("export-{write}")); super::materialize_export( &config, &snap, &dir, &fingerprint, Some(&store), prior.as_ref(), )?; let partitions = crate::partitions::Partitions::read(&dir)?.expect("partitions"); let packs = partitions.public().len() + partitions.cob().len(); assert!( packs <= 2 + 2 * (write as f64).log2().ceil() as usize, "{packs} packs after {write} writes" ); if write % 8 == 0 { let clone = root.path().join(format!("clone-{write}")); git( root.path(), &[ "-c", "transfer.fsckObjects=true", "clone", "-q", "--mirror", dir.to_str().unwrap(), clone.to_str().unwrap(), ], ); assert_eq!( git( &clone, &["rev-parse", "refs/cobs/org.tangled.track.ticket/1"] ), cob ); git(&clone, &["fsck", "--strict"]); for (name, tips) in super::FULL_REPLIES.iter().zip([ vec![git(&source, &["rev-parse", "main"])], vec![git(&source, &["rev-parse", "main"]), cob.clone()], ]) { let reply = std::fs::read(dir.join("smart").join(name))?; assert_whole_pack(root.path(), &source, &reply, &tips, "sha1"); } } let refs = exported_refs(&snap) .map(|(name, raw)| (name.to_string(), raw.to_string())) .collect(); let bases = super::fetch_replies(&dir)?.bases; prior = Some(ExportPrior { fingerprint, refs, bases, }); } Ok(())}31 unmodified lines323334353636373839404142434445464746484950515253505455565758302 unmodified linesExpand all31 unmodified lines let fingerprint = super::export_fingerprint(&snap, "main", prior); let dir = root.path().join(name); super::materialize_export(&config, &snap, &dir, &fingerprint, Some(&store), prior).unwrap(); assert_partitions_exact(root.path(), &source, &dir, &snap); // a fast-forward only adds packs, and the older ones must still lead the new list. // a fast-forward keeps the packs the replies frame and only merges the ones after them. let partitions = super::Partitions::read(&dir).unwrap().unwrap(); let grown = prior.is_some_and(|prior| super::refs_fast_forwarded(prior, &snap, &config).unwrap()); if let Some(prior) = prior.filter(|_| grown) { let entry = store .read_directory(super::EXPORT_CACHE_KIND, &prior.fingerprint) .unwrap(); let before = super::Partitions::read(&entry).unwrap().unwrap(); let tails = crate::consolidate::tails(&entry.join("smart")).unwrap(); let framed = [0, 1].map(|at| tails[0].packs()[at].max(tails[1].packs()[at])); assert!( partitions.public().starts_with(before.public()), partitions .public() .starts_with(&before.public()[..framed[0]]), "{name} packed anew" ); assert!( partitions.cob().starts_with(before.cob()), partitions.cob().starts_with(&before.cob()[..framed[1]]), "{name} packed anew" ); } let refs = exported_refs(&snap)302 unmodified linesExpand all