use std::path::Path;
use anyhow::Context as _;
use gix::ObjectId;
use knot_types::ObjectFormat;
use crate::git_export::write_unlinked;
use crate::partitions::{Holds, PartitionPack, Partitions};
pub(crate) const TAILS: &str = "tail-start";
const AT_LEAST: u64 = if cfg!(test) { 256 << 10 } else { 32 << 20 };
/// the public and cob partition packs come first, then the regions in order.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Tail {
start: u64,
count: u32,
public: usize,
cob: usize,
regions: Vec<String>,
}
impl Tail {
pub fn framed(start: u64, count: u32, public: usize, cob: usize) -> Self {
Self {
start,
count,
public,
cob,
regions: Vec::new(),
}
}
pub fn start(&self) -> u64 {
self.start
}
pub fn count(&self) -> u32 {
self.count
}
/// how many public and cob packs the reply holds whole
pub fn packs(&self) -> [usize; 2] {
[self.public, self.cob]
}
#[cfg(test)]
pub fn regions(&self) -> &[String] {
&self.regions
}
fn line(&self) -> String {
let mut line = format!("{} {} {} {}", self.start, self.count, self.public, self.cob);
for region in &self.regions {
line.push(' ');
line.push_str(region);
}
line.push('\n');
line
}
fn parse(line: &str) -> Option<Self> {
let mut fields = line.split(' ');
let mut number = || fields.next()?.parse::<u64>().ok();
let tail = Self {
start: number()?,
count: u32::try_from(number()?).ok()?,
public: usize::try_from(number()?).ok()?,
cob: usize::try_from(number()?).ok()?,
regions: Vec::new(),
};
let regions: Vec<String> = fields.map(str::to_string).collect();
regions
.iter()
.all(|region| is_region(region))
.then_some(Self { regions, ..tail })
}
pub fn due(&self, body: u64, added: u64) -> bool {
let tail = body.saturating_sub(self.start).saturating_add(added);
tail > (body / 100).max(AT_LEAST)
}
}
pub(crate) fn tails(dir: &Path) -> Option<[Tail; 2]> {
let text = std::fs::read_to_string(dir.join(TAILS)).ok()?;
let mut lines = text.lines().map(Tail::parse);
let tails = [lines.next()??, lines.next()??];
let regions = tails.iter().flat_map(|tail| &tail.regions);
let whole = regions.into_iter().all(|region| dir.join(region).is_file());
(lines.next().is_none() && whole).then_some(tails)
}
pub(crate) fn write_tails(
dir: &Path,
prior: Option<&Path>,
tails: &[Tail; 2],
) -> anyhow::Result<()> {
for region in tails.iter().flat_map(|tail| &tail.regions) {
let path = dir.join(region);
if !path.exists() {
let from = prior.context("a region without a prior")?.join(region);
if std::fs::hard_link(&from, &path).is_err() {
let bytes =
std::fs::read(&from).with_context(|| format!("read {}", from.display()))?;
write_unlinked(&path, &bytes)?;
}
}
}
let text: String = tails.iter().map(Tail::line).collect();
write_unlinked(&dir.join(TAILS), text.as_bytes())
}
fn is_region(name: &str) -> bool {
name.strip_prefix("tail-").is_some_and(|hex| {
matches!(hex.len(), 40 | 64) && hex.bytes().all(|b| b.is_ascii_hexdigit())
})
}
pub(crate) struct Consolidated {
pub(crate) entries: Vec<u8>,
pub(crate) tail: Tail,
region: (String, Vec<u8>),
}
impl Consolidated {
pub fn write_region(&self, dir: &Path) -> anyhow::Result<()> {
write_unlinked(&dir.join(&self.region.0), &self.region.1)
}
}
pub(crate) fn consolidate(
export_dir: &Path,
smart: &Path,
tail: &Tail,
tips: &[ObjectId],
format: ObjectFormat,
) -> anyhow::Result<Consolidated> {
let partitions = Partitions::read(export_dir)?.context("an export without partitions")?;
let base = Base::open(export_dir, smart, tail, &partitions, format)?;
let repo = knot_git::Repo::open(export_dir).context("open the export")?;
let segment = crate::partitions::unknown_reached(repo.git(), tips, &base)?;
anyhow::ensure!(!segment.is_empty(), "a tail of no objects");
// parents the base holds go in as `-` lines, which makes git use their trees as delta bases
let mut list = Vec::new();
for oid in segment.oids() {
let object = repo
.git()
.find_object(oid)
.with_context(|| format!("read {oid}"))?;
if object.kind != gix::object::Kind::Commit {
continue;
}
let commit = gix::objs::CommitRefIter::from_bytes(&object.data, format.kind());
for parent in commit.parent_ids().filter(|parent| base.holds(parent)) {
list.extend_from_slice(format!("-{parent}\n").as_bytes());
}
}
list.extend_from_slice(segment.object_list());
let dir = tempfile::tempdir_in(export_dir).context("a dir for the tail's pack")?;
let stem = std::path::absolute(dir.path().join("tail"))?;
let hash = crate::partitions::pack_objects(
export_dir,
&[stem.to_str().context("a path that isn't utf-8")?],
&list,
)?;
let pack = dir.path().join(format!(
"tail-{}.pack",
String::from_utf8_lossy(&hash).trim()
));
let (entries, mut objects) =
crate::packfile::rebase(&pack, format, tail.start, &|oid| base.offset(oid))?;
let count = u32::try_from(objects.len())
.ok()
.and_then(|objects| tail.count.checked_add(objects))
.context("too many objects")?;
objects.sort_unstable();
let mut region = Vec::with_capacity(objects.len() * (format.kind().len_in_bytes() + 8));
for (oid, offset) in &objects {
region.extend_from_slice(oid.as_bytes());
region.extend(offset.to_le_bytes());
}
let name = region_name(format, ®ion);
let mut regions = tail.regions.clone();
regions.push(name.clone());
Ok(Consolidated {
tail: Tail {
start: tail.start + entries.len() as u64,
count,
regions,
..tail.clone()
},
entries,
region: (name, region),
})
}
fn region_name(format: ObjectFormat, bytes: &[u8]) -> String {
let mut hasher = crate::reply_pack::Hasher::new(format);
hasher.update(bytes);
let hash = knot_types::lowercase_hex(&hasher.finish());
format!("tail-{hash}")
}
struct Base {
packs: Vec<(gix::odb::pack::index::File, u64)>,
regions: Vec<Region>,
}
impl Base {
fn open(
export_dir: &Path,
smart: &Path,
tail: &Tail,
partitions: &Partitions,
format: ObjectFormat,
) -> anyhow::Result<Self> {
let (public, cob) = (partitions.public(), partitions.cob());
let framed: Vec<&PartitionPack> = (public.get(..tail.public).into_iter().flatten())
.chain(cob.get(..tail.cob).into_iter().flatten())
.collect();
anyhow::ensure!(
framed.len() == tail.public + tail.cob,
"a tail after packs the partitions lack"
);
let mut at = 12u64;
let mut packs = Vec::new();
for pack in framed {
let idx = pack.path(export_dir, "idx");
let index = gix::odb::pack::index::File::at(&idx, format.kind())
.with_context(|| format!("open {}", idx.display()))?;
let len = std::fs::metadata(pack.path(export_dir, "pack"))?.len();
packs.push((index, at));
at += (len.checked_sub(12 + format.kind().len_in_bytes() as u64))
.context("a partition pack too short")?;
}
let regions: Vec<Region> = (tail.regions.iter())
.map(|name| Region::open(smart, name, format))
.collect::<anyhow::Result<_>>()?;
let spans: Vec<(u64, u64)> = (regions.iter())
.map(|region| region.span().context("an empty region"))
.collect::<anyhow::Result<_>>()?;
in_place(at, &spans, tail.start)?;
Ok(Self { packs, regions })
}
fn offset(&self, oid: &gix::hash::oid) -> Option<u64> {
let in_pack = self.packs.iter().find_map(|(index, start)| {
let at = index.lookup(oid)?;
Some(start + index.pack_offset_at_index(at) - 12)
});
in_pack.or_else(|| self.regions.iter().find_map(|region| region.offset(oid)))
}
}
impl Holds for Base {
fn holds(&self, oid: &gix::hash::oid) -> bool {
self.offset(oid).is_some()
}
}
fn in_place(mut at: u64, spans: &[(u64, u64)], start: u64) -> anyhow::Result<()> {
let ends = spans.iter().skip(1).map(|&(first, _)| first).chain([start]);
for (&(first, last), end) in spans.iter().zip(ends) {
anyhow::ensure!(first == at && last < end, "a region out of place");
at = end;
}
anyhow::ensure!(at == start, "a tail after a gap");
Ok(())
}
struct Region {
bytes: Vec<u8>,
width: usize,
}
impl Region {
fn open(smart: &Path, name: &str, format: ObjectFormat) -> anyhow::Result<Self> {
let path = smart.join(name);
let bytes = std::fs::read(&path).with_context(|| format!("read {}", path.display()))?;
let width = format.kind().len_in_bytes() + 8;
anyhow::ensure!(
bytes.len() % width == 0 && name == region_name(format, &bytes),
"{} isn't the region it's named",
path.display()
);
Ok(Self { bytes, width })
}
fn span(&self) -> Option<(u64, u64)> {
let hash = self.width - 8;
let offsets = (self.bytes.chunks_exact(self.width))
.map(|record| u64::from_le_bytes(record[hash..].try_into().expect("eight bytes")));
offsets.fold(None, |span, offset| match span {
None => Some((offset, offset)),
Some((first, last)) => Some((first.min(offset), last.max(offset))),
})
}
fn offset(&self, oid: &gix::hash::oid) -> Option<u64> {
let hash = self.width - 8;
let record = |at: usize| &self.bytes[at * self.width..(at + 1) * self.width];
let (mut low, mut high) = (0, self.bytes.len() / self.width);
while low < high {
let middle = (low + high) / 2;
match record(middle)[..hash].cmp(oid.as_bytes()) {
std::cmp::Ordering::Less => low = middle + 1,
std::cmp::Ordering::Greater => high = middle,
std::cmp::Ordering::Equal => {
return Some(u64::from_le_bytes(record(middle)[hash..].try_into().ok()?));
}
}
}
None
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_tail_is_due_past_its_bound() {
let tail = Tail::framed(1000, 3, 1, 0);
let small = 1000 + AT_LEAST;
assert!(!tail.due(small, 0));
assert!(tail.due(small, 1));
assert!(tail.due(small + 1, 0));
let body = 1000 * AT_LEAST;
let at = Tail::framed(body - body / 100, 3, 1, 0);
assert!(!at.due(body, 0));
assert!(at.due(body, 1));
assert!(!Tail::framed(body - AT_LEAST, 3, 1, 0).due(body, body / 100 - AT_LEAST));
}
#[test]
fn regions_are_in_place_end_to_end() {
let ok = |at, spans: &[(u64, u64)], start| super::in_place(at, spans, start).is_ok();
assert!(ok(100, &[], 100));
assert!(ok(100, &[(100, 150)], 200));
assert!(ok(100, &[(100, 150), (180, 200), (240, 260)], 300));
assert!(!ok(100, &[], 120), "a gap before the tail");
assert!(!ok(100, &[(101, 150)], 200), "a gap before a region");
assert!(!ok(100, &[(100, 190), (180, 200)], 250), "an overlap");
assert!(
!ok(100, &[(100, 150), (180, 250)], 250),
"past the tail start"
);
}
#[test]
fn tails_read_back_as_written() {
let dir = tempfile::tempdir().unwrap();
let region = format!("tail-{}", "a".repeat(40));
std::fs::write(dir.path().join(®ion), b"").unwrap();
let tails = [
Tail::framed(4096, 7, 2, 0),
Tail {
regions: vec![region.clone()],
..Tail::framed(8192, 9, 2, 1)
},
];
write_tails(dir.path(), None, &tails).unwrap();
assert_eq!(super::tails(dir.path()), Some(tails.clone()));
let other = tempfile::tempdir().unwrap();
write_tails(other.path(), Some(dir.path()), &tails).unwrap();
assert!(other.path().join(®ion).is_file(), "the region is linked");
std::fs::remove_file(dir.path().join(®ion)).unwrap();
assert_eq!(super::tails(dir.path()), None, "a region gone");
for bad in [
"4096 7 2 0\n",
"4096 7 2 0\n8192 9 2\n",
"4096 7 2 0\n8192 9 2 1 tail-x\n",
"4096 7 2 0\n8192 9 2 1\n8192 9 2 1\n",
"4096 99999999999 2 0\n8192 9 2 1\n",
] {
std::fs::write(dir.path().join(TAILS), bad).unwrap();
assert_eq!(super::tails(dir.path()), None, "{bad:?}");
}
}
}