use std::fs::File;
use std::io::{BufReader, BufWriter, Read as _, Write as _};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use anyhow::Context as _;
use gix::odb::pack::index;
use knot_types::ObjectFormat;
use crate::partitions::PartitionPack;
use crate::reply_pack::Hasher;
const MAX_SMALL_OFFSET: u64 = 0x7fff_ffff;
const OFS_DELTA: u8 = 6;
const REF_DELTA: u8 = 7;
static STAGING: AtomicU64 = AtomicU64::new(0);
/// staged as dot files, an unfinished pack never looks real.
struct PackWriter {
export_dir: PathBuf,
format: ObjectFormat,
staged: Staged,
out: BufWriter<File>,
hasher: Hasher,
declared: u32,
written: u32,
len: u64,
}
struct Staged(Vec<PathBuf>);
impl Drop for Staged {
fn drop(&mut self) {
for path in &self.0 {
let _ = std::fs::remove_file(path);
}
}
}
impl Staged {
fn file(&mut self, dir: &Path, ext: &str) -> anyhow::Result<File> {
let path = dir.join(format!(
".tmp-pack-{}-{}.{ext}",
std::process::id(),
STAGING.fetch_add(1, Ordering::Relaxed)
));
self.0.push(path.clone());
File::create(&path).with_context(|| format!("create {}", path.display()))
}
}
impl PackWriter {
fn new(export_dir: &Path, format: ObjectFormat, count: u32) -> anyhow::Result<Self> {
let mut staged = Staged(Vec::new());
let file = staged.file(&export_dir.join("objects/pack"), "pack")?;
let mut writer = Self {
export_dir: export_dir.to_path_buf(),
format,
staged,
out: BufWriter::with_capacity(1 << 20, file),
hasher: Hasher::new(format),
declared: count,
written: 0,
len: 0,
};
writer.write(&crate::reply_pack::pack_header(count))?;
Ok(writer)
}
fn write(&mut self, bytes: &[u8]) -> anyhow::Result<()> {
self.out.write_all(bytes).context("write the pack")?;
self.hasher.update(bytes);
self.len += bytes.len() as u64;
Ok(())
}
fn entry(&mut self, parts: &[&[u8]]) -> anyhow::Result<u32> {
let mut crc = flate2::Crc::new();
for part in parts {
crc.update(part);
self.write(part)?;
}
self.written += 1;
Ok(crc.sum())
}
fn finish(mut self, entries: &Filtered) -> anyhow::Result<PartitionPack> {
anyhow::ensure!(
self.written == self.declared && entries.len() == self.written,
"a pack declared {} objects, holds {} and indexes {}",
self.declared,
self.written,
entries.len()
);
let checksum = std::mem::replace(&mut self.hasher, Hasher::new(self.format)).finish();
self.out.write_all(&checksum).context("write the pack")?;
self.out.flush().context("write the pack")?;
crate::datastore::write::start_writeback(self.out.get_ref());
let checksum = gix::ObjectId::from_bytes_or_panic(&checksum);
let dir = self.export_dir.join("objects/pack");
let idx = self.staged.file(&dir, "idx")?;
entries.write_idx(&checksum, idx)?;
let rev = self.staged.file(&dir, "rev")?;
write_rev(entries.len(), |at| entries.offset(at), &checksum, rev)?;
let stem = format!("pack-{checksum}");
// the idx goes in last, git only picks up a pack once its idx exists.
for (at, ext) in [(0, "pack"), (2, "rev"), (1, "idx")] {
let to = dir.join(format!("{stem}.{ext}"));
std::fs::rename(&self.staged.0[at], &to)
.with_context(|| format!("install {}", to.display()))?;
}
self.staged.0.clear();
PartitionPack::new(&self.export_dir, &stem)
}
}
/// a delta whose base is dropped is written whole, read through `repo`.
pub(crate) fn filter_copy(
repo: &gix::Repository,
source: &Path,
keep: &dyn Fn(u32) -> bool,
export_dir: &Path,
format: ObjectFormat,
) -> anyhow::Result<Option<PartitionPack>> {
let index_path = source.with_extension("idx");
let index = index::File::at(&index_path, format.kind())
.with_context(|| format!("open {}", index_path.display()))?;
let count = index.num_objects();
let kept = (0..count).filter(|&at| keep(at)).count();
if kept == 0 {
return Ok(None);
}
let mut order: Vec<u32> = (0..count).collect();
order.sort_unstable_by_key(|&at| index.pack_offset_at_index(at));
let offsets: Vec<u64> = order
.iter()
.map(|&at| index.pack_offset_at_index(at))
.collect();
let file = File::open(source).with_context(|| format!("open {}", source.display()))?;
let end = (file.metadata()?.len())
.checked_sub(format.kind().len_in_bytes() as u64)
.with_context(|| format!("{} is too short for a pack", source.display()))?;
let mut input = BufReader::with_capacity(1 << 20, file);
let mut header = [0; 12];
input
.read_exact(&mut header)
.with_context(|| format!("read {}", source.display()))?;
anyhow::ensure!(
header == *crate::reply_pack::pack_header(count),
"{} isn't a version 2 pack of the {count} objects its index names",
source.display()
);
anyhow::ensure!(
offsets.first().is_none_or(|&first| first == 12),
"{}'s first entry isn't after its header",
source.display()
);
let mut writer = PackWriter::new(export_dir, format, u32::try_from(kept)?)?;
let mut written = vec![u64::MAX; count as usize];
let mut crcs = vec![0u32; count as usize];
let mut entry = Vec::new();
let mut header = Vec::new();
for (k, &at) in order.iter().enumerate() {
let next = offsets.get(k + 1).copied().unwrap_or(end);
let len = (next.checked_sub(offsets[k]))
.with_context(|| format!("{} ends inside an entry", source.display()))?;
entry.resize(usize::try_from(len)?, 0);
input
.read_exact(&mut entry)
.with_context(|| format!("read {}", source.display()))?;
if !keep(at) {
continue;
}
let parsed = Entry::parse(&entry, format)?;
let base = match parsed.base {
Base::None => None,
Base::Offset(distance) => {
let offset = offsets[k]
.checked_sub(distance)
.context("a delta base before the pack")?;
let base_k = offsets
.binary_search(&offset)
.map_err(|_| anyhow::anyhow!("a delta base at no entry's offset"))?;
Some(order[base_k])
}
Base::Id(oid) => Some(
index
.lookup(oid)
.context("a ref delta whose base isn't in its pack")?,
),
};
let here = writer.len;
crcs[at as usize] = match base.map(|base| written[base as usize]) {
None => writer.entry(&[&entry])?,
Some(u64::MAX) => {
let object = repo
.find_object(index.oid_at_index(at).to_owned())
.with_context(|| format!("read {}", index.oid_at_index(at)))?;
writer.entry(&[&whole_entry(object.kind, &object.data)?])?
}
Some(base_at) => {
header.clear();
header.extend_from_slice(&entry[..parsed.size_end]);
header[0] = (header[0] & 0x8f) | (OFS_DELTA << 4);
offset_varint(&mut header, here - base_at);
writer.entry(&[&header, &entry[parsed.data..]])?
}
};
written[at as usize] = here;
}
let entries = Filtered {
index: &index,
kept: (0..count)
.filter(|&at| written[at as usize] != u64::MAX)
.collect(),
written: &written,
crcs: &crcs,
};
writer.finish(&entries).map(Some)
}
pub(crate) type Placed = Vec<(gix::ObjectId, u64)>;
pub(crate) fn rebase(
pack: &Path,
format: ObjectFormat,
start: u64,
outside: &dyn Fn(&gix::hash::oid) -> Option<u64>,
) -> anyhow::Result<(Vec<u8>, Placed)> {
let index_path = pack.with_extension("idx");
let index = index::File::at(&index_path, format.kind())
.with_context(|| format!("open {}", index_path.display()))?;
let bytes = std::fs::read(pack).with_context(|| format!("read {}", pack.display()))?;
let count = index.num_objects();
let end = (bytes.len())
.checked_sub(format.kind().len_in_bytes())
.with_context(|| format!("{} is too short for a pack", pack.display()))?;
anyhow::ensure!(
bytes.starts_with(&crate::reply_pack::pack_header(count)),
"{} isn't a version 2 pack of the {count} objects its index names",
pack.display()
);
let mut order: Vec<u32> = (0..count).collect();
order.sort_unstable_by_key(|&at| index.pack_offset_at_index(at));
let offsets: Vec<u64> = order
.iter()
.map(|&at| index.pack_offset_at_index(at))
.collect();
anyhow::ensure!(
offsets.first().is_none_or(|&first| first == 12),
"{}'s first entry isn't after its header",
pack.display()
);
let mut out = Vec::with_capacity(end);
let mut placed = Vec::with_capacity(order.len());
let mut header = Vec::new();
for k in 0..order.len() {
let next = offsets.get(k + 1).map_or(end as u64, |&next| next);
let entry = (bytes.get(usize::try_from(offsets[k])?..usize::try_from(next)?))
.with_context(|| format!("{} ends inside an entry", pack.display()))?;
let parsed = Entry::parse(entry, format)?;
let here = start + out.len() as u64;
let base = match parsed.base {
Base::None => None,
Base::Offset(distance) => {
let offset = offsets[k]
.checked_sub(distance)
.context("a delta base before the pack")?;
let base_k = offsets
.binary_search(&offset)
.map_err(|_| anyhow::anyhow!("a delta base at no entry's offset"))?;
Some(*placed.get(base_k).context("a delta that is its own base")?)
}
Base::Id(oid) => Some(match index.lookup(oid) {
Some(base_at) => {
let base_k = offsets
.binary_search(&index.pack_offset_at_index(base_at))
.expect("an indexed offset");
*placed.get(base_k).context("a ref delta before its base")?
}
None => outside(oid).with_context(|| format!("no base {oid} before the pack"))?,
}),
};
match base {
None => out.extend_from_slice(entry),
Some(base) => {
header.clear();
header.extend_from_slice(&entry[..parsed.size_end]);
header[0] = (header[0] & 0x8f) | (OFS_DELTA << 4);
let distance = here.checked_sub(base).filter(|d| *d > 0);
offset_varint(
&mut header,
distance.context("a delta base after its delta")?,
);
out.extend_from_slice(&header);
out.extend_from_slice(&entry[parsed.data..]);
}
}
placed.push(here);
}
let objects = order
.iter()
.zip(&placed)
.map(|(&at, &offset)| (index.oid_at_index(at).to_owned(), offset))
.collect();
Ok((out, objects))
}
struct Entry<'a> {
size_end: usize,
base: Base<'a>,
data: usize,
}
enum Base<'a> {
None,
Offset(u64),
Id(&'a gix::hash::oid),
}
impl<'a> Entry<'a> {
fn parse(entry: &'a [u8], format: ObjectFormat) -> anyhow::Result<Self> {
let byte = |at: usize| entry.get(at).copied().context("an entry cut short");
let kind = (byte(0)? >> 4) & 7;
let mut at = 0;
while byte(at)? & 0x80 != 0 {
at += 1;
}
let size_end = at + 1;
let (base, data) = match kind {
OFS_DELTA => {
let mut at = size_end;
let mut b = byte(at)?;
let mut distance = u64::from(b & 0x7f);
while b & 0x80 != 0 {
at += 1;
b = byte(at)?;
distance = (distance.checked_add(1).and_then(|d| d.checked_mul(128)))
.context("an offset delta's distance past any pack")?
| u64::from(b & 0x7f);
}
(Base::Offset(distance), at + 1)
}
REF_DELTA => {
let end = size_end + format.kind().len_in_bytes();
let oid = entry.get(size_end..end).context("an entry cut short")?;
(Base::Id(gix::hash::oid::from_bytes_unchecked(oid)), end)
}
1..=4 => (Base::None, size_end),
_ => anyhow::bail!("an entry of type {kind}"),
};
Ok(Self {
size_end,
base,
data,
})
}
}
pub(crate) fn whole_entry(kind: gix::object::Kind, data: &[u8]) -> std::io::Result<Vec<u8>> {
let mut header = Vec::new();
size_header(&mut header, kind_code(kind), data.len() as u64);
let mut z = flate2::write::ZlibEncoder::new(header, flate2::Compression::default());
z.write_all(data)?;
z.finish()
}
fn size_header(out: &mut Vec<u8>, kind: u8, mut size: u64) {
out.push((kind << 4) | (size & 0x0f) as u8);
size >>= 4;
while size > 0 {
*out.last_mut().unwrap() |= 0x80;
out.push((size & 0x7f) as u8);
size >>= 7;
}
}
fn offset_varint(out: &mut Vec<u8>, mut distance: u64) {
let mut bytes = vec![(distance & 0x7f) as u8];
distance >>= 7;
while distance > 0 {
distance -= 1;
bytes.push(0x80 | (distance & 0x7f) as u8);
distance >>= 7;
}
out.extend(bytes.iter().rev());
}
fn kind_code(kind: gix::object::Kind) -> u8 {
match kind {
gix::object::Kind::Commit => 1,
gix::object::Kind::Tree => 2,
gix::object::Kind::Blob => 3,
gix::object::Kind::Tag => 4,
}
}
struct Filtered<'a> {
index: &'a index::File,
kept: Vec<u32>,
written: &'a [u64],
crcs: &'a [u32],
}
impl Filtered<'_> {
fn len(&self) -> u32 {
self.kept.len() as u32
}
fn oid(&self, at: u32) -> &gix::hash::oid {
self.index.oid_at_index(self.kept[at as usize])
}
fn crc32(&self, at: u32) -> u32 {
self.crcs[self.kept[at as usize] as usize]
}
fn offset(&self, at: u32) -> u64 {
self.written[self.kept[at as usize] as usize]
}
fn write_idx(&self, checksum: &gix::hash::oid, file: File) -> anyhow::Result<()> {
let mut out = Hashed::new(file, ObjectFormat::from_kind(checksum.kind()));
out.write(b"\xfftOc\0\0\0\x02")?;
let mut fanout = [0u32; 256];
for at in 0..self.len() {
fanout[usize::from(self.oid(at).as_bytes()[0])] += 1;
}
let mut total = 0u32;
for count in fanout {
total += count;
out.write(&total.to_be_bytes())?;
}
for at in 0..self.len() {
out.write(self.oid(at).as_bytes())?;
}
for at in 0..self.len() {
out.write(&self.crc32(at).to_be_bytes())?;
}
let mut large = Vec::new();
for at in 0..self.len() {
let offset = self.offset(at);
let small = if offset <= MAX_SMALL_OFFSET {
offset as u32
} else {
large.push(offset);
0x8000_0000 | u32::try_from(large.len() - 1).context("too many large offsets")?
};
out.write(&small.to_be_bytes())?;
}
for offset in large {
out.write(&offset.to_be_bytes())?;
}
out.write(checksum.as_bytes())?;
out.finish()
}
}
/// git's `index-pack --fix-thin` refers to bases by id, these packs by offset.
/// it reads one byte per entry, the type, not the whole pack.
pub(crate) fn has_ref_deltas(path: &Path, format: ObjectFormat) -> anyhow::Result<bool> {
let idx = path.with_extension("idx");
let index =
index::File::at(&idx, format.kind()).with_context(|| format!("open {}", idx.display()))?;
let mut offsets: Vec<u64> = (0..index.num_objects())
.map(|at| index.pack_offset_at_index(at))
.collect();
offsets.sort_unstable();
let file = File::open(path).with_context(|| format!("open {}", path.display()))?;
let mut pack = BufReader::with_capacity(8 << 20, file);
let mut at = 0u64;
let mut first = [0u8];
for offset in offsets {
let skip = (offset.checked_sub(at))
.and_then(|skip| i64::try_from(skip).ok())
.context("two entries at one offset")?;
pack.seek_relative(skip)
.and_then(|()| pack.read_exact(&mut first))
.with_context(|| format!("read {} at {offset}", path.display()))?;
at = offset + 1;
if (first[0] >> 4) & 7 == REF_DELTA {
return Ok(true);
}
}
Ok(false)
}
/// a linked pack can lack a `.rev` in its source, and one is written beside the idx.
pub(crate) fn write_rev_of(idx: &Path, format: ObjectFormat) -> anyhow::Result<()> {
let index =
index::File::at(idx, format.kind()).with_context(|| format!("open {}", idx.display()))?;
let dir = idx.parent().context("an index outside a directory")?;
let mut staged = Staged(Vec::new());
let file = staged.file(dir, "rev")?;
let offset = |at| index.pack_offset_at_index(at);
write_rev(index.num_objects(), offset, &index.pack_checksum(), file)?;
let to = idx.with_extension("rev");
std::fs::rename(&staged.0[0], &to).with_context(|| format!("install {}", to.display()))?;
staged.0.clear();
Ok(())
}
fn write_rev(
count: u32,
offset: impl Fn(u32) -> u64,
checksum: &gix::hash::oid,
file: File,
) -> anyhow::Result<()> {
let mut positions: Vec<u32> = (0..count).collect();
positions.sort_unstable_by_key(|&at| offset(at));
let mut out = Hashed::new(file, ObjectFormat::from_kind(checksum.kind()));
out.write(b"RIDX\0\0\0\x01")?;
let hash_id = crate::midx_rev::hash_id(ObjectFormat::from_kind(checksum.kind()));
out.write(&hash_id.to_be_bytes())?;
for at in positions {
out.write(&at.to_be_bytes())?;
}
out.write(checksum.as_bytes())?;
out.finish()
}
struct Hashed {
out: BufWriter<File>,
hasher: Hasher,
}
impl Hashed {
fn new(file: File, format: ObjectFormat) -> Self {
Self {
out: BufWriter::with_capacity(1 << 20, file),
hasher: Hasher::new(format),
}
}
fn write(&mut self, bytes: &[u8]) -> anyhow::Result<()> {
self.hasher.update(bytes);
self.out.write_all(bytes).context("write an index")
}
fn finish(mut self) -> anyhow::Result<()> {
let hash = self.hasher.finish();
self.out.write_all(&hash).context("write an index")?;
self.out.flush().context("write an index")
}
}
#[cfg(test)]
#[path = "tests/packfile.rs"]
mod tests;