use std::path::{Path, PathBuf};
use anyhow::Context as _;
use gix::bstr::BString;
use gix::hash::ObjectId;
use crate::git_export::{git_command, run_cmd, stdin_bytes};
use crate::store::RepoStore;
use super::partitions::{PartitionPack, Partitions, REF_DELTAS_KIND, Segment};
pub(super) struct Marked {
pub(super) packs: Vec<MarkedPack>,
pub(super) unpacked: Segment,
}
pub(super) struct MarkedPack {
pub(super) path: PathBuf,
pub(super) len: u32,
pub(super) bits: Vec<u64>,
pub(super) ref_deltas: bool,
}
impl MarkedPack {
pub(super) fn has(&self, at: u32) -> bool {
self.bits[at as usize / 64] & (1 << (at % 64)) != 0
}
/// a pack with ref deltas can't be linked because a reply frames deltas by offset.
pub(super) fn linkable(&self) -> bool {
!self.ref_deltas && self.bits.iter().map(|word| word.count_ones()).sum::<u32>() == self.len
}
}
impl Marked {
/// an object two packs hold is marked in the first one only.
pub(super) fn of(
source: &Path,
tips: &[ObjectId],
format: knot_types::ObjectFormat,
store: Option<&RepoStore>,
) -> anyhow::Result<Self> {
let dir = source.join("objects/pack");
let mut paths: Vec<PathBuf> = match std::fs::read_dir(&dir) {
Ok(entries) => entries
.map(|entry| entry.map(|entry| entry.path()))
.collect::<Result<_, _>>()?,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
Err(error) => return Err(error).with_context(|| format!("list {}", dir.display())),
};
paths.retain(|path| path.extension().is_some_and(|ext| ext == "pack"));
paths.sort();
let mut packs = Vec::new();
let mut indexes = Vec::new();
for path in paths {
let idx = path.with_extension("idx");
// a pack without its .idx is still arriving, since git installs the .idx last.
if !idx.is_file() {
continue;
}
let index = gix::odb::pack::index::File::at(&idx, format.kind())
.with_context(|| format!("open {}", idx.display()))?;
let len = index.num_objects();
packs.push(MarkedPack {
path,
len,
bits: vec![0; (len as usize).div_ceil(64)],
ref_deltas: false,
});
indexes.push(index);
}
let scanned: Vec<(PathBuf, String)> = (packs.iter().zip(&indexes))
.map(|(pack, index)| (pack.path.clone(), index.pack_checksum().to_string()))
.collect();
// the scan takes seconds on a big pack, which is why it runs beside the walk.
let (unpacked, ref_deltas) = std::thread::scope(|scope| {
let scan = scope.spawn(|| {
(scanned.iter())
.map(|(path, checksum)| ref_deltas(store, path, checksum, format))
.collect::<anyhow::Result<Vec<bool>>>()
});
let unpacked = mark(source, tips, format.kind(), &indexes, &mut packs);
(unpacked, scan.join().expect("the ref delta scan panicked"))
});
for (pack, ref_deltas) in packs.iter_mut().zip(ref_deltas?) {
pack.ref_deltas = ref_deltas;
}
Ok(Self {
packs,
unpacked: unpacked?,
})
}
}
fn ref_deltas(
store: Option<&RepoStore>,
path: &Path,
checksum: &str,
format: knot_types::ObjectFormat,
) -> anyhow::Result<bool> {
if let Some(filed) = store.and_then(|store| store.read_file(REF_DELTAS_KIND, checksum)) {
return Ok(filed);
}
let scanned = crate::packfile::has_ref_deltas(path, format)?;
if let Some(store) = store {
store.publish_file(REF_DELTAS_KIND, checksum, &scanned)?;
}
Ok(scanned)
}
pub(crate) fn filed_ref_deltas(
store: &RepoStore,
entry: &Path,
format: knot_types::ObjectFormat,
) -> Vec<PathBuf> {
let Ok(Some(partitions)) = Partitions::read(entry) else {
return Vec::new();
};
(partitions.public().iter().chain(partitions.cob()))
.filter_map(|pack| {
gix::odb::pack::index::File::at(pack.path(entry, "idx"), format.kind()).ok()
})
.map(|index| store.file_path(REF_DELTAS_KIND, &index.pack_checksum().to_string()))
.collect()
}
/// --use-bitmap-index answers from the bitmap instead of reading every tree.
pub(crate) const MARK_WALK: &[&str] = &[
"-c",
"core.packedGitLimit=256m",
"-c",
"core.packedGitWindowSize=32m",
"rev-list",
"--objects",
"--no-object-names",
"--use-bitmap-index",
"--stdin",
];
fn mark(
source: &Path,
tips: &[ObjectId],
kind: gix::hash::Kind,
indexes: &[gix::odb::pack::index::File],
packs: &mut [MarkedPack],
) -> anyhow::Result<Segment> {
// size it once, the walk names at most the packed objects plus loose ones.
let packed: u32 = indexes.iter().map(|index| index.num_objects()).sum();
Ok(match kind {
gix::hash::Kind::Sha1 => {
mark_held(reach::<20>(source, tips, packed as usize)?, indexes, packs)
}
gix::hash::Kind::Sha256 => {
mark_held(reach::<32>(source, tips, packed as usize)?, indexes, packs)
}
other => anyhow::bail!("no mark for {other:?} ids"),
})
}
pub(crate) fn reach<const N: usize>(
source: &Path,
tips: &[ObjectId],
capacity: usize,
) -> anyhow::Result<Vec<[u8; N]>> {
match bitmap_reach(source, tips)? {
Some(ids) => Ok(ids),
None => walk(source, tips, capacity),
}
}
/// gix can read through a replace ref by its own config, git's walk never does.
/// that's why a repo with one takes the walk.
pub(super) fn bitmap_reach<const N: usize>(
source: &Path,
tips: &[ObjectId],
) -> anyhow::Result<Option<Vec<[u8; N]>>> {
let repo = knot_git::Repo::open(source).context("open the source repo")?;
let refs = repo.references().context("list the source's refs")?;
if (refs.iter()).any(|record| record.name.as_str().starts_with("refs/replace/")) {
return Ok(None);
}
let wants: Vec<knot_types::Oid> = tips.iter().map(|tip| knot_types::Oid::from(*tip)).collect();
let none = knot_git::Haves::new(&[]);
// knot_git can't read git's bitmaps without a lookup table, or one with a bad checksum.
let reached = match crate::bitmap::reachable_via_bitmap(
&repo,
knot_git::Wants::new(&wants),
none,
) {
Ok(Some(reached)) => reached,
Ok(None) => return Ok(None),
Err(error) => {
tracing::warn!(source = %source.display(), "the mark walks, knot_git can't read the bitmap: {error}");
return Ok(None);
}
};
(reached.iter())
.map(|oid| {
(oid.object_id().as_bytes().try_into())
.with_context(|| format!("{oid} isn't an id of {N} bytes"))
})
.collect::<anyhow::Result<_>>()
.map(Some)
}
fn walk<const N: usize>(
source: &Path,
tips: &[ObjectId],
capacity: usize,
) -> anyhow::Result<Vec<[u8; N]>> {
let revs: String = tips.iter().map(|tip| format!("{tip}\n")).collect();
let mut walk = git_command(source);
walk.args(MARK_WALK);
let errors = tempfile::tempfile().context("create stderr file")?;
walk.stdin(stdin_bytes(revs.as_bytes())?)
.stdout(std::process::Stdio::piped())
.stderr(errors.try_clone()?);
let mut child = walk.spawn().context("spawn git rev-list")?;
let stdout = child.stdout.take().context("git rev-list's stdout")?;
let listed = std::io::BufReader::with_capacity(1 << 20, stdout);
let ids = read_ids(listed, capacity)?;
let status = child.wait().context("wait for git rev-list")?;
if !status.success() {
use std::io::{Read as _, Seek as _};
let mut errors = errors;
let mut text = String::new();
errors.rewind()?;
errors.read_to_string(&mut text)?;
anyhow::bail!("git rev-list --objects failed ({status}): {}", text.trim());
}
Ok(ids)
}
/// bare digests, not `ObjectId`s, which are sized for the longer hash.
pub(super) fn read_ids<const N: usize>(
mut reader: impl std::io::BufRead,
capacity: usize,
) -> anyhow::Result<Vec<[u8; N]>> {
let mut ids = Vec::with_capacity(capacity);
let mut line = Vec::new();
loop {
line.clear();
if reader
.read_until(b'\n', &mut line)
.context("read git rev-list")?
== 0
{
return Ok(ids);
}
let hex = line.strip_suffix(b"\n").unwrap_or(&line);
let oid =
ObjectId::from_hex(hex).with_context(|| format!("oid {:?}", BString::from(hex)))?;
let id = oid
.as_bytes()
.try_into()
.with_context(|| format!("{oid} isn't an id of {N} bytes"))?;
ids.push(id);
}
}
pub(super) fn mark_held<const N: usize>(
mut ids: Vec<[u8; N]>,
indexes: &[gix::odb::pack::index::File],
packs: &mut [MarkedPack],
) -> Segment {
// sorted, each index is walked once, where a lookup per id was most of the mark.
ids.sort_unstable();
for (index, pack) in indexes.iter().zip(packs.iter_mut()) {
let (mut at, len) = (0, index.num_objects());
ids.retain(|id| {
while at < len && index.oid_at_index(at).as_bytes() < id.as_slice() {
at += 1;
}
let held = at < len && index.oid_at_index(at).as_bytes() == id.as_slice();
if held {
pack.bits[at as usize / 64] |= 1 << (at % 64);
}
!held
});
}
let mut unpacked = Segment::default();
for id in &ids {
unpacked.push(gix::hash::oid::from_bytes_unchecked(id), None);
}
unpacked
}
/// a missing .rev is written here, or every git cut reusing the pack builds one in memory.
pub(super) fn link_pack(
path: &Path,
export_dir: &Path,
format: knot_types::ObjectFormat,
) -> anyhow::Result<PartitionPack> {
let stem = path
.file_stem()
.and_then(|stem| stem.to_str())
.context("a pack name that isn't utf-8")?;
let dir = export_dir.join("objects/pack");
for ext in ["pack", "idx", "rev"] {
let from = path.with_extension(ext);
let to = dir.join(format!("{stem}.{ext}"));
if ext == "rev" && !from.exists() {
crate::packfile::write_rev_of(&to.with_extension("idx"), format)?;
continue;
}
std::fs::hard_link(&from, &to)
.or_else(|_| {
let source = std::fs::File::open(&from)?;
let target = std::fs::File::create(&to)?;
crate::reply_pack::reflink_or_copy(&source, &target)?;
crate::datastore::write::start_writeback(&target);
std::io::Result::Ok(())
})
.with_context(|| format!("link {} to {}", from.display(), to.display()))?;
}
PartitionPack::new(export_dir, stem)
}
pub(super) fn pack_into(
source: &Path,
export_dir: &Path,
segment: &Segment,
) -> anyhow::Result<Option<PartitionPack>> {
if segment.is_empty() {
return Ok(None);
}
write_pack(source, export_dir, &[], segment.object_list()).map(Some)
}
pub(super) fn write_pack(
source: &Path,
export_dir: &Path,
args: &[&str],
stdin: &[u8],
) -> anyhow::Result<PartitionPack> {
let base = std::path::absolute(export_dir.join("objects/pack/pack"))?;
let base = base.to_str().context("a pack path that isn't utf-8")?;
let hash = pack_objects(source, &[args, &[base]].concat(), stdin)?;
PartitionPack::new(
export_dir,
&format!("pack-{}", String::from_utf8_lossy(&hash).trim()),
)
}
pub(crate) fn pack_objects(source: &Path, args: &[&str], stdin: &[u8]) -> anyhow::Result<Vec<u8>> {
let mut pack = git_command(source);
// a kernel-sized pack maps only part of itself at once, hence the window limits.
pack.args([
"-c",
"core.packedGitLimit=256m",
"-c",
"core.packedGitWindowSize=32m",
]);
pack.args(["pack-objects", "-q", "--delta-base-offset"]);
pack.args(args);
pack.stdin(stdin_bytes(stdin)?);
Ok(run_cmd(pack, "git pack-objects")?.stdout)
}