use std::os::unix::fs::FileExt as _;
use std::path::Path;
use anyhow::Context as _;
use knot_types::ObjectFormat;
const HEADER: u64 = 12;
const ENTRY: usize = 12;
/// this copies the reverse index out of the midx, since git writes no `.rev` for it.
/// a chunk that doesn't name each object once is refused.
pub(crate) fn complete(pack_dir: &Path, format: ObjectFormat) -> anyhow::Result<()> {
let path = pack_dir.join("multi-pack-index");
let midx = match std::fs::File::open(&path) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error).with_context(|| format!("open {}", path.display())),
};
let hash = format.kind().len_in_bytes();
let body = (midx.metadata()?.len())
.checked_sub(hash as u64)
.context("a truncated multi-pack index")?;
let mut header = [0; HEADER as usize];
midx.read_exact_at(&mut header, 0)
.context("read the multi-pack index's header")?;
anyhow::ensure!(&header[..4] == b"MIDX", "not a multi-pack index");
anyhow::ensure!(
u32::from(header[5]) == hash_id(format),
"a multi-pack index of another hash"
);
anyhow::ensure!(header[7] == 0, "a multi-pack index with a base");
let mut table = vec![0; (usize::from(header[6]) + 1) * ENTRY];
midx.read_exact_at(&mut table, HEADER)
.context("read the multi-pack index's chunks")?;
let entries: Vec<(&[u8], u64)> = table
.chunks_exact(ENTRY)
.map(|entry| {
(
&entry[..4],
u64::from_be_bytes(entry[4..].try_into().unwrap()),
)
})
.collect();
let chunk = |id: &[u8; 4]| {
entries
.windows(2)
.find(|pair| pair[0].0 == id)
.map(|pair| pair[0].1..pair[1].1)
};
let Some(ridx) = chunk(b"RIDX") else {
return Ok(());
};
let fanout = chunk(b"OIDF").context("a multi-pack index without its fanout")?;
anyhow::ensure!(
fanout.end.checked_sub(fanout.start) == Some(256 * 4),
"a short fanout"
);
let mut objects = [0; 4];
midx.read_exact_at(&mut objects, fanout.start + 255 * 4)?;
let objects = u32::from_be_bytes(objects);
anyhow::ensure!(
ridx.start <= ridx.end
&& ridx.end <= body
&& ridx.end - ridx.start == 4 * u64::from(objects),
"the RIDX chunk isn't one position per object"
);
let mut checksum = vec![0; hash];
midx.read_exact_at(&mut checksum, body)?;
let rev = pack_dir.join(format!(
"multi-pack-index-{}.rev",
gix::ObjectId::from_bytes_or_panic(&checksum)
));
if rev.exists() {
return Ok(());
}
let mut out = Vec::with_capacity(HEADER as usize + 4 * objects as usize + 2 * hash);
out.extend_from_slice(b"RIDX");
out.extend_from_slice(&1u32.to_be_bytes());
out.extend_from_slice(&hash_id(format).to_be_bytes());
out.resize(HEADER as usize + 4 * objects as usize, 0);
midx.read_exact_at(&mut out[HEADER as usize..], ridx.start)
.context("read the RIDX chunk")?;
anyhow::ensure!(
each_once(&out[HEADER as usize..], objects),
"the RIDX chunk names a position twice or past the index"
);
out.extend_from_slice(&checksum);
let mut hasher = crate::reply_pack::Hasher::new(format);
hasher.update(&out);
out.extend_from_slice(&hasher.finish());
crate::datastore::write::publish(&rev, &out, crate::datastore::Durability::Rename)
.with_context(|| format!("write {}", rev.display()))
}
/// `1` is sha1 and `2` is sha256, in a midx and a `.rev` both.
pub(crate) fn hash_id(format: ObjectFormat) -> u32 {
if format == ObjectFormat::SHA1 { 1 } else { 2 }
}
fn each_once(table: &[u8], objects: u32) -> bool {
let mut seen = vec![0u64; (objects as usize).div_ceil(64)];
table.chunks_exact(4).all(|entry| {
let at = u32::from_be_bytes(entry.try_into().unwrap());
let (word, bit) = (at as usize / 64, 1u64 << (at % 64));
let fresh = at < objects && seen[word] & bit == 0;
if fresh {
seen[word] |= bit;
}
fresh
})
}
#[cfg(test)]
pub(crate) mod tests {
use std::collections::BTreeSet;
use std::io::Write as _;
use std::path::{Path, PathBuf};
use super::*;
pub(crate) fn git(repo: &Path, args: &[&str], stdin: &str) -> String {
let mut child = crate::test_git()
.current_dir(repo)
.args(args)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.unwrap();
child
.stdin
.take()
.unwrap()
.write_all(stdin.as_bytes())
.unwrap();
let output = child.wait_with_output().unwrap();
assert!(
output.status.success(),
"git {args:?}: {}",
String::from_utf8_lossy(&output.stderr)
);
String::from_utf8(output.stdout).unwrap().trim().to_string()
}
pub(crate) fn push(repo: &Path, branch: &str, name: &str) {
let blob = git(
repo,
&["hash-object", "-w", "--stdin"],
&format!("{name}\n"),
);
let tree = git(repo, &["mktree"], &format!("100644 blob {blob}\t{name}\n"));
let parent = git(
repo,
&["for-each-ref", "--format=%(objectname)", branch],
"",
);
let mut args = vec!["commit-tree", &tree, "-m", name];
if !parent.is_empty() {
args.extend(["-p", &parent]);
}
let commit = git(repo, &args, "");
git(repo, &["update-ref", branch, &commit], "");
git(repo, &["repack", "-dq"], "");
}
fn bitmapped(dir: &Path) -> PathBuf {
let repo = dir.join("repo.git");
std::fs::create_dir(&repo).unwrap();
git(&repo, &["init", "-q", "--bare", "-b", "main"], "");
for name in ["a", "b", "c", "d"] {
push(&repo, "refs/heads/main", name);
}
git(&repo, &["multi-pack-index", "write", "--bitmap"], "");
repo
}
fn previous(repo: &Path) -> Option<crate::bitmap::BitmapIndex> {
crate::bitmap::open_bitmap_index(&knot_git::Repo::open(repo).unwrap()).unwrap()
}
fn objects(repo: &Path, bitmap: bool) -> BTreeSet<String> {
let mut args = vec!["rev-list", "--objects", "--no-object-names", "main"];
if bitmap {
args.push("--use-bitmap-index");
}
git(repo, &args, "").lines().map(str::to_string).collect()
}
#[test]
fn a_bitmap_git_wrote_is_the_previous_of_knot2s() {
let dir = tempfile::tempdir().unwrap();
let repo = bitmapped(dir.path());
assert!(previous(&repo).is_none(), "knot_git reads no RIDX chunk");
complete(&repo.join("objects/pack"), ObjectFormat::SHA1).unwrap();
let previous = previous(&repo).expect("the bitmap git wrote");
push(&repo, "refs/heads/main", "e");
git(&repo, &["repack", "-dq", "--geometric=2"], "");
git(&repo, &["multi-pack-index", "write"], "");
let opened = knot_git::Repo::open(&repo).unwrap();
assert!(crate::bitmap::write_midx_bitmap(&opened, Some(&previous)).unwrap());
assert_eq!(objects(&repo, true), objects(&repo, false));
git(&repo, &["multi-pack-index", "verify"], "");
}
#[test]
fn a_ridx_chunk_short_or_repeating_is_refused() {
type Damage = fn(&std::fs::File, u64, u64);
let damages: [(&str, Damage); 2] = [
("short", |midx, entry, _| {
let mut start = [0; 8];
midx.read_exact_at(&mut start, entry + 4).unwrap();
let start = u64::from_be_bytes(start) + 4;
midx.write_all_at(&start.to_be_bytes(), entry + 4).unwrap();
}),
("repeating", |midx, _, start| {
let mut first = [0; 4];
midx.read_exact_at(&mut first, start).unwrap();
midx.write_all_at(&first, start + 4).unwrap();
}),
];
for (name, damage) in damages {
let dir = tempfile::tempdir().unwrap();
let repo = bitmapped(dir.path());
let pack_dir = repo.join("objects/pack");
let path = pack_dir.join("multi-pack-index");
let bytes = std::fs::read(&path).unwrap();
let entry = (0..usize::from(bytes[6]))
.map(|n| HEADER as usize + n * ENTRY)
.find(|at| &bytes[*at..at + 4] == b"RIDX")
.unwrap();
let start = u64::from_be_bytes(bytes[entry + 4..entry + ENTRY].try_into().unwrap());
let midx = std::fs::OpenOptions::new()
.write(true)
.read(true)
.open(&path)
.unwrap();
damage(&midx, entry as u64, start);
assert!(complete(&pack_dir, ObjectFormat::SHA1).is_err(), "{name}");
assert!(previous(&repo).is_none(), "{name}");
}
}
}