use super::fetch_cobs::client_result;
use std::collections::BTreeMap;
use super::*;
use crate::git_export::{ExportPrior, FetchReplies};
use crate::snapshot::{PublicRef, RepoSnapshot};
use knot_types::{ObjectFormat, OwnerDid, RepoDid, RepoRkey};
pub(super) fn snapshot_of(source: &Path, did: &str, object_format: ObjectFormat) -> RepoSnapshot {
let public_refs = git_in(
source,
&[
"for-each-ref",
"--format=%(objectname) %(*objectname) %(refname)",
"refs/heads",
"refs/tags",
],
)
.lines()
.map(|line| {
let mut fields = line.splitn(3, ' ');
let (raw, peeled, name) = (fields.next(), fields.next(), fields.next());
PublicRef {
name: name.unwrap().into(),
raw: raw.unwrap().into(),
peeled: peeled.filter(|peeled| !peeled.is_empty()).map(Into::into),
}
})
.collect();
RepoSnapshot {
repo_did: RepoDid::new(did).unwrap(),
owner: OwnerDid::new("did:plc:owner").unwrap(),
rkey: RepoRkey::new("demo").unwrap(),
name: None,
head_capture: crate::snapshot::HeadCapture {
resolved: Some(git_in(source, &["rev-parse", "main"])),
symbolic: Some("refs/heads/main".into()),
unborn: false,
},
public_refs,
cob_refs: git_in(
source,
&[
"for-each-ref",
"--format=%(refname) %(objectname)",
"refs/cobs",
],
)
.lines()
.map(|line| {
let (name, raw) = line.split_once(' ').unwrap();
(name.into(), raw.into())
})
.collect(),
object_format,
}
}
/// hex hashes don't compress, which keeps the full pack bigger than the cut replies.
pub(super) fn ballast(source: &Path) {
let big: String = (0..4096u64)
.map(|i| format!("{:x}\n", i.wrapping_mul(0x9e37_79b9_7f4a_7c15)))
.collect();
std::fs::write(source.join("big.txt"), big).unwrap();
}
pub(super) fn commit(source: &Path, path: &str) {
std::fs::write(source.join(path), format!("{path}\n").repeat(50)).unwrap();
git_in(source, &["add", "-A"]);
git_in(source, &["commit", "-q", "-m", path]);
}
pub(super) fn client(dir: &Path, args: &[&str], trace: &Path) -> String {
let output = crate::test_git()
.current_dir(dir)
.args(["-c", "protocol.version=2", "-c", "fetch.unpackLimit=1"])
.args(["-c", &format!("http.extraHeader={FORWARDED_FOR}")])
.args(args)
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_TRACE_PACKET", trace)
.output()
.unwrap();
assert!(
output.status.success(),
"git {args:?}: {}",
String::from_utf8_lossy(&output.stderr)
);
String::from_utf8(output.stdout).unwrap()
}
pub(super) fn packs(repo: &Path) -> BTreeMap<String, u32> {
let dir = repo.join("objects/pack");
let dir = if dir.is_dir() {
dir
} else {
repo.join(".git/objects/pack")
};
std::fs::read_dir(dir)
.unwrap()
.flatten()
.filter(|entry| entry.path().extension().is_some_and(|ext| ext == "idx"))
.map(|entry| {
let idx = std::fs::read(entry.path()).unwrap();
// the v2 idx object count is the last fanout entry, at byte 8 + 255 * 4.
let count = u32::from_be_bytes(idx[8 + 255 * 4..8 + 256 * 4].try_into().unwrap());
(entry.file_name().to_string_lossy().into_owned(), count)
})
.collect()
}
pub(super) fn fetched_objects(repo: &Path, fetch: impl FnOnce()) -> u32 {
let before = packs(repo);
fetch();
let added: Vec<u32> = packs(repo)
.into_iter()
.filter(|(name, _)| !before.contains_key(name))
.map(|(_, count)| count)
.collect();
assert_eq!(added.len(), 1, "one pack fetched");
added[0]
}
pub(super) fn heads_and_tags(repo: &Path) -> String {
git_in(
repo,
&[
"for-each-ref",
"--format=%(objectname) %(refname)",
"refs/heads",
"refs/tags",
],
)
}
pub(super) fn new_objects(source: &Path, tips: &[&str], since: &[String]) -> u32 {
let mut args = vec!["rev-list", "--objects"];
args.extend(tips);
args.push("--not");
args.extend(since.iter().map(String::as_str));
git_in(source, &args).lines().count() as u32
}
pub(super) fn fetch_request(args: &[String], object_format: ObjectFormat) -> Vec<u8> {
let mut body = pkt("command=fetch\n");
body.extend(pkt("agent=knot/0\n"));
if object_format != ObjectFormat::SHA1 {
body.extend(pkt(&format!(
"object-format={}\n",
object_format.capability()
)));
}
body.extend_from_slice(b"0001");
for arg in args {
body.extend(pkt(&format!("{arg}\n")));
}
body.extend_from_slice(b"0000");
body
}
pub(super) fn knot_pack_args(wants: &[String], haves: &[String]) -> Vec<String> {
let mut args = vec!["no-progress".to_string(), "ofs-delta".to_string()];
args.extend(wants.iter().map(|want| format!("want {want}")));
args.extend(haves.iter().map(|have| format!("have {have}")));
args.push("done".into());
args
}
pub(super) fn packfile(reply: &[u8]) -> Vec<u8> {
let mut pack = Vec::new();
let mut in_pack = false;
let mut rest = reply;
while rest.len() >= 4 {
let len = usize::from_str_radix(std::str::from_utf8(&rest[..4]).unwrap(), 16).unwrap();
if len < 4 {
rest = &rest[4..];
continue;
}
let payload = &rest[4..len];
if in_pack {
if payload[0] == 1 {
pack.extend_from_slice(&payload[1..]);
}
} else {
in_pack = payload == b"packfile\n";
}
rest = &rest[len..];
}
pack
}
pub(super) struct Published {
pub(super) generation: PathBuf,
pub(super) export: PathBuf,
pub(super) prior: ExportPrior,
pub(super) replies: FetchReplies,
}
pub(super) fn publish(
config: &ValidatedConfig,
manifest: &mut GenerationManifest,
generation: &Path,
source: &Path,
did: &str,
object_format: ObjectFormat,
prior: Option<&ExportPrior>,
) -> Published {
let snapshot = snapshot_of(source, did, object_format);
let export = generation.join("www/_git").join(repo_token(did));
let store = crate::store::RepoStore::open(&generation.parent().unwrap().join("repo-store"));
let fingerprint = crate::git_export::export_fingerprint(&snapshot, "main", prior);
crate::git_export::materialize_export(
config,
&snapshot,
&export,
&fingerprint,
Some(&store),
prior,
)
.unwrap();
let replies = crate::git_export::fetch_replies(&export).unwrap();
let public_refs: Vec<(String, String)> = snapshot
.public_refs
.iter()
.map(|r| (r.name.clone(), r.raw.clone()))
.collect();
let mut entry = repo_entry(did);
entry.object_format = object_format.capability().into();
entry.public_refs = public_refs.clone();
entry.fetch_bases = replies.bases.clone();
entry.shallow_branches = replies.shallow_branches.clone();
entry.shallow_head = replies.shallow_head.clone();
entry.moved_wants = replies.moved_wants.clone();
manifest.repos = vec![entry];
crate::atproto::emit(generation, &[]).unwrap();
crate::blobs::emit(
generation.parent().unwrap(),
generation,
&[],
&Default::default(),
)
.unwrap();
super::super::emit_at(config, manifest, generation, generation).unwrap();
Published {
generation: generation.to_path_buf(),
export,
prior: ExportPrior {
fingerprint,
refs: public_refs.into_iter().collect(),
bases: replies.bases.clone(),
},
replies,
}
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn only_the_newest_base_carries_the_tags_aside() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:defghijklmnopqrstuvwxyzab";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = (temp.path().join("repos/plc"))
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
ballast(&source);
commit(&source, "a.txt");
git_in(&source, &["tag", "-a", "v1", "-m", "v1"]);
commit(&source, "b.txt");
let mut prior = None;
let mut heads = Vec::new();
for (n, path) in ["c.txt", "d.txt"].into_iter().enumerate() {
let published = publish(
&config,
&mut manifest,
&temp.path().join(format!("gen-{n}")),
&source,
did,
ObjectFormat::SHA1,
prior.as_ref(),
);
let url = format!("file://{}", published.export.display());
for name in [format!("plain-{n}"), format!("no-tags-{n}")] {
let mut args = vec!["clone", "-q"];
args.extend(name.starts_with("no-tags").then_some("--no-tags"));
git_in(
temp.path(),
&[args, vec![url.as_str(), name.as_str()]].concat(),
);
}
heads.push(vec![git_in(&source, &["rev-parse", "main"])]);
prior = Some(published.prior);
commit(&source, path);
git_in(&source, &["tag", "-a", &format!("v{}", n + 2), "-m", "v"]);
}
let third = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
ObjectFormat::SHA1,
prior.as_ref(),
);
assert_eq!(third.replies.bases, [heads[1].clone(), heads[0].clone()]);
assert!(!third.replies.moved_wants.is_empty());
let (port, _nginx) = start_nginx(&nginx_binary, &third.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let every: Vec<String> = (git_in(&source, &["tag"]).lines())
.map(|tag| format!("refs/tags/{tag}"))
.chain(["main".to_string()])
.collect();
let every: Vec<&str> = every.iter().map(String::as_str).collect();
let full = new_objects(&source, &every, &[]);
let trace = temp.path().join("fetch.trace");
let strict = ["-c", "fetch.fsckObjects=true"];
for (n, base) in [(1, &heads[1]), (0, &heads[0])] {
let moved = new_objects(&source, &["main", "refs/tags/v2", "refs/tags/v3"], base);
for (name, fetch) in [
(format!("plain-{n}"), vec!["fetch", "-q", "origin"]),
(
format!("no-tags-{n}"),
vec!["fetch", "-q", "--tags", "origin"],
),
] {
let dir = temp.path().join(&name);
git_in(&dir, &["remote", "set-url", "origin", &remote]);
let received = fetched_objects(&dir, || {
client(&dir, &[&strict[..], &fetch].concat(), &trace);
});
match (name.starts_with("plain"), n) {
(true, _) => assert_eq!(received, moved, "{name}"),
(false, 1) => assert!(moved < received && received < full, "{name}: {received}"),
(false, _) => assert_eq!(received, full, "{name}"),
}
git_in(&dir, &["fsck", "--strict"]);
}
}
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn fetches_of_what_moved_get_none_of_the_tags_aside() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:cdefghijklmnopqrstuvwxyz";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = (temp.path().join("repos/plc"))
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(source.join("old")).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
commit(&source, "old/x.txt");
let tree = git_in(&source, &["rev-parse", "HEAD^{tree}"]);
git_in(&source, &["tag", "-a", "t-tree", "-m", "a tree", &tree]);
git_in(&source, &["rm", "-q", "-r", "old"]);
commit(&source, "a.txt");
git_in(&source, &["tag", "-a", "v1", "-m", "v1"]);
git_in(&source, &["tag", "light"]);
git_in(&source, &["checkout", "-q", "-b", "feature"]);
commit(&source, "feature.txt");
git_in(&source, &["checkout", "-q", "main"]);
commit(&source, "b.txt");
let first = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
let first_heads: Vec<String> = ["main", "feature"]
.map(|branch| git_in(&source, &["rev-parse", branch]))
.into();
let url = format!("file://{}", first.export.display());
for (name, args) in [
("git", vec!["clone", "-q"]),
("mirror", vec!["clone", "-q", "--mirror"]),
("no-tags", vec!["clone", "-q", "--no-tags"]),
] {
git_in(temp.path(), &[args, vec![url.as_str(), name]].concat());
}
let (ok, stderr) = client_result(temp.path(), "gix", &["clone", &url, "gix"]);
assert!(ok, "gix clone: {stderr}");
commit(&source, "c.txt");
git_in(&source, &["tag", "-a", "v2", "-m", "v2"]);
let second = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
ObjectFormat::SHA1,
Some(&first.prior),
);
assert!(!second.replies.moved_wants.is_empty());
let (_port, nginx) = start_nginx(&nginx_binary, &second.generation);
let remote = format!("http://127.0.0.1:{_port}/{did}");
let moved = new_objects(&source, &["main", "refs/tags/v2"], &first_heads);
let tags = git_in(&source, &["tag"]);
let every: Vec<String> = tags.lines().map(|tag| format!("refs/tags/{tag}")).collect();
let every: Vec<&str> = every.iter().map(String::as_str).chain(["main"]).collect();
let with_aside = new_objects(&source, &every, &first_heads);
assert!(with_aside > moved);
let trace = temp.path().join("fetch.trace");
let strict = ["-c", "fetch.fsckObjects=true"];
for (name, fetch, objects) in [
("git", vec!["fetch", "-q", "origin"], moved),
(
"mirror",
vec!["fetch", "-q", "--prune", &remote, "+refs/*:refs/*"],
moved,
),
(
"no-tags",
vec!["fetch", "-q", "--tags", "origin"],
with_aside,
),
] {
let dir = temp.path().join(name);
git_in(&dir, &["remote", "set-url", "origin", &remote]);
let received = fetched_objects(&dir, || {
client(&dir, &[&strict[..], &fetch].concat(), &trace);
});
assert_eq!(received, objects, "{name}");
git_in(&dir, &["fsck", "--strict"]);
}
let dir = temp.path().join("gix");
git_in(&dir, &["remote", "set-url", "origin", &remote]);
let received = fetched_objects(&dir, || {
let (ok, stderr) = client_result(&dir, "gix", &["fetch"]);
assert!(ok, "gix fetch: {stderr}");
});
assert_eq!(received, moved, "gix");
git_in(&dir, &["fsck", "--strict"]);
drop(nginx);
git_in(&source, &["tag", "-a", "v0", "-m", "v0", "main~2"]);
git_in(&source, &["tag", "-a", "t-tree-2", "-m", "a tree", &tree]);
let third = publish(
&config,
&mut manifest,
&temp.path().join("gen-3"),
&source,
did,
ObjectFormat::SHA1,
Some(&second.prior),
);
for tag in ["v0", "t-tree-2"] {
let oid = git_in(&source, &["rev-parse", &format!("refs/tags/{tag}")]);
assert!(!third.replies.moved_wants.contains(&oid), "{tag}");
}
let (port, _nginx) = start_nginx(&nginx_binary, &third.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let dir = temp.path().join("git");
git_in(&dir, &["remote", "set-url", "origin", &remote]);
client(
&dir,
&[&strict[..], &["fetch", "-q", "--tags", "origin"]].concat(),
&trace,
);
git_in(&dir, &["fsck", "--strict"]);
for tag in ["v0", "t-tree-2"] {
let name = format!("refs/tags/{tag}");
assert_eq!(
git_in(&dir, &["rev-parse", &name]),
git_in(&source, &["rev-parse", &name]),
"{tag}"
);
}
}
#[test]
fn fetch_map_names_only_replies_the_export_has() {
let temp = tempfile::tempdir().unwrap();
let (_, mut manifest, _) = test_fixture(temp.path());
manifest.repos[0].public_refs = vec![("refs/heads/main".into(), "1".repeat(40))];
let rows = |manifest: &GenerationManifest| {
let map = super::super::maps::fetch_map(manifest)
.unwrap()
.block()
.to_string();
map.lines().filter(|line| line.contains("~^")).count()
};
assert_eq!(rows(&manifest), 0);
manifest.repos[0].shallow_branches = vec!["3".repeat(40)];
manifest.repos[0].fetch_bases = vec![vec!["2".repeat(40)]];
assert_eq!(rows(&manifest), 3);
manifest.repos[0].shallow_head = Some("1".repeat(40));
assert_eq!(rows(&manifest), 4);
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn fetches_get_the_pack_cut_for_what_the_client_has() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
for (format, object_format, did) in [
(
"sha1",
ObjectFormat::SHA1,
"did:plc:abcdefghijklmnopqrstuvwx",
),
(
"sha256",
ObjectFormat::SHA256,
"did:plc:bcdefghijklmnopqrstuvwxy",
),
] {
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = temp
.path()
.join("repos/plc")
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(
&source,
&[
"init",
"-q",
"-b",
"main",
&format!("--object-format={format}"),
],
);
// history outweighs the pushes, and every base still fits under the size bound.
let history: String = (0..20_000u64)
.map(|n| format!("{:x}\n", n.wrapping_mul(0x9e37_79b9_7f4a_7c15)))
.collect();
std::fs::write(source.join("history.txt"), history).unwrap();
commit(&source, "a.txt");
commit(&source, "b.txt");
git_in(&source, &["checkout", "-q", "-b", "feature"]);
commit(&source, "feature.txt");
git_in(&source, &["checkout", "-q", "main"]);
commit(&source, "b2.txt");
git_in(&source, &["tag", "-a", "v1", "-m", "v1"]);
let first = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
object_format,
None,
);
assert_eq!(first.replies.bases, Vec::<Vec<String>>::new());
let first_heads: Vec<String> = ["main", "feature"]
.map(|branch| git_in(&source, &["rev-parse", branch]))
.into();
let first_clone = temp.path().join("first");
git_in(
temp.path(),
&[
"clone",
"-q",
&format!("file://{}", first.export.display()),
first_clone.to_str().unwrap(),
],
);
commit(&source, "c.txt");
commit(&source, "d.txt");
let second = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
object_format,
Some(&first.prior),
);
let mut first_base = first_heads.clone();
first_base.sort();
assert_eq!(second.replies.bases, vec![first_base.clone()], "{format}");
assert_eq!(second.replies.shallow_branches.len(), 1, "{format}");
let (port, nginx) = start_nginx(&nginx_binary, &second.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let smart = |name: &str| std::fs::read(second.export.join("smart").join(name)).unwrap();
let main_tip = git_in(&source, &["rev-parse", "main"]);
git_in(&first_clone, &["remote", "set-url", "origin", &remote]);
let trace = temp.path().join("fetch.trace");
let received = fetched_objects(&first_clone, || {
client(&first_clone, &["fetch", "-q", "origin"], &trace);
});
assert_eq!(
received,
new_objects(&source, &[&main_tip], &first_heads),
"{format}: the fetch carried only what the first state lacks"
);
let trace = std::fs::read_to_string(&trace).unwrap();
assert_eq!(
trace.matches("fetch> command=fetch").count(),
1,
"{format}: {trace}"
);
assert!(trace.contains("fetch< ready"), "{format}: {trace}");
git_in(&first_clone, &["fsck", "--strict"]);
for branch in ["main", "feature"] {
assert_eq!(
git_in(&first_clone, &["rev-parse", &format!("origin/{branch}")]),
git_in(&source, &["rev-parse", branch]),
"{format}"
);
}
let wants: Vec<String> = manifest.repos[0]
.public_refs
.iter()
.map(|(_, oid)| oid.clone())
.collect();
let width = main_tip.len();
let mut haves: Vec<String> = (0..10_000).map(|n| format!("{n:0width$x}")).collect();
haves.extend(first_heads.iter().cloned());
let body = fetch_request(&knot_pack_args(&wants, &haves), object_format);
assert!(body.len() > 256 * 1024);
let (status, _, reply) = http(port, "POST", &format!("/{did}/git-upload-pack"), &[], &body);
assert_eq!(status, 200, "{format}");
assert_eq!(reply, smart("have-0-done"), "{format}: knot-pack shape");
let unpacked = temp.path().join("unpacked");
git_in(
temp.path(),
&[
"init",
"-q",
"--bare",
&format!("--object-format={format}"),
unpacked.to_str().unwrap(),
],
);
let pack_path = temp.path().join("incremental.pack");
std::fs::write(&pack_path, packfile(&reply)).unwrap();
let indexed = crate::test_git()
.current_dir(&unpacked)
.args(["index-pack", "--stdin"])
.stdin(std::fs::File::open(&pack_path).unwrap())
.output()
.unwrap();
assert!(
indexed.status.success(),
"{}",
String::from_utf8_lossy(&indexed.stderr)
);
let tips: Vec<&str> = wants.iter().map(String::as_str).collect();
assert_eq!(
packs(&unpacked).into_values().sum::<u32>(),
new_objects(&source, &tips, &first_heads),
"{format}"
);
let post = |args: &[String]| {
let body = fetch_request(args, object_format);
http(port, "POST", &format!("/{did}/git-upload-pack"), &[], &body).2
};
let stranger = vec!["1".repeat(width)];
let longer: Vec<String> = first_heads.iter().map(|tip| format!("{tip}0")).collect();
assert_eq!(
post(&knot_pack_args(&wants, &longer)),
smart("fetch-done"),
"{format}: a have is the whole oid"
);
assert_eq!(
post(&knot_pack_args(&wants, &stranger)),
smart("fetch-done"),
"{format}"
);
for (extra, canned) in [
(format!("shallow {}", first_heads[0]), "fetch-done"),
("deepen 2".to_string(), "fetch-done"),
("deepen 1".to_string(), "fetch-done"),
("deepen 2147483647".to_string(), "err-unshallow"),
("wait-for-done".to_string(), "fetch-nodone"),
] {
let mut args = knot_pack_args(&wants, &first_heads);
args.insert(0, extra.clone());
if canned == "fetch-nodone" {
args.pop();
}
assert_eq!(post(&args), smart(canned), "{format}: {extra}");
}
let shallow = temp.path().join("shallow");
let trace = temp.path().join("shallow.trace");
client(
temp.path(),
&[
"clone",
"-q",
"--depth=1",
&remote,
shallow.to_str().unwrap(),
],
&trace,
);
assert_eq!(
git_in(&shallow, &["rev-list", "--count", "HEAD"]),
"1",
"{format}"
);
assert_eq!(git_in(&shallow, &["rev-parse", "HEAD"]), main_tip);
assert_eq!(
std::fs::read_to_string(shallow.join(".git/shallow")).unwrap(),
format!("{main_tip}\n")
);
let feature_tip = git_in(&source, &["rev-parse", "feature"]);
assert!(
crate::test_git()
.current_dir(&shallow)
.args(["cat-file", "-e", &feature_tip])
.status()
.is_ok_and(|status| !status.success()),
"{format}"
);
git_in(&shallow, &["fsck", "--strict"]);
let both = temp.path().join("both");
git_in(
temp.path(),
&[
"init",
"-q",
&format!("--object-format={format}"),
both.to_str().unwrap(),
],
);
client(
&both,
&["fetch", "-q", "--depth=1", &remote, "main", "feature"],
&trace,
);
git_in(&both, &["cat-file", "-e", &main_tip]);
git_in(&both, &["cat-file", "-e", &feature_tip]);
git_in(&both, &["fsck", "--strict"]);
let filtered = temp.path().join("filtered");
client(
temp.path(),
&[
"clone",
"-q",
"--depth=1",
"--filter=blob:none",
&remote,
filtered.to_str().unwrap(),
],
&trace,
);
assert_eq!(
git_in(&filtered, &["rev-list", "--count", "HEAD"]),
"1",
"{format}"
);
git_in(&filtered, &["fsck", "--strict"]);
let deeper = temp.path().join("deeper");
client(
temp.path(),
&[
"clone",
"-q",
"--depth=2",
&remote,
deeper.to_str().unwrap(),
],
&trace,
);
assert_eq!(
git_in(&deeper, &["rev-list", "--count", "HEAD"]),
"5",
"{format}"
);
assert!(!deeper.join(".git/shallow").exists());
let first_main = &first_heads[0];
let pinned = temp.path().join("pinned");
git_in(
temp.path(),
&[
"init",
"-q",
&format!("--object-format={format}"),
pinned.to_str().unwrap(),
],
);
client(
&pinned,
&["fetch", "-q", "--depth=1", &remote, first_main],
&trace,
);
git_in(&pinned, &["cat-file", "-e", first_main]);
git_in(&pinned, &["fsck", "--strict"]);
let full_clone = temp.path().join("full");
client(
temp.path(),
&[
"clone",
"-q",
"--bare",
&remote,
full_clone.to_str().unwrap(),
],
&trace,
);
git_in(&full_clone, &["fsck", "--strict"]);
assert_eq!(heads_and_tags(&full_clone), heads_and_tags(&source));
drop(nginx);
let second_clone = temp.path().join("second");
git_in(
temp.path(),
&[
"clone",
"-q",
&format!("file://{}", second.export.display()),
second_clone.to_str().unwrap(),
],
);
git_in(&source, &["reset", "-q", "--hard", "HEAD~2"]);
commit(&source, "rewritten.txt");
git_in(&source, &["branch", "-D", "feature"]);
let third = publish(
&config,
&mut manifest,
&temp.path().join("gen-3"),
&source,
did,
object_format,
Some(&second.prior),
);
assert_eq!(
third.replies.bases,
vec![vec![first_heads[0].clone()]],
"{format}"
);
let (port, _nginx) = start_nginx(&nginx_binary, &third.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
for clone in [&first_clone, &second_clone] {
git_in(clone, &["remote", "set-url", "origin", &remote]);
client(
clone,
&["fetch", "-q", "--prune", "--force", "origin"],
&trace,
);
git_in(clone, &["fsck", "--strict"]);
assert_eq!(
git_in(clone, &["rev-parse", "origin/main"]),
git_in(&source, &["rev-parse", "main"]),
"{format}"
);
git_in(clone, &["checkout", "-q", "--detach", "origin/main"]);
assert!(clone.join("rewritten.txt").is_file());
}
let puller = temp.path().join("puller");
client(
temp.path(),
&[
"clone",
"-q",
"--depth=1",
&remote,
puller.to_str().unwrap(),
],
&trace,
);
drop(_nginx);
let rewritten = git_in(&source, &["rev-parse", "main"]);
for path in ["e.txt", "f.txt", "g.txt"] {
commit(&source, path);
}
let fourth = publish(
&config,
&mut manifest,
&temp.path().join("gen-4"),
&source,
did,
object_format,
Some(&third.prior),
);
assert_eq!(
fourth.replies.bases,
vec![vec![rewritten.clone()], vec![first_heads[0].clone()]],
"{format}"
);
let (port, _nginx) = start_nginx(&nginx_binary, &fourth.generation);
let tip = git_in(&source, &["rev-parse", "main"]);
let body = fetch_request(
&knot_pack_args(&[tip.clone()], &[first_heads[0].clone(), rewritten]),
object_format,
);
let (_, _, reply) = http(port, "POST", &format!("/{did}/git-upload-pack"), &[], &body);
assert_eq!(
reply,
std::fs::read(fourth.export.join("smart/moved-0-done")).unwrap(),
"{format}"
);
// a `--depth=1` pull sends its own shallow commit, which the canned reply can't serve.
let remote = format!("http://127.0.0.1:{port}/{did}");
git_in(&puller, &["remote", "set-url", "origin", &remote]);
client(
&puller,
&["pull", "-q", "--ff-only", "--depth=1", "origin", "main"],
&trace,
);
assert_eq!(git_in(&puller, &["rev-parse", "HEAD"]), tip, "{format}");
git_in(&puller, &["fsck", "--strict"]);
}
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn tags_of_trees_blobs_and_tags_are_fetched() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:defghijklmnopqrstuvwxyza";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = temp
.path()
.join("repos/plc")
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
// git's `^` doesn't exclude a tagged tree. the tag on the root tree still carries the ballast.
ballast(&source);
commit(&source, "seed.txt");
git_in(&source, &["rm", "-q", "big.txt"]);
std::fs::create_dir_all(source.join("d")).unwrap();
std::fs::write(source.join("d/sub.txt"), "sub\n").unwrap();
commit(&source, "a.txt");
let blob = git_in(&source, &["rev-parse", "HEAD:a.txt"]);
git_in(
&source,
&["tag", "-a", "tree-tag", "-m", "tree", "HEAD^{tree}"],
);
git_in(
&source,
&["tag", "-a", "subtree-tag", "-m", "subtree", "HEAD:d"],
);
git_in(&source, &["tag", "blob-light", &blob]);
git_in(
&source,
&["tag", "-a", "tag-of-tag", "-m", "tag", "tree-tag"],
);
let first = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
commit(&source, "b.txt");
let second = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
ObjectFormat::SHA1,
Some(&first.prior),
);
assert!(!second.replies.bases.is_empty());
let (port, _nginx) = start_nginx(&nginx_binary, &second.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let trace = temp.path().join("tags.trace");
let every_tag = |clone: &Path| {
git_in(clone, &["fsck", "--strict"]);
for tag in ["tree-tag", "subtree-tag", "blob-light", "tag-of-tag"] {
let name = format!("refs/tags/{tag}");
let oid = git_in(&source, &["rev-parse", &name]);
assert_eq!(git_in(clone, &["rev-parse", &name]), oid, "{tag}");
assert_eq!(
git_in(clone, &["cat-file", "-t", &oid]),
git_in(&source, &["cat-file", "-t", &oid]),
"{tag}"
);
}
};
let full = temp.path().join("full");
client(
temp.path(),
&["clone", "-q", &remote, full.to_str().unwrap()],
&trace,
);
every_tag(&full);
let shallow = temp.path().join("shallow");
client(
temp.path(),
&[
"clone",
"-q",
"--depth=1",
&remote,
shallow.to_str().unwrap(),
],
&trace,
);
client(
&shallow,
&[
"fetch",
"-q",
"--depth=1",
&remote,
"refs/tags/*:refs/tags/*",
],
&trace,
);
every_tag(&shallow);
let old = temp.path().join("old");
git_in(
temp.path(),
&[
"clone",
"-q",
&format!("file://{}", first.export.display()),
old.to_str().unwrap(),
],
);
git_in(&old, &["remote", "set-url", "origin", &remote]);
client(&old, &["fetch", "-q", "origin"], &trace);
every_tag(&old);
assert_eq!(
git_in(&old, &["rev-parse", "origin/main"]),
git_in(&source, &["rev-parse", "main"])
);
}