use std::collections::{BTreeMap, BTreeSet};
use super::fetch::*;
use super::*;
use crate::git_export::ExportPrior;
use knot_types::ObjectFormat;
fn cob_ref(source: &Path, name: &str, text: &str) {
let file = source.join(".git/cob-scratch");
std::fs::write(&file, text).unwrap();
let blob = git_in(source, &["hash-object", "-w", file.to_str().unwrap()]);
git_in(source, &["read-tree", "--empty"]);
git_in(
source,
&[
"update-index",
"--add",
"--cacheinfo",
&format!("100644,{blob},cob.txt"),
],
);
let tree = git_in(source, &["write-tree"]);
git_in(source, &["read-tree", "HEAD"]);
let commit = git_in(source, &["commit-tree", &tree, "-m", name]);
git_in(source, &["update-ref", name, &commit]);
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn a_repo_with_cobs_gets_the_incremental_reply() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:cdefghijklmnopqrstuvwxyz";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = temp
.path()
.join("repos/plc")
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
commit(&source, "a.txt");
git_in(&source, &["checkout", "-q", "-b", "feature"]);
commit(&source, "feature.txt");
git_in(&source, &["checkout", "-q", "main"]);
commit(&source, "b.txt");
for n in 0..6u64 {
let text: String = (0..60u64)
.map(|i| format!("{:x}\n", (n * 100 + i).wrapping_mul(0x9e37_79b9_7f4a_7c15)))
.collect();
cob_ref(
&source,
&format!("refs/cobs/org.tangled.track.ticket/{n}"),
&text,
);
}
let first = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
let clone = temp.path().join("clone");
git_in(
temp.path(),
&[
"clone",
"-q",
&format!("file://{}", first.export.display()),
clone.to_str().unwrap(),
],
);
commit(&source, "c.txt");
let second = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
ObjectFormat::SHA1,
Some(&first.prior),
);
let (port, _nginx) = start_nginx(&nginx_binary, &second.generation);
git_in(
&clone,
&[
"remote",
"set-url",
"origin",
&format!("http://127.0.0.1:{port}/{did}"),
],
);
let trace = temp.path().join("fetch.trace");
client(&clone, &["fetch", "-q", "origin"], &trace);
let trace = std::fs::read_to_string(&trace).unwrap();
assert!(trace.contains("fetch< ready"), "{trace}");
git_in(&clone, &["fsck", "--strict"]);
assert_eq!(
git_in(&clone, &["rev-parse", "origin/main"]),
git_in(&source, &["rev-parse", "main"])
);
}
/// the cgi takes gzipped fetches, because nginx can't read them.
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn a_gzipped_fetch_goes_to_the_cgi_when_there_is_one() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:efghijklmnopqrstuvwxyzab";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = temp
.path()
.join("repos/plc")
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
commit(&source, "a.txt");
let post = |generation: &Path| {
let (port, _nginx) = start_nginx(&nginx_binary, generation);
let headers = [("Content-Encoding", "gzip"), ("Git-Protocol", "version=2")];
let (status, _, body) = http(
port,
"POST",
&format!("/{did}/git-upload-pack"),
&headers,
b"not even gzip",
);
(status, String::from_utf8(body).unwrap())
};
let without = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
let conf = std::fs::read_to_string(without.generation.join("nginx/knot-static.conf")).unwrap();
assert!(
!conf.contains("/_ks_inflate {"),
"no location without a cgi"
);
let (status, body) = post(&without.generation);
assert_eq!(status, 415, "{body}");
assert!(
body.contains("gzipped fetches need this knot's fcgiwrap"),
"{body}"
);
config.writes.fastcgi_socket = Some("unix:/dev/null".into());
let with = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
ObjectFormat::SHA1,
None,
);
let conf = std::fs::read_to_string(with.generation.join("nginx/knot-static.conf")).unwrap();
assert!(
conf.contains("limit_req zone=ks_inflate_all burst=1500;"),
"{conf}"
);
let (status, body) = post(&with.generation);
assert_eq!(status, 503, "{body}");
}
fn sibling_branches(source: &Path, count: usize) {
let main = git_in(source, &["rev-parse", "main"]);
let stream: String = (0..count)
.map(|n| {
format!(
"commit refs/heads/b{n}\ncommitter t <t@t> 0 +0000\ndata {}\nb{n}\nfrom {main}\n\n",
format!("b{n}\n").len()
)
})
.collect();
let mut import = crate::test_git()
.current_dir(source)
.args(["fast-import", "--quiet"])
.stdin(Stdio::piped())
.spawn()
.unwrap();
import
.stdin
.take()
.unwrap()
.write_all(stream.as_bytes())
.unwrap();
assert!(import.wait().unwrap().success());
}
#[test]
fn every_reply_cut_for_a_base_is_routed() {
for cobs in [0, 98] {
replies_match_rows(cobs);
}
}
fn replies_match_rows(cobs: usize) {
let did = "did:plc:efghijklmnopqrstuvwxyzabc";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = (temp.path().join("repos/plc"))
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
ballast(&source);
commit(&source, "a.txt");
git_in(&source, &["tag", "-a", "v1", "-m", "v1"]);
for n in 0..cobs {
cob_ref(
&source,
&format!("refs/cobs/org.tangled.track.ticket/{n}"),
&format!("{n}\n"),
);
}
let mut prior: Option<ExportPrior> = None;
let mut last = None;
for (n, path) in ["b.txt", "c.txt", "d.txt"].into_iter().enumerate() {
commit(&source, path);
let published = publish(
&config,
&mut manifest,
&temp.path().join(format!("gen-{n}")),
&source,
did,
ObjectFormat::SHA1,
prior.as_ref(),
);
prior = Some(published.prior.clone());
last = Some(published);
}
let published = last.unwrap();
assert_eq!(published.replies.bases.len(), 2, "{cobs}");
// past ~95 wants no row has room, and there is no `moved-wants`.
let moved = std::fs::read_to_string(published.export.join("smart/moved-wants"));
assert_eq!(moved.is_ok(), cobs == 0);
let reply = |name: &str| {
(name.starts_with("have-") || name.starts_with("moved-")) && name != "moved-wants"
};
let map = super::super::maps::fetch_map(&manifest)
.unwrap()
.block()
.to_string();
let routed: BTreeSet<String> = (map.lines())
.filter_map(|row| row.trim().trim_end_matches(';').rsplit_once(' '))
.inspect(|(word, _)| assert!(word.len() < 4096, "{}", word.len()))
.map(|(_, name)| name.to_string())
.filter(|name| reply(name))
.collect();
let cut: BTreeSet<String> = std::fs::read_dir(published.export.join("smart"))
.unwrap()
.map(|entry| entry.unwrap().file_name().into_string().unwrap())
.filter(|name| reply(name))
.collect();
assert!(cut.contains("have-0-done"), "{cut:?}");
assert_eq!(cut, routed, "{cobs} cob refs");
}
/// a base whose tips don't fit in one 4096 byte nginx word gets no row.
#[test]
fn fetch_map_leaves_out_a_base_nginx_cant_read() {
let temp = tempfile::tempdir().unwrap();
let (_, mut manifest, _) = test_fixture(temp.path());
let tips: Vec<String> = (0..10_000u32).map(|n| format!("{n:040x}")).collect();
let repo = &mut manifest.repos[0];
repo.public_refs = tips
.iter()
.map(|tip| ("refs/heads/x".into(), tip.clone()))
.collect();
repo.fetch_bases = vec![tips.clone(), tips[..2].to_vec()];
let map = super::super::maps::fetch_map(&manifest).unwrap().block();
let rows: Vec<String> = map.to_string().lines().map(str::to_string).collect();
for row in &rows {
let word = row
.trim()
.rsplit_once(' ')
.map_or(row.as_str(), |(word, _)| word);
assert!(word.len() < 4096, "{}", word.len());
}
let haves: Vec<&String> = rows.iter().filter(|row| row.contains("have-")).collect();
assert_eq!(haves.len(), 2);
assert!(haves.iter().all(|row| row.contains(" have-1")), "{haves:?}");
}
pub(super) fn client_result(dir: &Path, program: &str, args: &[&str]) -> (bool, String) {
let mut command = match program {
"git" => {
let mut command = crate::test_git();
command.args(["-c", &format!("http.extraHeader={FORWARDED_FOR}")]);
command
}
_ => {
let mut command = std::process::Command::new(program);
crate::git_export::without_repo_env(&mut command);
let header = format!("'http.extraHeader={FORWARDED_FOR}'");
command.env("GIT_CONFIG_PARAMETERS", header);
command
}
};
let output = command
.current_dir(dir)
.args(args)
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_COMMITTER_NAME", "t")
.env("GIT_COMMITTER_EMAIL", "t@t")
.output()
.unwrap();
let stderr = String::from_utf8_lossy(&output.stderr).into_owned();
(output.status.success(), stderr)
}
/// the server doesn't advertise `filter`, and git fetches without it.
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn depth_one_fetches_get_their_branchs_reply_and_the_rest_a_whole_pack_or_an_error() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:fghijklmnopqrstuvwxyzabc";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = temp
.path()
.join("repos/plc")
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(source.join(".tangled/workflows")).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
for name in ["a.txt", "b.txt", ".tangled/workflows/ci.yml"] {
commit(&source, name);
}
git_in(&source, &["tag", "-a", "v1", "-m", "v1"]);
git_in(&source, &["checkout", "-q", "-b", "feature"]);
commit(&source, "feature.txt");
git_in(&source, &["checkout", "-q", "main"]);
commit(&source, "c.txt");
let published = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
let main = git_in(&source, &["rev-parse", "main"]);
let feature = git_in(&source, &["rev-parse", "feature"]);
assert_eq!(published.replies.shallow_branches, [feature.clone()]);
let (port, _nginx) = start_nginx(&nginx_binary, &published.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let shallow_of = |repo: &Path| {
std::fs::read_to_string(repo.join(".git/shallow"))
.map(|text| text.lines().map(str::to_string).collect::<Vec<_>>())
.unwrap_or_default()
};
let objects = |repo: &Path, tip: &str| {
let mut oids: Vec<String> =
git_in(repo, &["rev-list", "--objects", "--no-object-names", tip])
.lines()
.map(str::to_string)
.collect();
oids.sort();
oids
};
let depth_one = |tip: &str| {
let mut oids: Vec<String> = git_in(
&source,
&[
"rev-list",
"--objects",
"--no-object-names",
"--no-walk",
tip,
],
)
.lines()
.map(str::to_string)
.collect();
oids.extend(
git_in(&source, &["ls-tree", "-r", "-t", "--object-only", tip])
.lines()
.map(str::to_string),
);
oids.sort();
oids.dedup();
oids
};
let fresh = |name: &str| {
let dir = temp.path().join(name);
std::fs::create_dir_all(&dir).unwrap();
git_in(&dir, &["init", "-q"]);
git_in(&dir, &["remote", "add", "origin", &remote]);
dir
};
let sparse = fresh("sparse");
git_in(
&sparse,
&["sparse-checkout", "set", "--no-cone", ".tangled/workflows"],
);
let (ok, stderr) = client_result(
&sparse,
"git",
&[
"fetch",
"-q",
"--depth=1",
"--filter=tree:0",
"origin",
&main,
],
);
assert!(ok, "{stderr}");
assert!(
stderr.contains("filtering not recognized by server"),
"{stderr}"
);
assert_eq!(shallow_of(&sparse), [main.clone()]);
git_in(&sparse, &["checkout", "-q", &main]);
assert!(sparse.join(".tangled/workflows/ci.yml").exists());
let ci = fresh("ci");
let (ok, stderr) = client_result(
&ci,
"git",
&["fetch", "-q", "--depth=1", "origin", &feature],
);
assert!(ok, "{stderr}");
assert_eq!(shallow_of(&ci), [feature.clone()]);
assert_eq!(
objects(&ci, &feature),
depth_one(&feature),
"only the branch's depth-1 objects"
);
git_in(&ci, &["fsck", "--strict"]);
let contains = fresh("contains");
let (ok, stderr) = client_result(
&contains,
"git",
&[
"fetch",
"-q",
"--filter=tree:0",
"--depth=100",
"origin",
"main",
],
);
assert!(ok, "{stderr}");
assert!(shallow_of(&contains).is_empty());
assert_eq!(objects(&contains, &main), objects(&source, &main));
let every = temp.path().join("every");
let (ok, stderr) = client_result(
temp.path(),
"git",
&[
"clone",
"-q",
"--depth=1",
"--no-single-branch",
&remote,
every.to_str().unwrap(),
],
);
assert!(ok, "{stderr}");
assert!(shallow_of(&every).is_empty());
git_in(&every, &["fsck", "--strict"]);
for (name, args) in [
(
"gix-depth",
vec!["clone", "--depth", "1", remote.as_str(), "c"],
),
(
"gix-branch",
vec![
"clone",
"--depth",
"1",
"--revision",
feature.as_str(),
remote.as_str(),
"c",
],
),
] {
let dir = temp.path().join(name);
std::fs::create_dir_all(&dir).unwrap();
let (ok, stderr) = client_result(&dir, "gix", &args);
assert!(ok, "{name}: {stderr}");
git_in(&dir.join("c"), &["fsck", "--connectivity-only"]);
git_in(&dir.join("c"), &["cat-file", "-e", &feature]);
}
// a depth-1 clone can't unshallow here, the server says to clone again.
let head = temp.path().join("head");
let (ok, stderr) = client_result(
temp.path(),
"git",
&["clone", "-q", "--depth=1", &remote, head.to_str().unwrap()],
);
assert!(ok, "{stderr}");
assert_eq!(shallow_of(&head), [main.clone()]);
let (ok, stderr) = client_result(&head, "git", &["fetch", "-q", "--unshallow", "origin"]);
assert!(
!ok && stderr.contains("clone again without --depth"),
"{stderr}"
);
let v0 = temp.path().join("v0");
let (ok, stderr) = client_result(
temp.path(),
"git",
&[
"-c",
"protocol.version=0",
"clone",
"-q",
"--depth=1",
&remote,
v0.to_str().unwrap(),
],
);
assert!(!ok && stderr.contains("need git protocol v2"), "{stderr}");
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn depth_one_replies_built_from_the_prior_clone_at_depth_one() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:fghijklmnopqrstuvwxyzabc";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = temp
.path()
.join("repos/plc")
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
// the files hardly compress, and the exports can share whole blocks.
for n in 0..40u64 {
let text: String = (0..2048u64)
.map(|i| format!("{:x}\n", (n << 32 | i).wrapping_mul(0x9e37_79b9_7f4a_7c15)))
.collect();
std::fs::write(source.join(format!("f-{n:02}.txt")), text).unwrap();
}
git_in(&source, &["add", "-A"]);
git_in(&source, &["commit", "-q", "-m", "files"]);
git_in(&source, &["checkout", "-q", "-b", "feature"]);
commit(&source, "feature.txt");
git_in(&source, &["checkout", "-q", "main"]);
let first = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
git_in(&source, &["rm", "-q", "f-00.txt"]);
commit(&source, "new.txt");
git_in(&source, &["checkout", "-q", "feature"]);
commit(&source, "feature-2.txt");
git_in(&source, &["checkout", "-q", "main"]);
let published = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
ObjectFormat::SHA1,
Some(&first.prior),
);
let main = git_in(&source, &["rev-parse", "main"]);
let feature = git_in(&source, &["rev-parse", "feature"]);
for name in ["shallow-head", "shallow-0"] {
let (entries, _) = crate::depth_one::tests::map_of(
&published.export.join("smart").join(name),
ObjectFormat::SHA1,
)
.expect("a map");
assert!(
![main.as_str(), feature.as_str()].contains(&entries[0].as_str()),
"{name} built"
);
}
let (port, _nginx) = start_nginx(&nginx_binary, &published.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let head = temp.path().join("head");
let (ok, stderr) = client_result(
temp.path(),
"git",
&["clone", "-q", "--depth=1", &remote, head.to_str().unwrap()],
);
assert!(ok, "{stderr}");
git_in(&head, &["fsck", "--strict"]);
assert_eq!(git_in(&head, &["rev-parse", "HEAD"]), main);
let ci = temp.path().join("ci");
std::fs::create_dir_all(&ci).unwrap();
git_in(&ci, &["init", "-q"]);
let (ok, stderr) = client_result(&ci, "git", &["fetch", "-q", "--depth=1", &remote, &feature]);
assert!(ok, "{stderr}");
git_in(&ci, &["fsck", "--strict"]);
for (name, tip) in [("gix-main", &main), ("gix-feature", &feature)] {
let dir = temp.path().join(name);
std::fs::create_dir_all(&dir).unwrap();
let args = ["clone", "--depth", "1", "--revision", tip, &remote, "c"];
let (ok, stderr) = client_result(&dir, "gix", &args);
assert!(ok, "{name}: {stderr}");
git_in(&dir.join("c"), &["fsck", "--strict"]);
git_in(&dir.join("c"), &["cat-file", "-e", tip]);
}
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn consolidated_full_replies_clone_over_v0_and_v2() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:fghijklmnopqrstuvwxyzabc";
for cobs in [0, 1] {
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = (temp.path().join("repos/plc"))
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
// it hardly compresses, and two pushes of it pass the size bound.
let big: String = (0..20_000u64)
.map(|i| {
format!(
"{:016x}\n",
i.wrapping_mul(0x9e37_79b9_7f4a_7c15).rotate_left(17)
)
})
.collect();
let mut prior = None;
let mut exports = Vec::new();
let mut published = None;
for generation in 1..=3 {
std::fs::write(source.join("big.txt"), format!("{big}{generation}\n")).unwrap();
commit(&source, &format!("{generation}.txt"));
if generation == 1 {
for n in 0..cobs {
cob_ref(
&source,
&format!("refs/cobs/org.tangled.track.ticket/{n}"),
&format!("{n}\n"),
);
}
}
let next = publish(
&config,
&mut manifest,
&temp.path().join(format!("gen-{generation}")),
&source,
did,
ObjectFormat::SHA1,
prior.as_ref(),
);
prior = Some(next.prior.clone());
exports.push(next.export.clone());
published = Some(next);
}
let published = published.unwrap();
let tails = |generation: usize| {
let dir = exports[generation - 1].join("smart");
let tails = crate::consolidate::tails(&dir).expect("tails");
tails.map(|tail| tail.regions().len())
};
assert_eq!(tails(2), [0, 0], "{cobs} cobs");
assert_eq!(tails(3), [1, 1], "{cobs} cobs: the third push consolidates");
let (port, _nginx) = start_nginx(&nginx_binary, &published.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let main = git_in(&source, &["rev-parse", "main"]);
for (name, program, args) in [
(
"git-v0",
"git",
vec!["-c", "protocol.version=0", "clone", "-q"],
),
("git-v2", "git", vec!["clone", "-q"]),
("gix-v0", "gix", vec!["-c", "protocol.version=0", "clone"]),
("gix-v2", "gix", vec!["clone"]),
] {
let dir = temp.path().join(name);
let args = [args, vec![remote.as_str(), dir.to_str().unwrap()]].concat();
let (ok, stderr) = client_result(temp.path(), program, &args);
assert!(ok, "{name} with {cobs} cobs: {stderr}");
git_in(&dir, &["fsck", "--strict"]);
assert_eq!(git_in(&dir, &["rev-parse", "HEAD"]), main, "{name}");
}
}
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn full_replies_framed_alike_clone_over_v0_and_v2() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:fghijklmnopqrstuvwxyzabc";
for cobs in [0, 1] {
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = (temp.path().join("repos/plc"))
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
let big: String = (0..32_768u64)
.map(|i| format!("{:x}\n", i.wrapping_mul(0x9e37_79b9_7f4a_7c15)))
.collect();
std::fs::write(source.join("big.txt"), big).unwrap();
commit(&source, "a.txt");
git_in(&source, &["tag", "-a", "v1", "-m", "v1"]);
for n in 0..cobs {
cob_ref(
&source,
&format!("refs/cobs/org.tangled.track.ticket/{n}"),
&format!("{n}\n"),
);
}
let published = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
let v0 = std::fs::read(published.export.join("smart/upload-pack")).unwrap();
assert!(v0.starts_with(b"0008NAK\n0005\x02"), "{cobs} cobs");
let (port, _nginx) = start_nginx(&nginx_binary, &published.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let main = git_in(&source, &["rev-parse", "main"]);
for (name, program, args) in [
(
"git-v0",
"git",
vec!["-c", "protocol.version=0", "clone", "-q"],
),
("git-v2", "git", vec!["clone", "-q"]),
("gix-v0", "gix", vec!["-c", "protocol.version=0", "clone"]),
] {
let dir = temp.path().join(name);
let args = [args, vec![remote.as_str(), dir.to_str().unwrap()]].concat();
let (ok, stderr) = client_result(temp.path(), program, &args);
assert!(ok, "{name} with {cobs} cobs: {stderr}");
git_in(&dir, &["fsck", "--strict"]);
assert_eq!(git_in(&dir, &["rev-parse", "HEAD"]), main, "{name}");
assert!(
!git_in(&dir, &["for-each-ref"]).contains("refs/cobs"),
"{name}"
);
}
}
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn a_reply_with_spare_slots_clones_with_what_it_reaches_and_its_fillers() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:ghijklmnopqrstuvwxyzabcd";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = (temp.path().join("repos/plc"))
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
// incompressible and past `spare::THRESHOLD`, which is what reserves spare slots.
let mut x = 0x9e37_79b9_7f4a_7c15u64;
let big: Vec<u8> = (0..crate::spare::THRESHOLD + (1 << 20))
.map(|_| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x as u8
})
.collect();
std::fs::write(source.join("big.bin"), big).unwrap();
commit(&source, "a.txt");
let pushes: [&dyn Fn(); 3] = [
&|| {},
&|| commit(&source, "b.txt"),
// 1100 files, more than the fewest slots one reservation makes.
&|| {
for n in 0..1100 {
std::fs::write(source.join(format!("many-{n:04}.txt")), format!("{n}\n")).unwrap();
}
commit(&source, "c.txt");
},
];
let mut prior = None;
let mut nonces = Vec::new();
for (step, push) in pushes.iter().enumerate() {
push();
let published = publish(
&config,
&mut manifest,
&temp.path().join(format!("gen-{step}")),
&source,
did,
ObjectFormat::SHA1,
prior.as_ref(),
);
prior = Some(published.prior.clone());
let spares = crate::spare::read(&published.export.join("smart"));
let fillers = |at: usize| -> BTreeSet<String> {
let spare = spares[at].clone().expect("spare slots");
(spare.ids(ObjectFormat::SHA1).iter())
.map(|id| knot_types::lowercase_hex(&id))
.collect()
};
nonces.push(spares[0].clone().unwrap().nonce);
let reached: BTreeSet<String> = git_in(&source, &["rev-list", "--objects", "--all"])
.lines()
.map(|line| line[..40].to_string())
.collect();
let (port, _nginx) = start_nginx(&nginx_binary, &published.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let main = git_in(&source, &["rev-parse", "main"]);
for (name, program, args, at) in [
(
"git-v0",
"git",
vec!["-c", "protocol.version=0", "clone", "-q"],
0,
),
("git-v2", "git", vec!["clone", "-q"], 1),
("gix-v2", "gix", vec!["clone"], 1),
] {
let dir = temp.path().join(format!("{name}-{step}"));
let args = [args, vec![remote.as_str(), dir.to_str().unwrap()]].concat();
let (ok, stderr) = client_result(temp.path(), program, &args);
assert!(ok, "{name} after push {step}: {stderr}");
git_in(&dir, &["fsck", "--strict", "--no-dangling"]);
assert_eq!(git_in(&dir, &["rev-parse", "HEAD"]), main, "{name}");
let held: BTreeSet<String> = git_in(
&dir,
&[
"cat-file",
"--batch-all-objects",
"--batch-check=%(objectname)",
],
)
.lines()
.map(str::to_string)
.collect();
let fillers = fillers(at);
let expected: BTreeSet<String> = reached.union(&fillers).cloned().collect();
assert_eq!(held, expected, "{name} after push {step}");
let dangling: BTreeSet<String> = git_in(
&dir,
&["fsck", "--dangling", "--no-reflogs", "--no-progress"],
)
.lines()
.map(|line| {
let id = line.strip_prefix("dangling blob ");
id.unwrap_or_else(|| panic!("{name}: {line}")).to_string()
})
.collect();
assert_eq!(dangling, fillers, "{name} after push {step}");
}
}
assert_eq!(nonces[1], nonces[0]);
assert_ne!(nonces[2], nonces[1]);
}
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn a_repo_with_many_refs_activates_and_picks_the_replies() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:fghijklmnopqrstuvwxyzabc";
for count in [1_000] {
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = temp
.path()
.join("repos/plc")
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
commit(&source, "a.txt");
sibling_branches(&source, count);
let first = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
let old_main = git_in(&source, &["rev-parse", "main"]);
commit(&source, "b.txt");
let second = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
ObjectFormat::SHA1,
Some(&first.prior),
);
assert!(second.replies.bases.is_empty(), "{count}");
let (port, _nginx) = start_nginx(&nginx_binary, &second.generation);
let remote = format!("http://127.0.0.1:{port}/{did}");
let smart = |name: &str| std::fs::read(second.export.join("smart").join(name)).unwrap();
let post = |args: &[String]| {
let body = fetch_request(args, ObjectFormat::SHA1);
http(port, "POST", &format!("/{did}/git-upload-pack"), &[], &body).2
};
let tip = |branch: &str| git_in(&source, &["rev-parse", branch]);
let depth_one = |wants: &[String]| {
let mut args = vec!["deepen 1".to_string()];
args.extend(wants.iter().map(|want| format!("want {want}")));
args.push("done".into());
post(&args)
};
let last = format!("b{}", count - 1);
assert!(second.replies.shallow_branches.is_empty(), "{count}");
assert_eq!(depth_one(&[tip(&last)]), smart("fetch-done"), "{count}");
assert_eq!(
depth_one(&[tip("b0"), tip(&last)]),
smart("fetch-done"),
"{count}"
);
assert_eq!(
depth_one(&[old_main.clone()]),
smart("fetch-done"),
"{count}"
);
let wants: Vec<String> = manifest.repos[0]
.public_refs
.iter()
.map(|(_, oid)| oid.clone())
.collect();
let haves: Vec<String> = first.prior.refs.values().cloned().collect();
assert_eq!(
post(&knot_pack_args(&wants, &haves)),
smart("fetch-done"),
"{count}"
);
let trace = temp.path().join("trace");
let shallow = temp.path().join("shallow");
client(
temp.path(),
&[
"clone",
"-q",
"--depth=1",
"-b",
&last,
&remote,
shallow.to_str().unwrap(),
],
&trace,
);
assert_eq!(git_in(&shallow, &["rev-parse", "HEAD"]), tip(&last));
git_in(&shallow, &["fsck", "--strict"]);
// git gzips a request for every ref, and only the cgi reads gzip.
let spread = temp.path().join("spread");
git_in(temp.path(), &["init", "-q", spread.to_str().unwrap()]);
client(
&spread,
&["fetch", "-q", "--depth=1", &remote, "b0", &last],
&trace,
);
git_in(&spread, &["cat-file", "-e", &tip("b0")]);
git_in(&spread, &["cat-file", "-e", &tip(&last)]);
git_in(&spread, &["fsck", "--strict"]);
}
}
/// after a reload, a new worker can get a loopback request the old map sent.
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn the_loopback_serves_the_generation_whose_map_chose_the_reply() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let temp = tempfile::tempdir().unwrap();
let (config, manifest, _) = test_fixture(temp.path());
let token = repo_token(&manifest.repos[0].repo_did);
for name in ["gen-1", "gen-2"] {
let generation = temp.path().join(name);
let smart = generation.join("www/_git").join(&token).join("smart");
std::fs::create_dir_all(&smart).unwrap();
std::fs::write(smart.join("have-0"), format!("{name} have-0")).unwrap();
crate::atproto::emit(&generation, &[]).unwrap();
crate::blobs::emit(temp.path(), &generation, &[], &Default::default()).unwrap();
super::super::emit_at(&config, &manifest, &generation, &generation).unwrap();
}
let old_conf =
std::fs::read_to_string(temp.path().join("gen-1/nginx/knot-static.conf")).unwrap();
assert!(
old_conf.contains(&format!("/_ks_git/gen-1/{token};")),
"{old_conf}"
);
let (port, _nginx) = start_nginx(&nginx_binary, &temp.path().join("gen-2"));
let get = |generation: &str| {
let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap();
write!(
stream,
"GET /_ks_git/{generation}/{token} HTTP/1.1\r\nHost: knot.test\r\nX-Ks-Canned: have-0\r\nConnection: close\r\n\r\n"
)
.unwrap();
let mut response = String::new();
stream.read_to_string(&mut response).unwrap();
response
};
for name in ["gen-1", "gen-2"] {
let response = get(name);
assert!(response.starts_with("HTTP/1.1 200 "), "{response}");
assert!(
response.ends_with(&format!("\r\n\r\n{name} have-0")),
"{response}"
);
}
let gone = get("gen-0");
assert!(gone.starts_with("HTTP/1.1 404 "), "{gone}");
let headers = [("X-Ks-Canned", "have-0")];
let (status, _, _) = http(
port,
"GET",
&format!("/_ks_git/gen-1/{token}"),
&headers,
&[],
);
assert_eq!(status, 403, "forwarded from outside");
}
/// where git-ssh has no reply, the test expects nginx's 400.
#[test]
#[ignore = "requires nginx; set NGINX_BINARY"]
fn git_ssh_picks_the_reply_nginx_serves() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let did = "did:plc:fghijklmnopqrstuvwxyzabc";
let temp = tempfile::tempdir().unwrap();
let (mut config, mut manifest, _) = test_fixture(temp.path());
config.scan_path = temp.path().join("repos");
let source = temp
.path()
.join("repos/plc")
.join(&did[8..10])
.join(&did[10..]);
std::fs::create_dir_all(&source).unwrap();
git_in(&source, &["init", "-q", "-b", "main"]);
commit(&source, "a.txt");
git_in(&source, &["tag", "-a", "v1", "-m", "v1"]);
git_in(&source, &["checkout", "-q", "-b", "feature"]);
commit(&source, "feature.txt");
git_in(&source, &["checkout", "-q", "main"]);
commit(&source, "b.txt");
let first = publish(
&config,
&mut manifest,
&temp.path().join("gen-1"),
&source,
did,
ObjectFormat::SHA1,
None,
);
commit(&source, "c.txt");
let second = publish(
&config,
&mut manifest,
&temp.path().join("gen-2"),
&source,
did,
ObjectFormat::SHA1,
Some(&first.prior),
);
assert!(!second.replies.bases.is_empty() && !second.replies.shallow_branches.is_empty());
let (port, _nginx) = start_nginx(&nginx_binary, &second.generation);
let maps = super::super::git_maps(&manifest).unwrap();
let main = git_in(&source, &["rev-parse", "main"]);
let feature = git_in(&source, &["rev-parse", "feature"]);
let tag = git_in(&source, &["rev-parse", "v1"]);
let old: Vec<String> = second.replies.bases[0].clone();
let wants: Vec<String> = manifest.repos[0]
.public_refs
.iter()
.map(|(_, oid)| oid.clone())
.collect();
let v2 = |lines: &[String]| fetch_request(lines, ObjectFormat::SHA1);
let bare = |lines: &[&str]| {
let mut body = pkt("command=fetch\n");
body.extend(pkt("agent=git/2.55.0\n"));
body.extend_from_slice(b"0001");
for line in lines {
body.extend(pkt(line));
}
body.extend_from_slice(b"0000");
body
};
let s = |items: &[&str]| {
items
.iter()
.map(|item| item.to_string())
.collect::<Vec<_>>()
};
let want = |oid: &str| format!("want {oid}\n");
let mut bodies: Vec<(String, Vec<u8>)> = vec![
("ls-refs".into(), {
let mut body = pkt("command=ls-refs\n");
body.extend_from_slice(b"0001");
body.extend(pkt("peel\n"));
body.extend(pkt("ref-prefix refs/heads/\n"));
body.extend_from_slice(b"0000");
body
}),
("ls-refs head".into(), {
let mut body = pkt("command=ls-refs\n");
body.extend_from_slice(b"0001");
body.extend(pkt("ref-prefix HEAD\n"));
body.extend_from_slice(b"0000");
body
}),
("git fetch of the cob refs".into(), {
let mut body = pkt("command=ls-refs\n");
body.extend_from_slice(b"0001");
body.extend(pkt("ref-prefix refs/heads/\n"));
body.extend(pkt("ref-prefix refs/cobs/\n"));
body.extend(pkt("ref-prefix refs/tngl/keep/\n"));
body.extend(pkt("ref-prefix HEAD\n"));
body.extend_from_slice(b"0000");
body
}),
("git fetch of refs/*".into(), {
let mut body = pkt("command=ls-refs\n");
body.extend_from_slice(b"0001");
body.extend(pkt("ref-prefix refs/\n"));
body.extend(pkt("ref-prefix HEAD\n"));
body.extend_from_slice(b"0000");
body
}),
("clone".into(), v2(&knot_pack_args(&wants, &[]))),
(
"negotiation".into(),
v2(&s(&[
"ofs-delta",
&format!("want {main}"),
&format!("have {main}"),
])),
),
(
"have-0".into(),
v2(&knot_pack_args(&wants, &old)[..knot_pack_args(&wants, &old).len() - 1]),
),
("have-0 done".into(), v2(&knot_pack_args(&wants, &old))),
(
"unknown have".into(),
v2(&knot_pack_args(&wants, &["1".repeat(40)])),
),
(
"depth 1 head".into(),
bare(&["ofs-delta", "deepen 1", &want(&main), "done\n"]),
),
(
"depth 1 head, newline".into(),
bare(&["ofs-delta", "deepen 1\n", &want(&main), "done\n"]),
),
(
"depth 1 feature (ci)".into(),
bare(&["ofs-delta", "deepen 1", &want(&feature), "done\n"]),
),
(
"sparse depth 1 + filter".into(),
bare(&[
"ofs-delta",
"deepen 1",
"filter tree:0",
&want(&main),
"done\n",
]),
),
(
"depth 100 + filter".into(),
bare(&[
"ofs-delta",
"deepen 100",
"filter tree:0",
&want(&main),
"done\n",
]),
),
(
"depth 1 tag".into(),
bare(&["ofs-delta", "deepen 1", &want(&tag), "done\n"]),
),
(
"depth 1 two branches".into(),
bare(&[
"ofs-delta",
"deepen 1",
&want(&main),
&want(&feature),
"done\n",
]),
),
(
"depth 1 from a shallow clone".into(),
bare(&[
"ofs-delta",
&format!("shallow {main}"),
"deepen 1",
&want(&main),
"done\n",
]),
),
(
"unshallow".into(),
bare(&[
"ofs-delta",
&format!("shallow {main}"),
"deepen 2147483647",
&want(&main),
&format!("have {main}"),
]),
),
(
"shallow-since".into(),
bare(&[
"ofs-delta",
"deepen-since 946684800",
&want(&main),
"done\n",
]),
),
("v0 clone".into(), {
let mut body = pkt(&format!("want {main} ofs-delta side-band-64k\n"));
body.extend_from_slice(b"0000");
body.extend(pkt("done\n"));
body
}),
("v0 round".into(), {
let mut body = pkt(&format!("want {main} ofs-delta side-band-64k\n"));
body.extend_from_slice(b"0000");
body
}),
("v0 depth 1".into(), {
let mut body = pkt(&format!("want {main} ofs-delta side-band-64k\n"));
body.extend(pkt("deepen 1"));
body.extend_from_slice(b"0000");
body
}),
("not git".into(), b"hello".to_vec()),
];
bodies.push(("empty".into(), Vec::new()));
let mut picked = BTreeMap::new();
for (label, body) in &bodies {
let (status, _, reply) = http(port, "POST", &format!("/{did}/git-upload-pack"), &[], body);
match maps.reply(did, body).unwrap() {
Some(name) => {
assert_eq!(status, 200, "{label}");
let file = std::fs::read(second.export.join("smart").join(&name)).unwrap();
assert!(reply == file, "{label}: nginx didn't serve {name}");
picked.insert(name, label.clone());
}
None => assert_eq!(status, 400, "{label}"),
}
}
for (label, name) in [
("ls-refs", "ls-refs"),
("ls-refs head", "ls-refs-head"),
("git fetch of the cob refs", "ls-refs-head-cobs"),
("git fetch of refs/*", "ls-refs-head-cobs"),
] {
let (_, body) = bodies.iter().find(|(l, _)| l == label).unwrap();
assert_eq!(
maps.reply(did, body).unwrap().as_deref(),
Some(name),
"{label}"
);
}
for name in [
"shallow-head",
"shallow-0",
"have-0",
"have-0-done",
"fetch-done",
"fetch-nodone",
"err-unshallow",
"err-v0-shallow",
"upload-pack",
"v0-nodone",
"ls-refs",
"ls-refs-head",
"ls-refs-head-cobs",
] {
assert!(
picked.contains_key(name),
"no request picked {name}: {picked:?}"
);
}
}