use std::collections::{BTreeMap, BTreeSet};
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use knot_types::{Oid, RepoDid};
use super::{http, start_nginx, test_fixture};
use crate::git_xrpc::paths::{self, Kind};
use crate::git_xrpc::{RepoInput, Rows, SealedRepo};
use crate::plan::repo_token;
use crate::store::RepoStore;
use crate::test_rng::Rng;
use crate::xrpc::{form_encoded, uri_component};
const DID: &str = "did:plc:pathfuzzfixture000000000";
const EPOCH: i64 = 1_700_000_000;
const SECRET: &str = "SECRET-DO-NOT-SERVE";
/// longer request lines run into nginx's 8k header buffer and get a 414 before any routing.
const MAX_TARGET: usize = 6000;
/// the longest keys git_xrpc writes.
const MAX_KEY_BYTES: usize = 400;
type Dir = BTreeMap<String, Node>;
#[derive(Clone)]
enum Node {
File(Vec<u8>),
Link(String),
Module,
Dir(Dir),
}
/// nginx refuses maps with case twins, so these only run with FUZZ_TWINS.
fn twins() -> bool {
std::env::var_os("FUZZ_TWINS").is_some()
}
fn fold_case(dir: &mut Dir) {
let mut seen = std::collections::BTreeSet::new();
dir.retain(|name, _| seen.insert(name.to_ascii_lowercase()));
for node in dir.values_mut() {
if let Node::Dir(inner) = node {
fold_case(inner);
}
}
}
fn env<T: std::str::FromStr>(name: &str) -> Option<T> {
std::env::var(name).ok()?.parse().ok()
}
fn git(dir: &Path, args: &[&str], stdin: &[u8]) -> Vec<u8> {
let mut child = crate::test_git()
.current_dir(dir)
.args(args)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("spawn git");
let mut input = child.stdin.take().unwrap();
let stdin = stdin.to_vec();
let writer = std::thread::spawn(move || {
let _ = input.write_all(&stdin);
});
let output = child.wait_with_output().expect("wait for git");
writer.join().unwrap();
assert!(
output.status.success(),
"git {args:?}: {}",
String::from_utf8_lossy(&output.stderr)
);
output.stdout
}
const PLAIN: [&str; 8] = [
"a", "b.txt", "lib.rs", "main", "Makefile", "README", "src", "x",
];
const COMMA: [&str; 7] = [
"a,b",
"x,blob",
"y,tree.json",
"z,entry.json",
",",
"w,blame.json",
"q,",
];
const DOTS: [&str; 7] = ["...", ".x", "..x", "x..", ".hidden", "a.b.c", "x."];
const PERCENT: [&str; 9] = [
"100%", "%2F", "a%2Fb", "%40", "%2B", "%252F", "%zz", "%", "%2f",
];
const SPACE: [&str; 5] = ["a b", " lead", "trail ", "two spaces", " "];
const PUNCT: [&str; 6] = ["a~b", "(x)", "!bang", "star*", "it's", "~"];
const SYMBOLS: [&str; 9] = [
"a&b",
"k=v",
"#hash",
"what?",
"a=b&path=c",
"a\\b",
"&",
"=",
"a;b",
];
const UNICODE: [&str; 7] = ["héllo", "日本語", "ß", "İ", "é", "emoji\u{1F600}", "ñ"];
fn gen_name(rng: &mut Rng) -> String {
let word = |rng: &mut Rng| -> String {
(0..rng.range(1, 8))
.map(|_| (b'a' + rng.below(26) as u8) as char)
.collect()
};
match rng.below(15) {
0 | 1 => rng.pick(&PLAIN).to_string(),
2 => format!("{}.{}", word(rng), word(rng)),
3 => word(rng),
4 => rng.pick(&COMMA).to_string(),
5 => rng.pick(&DOTS).to_string(),
6 => rng.pick(&PERCENT).to_string(),
7 => match rng.below(3) {
0 => "+".repeat(rng.range(1, 12)),
1 => "@".repeat(rng.range(1, 12)),
_ => ["c++", "a+b", "@scope", "a@b"][rng.below(4)].to_string(),
},
8 => rng.pick(&SPACE).to_string(),
9 => rng.pick(&PUNCT).to_string(),
10 => rng.pick(&SYMBOLS).to_string(),
11 => rng.pick(&UNICODE).to_string(),
12 => {
let len = rng.range(200, 255);
let mut name = format!("{}-", word(rng));
while name.len() < len {
name.push((b'a' + rng.below(26) as u8) as char);
}
name
}
13 => format!(
"{}{}",
rng.pick(&PLAIN),
rng.pick(&["%2F", "@", "+", "%", ","])
),
_ => word(rng).to_uppercase(),
}
}
fn usable(name: &str) -> bool {
!name.is_empty()
&& name != "."
&& name != ".."
&& !name.to_ascii_lowercase().starts_with(".git")
}
/// gix-blame panics on random bytes inside the seal.
fn text_only() -> bool {
std::env::var_os("FUZZ_TEXT_ONLY").is_some()
}
fn gen_content(rng: &mut Rng) -> Vec<u8> {
match rng.below(6) {
0 => Vec::new(),
1 if !text_only() => (0..rng.range(1, 64))
.map(|_| rng.below(256) as u8)
.collect(),
2 => b"same\n".to_vec(),
_ => format!("line {}\nline {}\n", rng.next(), rng.next()).into_bytes(),
}
}
fn gen_node(rng: &mut Rng, depth: usize) -> Node {
match rng.below(100) {
0..=29 if depth > 0 => Node::Dir(gen_dir(rng, depth - 1)),
30..=37 => Node::Link(
["../../etc", "/etc/passwd", "target", "..", "../x/../y"][rng.below(5)].to_string(),
),
38..=41 => Node::Module,
_ => Node::File(gen_content(rng)),
}
}
fn insert(dir: &mut Dir, name: String, node: Node) {
if usable(&name) {
dir.entry(name).or_insert(node);
}
}
fn gen_dir(rng: &mut Rng, depth: usize) -> Dir {
let mut dir = Dir::new();
add_entries(rng, &mut dir, depth);
dir
}
fn add_entries(rng: &mut Rng, dir: &mut Dir, depth: usize) {
for _ in 0..rng.range(1, 5) {
let name = gen_name(rng);
let node = gen_node(rng, depth);
if twins() && rng.odds(20) {
let twin = match rng.below(3) {
0 => name.to_ascii_uppercase(),
1 => name.to_ascii_lowercase(),
_ => name
.replace('\u{e9}', "e\u{301}")
.replace('\u{f1}', "n\u{303}"),
};
if twin != name {
insert(dir, twin, node.clone());
}
}
insert(dir, name, node);
}
}
fn deep(rng: &mut Rng, long: bool) -> (String, Node) {
let levels = *rng.pick(&[3, 10, 31, 32, 33, 34, 40, 45]);
let names = ["a", "b", "%2F", "+", "@", "d", "x,y"];
let mut node = Node::File(gen_content(rng));
let leaf = format!("leaf{}", rng.below(3));
let mut chain = vec![leaf];
for _ in 0..levels {
chain.push(rng.pick(&names).to_string());
}
if long {
chain.truncate(4);
for name in chain.iter_mut().skip(1) {
*name = format!(
"{}{}",
rng.pick(&["p", "q"]),
"n".repeat(rng.range(190, 230))
);
}
}
let top = chain.pop().unwrap();
for name in chain.into_iter() {
node = Node::Dir(BTreeMap::from([(name, node)]));
}
(top, node)
}
fn mutate(rng: &mut Rng, dir: &mut Dir, depth: usize) {
let names: Vec<String> = dir.keys().cloned().collect();
for name in names {
if rng.odds(10) {
dir.remove(&name);
} else if let Some(node) = dir.get_mut(&name) {
match node {
Node::File(bytes) if rng.odds(15) => *bytes = gen_content(rng),
Node::Dir(inner) if rng.odds(60) => mutate(rng, inner, depth),
_ => {}
}
}
}
if rng.odds(40) {
add_entries(rng, dir, depth);
}
}
fn quote(path: &str) -> String {
let mut out = String::from("\"");
for byte in path.bytes() {
match byte {
b'"' | b'\\' => {
out.push('\\');
out.push(byte as char);
}
0x20..=0x7e => out.push(byte as char),
_ => out.push_str(&format!("\\{byte:03o}")),
}
}
out + "\""
}
fn data(stream: &mut Vec<u8>, bytes: &[u8]) {
stream.extend(format!("data {}\n", bytes.len()).as_bytes());
stream.extend(bytes);
stream.push(b'\n');
}
fn files(
dir: &Dir,
prefix: &str,
into: &mut Vec<(String, Node)>,
listed: &mut Vec<(String, bool)>,
) {
for (name, node) in dir {
let path = format!("{prefix}{name}");
match node {
Node::Dir(inner) => {
listed.push((path.clone(), true));
files(inner, &format!("{path}/"), into, listed);
}
_ => {
listed.push((path.clone(), false));
into.push((path, node.clone()));
}
}
}
}
struct Built {
dir: PathBuf,
listed: Vec<(String, bool)>,
commits: Vec<String>,
branch_names: Vec<String>,
held: Vec<(String, String, Vec<u8>)>,
}
fn build(rng: &mut Rng, dir: &Path) -> Built {
let repo = dir.join("repo.git");
std::fs::create_dir_all(&repo).unwrap();
git(&repo, &["init", "-q", "--bare", "-b", "main"], b"");
// a mac's git init turns these on, and that merges the case and unicode twins
for key in ["core.ignorecase", "core.precomposeunicode"] {
git(&repo, &["config", key, "false"], b"");
}
let mut root = gen_dir(rng, 3);
for _ in 0..rng.range(1, 3) {
let (name, node) = deep(rng, false);
insert(&mut root, name, node);
}
if rng.odds(30) {
let (name, node) = deep(rng, true);
insert(&mut root, name, node);
}
let count = rng.range(2, 4);
if !twins() {
fold_case(&mut root);
}
let mut trees = vec![root];
for _ in 1..count {
let mut next = trees.last().unwrap().clone();
mutate(rng, &mut next, 2);
if next.is_empty() {
next = gen_dir(rng, 2);
}
if !twins() {
fold_case(&mut next);
}
trees.push(next);
}
let mut branches: Vec<(String, usize)> = vec![("main".into(), count)];
let wanted = [
"feature/x",
"a%2Fb",
"r+s",
"at@x",
"plain",
"x.y",
"UPPER",
"dev",
];
for _ in 0..rng.range(0, 3) {
let name = rng.pick(&wanted).to_string();
if !branches.iter().any(|(have, _)| *have == name) {
branches.push((name, rng.range(1, count)));
}
}
let mut stream = Vec::new();
let mut listed = Vec::new();
let mut held_by_mark: Vec<(usize, String, Vec<u8>)> = Vec::new();
let mut marks = 0;
let mut write =
|stream: &mut Vec<u8>, refname: &str, tree: &Dir, from: Option<usize>, mark: usize| {
stream.extend(
format!(
"commit {refname}\nmark :{mark}\ncommitter t <t@t> {} +0000\n",
EPOCH + 100 * mark as i64
)
.as_bytes(),
);
data(stream, format!("commit {mark}\n").as_bytes());
if let Some(from) = from {
stream.extend(format!("from :{from}\n").as_bytes());
}
stream.extend(b"deleteall\n");
let mut flat = Vec::new();
files(tree, "", &mut flat, &mut listed);
for (path, node) in flat {
match node {
Node::File(bytes) => {
stream.extend(format!("M 100644 inline {}\n", quote(&path)).as_bytes());
data(stream, &bytes);
held_by_mark.push((mark, path, bytes));
}
Node::Link(target) => {
stream.extend(format!("M 120000 inline {}\n", quote(&path)).as_bytes());
data(stream, target.as_bytes());
held_by_mark.push((mark, path, target.into_bytes()));
}
Node::Module => stream.extend(
format!(
"M 160000 {} {}\n",
"1234567890abcdef1234567890abcdef12345678",
quote(&path)
)
.as_bytes(),
),
Node::Dir(_) => unreachable!(),
}
}
stream.push(b'\n');
};
for number in 1..=count {
marks += 1;
write(
&mut stream,
"refs/heads/main",
&trees[number - 1],
(number > 1).then(|| number - 1),
marks,
);
}
let mut tips: BTreeMap<String, usize> = BTreeMap::from([("main".to_string(), count)]);
for (name, at) in branches.iter().skip(1) {
marks += 1;
let mut tree = trees[at - 1].clone();
mutate(rng, &mut tree, 1);
if tree.is_empty() {
tree = gen_dir(rng, 1);
}
if !twins() {
fold_case(&mut tree);
}
write(
&mut stream,
&format!("refs/heads/{name}"),
&tree,
Some(*at),
marks,
);
tips.insert(name.clone(), marks);
}
for name in ["v1", "v2", "rel+1", "r%2Fx"].iter().take(rng.range(0, 4)) {
let at = rng.range(1, count);
if rng.odds(50) {
stream.extend(format!("reset refs/tags/{name}\nfrom :{at}\n\n").as_bytes());
} else {
stream.extend(
format!(
"tag {name}\nfrom :{at}\ntagger t <t@t> {} +0000\n",
EPOCH + 5000
)
.as_bytes(),
);
data(&mut stream, b"tag\n");
}
}
let marks_file = dir.join("marks");
git(
&repo,
&[
"fast-import",
"--quiet",
&format!("--export-marks={}", marks_file.display()),
],
&stream,
);
let exported = std::fs::read_to_string(&marks_file).unwrap();
let oid_of = |mark: usize| -> String {
exported
.lines()
.find_map(|line| line.strip_prefix(&format!(":{mark} ")))
.unwrap()
.to_string()
};
listed.sort();
listed.dedup();
Built {
dir: repo,
listed,
commits: (1..=marks).map(oid_of).collect(),
branch_names: tips.into_keys().collect(),
held: held_by_mark
.into_iter()
.map(|(mark, path, bytes)| (oid_of(mark), path, bytes))
.collect(),
}
}
fn check_blobs(built: &Built, rows: &Rows, pool: &Path) -> Result<usize, String> {
let mut checked = 0;
// a literal `%40` and an `@` both end up with the row key `%40`
let mut owners: BTreeMap<String, usize> = BTreeMap::new();
for (path, _) in &built.listed {
for spelling in crate::xrpc::spellings(path) {
*owners.entry(spelling).or_default() += 1;
}
}
for (commit, path, bytes) in &built.held {
if owners[path] > 1 {
continue;
}
if let Some(file) = rows.blob.get(&format!("{commit}|{path}")) {
let held = std::fs::read(pool.join(file))
.map_err(|error| format!("read blob row of {path}: {error}"))?;
if &held != bytes {
return Err(format!(
"blob row of {commit}:{path} holds {} bytes, git holds {}",
held.len(),
bytes.len()
));
}
checked += 1;
}
}
let queries: String = built
.held
.iter()
.map(|(commit, path, _)| format!("{commit}:{path}\n"))
.collect();
let out = git(&built.dir, &["cat-file", "--batch"], queries.as_bytes());
let mut at = 0;
for (commit, path, bytes) in &built.held {
let end = at
+ out[at..]
.iter()
.position(|&byte| byte == b'\n')
.ok_or("short cat-file")?;
let head = String::from_utf8_lossy(&out[at..end]).into_owned();
let size: usize = head
.rsplit(' ')
.next()
.unwrap()
.parse()
.map_err(|_| format!("cat-file {commit}:{path}: {head}"))?;
let got = &out[end + 1..end + 1 + size];
if got != bytes.as_slice() {
return Err(format!("the model disagrees with git at {commit}:{path}"));
}
at = end + 1 + size + 1;
}
Ok(checked)
}
fn wire_safe(text: &str) -> String {
let mut out = String::new();
for ch in text.chars() {
match ch {
'\u{0}'..=' ' | '#' | '\u{7f}' => out.push_str(&format!("%{:02X}", ch as u32)),
_ => out.push(ch),
}
}
out
}
fn lower_hex(text: &str) -> String {
let bytes = text.as_bytes();
let mut out = String::new();
let mut at = 0;
while at < bytes.len() {
if bytes[at] == b'%' && at + 2 < bytes.len() && bytes[at + 1].is_ascii_hexdigit() {
out.push('%');
out.push((bytes[at + 1] as char).to_ascii_lowercase());
out.push((bytes[at + 2] as char).to_ascii_lowercase());
at += 3;
} else {
out.push(bytes[at] as char);
at += 1;
}
}
out
}
fn mixed(rng: &mut Rng, path: &str) -> String {
let mut out = String::new();
for ch in path.chars() {
match ch {
'/' if rng.odds(50) => out.push_str("%2F"),
'/' => out.push('/'),
_ => out.push_str(&uri_component(&ch.to_string())),
}
}
out
}
fn partial(rng: &mut Rng, path: &str) -> String {
let mut out = String::new();
for ch in path.chars() {
if ch.is_ascii() && rng.odds(15) {
out.push_str(&format!("%{:02X}", ch as u32));
} else {
out.push(ch);
}
}
out
}
fn spell(rng: &mut Rng, path: &str) -> String {
let spelled = match rng.below(10) {
0..=2 => path.to_string(),
3 => form_encoded(path),
4 | 5 => uri_component(path),
6 => mixed(rng, path),
7 => lower_hex(&uri_component(path)),
8 => uri_component(&uri_component(path)),
_ => partial(rng, path),
};
wire_safe(&spelled)
}
fn hostile(rng: &mut Rng, base: &str, evil: &[String]) -> String {
let target = if rng.odds(40) {
rng.pick(evil).clone()
} else {
base.to_string()
};
let slash = ["/", "%2F", "%2f", "%252F", "%C0%AF", "%5C", "\\"];
let dots = [
"..",
"%2E%2E",
"%2e%2e",
".%2E",
"%252E%252E",
"%C0%AE%C0%AE",
"..;",
"..%00",
".",
"%2E",
];
let up = |rng: &mut Rng| format!("{}{}", rng.pick(&dots), rng.pick(&slash));
let (lead, tail) = match rng.below(14) {
0 => (up(rng), String::new()),
1 => (format!("a{}{}", rng.pick(&slash), up(rng)), String::new()),
2 => (rng.pick(&slash).to_string(), String::new()),
3 => (String::new(), rng.pick(&slash).to_string()),
4 => (String::new(), "%00".into()),
5 => (String::new(), "%00.json".into()),
6 => ("//".into(), String::new()),
7 => (String::new(), "//".into()),
8 => (up(rng).repeat(rng.range(2, 5)), String::new()),
9 => (
String::new(),
format!("{}{}", rng.pick(&slash), rng.pick(&dots)),
),
10 => (
format!("{}{}", rng.pick(&dots), rng.pick(&slash)),
String::new(),
),
11 => ("%EF%BC%8F".into(), String::new()),
12 => (String::new(), "%".into()),
_ => (up(rng), up(rng)),
};
let body = if rng.odds(50) {
target.clone()
} else {
spell(rng, &target)
};
wire_safe(&format!("{lead}{body}{tail}"))
}
fn mutated(rng: &mut Rng, path: &str, other: &str) -> String {
match rng.below(7) {
0 => {
let letters: Vec<usize> = path
.char_indices()
.filter(|(_, ch)| ch.is_ascii_alphabetic())
.map(|(at, _)| at)
.collect();
match letters.is_empty() {
true => format!("{path}x"),
false => {
let at = *rng.pick(&letters);
let flipped = path.as_bytes()[at] ^ 0x20;
format!("{}{}{}", &path[..at], flipped as char, &path[at + 1..])
}
}
}
1 => format!("{path}x"),
2 => path
.chars()
.take(path.chars().count().saturating_sub(1))
.collect(),
3 => format!("{path}/{}", other.rsplit('/').next().unwrap_or("")),
4 => other.to_string(),
5 => path.replacen('/', "//", 1),
_ => format!("{}{path}", other.rsplit('/').next().unwrap_or("")),
}
}
#[derive(Default)]
struct Lower {
commit: BTreeMap<String, String>,
/// first the residual rows the map serves, then the rest, which come from the layout.
tree: (BTreeMap<String, String>, BTreeMap<String, String>),
entry: (BTreeMap<String, String>, BTreeMap<String, String>),
blob: (BTreeMap<String, String>, BTreeMap<String, String>),
blame: (BTreeMap<String, String>, BTreeMap<String, String>),
log: (BTreeMap<String, String>, BTreeMap<String, String>),
}
#[derive(Clone, Copy)]
enum Mode {
Exact,
Residual,
Disk,
}
fn upper_hex(text: &str) -> String {
let bytes = text.as_bytes();
let mut out = String::new();
let mut at = 0;
while at < bytes.len() {
if bytes[at] == b'%'
&& at + 2 < bytes.len()
&& bytes[at + 1].is_ascii_hexdigit()
&& bytes[at + 2].is_ascii_hexdigit()
{
out.push('%');
out.push((bytes[at + 1] as char).to_ascii_uppercase());
out.push((bytes[at + 2] as char).to_ascii_uppercase());
at += 3;
} else {
let ch = text[at..].chars().next().unwrap();
out.push(ch);
at += ch.len_utf8();
}
}
out
}
fn split_lowered(
rows: &BTreeMap<String, String>,
residual: &BTreeMap<String, String>,
) -> (BTreeMap<String, String>, BTreeMap<String, String>) {
let laid = rows
.iter()
.filter(|(key, _)| !residual.contains_key(*key))
.map(|(key, file)| (key.to_ascii_lowercase(), file.clone()))
.collect();
(lowered(residual), laid)
}
fn lowered(rows: &BTreeMap<String, String>) -> BTreeMap<String, String> {
rows.iter()
.map(|(key, file)| (key.to_ascii_lowercase(), file.clone()))
.collect()
}
struct Seed<'a> {
rng: Rng,
built: &'a Built,
sealed: &'a SealedRepo,
refs: Vec<String>,
evil: Vec<String>,
insensitive: bool,
generation: PathBuf,
port: u16,
lower: Lower,
log_keys: Vec<String>,
residual: Rows,
prefixes: BTreeMap<String, BTreeSet<String>>,
layout: paths::Layout,
stats: BTreeMap<&'static str, usize>,
mismatches: Vec<String>,
}
enum Want {
Hit(PathBuf),
/// keys over MAX_KEY_BYTES stay out of the map, so either answer is fine.
Dropped(PathBuf),
Miss(&'static str),
}
/// `$arg_name` takes the first `name=` in the query, and the name ignores ascii case.
fn arg(query: &str, name: &str) -> String {
query
.split('&')
.find_map(|pair| {
let (key, value) = pair.split_once('=')?;
key.eq_ignore_ascii_case(name).then(|| value.to_string())
})
.unwrap_or_default()
}
impl Seed<'_> {
/// the repo and ref come through maps, which fold ascii case.
fn want(&self, nsid: &str, query: &str, mode: Mode) -> Want {
let (repo, at, path) = (arg(query, "repo"), arg(query, "ref"), arg(query, "path"));
let blame = nsid == "getBlame" && path.is_empty();
if (nsid == "listCommits" || nsid == "getBlame") && at.is_empty() || blame {
return Want::Miss("invalid_request");
}
if repo.is_empty() {
return Want::Miss("invalid_request");
}
if !repo.eq_ignore_ascii_case(DID) {
return Want::Miss("git_repo_not_found");
}
let Some(commit) = self.lower.commit.get(&at.to_ascii_lowercase()) else {
return Want::Miss("git_ref_not_found");
};
if matches!(nsid, "getEntry" | "getBlob") && path.is_empty() {
return Want::Miss("invalid_request");
}
let rows = &self.sealed.rows();
let (map, lower, missing) = match nsid {
"getTree" => (&rows.tree, &self.lower.tree, "git_path_not_found"),
"getEntry" => (&rows.entry, &self.lower.entry, "git_entry_not_found"),
"getBlob" => (&rows.blob, &self.lower.blob, "git_file_not_found"),
"listCommits" => (&rows.log, &self.lower.log, "git_not_projected"),
_ => (&rows.blame, &self.lower.blame, "git_not_projected"),
};
let key = match nsid {
"listCommits" => format!(
"{commit}|{path}|{}|{}",
arg(query, "limit"),
arg(query, "cursor")
),
_ => format!("{commit}|{path}"),
};
let found = match mode {
// %2f and %2F decode alike, the uppercase spelling is tried too
Mode::Exact => map
.get(&key)
.or_else(|| map.get(&format!("{commit}|{}", upper_hex(&path)))),
Mode::Residual => lower.0.get(&key.to_ascii_lowercase()),
Mode::Disk => lower.1.get(&key.to_ascii_lowercase()),
};
match found {
None => Want::Miss(missing),
Some(file) => {
let file = self.sealed.pool.join(file);
let old_key = repo_token(DID).len() + 1 + key.len();
match old_key > MAX_KEY_BYTES {
true => Want::Dropped(file),
false => Want::Hit(file),
}
}
}
}
fn error(&self, name: &str) -> (u16, Vec<u8>) {
let status = crate::nginx::ERRORS
.iter()
.find(|(have, ..)| *have == name)
.unwrap()
.1;
let body = std::fs::read(self.generation.join(format!("www/_errors/{name}.json"))).unwrap();
(status, body)
}
fn matches(
&self,
nsid: &str,
want: &Want,
answer: &(u16, String, Vec<u8>),
) -> Result<(), String> {
let (status, head, body) = answer;
let hit = |file: &Path| -> Result<(), String> {
let expected = std::fs::read(file).unwrap();
if *status != 200 || *body != expected {
return Err(format!(
"want 200 with {} bytes of {}, got {status} with {} bytes: {}",
expected.len(),
file.display(),
body.len(),
preview(body)
));
}
let kind = if nsid == "getBlob" {
"application/octet-stream"
} else {
"application/json"
};
match head
.to_ascii_lowercase()
.contains(&format!("content-type: {kind}"))
{
true => Ok(()),
false => Err(format!("wrong content type: {head}")),
}
};
match want {
Want::Hit(file) => hit(file),
Want::Dropped(file) => match status {
200 => hit(file),
_ => Ok(()),
},
Want::Miss(name) => {
let (code, expected) = self.error(name);
match *status == code && *body == expected {
true => Ok(()),
false => Err(format!(
"want {code} {name}, got {status} with {} bytes: {}",
body.len(),
preview(body)
)),
}
}
}
}
fn check(&mut self, nsid: &str, query: &str) {
let target = format!("/xrpc/org.tangled.git.{nsid}?{query}");
if target.len() > MAX_TARGET {
return;
}
let answer = http(self.port, "GET", &target, &[], b"");
let fail = |this: &mut Self, message: String| {
if this.mismatches.len() < 8 {
this.mismatches.push(format!("{target}: {message}"));
}
};
if String::from_utf8_lossy(&answer.2).contains(SECRET) {
return fail(self, "served a planted secret".into());
}
let exact = self.want(nsid, query, Mode::Exact);
let Err(message) = self.matches(nsid, &exact, &answer) else {
let kind = match exact {
Want::Hit(_) | Want::Dropped(_) if answer.0 == 200 => "hit",
_ => "miss",
};
*self.stats.entry(kind).or_default() += 1;
return;
};
let mut folds = vec![("map fold", Mode::Residual)];
if self.insensitive {
folds.push(("disk fold", Mode::Disk));
}
for (label, mode) in folds {
let folded = self.want(nsid, query, mode);
if matches!(folded, Want::Hit(_) | Want::Dropped(_))
&& self.matches(nsid, &folded, &answer).is_ok()
{
*self.stats.entry(label).or_default() += 1;
return;
}
}
// the map drops case twins and an insensitive disk can't hold laid-out ones, so a 404 is fine
if matches!(exact, Want::Hit(_)) && answer.0 != 200 {
let (in_map, in_rows) = self.twins(nsid, query);
if in_map > 1 || self.insensitive && (in_rows > 1 || self.folded_parent(query)) {
*self.stats.entry("twin 404").or_default() += 1;
return;
}
}
fail(self, message);
}
fn folded_parent(&self, query: &str) -> bool {
let path = unescape(&arg(query, "path"));
let mut end = 0;
for part in path.split('/') {
end += part.len();
let prefix = path[..end].to_ascii_lowercase();
if self
.prefixes
.get(&prefix)
.is_some_and(|spellings| spellings.len() > 1)
{
return true;
}
end += 1;
}
false
}
fn twins(&self, nsid: &str, query: &str) -> (usize, usize) {
let Some(commit) = self
.lower
.commit
.get(&arg(query, "ref").to_ascii_lowercase())
else {
return (0, 0);
};
let (rows, residual) = (&self.sealed.rows(), &self.residual);
let (all, map, key) = match nsid {
"getTree" => (
&rows.tree,
&residual.tree,
format!("{commit}|{}", arg(query, "path")),
),
"getEntry" => (
&rows.entry,
&residual.entry,
format!("{commit}|{}", arg(query, "path")),
),
"getBlob" => (
&rows.blob,
&residual.blob,
format!("{commit}|{}", arg(query, "path")),
),
"getBlame" => (
&rows.blame,
&residual.blame,
format!("{commit}|{}", arg(query, "path")),
),
_ => (
&rows.log,
&residual.log,
format!(
"{commit}|{}|{}|{}",
arg(query, "path"),
arg(query, "limit"),
arg(query, "cursor")
),
),
};
let key = key.to_ascii_lowercase();
let folding = |rows: &BTreeMap<String, String>| {
rows.keys()
.filter(|have| have.to_ascii_lowercase() == key)
.count()
};
(folding(map), folding(all))
}
fn pick_ref(&mut self) -> String {
let rng = &mut self.rng;
let name = rng.pick(&self.built.branch_names).clone();
let wire = match rng.below(12) {
0..=4 => rng.pick(&self.refs).clone(),
5 => format!("refs/heads/{name}"),
6 => spell(rng, &format!("refs/heads/{name}")),
7 => spell(rng, &name),
8 => rng.pick(&self.built.commits).clone(),
9 => hostile(rng, &name, &self.evil),
10 => name.to_ascii_uppercase(),
_ => String::new(),
};
wire_safe(&wire)
}
fn pick_path(&mut self) -> String {
let rng = &mut self.rng;
let listed = &self.built.listed;
let real = if listed.is_empty() {
String::new()
} else {
rng.pick(listed).0.clone()
};
let other = if listed.is_empty() {
String::new()
} else {
rng.pick(listed).0.clone()
};
match rng.below(100) {
0..=44 => spell(rng, &real),
45..=59 => {
let changed = mutated(rng, &real, &other);
spell(rng, &changed)
}
60..=74 => hostile(rng, &real, &self.evil),
75..=79 => String::new(),
80..=84 => {
let evil = rng.pick(&self.evil).clone();
spell(rng, &evil)
}
_ => spell(rng, &real),
}
}
fn request(&mut self) {
let nsid = *self
.rng
.pick(&["getTree", "getEntry", "getBlob", "getBlame", "listCommits"]);
let repo = match self.rng.below(100) {
0..=3 => String::new(),
4..=7 => "did:plc:nobody".to_string(),
_ => DID.to_string(),
};
let mut at = self.pick_ref();
let mut path = self.pick_path();
let mut pairs = vec![format!("repo={repo}")];
if nsid == "listCommits" && self.rng.odds(70) {
let key = self.rng.pick(&self.log_keys).clone();
let mut parts = key.splitn(4, '|');
let (oid, logged) = (parts.next().unwrap(), parts.next().unwrap());
at = oid.to_string();
path = spell(&mut self.rng, logged);
for name in ["limit", "cursor"] {
let value = spell(&mut self.rng, parts.next().unwrap());
pairs.push(format!("{name}={value}"));
}
}
pairs.push(format!("ref={at}"));
if !self.rng.odds(4) {
pairs.push(format!("path={path}"));
}
if self.rng.odds(8) {
let second = self.pick_path();
pairs.push(format!("path={second}"));
}
if self.rng.odds(4) {
let second = self.pick_ref();
pairs.push(format!("ref={second}"));
}
if self.rng.odds(3) {
pairs.push("PATH=zz".into());
}
for at in (1..pairs.len()).rev() {
if self.rng.odds(40) {
pairs.swap(at, self.rng.below(at + 1));
}
}
self.check(nsid, &pairs.join("&"));
}
}
fn suffix(kind: Kind) -> String {
kind.file("", (kind == Kind::Log).then_some(("", "")))
}
fn preview(body: &[u8]) -> String {
let text = String::from_utf8_lossy(body);
text.chars().take(80).collect::<String>().replace('\n', " ")
}
fn plant(pool: &Path, sealed: &SealedRepo, outside: &Path) -> Vec<String> {
std::fs::write(outside.join("secret"), format!("{SECRET} {}", sealed.key)).unwrap();
std::fs::create_dir_all(outside.join("dir")).unwrap();
let secret = outside.join("secret");
for kind in Kind::ALL {
std::fs::copy(
&secret,
outside.join("dir").join(format!("x{}", suffix(kind))),
)
.unwrap();
}
let link = |at: &Path, name: &str, to: &Path| {
std::os::unix::fs::symlink(to, at.join(name)).unwrap();
};
let commits = pool.join("paths/c");
let seals = paths::seal_dir(pool, &sealed.paths);
let mut dirs: Vec<PathBuf> = std::fs::read_dir(&commits)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.is_dir())
.collect();
dirs.push(commits.clone());
dirs.push(seals);
for dir in &dirs {
for kind in Kind::ALL {
let ext = suffix(kind);
link(dir, &format!("zz-evil{ext}"), &secret);
link(dir, &format!("zz-up{ext}"), &secret);
link(
dir,
&format!("zz-rel{ext}"),
Path::new("../../../../../../../../../tmp/zz-nope"),
);
}
link(dir, "zz-dir", &outside.join("dir"));
}
[
"zz-evil",
"zz-up",
"zz-rel",
"zz-dir/x",
"zz-dir",
"../zz-up",
"../../zz-up",
]
.map(String::from)
.to_vec()
}
fn run_seed(seed: u64, nginx_binary: &Path, dir: &Path) -> Result<String, String> {
let mut rng = Rng(seed);
let built = build(&mut rng, dir);
let (mut config, mut manifest, generation) = test_fixture(dir);
config.serve.limits.reads = None;
config.serve.limits.read_connections = None;
manifest.repos[0].repo_did = DID.into();
let served = dir.join("repos/key");
let input = RepoInput {
did: RepoDid::new(DID).unwrap(),
name: "fixture".into(),
export_dir: built.dir.clone(),
served: RepoStore::served(&served),
shared_blobs: dir.join("git-xrpc/blobs"),
patches: Some(
built
.commits
.windows(2)
.take(rng.range(0, 2))
.map(|pair| {
(
Oid::from_hex(&pair[0]).unwrap(),
Oid::from_hex(&pair[1]).unwrap(),
)
})
.collect(),
),
refs_digest: format!("fuzz-{seed}"),
verify: false,
};
let sealed = crate::git_xrpc::seal(&input).map_err(|error| format!("seal: {error:#}"))?;
let folds = paths::folds(&sealed.pool).unwrap();
let (layout, residual) = paths::split(&sealed.rows(), folds);
let blobs = check_blobs(&built, &sealed.rows(), &sealed.pool)?;
crate::atproto::emit(&generation, &[]).unwrap();
crate::blobs::emit(dir, &generation, &[], &Default::default()).unwrap();
crate::git_xrpc::emit(&generation, std::slice::from_ref(&sealed)).unwrap();
let repo_map = generation.join("nginx/maps/60_atproto.conf");
let text = std::fs::read_to_string(&repo_map).unwrap().replace(
"default \"\";",
&format!("default \"\";\n {DID} {};", repo_token(DID)),
);
std::fs::write(&repo_map, text).unwrap();
super::super::emit_at(&config, &manifest, &generation, &generation).unwrap();
let outside = dir.join("outside");
std::fs::create_dir_all(&outside).unwrap();
let evil = plant(&sealed.pool, &sealed, &outside);
let (port, _nginx) = start_nginx(nginx_binary, &generation);
let mut refs: Vec<String> = sealed.rows().commit.keys().cloned().collect();
refs.retain(|spelling| !spelling.is_empty());
let mut this = Seed {
rng,
built: &built,
sealed: &sealed,
refs,
evil: evil.clone(),
insensitive: folds,
generation,
port,
lower: Lower {
commit: lowered(&sealed.rows().commit),
tree: split_lowered(&sealed.rows().tree, &residual.tree),
entry: split_lowered(&sealed.rows().entry, &residual.entry),
blob: split_lowered(&sealed.rows().blob, &residual.blob),
blame: split_lowered(&sealed.rows().blame, &residual.blame),
log: split_lowered(&sealed.rows().log, &residual.log),
},
log_keys: sealed.rows().log.keys().cloned().collect(),
residual: residual.clone(),
prefixes: prefixes(&built.listed),
layout,
stats: BTreeMap::new(),
mismatches: Vec::new(),
};
let lead: Vec<String> = this.refs.iter().take(3).cloned().collect();
for at in &lead {
for nsid in ["getTree", "getEntry", "getBlob", "getBlame", "listCommits"] {
for path in &evil {
for spelled in [path.clone(), uri_component(path), path.replace('/', "%2F")] {
this.check(
nsid,
&format!("repo={DID}&ref={}&path={spelled}", uri_component(at)),
);
}
}
}
}
let requests: usize = env("FUZZ_REQUESTS").unwrap_or(300);
for _ in 0..requests {
this.request();
}
if !this.mismatches.is_empty() {
return Err(this.mismatches.join("\n "));
}
let laid: usize = layout_len(&this.layout);
Ok(format!(
"seed={seed} commits={} laid={laid} residual={} blobs={blobs} {:?}",
built.commits.len(),
residual.tree.len() + residual.entry.len() + residual.blob.len() + residual.blame.len(),
this.stats
))
}
fn prefixes(listed: &[(String, bool)]) -> BTreeMap<String, BTreeSet<String>> {
let mut out: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
for (path, _) in listed {
let mut end = 0;
for part in path.split('/') {
end += part.len();
out.entry(path[..end].to_ascii_lowercase())
.or_default()
.insert(path[..end].to_string());
end += 1;
}
}
out
}
fn unescape(text: &str) -> String {
let bytes = text.as_bytes();
let mut out = Vec::new();
let mut at = 0;
while at < bytes.len() {
let hex = |at: usize| bytes.get(at).and_then(|byte| (*byte as char).to_digit(16));
match (bytes[at], hex(at + 1), hex(at + 2)) {
(b'%', Some(high), Some(low)) => {
out.push((high * 16 + low) as u8);
at += 3;
}
(byte, ..) => {
out.push(byte);
at += 1;
}
}
}
String::from_utf8_lossy(&out).into_owned()
}
fn layout_len(layout: &paths::Layout) -> usize {
layout.values().map(BTreeMap::len).sum()
}
#[test]
#[ignore = "a fuzzer: requires nginx (NGINX_BINARY); FUZZ_SEED, FUZZ_SEEDS, FUZZ_REQUESTS, FUZZ_KEEP"]
fn fuzz_paths() {
let nginx_binary = PathBuf::from(
std::env::var_os("NGINX_BINARY").expect("NGINX_BINARY must name the real nginx binary"),
);
let first: u64 = env("FUZZ_SEED").unwrap_or(0);
let count: u64 = env("FUZZ_SEEDS").unwrap_or(1);
let mut failed = Vec::new();
for seed in first..first + count {
let dir = tempfile::tempdir().unwrap();
let ran = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
run_seed(seed, &nginx_binary, dir.path())
}));
let result = ran.unwrap_or_else(|panic| {
Err(panic
.downcast_ref::<String>()
.cloned()
.or_else(|| panic.downcast_ref::<&str>().map(|text| text.to_string()))
.unwrap_or_default())
});
match result {
Ok(report) => println!("FUZZ ok {report}"),
Err(error) => {
let kept = std::env::var_os("FUZZ_KEEP").map(|_| dir.keep());
println!("FUZZ FAIL seed={seed} kept={kept:?}: {error}");
failed.push(seed);
}
}
}
assert!(failed.is_empty(), "failing seeds: {failed:?}");
}