@@ -0,0 +1,687 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use serde::Serialize; + +use crate::git_xrpc::Rows; +use crate::manifest::GenerationManifest; +use crate::plan::{repo_key, repo_token}; +use crate::store::RepoStore; + +pub(crate) type Params = BTreeMap; +type Key = (&'static str, Params); +type Group = (Vec, BTreeSet<(String, String)>); + +#[derive(Serialize, Debug, Clone, PartialEq)] +pub(crate) struct Line { + pub(crate) op: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) nsid: Option<&'static str>, + pub(crate) params: Params, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub(crate) any: Vec, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub(crate) default: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) old: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) new: Option, +} + +impl Line { + fn of(op: &'static str, (nsid, params): Key, old: Option, new: Option) -> Self { + Self { + op, + nsid: Some(nsid), + params, + any: Vec::new(), + default: false, + old, + new, + } + } + + pub(crate) fn repo(&self) -> Option<&str> { + self.params + .get("repo") + .or_else(|| self.params.get("repos")) + .or_else(|| self.params.get("subject")) + .map(String::as_str) + } +} + +#[derive(Default)] +pub(crate) struct Answers { + exact: BTreeMap, + bucketed: BTreeMap, + defaults: BTreeMap<&'static str, String>, +} + +impl Answers { + fn put(&mut self, nsid: &'static str, params: Params, file: String) { + self.exact.entry((nsid, params)).or_insert(file); + } +} + +fn is_oid(value: &str) -> bool { + matches!(value.len(), 40 | 64) && value.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// a page on a new oid could not be in the old generation +fn names_new_oid(params: &Params) -> bool { + ["ref", "base", "head"] + .iter() + .any(|name| params.get(*name).is_some_and(|value| is_oid(value))) +} + +pub(crate) fn compare(old: &Answers, new: &Answers, lines: &mut Vec) { + merge(&old.exact, &new.exact, |key, old, new| { + let mut line = match (old, new) { + (None, Some(_)) if names_new_oid(&key.1) => return, + (None, Some(new)) => Line::of("added", key.clone(), None, Some(new.clone())), + (Some(old), None) => Line::of("removed", key.clone(), Some(old.clone()), None), + (Some(old), Some(new)) if old != new => { + Line::of("changed", key.clone(), Some(old.clone()), Some(new.clone())) + } + _ => return, + }; + line.any = crate::xrpc::social::ignored_params(key.0) + .iter() + .map(|name| name.to_string()) + .collect(); + lines.push(line); + }); + merge(&old.bucketed, &new.bucketed, |key, old, new| { + let op = match (old, new) { + (None, Some(_)) => "added", + (Some(_), None) => "removed", + (Some(old), Some(new)) if old.1 != new.1 => "changed", + _ => return, + }; + let mut line = Line::of(op, key.clone(), None, None); + line.any = new.or(old).map(|(any, _)| any.clone()).unwrap_or_default(); + lines.push(line); + }); + merge(&old.defaults, &new.defaults, |nsid, old, new| { + if old != new { + let mut line = Line::of("changed", (nsid, Params::new()), old.cloned(), new.cloned()); + line.default = true; + lines.push(line); + } + }); +} + +fn merge( + old: &BTreeMap, + new: &BTreeMap, + mut each: impl FnMut(&K, Option<&V>, Option<&V>), +) { + let keys: BTreeSet<&K> = old.keys().chain(new.keys()).collect(); + for key in keys { + each(key, old.get(key), new.get(key)); + } +} + +pub(crate) fn decode(spelling: &str) -> Option { + let mut bytes = Vec::with_capacity(spelling.len()); + let mut rest = spelling.as_bytes(); + while let Some((&byte, tail)) = rest.split_first() { + match byte { + b'+' => bytes.push(b' '), + b'%' if tail.len() >= 2 => { + let hex = std::str::from_utf8(&tail[..2]).ok()?; + bytes.push(u8::from_str_radix(hex, 16).ok()?); + rest = &tail[2..]; + continue; + } + _ => bytes.push(byte), + } + rest = tail; + } + let value = String::from_utf8(bytes).ok()?; + crate::xrpc::spellings(&value) + .iter() + .any(|s| s == spelling) + .then_some(value) +} + +fn canonical_did(value: String) -> String { + match value + .strip_prefix("at://") + .and_then(|rest| rest.strip_suffix("/org.tangled.repo.manifest/self")) + { + Some(did) if !did.contains('/') => did.to_string(), + _ => value, + } +} + +fn params<'a>(pairs: impl IntoIterator)>) -> Params { + pairs + .into_iter() + .filter_map(|(name, value)| Some((name.to_string(), value?.to_string()))) + .filter(|(_, value)| !value.is_empty()) + .collect() +} + +fn ref_names(commit: &BTreeMap) -> BTreeMap<&str, BTreeSet>> { + let decoded: BTreeMap = commit + .iter() + .filter_map(|(spelling, oid)| Some((decode(spelling)?, oid.as_str()))) + .collect(); + let mut names: BTreeMap<&str, BTreeSet>> = BTreeMap::new(); + for (name, &oid) in &decoded { + let name = match name.as_str() { + "" => None, + full => { + let short = full + .strip_prefix("refs/heads/") + .or_else(|| full.strip_prefix("refs/tags/")); + match short { + Some(short) if decoded.get(short) == Some(&oid) => Some(short.to_string()), + _ => Some(full.to_string()), + } + } + }; + names.entry(oid).or_default().insert(name); + } + names +} + +/// `org.tangled.git.getArchive` is left out because no page embeds an archive +pub(crate) fn git_answers(did: &str, rows: &Rows, other: Option<&Rows>, answers: &mut Answers) { + let dir = format!("_git_xrpc/{}", repo_token(did)); + let file = |name: &str| format!("{dir}/{name}"); + let refs = ref_names(&rows.commit); + let other_refs = other.map(|other| ref_names(&other.commit)); + let at = |oid: &str| refs.get(oid).into_iter().flatten(); + let same = |theirs: Option<&BTreeMap>, key: &str, name: &str, oids: &[&str]| { + let (Some(theirs), Some(other_refs)) = (theirs, &other_refs) else { + return false; + }; + theirs.get(key).is_some_and(|other| other == name) + && oids.iter().all(|oid| refs.get(oid) == other_refs.get(oid)) + }; + let repo = Some(did); + + for (nsid, kind, theirs) in [ + ( + "org.tangled.git.listRefs", + &rows.refs, + other.map(|o| &o.refs), + ), + ( + "org.tangled.git.listBranches", + &rows.branches, + other.map(|o| &o.branches), + ), + ( + "org.tangled.git.listTags", + &rows.tags, + other.map(|o| &o.tags), + ), + ] { + for (key, name) in kind { + if same(theirs, key, name, &[]) { + continue; + } + let Some((limit, cursor)) = key.split_once('|') else { + continue; + }; + let (Some(limit), Some(cursor)) = (decode(limit), decode(cursor)) else { + continue; + }; + let params = params([ + ("repo", repo), + ("limit", Some(&limit)), + ("cursor", Some(&cursor)), + ]); + answers.put(nsid, params, file(name)); + } + } + for (nsid, kind, theirs, arg, prefix) in [ + ( + "org.tangled.git.getBranch", + &rows.branch, + other.map(|o| &o.branch), + "name", + "refs/heads/", + ), + ( + "org.tangled.git.getTag", + &rows.tag, + other.map(|o| &o.tag), + "tag", + "refs/tags/", + ), + ] { + for (key, name) in kind { + if same(theirs, key, name, &[]) { + continue; + } + let Some(value) = decode(key) else { continue }; + let value = value.strip_prefix(prefix).unwrap_or(&value); + answers.put( + nsid, + params([("repo", repo), (arg, Some(value))]), + file(name), + ); + } + } + // getBlame answers an empty ref with an error + for (nsid, kind, theirs, needs_ref) in [ + ( + "org.tangled.git.getTree", + &rows.tree, + other.map(|o| &o.tree), + false, + ), + ( + "org.tangled.git.getEntry", + &rows.entry, + other.map(|o| &o.entry), + false, + ), + ( + "org.tangled.git.getBlob", + &rows.blob, + other.map(|o| &o.blob), + false, + ), + ( + "org.tangled.git.getBlame", + &rows.blame, + other.map(|o| &o.blame), + true, + ), + ] { + for (key, name) in kind { + let Some((oid, path)) = key.split_once('|') else { + continue; + }; + if same(theirs, key, name, &[oid]) { + continue; + } + let Some(path) = decode(path) else { continue }; + for spelled in at(oid).filter(|spelled| !needs_ref || spelled.is_some()) { + let params = params([ + ("repo", repo), + ("ref", spelled.as_deref()), + ("path", Some(&path)), + ]); + answers.put(nsid, params, file(name)); + } + } + } + for (key, name) in &rows.log { + let Some((oid, rest)) = key.split_once('|') else { + continue; + }; + if same(other.map(|o| &o.log), key, name, &[oid]) { + continue; + } + let mut tail = rest.rsplitn(3, '|'); + let (Some(cursor), Some(limit), Some(path)) = (tail.next(), tail.next(), tail.next()) + else { + continue; + }; + let (Some(path), Some(limit), Some(cursor)) = (decode(path), decode(limit), decode(cursor)) + else { + continue; + }; + for spelled in at(oid).flatten() { + let params = params([ + ("repo", repo), + ("ref", Some(spelled)), + ("path", Some(&path)), + ("limit", Some(&limit)), + ("cursor", Some(&cursor)), + ]); + answers.put("org.tangled.git.listCommits", params, file(name)); + } + } + for (oid, name) in &rows.languages { + if same(other.map(|o| &o.languages), oid, name, &[oid]) { + continue; + } + for spelled in at(oid) { + let params = params([("repo", repo), ("ref", spelled.as_deref())]); + answers.put("org.tangled.git.listLanguages", params, file(name)); + } + } + for (nsid, repo_param, kind, theirs) in [ + ( + "org.tangled.git.getDiff", + "repo", + &rows.diff, + other.map(|o| &o.diff), + ), + ( + "org.tangled.temp.git.getMergeBase", + "repos", + &rows.merge_base, + other.map(|o| &o.merge_base), + ), + ] { + for (key, name) in kind { + let Some((base, head)) = key.split_once('|') else { + continue; + }; + if name == crate::git_xrpc::NO_MERGE_BASE || same(theirs, key, name, &[base, head]) { + continue; + } + for base in at(base).flatten() { + for head in at(head).flatten() { + let params = params([ + (repo_param, repo), + ("base", Some(base)), + ("head", Some(head)), + ]); + answers.put(nsid, params, file(name)); + } + } + } + } +} + +#[derive(Debug, PartialEq)] +pub(crate) struct MapBlock { + pub(crate) source: String, + pub(crate) target: String, + pub(crate) default: String, + pub(crate) rows: Vec<(String, String)>, +} + +/// only reads what `nginx::conf::Block` prints, not general nginx config +pub(crate) fn parse_maps(text: &str) -> Vec { + let mut maps = Vec::new(); + let mut open: Option = None; + for line in text.lines() { + let line = line.trim(); + if line == "}" { + maps.extend(open.take()); + continue; + } + let words = words(line.trim_end_matches('{')); + match (&mut open, words.as_slice()) { + (None, [map, source, target]) if map == "map" && line.ends_with('{') => { + open = Some(MapBlock { + source: source.clone(), + target: target.clone(), + default: String::new(), + rows: Vec::new(), + }); + } + (Some(map), [name, value]) if name == "default" => map.default = value.clone(), + (Some(map), [key, value]) => map.rows.push((key.clone(), value.clone())), + _ => {} + } + } + maps +} + +fn words(line: &str) -> Vec { + let mut words = Vec::new(); + let mut chars = line.trim_end_matches(';').chars().peekable(); + while let Some(&c) = chars.peek() { + if c.is_whitespace() { + chars.next(); + continue; + } + let mut word = String::new(); + if c == '"' { + chars.next(); + while let Some(c) = chars.next() { + match c { + '"' => break, + '\\' => match chars.next() { + Some('n') => word.push('\n'), + Some('r') => word.push('\r'), + Some('t') => word.push('\t'), + Some(other) => word.push(other), + None => {} + }, + _ => word.push(c), + } + } + } else { + while let Some(&c) = chars.peek() { + if c.is_whitespace() { + break; + } + word.push(c); + chars.next(); + } + } + words.push(word); + } + words +} + +pub(crate) const MAP_FILES: [&str; 3] = [ + crate::xrpc::MAPS_FILE, + crate::xrpc::RECORDS_MAPS_FILE, + crate::xrpc::social::MAPS_FILE, +]; + +pub(crate) const RECORD: &str = "com.atproto.repo.getRecord"; + +fn record_routes() -> [(&'static str, &'static str); 4] { + [ + (RECORD, "$ks_atp_rec"), + ("com.atproto.repo.listRecords", "$ks_atp_lr"), + ("com.atproto.repo.listRecords", "$ks_atp_lr_one"), + ("org.tangled.repo.listRecordVersions", "$ks_atp_ver"), + ] +} + +enum Part { + Args(Vec), + Repo, + Uri, + Bucket(Vec), +} + +fn arg_names(source: &str) -> Option> { + let names: Vec = source.split("$arg_").skip(1).map(str::to_string).collect(); + (source.starts_with("$arg_") && names.iter().all(|name| !name.contains('$'))).then_some(names) +} + +fn bucket_args(var: &str, sources: &BTreeMap<&str, &str>, depth: usize) -> Vec { + let Some(source) = sources.get(var).filter(|_| depth < 4) else { + return Vec::new(); + }; + let mut names = Vec::new(); + for part in source.split('|') { + match arg_names(part) { + Some(args) => names.extend(args), + None => names.extend(bucket_args(part, sources, depth + 1)), + } + } + names +} + +pub(crate) fn social_answers(maps: &[MapBlock], dids: &[&str], answers: &mut Answers) { + let tokens: BTreeMap = dids.iter().map(|did| (repo_token(did), *did)).collect(); + let sources: BTreeMap<&str, &str> = maps + .iter() + .map(|map| (map.target.as_str(), map.source.as_str())) + .collect(); + let routes: BTreeMap<&str, &'static str> = crate::xrpc::social::route_vars() + .chain(record_routes()) + .map(|(nsid, var)| (var, nsid)) + .collect(); + for map in maps { + let Some(&nsid) = routes.get(map.target.as_str()) else { + continue; + }; + if !map.default.is_empty() && !map.default.starts_with('$') { + answers + .defaults + .insert(nsid, format!("_atproto/{}", map.default)); + } + let parts: Vec = map + .source + .split('|') + .map(|part| match part { + "$ks_atp_repo" => Part::Repo, + "$ks_atp_uri" => Part::Uri, + _ => match arg_names(part) { + Some(names) => Part::Args(names), + None => Part::Bucket(bucket_args(part, &sources, 0)), + }, + }) + .collect(); + for (key, file) in &map.rows { + let values: Vec<&str> = key.split('|').collect(); + if values.len() != parts.len() { + continue; + } + let mut spelled = vec![Params::new()]; + let mut any = Vec::new(); + let mut buckets = Vec::new(); + let mut token = None; + for (part, value) in parts.iter().zip(&values) { + let name_value: Vec<(String, String)> = match part { + Part::Repo => { + let Some(did) = tokens.get(*value) else { + spelled.clear(); + break; + }; + token = Some(*value); + vec![("repo".into(), did.to_string())] + } + Part::Uri => vec![("uri".into(), value.to_string())], + Part::Args(names) => { + let Some(decoded) = decode(value) else { + spelled.clear(); + break; + }; + let decoded = canonical_did(decoded); + if decoded.is_empty() { + continue; + } + names + .iter() + .map(|name| (name.clone(), decoded.clone())) + .collect() + } + Part::Bucket(names) => { + any.extend(names.iter().cloned()); + buckets.push(value.to_string()); + continue; + } + }; + spelled = spelled + .iter() + .flat_map(|params| { + name_value.iter().map(move |(name, value)| { + let mut params = params.clone(); + params.insert(name.clone(), value.clone()); + params + }) + }) + .collect(); + } + let file = match (nsid, token) { + (RECORD, Some(token)) => format!("_atproto/{token}/{file}"), + _ => format!("_atproto/{file}"), + }; + for params in spelled { + if any.is_empty() { + answers.put(nsid, params, file.clone()); + } else { + let group = answers.bucketed.entry((nsid, params)).or_default(); + for name in &any { + if !group.0.contains(name) { + group.0.push(name.clone()); + } + } + group.1.insert((buckets.join("|"), file.clone())); + } + } + } + } +} + +pub(crate) fn read_maps(generation_dir: &Path) -> anyhow::Result> { + let mut maps = Vec::new(); + for name in MAP_FILES { + let path = generation_dir.join("nginx/maps").join(name); + maps.extend(parse_maps(&std::fs::read_to_string(&path)?)); + } + Ok(maps) +} + +fn seal_key<'a>(manifest: &'a GenerationManifest, did: &str) -> Option<&'a str> { + manifest.profile["git_xrpc_entries"][did].as_str() +} + +pub(crate) fn diff( + output_root: &Path, + old: (&GenerationManifest, &[MapBlock]), + new: (&GenerationManifest, &[MapBlock]), +) -> Vec { + let mut lines = Vec::new(); + let old_dids: BTreeSet<&str> = old.0.repos.iter().map(|r| r.repo_did.as_str()).collect(); + for entry in &new.0.repos { + let did = entry.repo_did.as_str(); + if !old_dids.contains(did) { + continue; + } + let (Some(was), Some(now)) = (seal_key(old.0, did), seal_key(new.0, did)) else { + lines.push(reset(did)); + continue; + }; + if was == now { + continue; + } + let served = RepoStore::served(&output_root.join("repos").join(repo_key(did))); + let trace = crate::generate::Tracer::new(); + let (Some(was), Some(now)) = trace.time("hooks: read seals", || { + ( + crate::git_xrpc::sealed_rows(&served, was), + crate::git_xrpc::sealed_rows(&served, now), + ) + }) else { + lines.push(reset(did)); + continue; + }; + let (mut before, mut after) = (Answers::default(), Answers::default()); + trace.time("hooks: expand git rows", || { + git_answers(did, &was, Some(&now), &mut before); + git_answers(did, &now, Some(&was), &mut after); + }); + trace.time("hooks: compare", || compare(&before, &after, &mut lines)); + } + if old.1 != new.1 { + let dids = |manifest: &GenerationManifest| -> Vec { + manifest.repos.iter().map(|r| r.repo_did.clone()).collect() + }; + let (old_dids, new_dids) = (dids(old.0), dids(new.0)); + let (mut before, mut after) = (Answers::default(), Answers::default()); + social_answers( + old.1, + &old_dids.iter().map(String::as_str).collect::>(), + &mut before, + ); + social_answers( + new.1, + &new_dids.iter().map(String::as_str).collect::>(), + &mut after, + ); + compare(&before, &after, &mut lines); + } + lines +} + +fn reset(did: &str) -> Line { + Line { + op: "reset", + nsid: None, + params: params([("repo", Some(did))]), + any: Vec::new(), + default: false, + old: None, + new: None, + } +}