use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use serde::Serialize;
use crate::git_xrpc::Rows;
use crate::manifest::GenerationManifest;
use crate::plan::{repo_key, repo_token};
use crate::store::RepoStore;
pub(crate) type Params = BTreeMap<String, String>;
type Key = (&'static str, Params);
type Group = (Vec<String>, BTreeSet<(String, String)>);
#[derive(Serialize, Debug, Clone, PartialEq)]
pub(crate) struct Line {
pub(crate) op: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) nsid: Option<&'static str>,
pub(crate) params: Params,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub(crate) any: Vec<String>,
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub(crate) default: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) old: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) new: Option<String>,
}
impl Line {
fn of(op: &'static str, (nsid, params): Key, old: Option<String>, new: Option<String>) -> Self {
Self {
op,
nsid: Some(nsid),
params,
any: Vec::new(),
default: false,
old,
new,
}
}
pub(crate) fn repo(&self) -> Option<&str> {
self.params
.get("repo")
.or_else(|| self.params.get("repos"))
.or_else(|| self.params.get("subject"))
.map(String::as_str)
}
}
#[derive(Default)]
pub(crate) struct Answers {
exact: BTreeMap<Key, String>,
bucketed: BTreeMap<Key, Group>,
defaults: BTreeMap<&'static str, String>,
}
impl Answers {
fn put(&mut self, nsid: &'static str, params: Params, file: String) {
self.exact.entry((nsid, params)).or_insert(file);
}
}
fn is_oid(value: &str) -> bool {
matches!(value.len(), 40 | 64) && value.bytes().all(|b| b.is_ascii_hexdigit())
}
/// a page on a new oid could not be in the old generation
fn names_new_oid(params: &Params) -> bool {
["ref", "base", "head"]
.iter()
.any(|name| params.get(*name).is_some_and(|value| is_oid(value)))
}
pub(crate) fn compare(old: &Answers, new: &Answers, lines: &mut Vec<Line>) {
merge(&old.exact, &new.exact, |key, old, new| {
let mut line = match (old, new) {
(None, Some(_)) if names_new_oid(&key.1) => return,
(None, Some(new)) => Line::of("added", key.clone(), None, Some(new.clone())),
(Some(old), None) => Line::of("removed", key.clone(), Some(old.clone()), None),
(Some(old), Some(new)) if old != new => {
Line::of("changed", key.clone(), Some(old.clone()), Some(new.clone()))
}
_ => return,
};
line.any = crate::xrpc::social::ignored_params(key.0)
.iter()
.map(|name| name.to_string())
.collect();
lines.push(line);
});
merge(&old.bucketed, &new.bucketed, |key, old, new| {
let op = match (old, new) {
(None, Some(_)) => "added",
(Some(_), None) => "removed",
(Some(old), Some(new)) if old.1 != new.1 => "changed",
_ => return,
};
let mut line = Line::of(op, key.clone(), None, None);
line.any = new.or(old).map(|(any, _)| any.clone()).unwrap_or_default();
lines.push(line);
});
merge(&old.defaults, &new.defaults, |nsid, old, new| {
if old != new {
let mut line = Line::of("changed", (nsid, Params::new()), old.cloned(), new.cloned());
line.default = true;
lines.push(line);
}
});
}
fn merge<K: Ord, V>(
old: &BTreeMap<K, V>,
new: &BTreeMap<K, V>,
mut each: impl FnMut(&K, Option<&V>, Option<&V>),
) {
let keys: BTreeSet<&K> = old.keys().chain(new.keys()).collect();
for key in keys {
each(key, old.get(key), new.get(key));
}
}
pub(crate) fn decode(spelling: &str) -> Option<String> {
let mut bytes = Vec::with_capacity(spelling.len());
let mut rest = spelling.as_bytes();
while let Some((&byte, tail)) = rest.split_first() {
match byte {
b'+' => bytes.push(b' '),
b'%' if tail.len() >= 2 => {
let hex = std::str::from_utf8(&tail[..2]).ok()?;
bytes.push(u8::from_str_radix(hex, 16).ok()?);
rest = &tail[2..];
continue;
}
_ => bytes.push(byte),
}
rest = tail;
}
let value = String::from_utf8(bytes).ok()?;
crate::xrpc::spellings(&value)
.iter()
.any(|s| s == spelling)
.then_some(value)
}
fn canonical_did(value: String) -> String {
match value
.strip_prefix("at://")
.and_then(|rest| rest.strip_suffix("/org.tangled.repo.manifest/self"))
{
Some(did) if !did.contains('/') => did.to_string(),
_ => value,
}
}
fn params<'a>(pairs: impl IntoIterator<Item = (&'a str, Option<&'a str>)>) -> Params {
pairs
.into_iter()
.filter_map(|(name, value)| Some((name.to_string(), value?.to_string())))
.filter(|(_, value)| !value.is_empty())
.collect()
}
fn ref_names(commit: &BTreeMap<String, String>) -> BTreeMap<&str, BTreeSet<Option<String>>> {
let decoded: BTreeMap<String, &str> = commit
.iter()
.filter_map(|(spelling, oid)| Some((decode(spelling)?, oid.as_str())))
.collect();
let mut names: BTreeMap<&str, BTreeSet<Option<String>>> = BTreeMap::new();
for (name, &oid) in &decoded {
let name = match name.as_str() {
"" => None,
full => {
let short = full
.strip_prefix("refs/heads/")
.or_else(|| full.strip_prefix("refs/tags/"));
match short {
Some(short) if decoded.get(short) == Some(&oid) => Some(short.to_string()),
_ => Some(full.to_string()),
}
}
};
names.entry(oid).or_default().insert(name);
}
names
}
/// `org.tangled.git.getArchive` is left out because no page embeds an archive
pub(crate) fn git_answers(did: &str, rows: &Rows, other: Option<&Rows>, answers: &mut Answers) {
let dir = format!("_git_xrpc/{}", repo_token(did));
let file = |name: &str| format!("{dir}/{name}");
let refs = ref_names(&rows.commit);
let other_refs = other.map(|other| ref_names(&other.commit));
let at = |oid: &str| refs.get(oid).into_iter().flatten();
let same = |theirs: Option<&BTreeMap<String, String>>, key: &str, name: &str, oids: &[&str]| {
let (Some(theirs), Some(other_refs)) = (theirs, &other_refs) else {
return false;
};
theirs.get(key).is_some_and(|other| other == name)
&& oids.iter().all(|oid| refs.get(oid) == other_refs.get(oid))
};
let repo = Some(did);
for (nsid, kind, theirs) in [
(
"org.tangled.git.listRefs",
&rows.refs,
other.map(|o| &o.refs),
),
(
"org.tangled.git.listBranches",
&rows.branches,
other.map(|o| &o.branches),
),
(
"org.tangled.git.listTags",
&rows.tags,
other.map(|o| &o.tags),
),
] {
for (key, name) in kind {
if same(theirs, key, name, &[]) {
continue;
}
let Some((limit, cursor)) = key.split_once('|') else {
continue;
};
let (Some(limit), Some(cursor)) = (decode(limit), decode(cursor)) else {
continue;
};
let params = params([
("repo", repo),
("limit", Some(&limit)),
("cursor", Some(&cursor)),
]);
answers.put(nsid, params, file(name));
}
}
for (nsid, kind, theirs, arg, prefix) in [
(
"org.tangled.git.getBranch",
&rows.branch,
other.map(|o| &o.branch),
"name",
"refs/heads/",
),
(
"org.tangled.git.getTag",
&rows.tag,
other.map(|o| &o.tag),
"tag",
"refs/tags/",
),
] {
for (key, name) in kind {
if same(theirs, key, name, &[]) {
continue;
}
let Some(value) = decode(key) else { continue };
let value = value.strip_prefix(prefix).unwrap_or(&value);
answers.put(
nsid,
params([("repo", repo), (arg, Some(value))]),
file(name),
);
}
}
// getBlame answers an empty ref with an error
for (nsid, kind, theirs, needs_ref) in [
(
"org.tangled.git.getTree",
&rows.tree,
other.map(|o| &o.tree),
false,
),
(
"org.tangled.git.getEntry",
&rows.entry,
other.map(|o| &o.entry),
false,
),
(
"org.tangled.git.getBlob",
&rows.blob,
other.map(|o| &o.blob),
false,
),
(
"org.tangled.git.getBlame",
&rows.blame,
other.map(|o| &o.blame),
true,
),
] {
for (key, name) in kind {
let Some((oid, path)) = key.split_once('|') else {
continue;
};
if same(theirs, key, name, &[oid]) {
continue;
}
let Some(path) = decode(path) else { continue };
for spelled in at(oid).filter(|spelled| !needs_ref || spelled.is_some()) {
let params = params([
("repo", repo),
("ref", spelled.as_deref()),
("path", Some(&path)),
]);
answers.put(nsid, params, file(name));
}
}
}
for (key, name) in &rows.log {
let Some((oid, rest)) = key.split_once('|') else {
continue;
};
if same(other.map(|o| &o.log), key, name, &[oid]) {
continue;
}
let mut tail = rest.rsplitn(3, '|');
let (Some(cursor), Some(limit), Some(path)) = (tail.next(), tail.next(), tail.next())
else {
continue;
};
let (Some(path), Some(limit), Some(cursor)) = (decode(path), decode(limit), decode(cursor))
else {
continue;
};
for spelled in at(oid).flatten() {
let params = params([
("repo", repo),
("ref", Some(spelled)),
("path", Some(&path)),
("limit", Some(&limit)),
("cursor", Some(&cursor)),
]);
answers.put("org.tangled.git.listCommits", params, file(name));
}
}
for (oid, name) in &rows.languages {
if same(other.map(|o| &o.languages), oid, name, &[oid]) {
continue;
}
for spelled in at(oid) {
let params = params([("repo", repo), ("ref", spelled.as_deref())]);
answers.put("org.tangled.git.listLanguages", params, file(name));
}
}
for (nsid, repo_param, kind, theirs) in [
(
"org.tangled.git.getDiff",
"repo",
&rows.diff,
other.map(|o| &o.diff),
),
(
"org.tangled.temp.git.getMergeBase",
"repos",
&rows.merge_base,
other.map(|o| &o.merge_base),
),
] {
for (key, name) in kind {
let Some((base, head)) = key.split_once('|') else {
continue;
};
if name == crate::git_xrpc::NO_MERGE_BASE || same(theirs, key, name, &[base, head]) {
continue;
}
for base in at(base).flatten() {
for head in at(head).flatten() {
let params = params([
(repo_param, repo),
("base", Some(base)),
("head", Some(head)),
]);
answers.put(nsid, params, file(name));
}
}
}
}
}
#[derive(Debug, PartialEq)]
pub(crate) struct MapBlock {
pub(crate) source: String,
pub(crate) target: String,
pub(crate) default: String,
pub(crate) rows: Vec<(String, String)>,
}
/// only reads what `nginx::conf::Block` prints, not general nginx config
pub(crate) fn parse_maps(text: &str) -> Vec<MapBlock> {
let mut maps = Vec::new();
let mut open: Option<MapBlock> = None;
for line in text.lines() {
let line = line.trim();
if line == "}" {
maps.extend(open.take());
continue;
}
let words = words(line.trim_end_matches('{'));
match (&mut open, words.as_slice()) {
(None, [map, source, target]) if map == "map" && line.ends_with('{') => {
open = Some(MapBlock {
source: source.clone(),
target: target.clone(),
default: String::new(),
rows: Vec::new(),
});
}
(Some(map), [name, value]) if name == "default" => map.default = value.clone(),
(Some(map), [key, value]) => map.rows.push((key.clone(), value.clone())),
_ => {}
}
}
maps
}
fn words(line: &str) -> Vec<String> {
let mut words = Vec::new();
let mut chars = line.trim_end_matches(';').chars().peekable();
while let Some(&c) = chars.peek() {
if c.is_whitespace() {
chars.next();
continue;
}
let mut word = String::new();
if c == '"' {
chars.next();
while let Some(c) = chars.next() {
match c {
'"' => break,
'\\' => match chars.next() {
Some('n') => word.push('\n'),
Some('r') => word.push('\r'),
Some('t') => word.push('\t'),
Some(other) => word.push(other),
None => {}
},
_ => word.push(c),
}
}
} else {
while let Some(&c) = chars.peek() {
if c.is_whitespace() {
break;
}
word.push(c);
chars.next();
}
}
words.push(word);
}
words
}
pub(crate) const MAP_FILES: [&str; 3] = [
crate::xrpc::MAPS_FILE,
crate::xrpc::RECORDS_MAPS_FILE,
crate::xrpc::social::MAPS_FILE,
];
pub(crate) const RECORD: &str = "com.atproto.repo.getRecord";
fn record_routes() -> [(&'static str, &'static str); 4] {
[
(RECORD, "$ks_atp_rec"),
("com.atproto.repo.listRecords", "$ks_atp_lr"),
("com.atproto.repo.listRecords", "$ks_atp_lr_one"),
("org.tangled.repo.listRecordVersions", "$ks_atp_ver"),
]
}
enum Part {
Args(Vec<String>),
Repo,
Uri,
Bucket(Vec<String>),
}
fn arg_names(source: &str) -> Option<Vec<String>> {
let names: Vec<String> = source.split("$arg_").skip(1).map(str::to_string).collect();
(source.starts_with("$arg_") && names.iter().all(|name| !name.contains('$'))).then_some(names)
}
fn bucket_args(var: &str, sources: &BTreeMap<&str, &str>, depth: usize) -> Vec<String> {
let Some(source) = sources.get(var).filter(|_| depth < 4) else {
return Vec::new();
};
let mut names = Vec::new();
for part in source.split('|') {
match arg_names(part) {
Some(args) => names.extend(args),
None => names.extend(bucket_args(part, sources, depth + 1)),
}
}
names
}
pub(crate) fn social_answers(maps: &[MapBlock], dids: &[&str], answers: &mut Answers) {
let tokens: BTreeMap<String, &str> = dids.iter().map(|did| (repo_token(did), *did)).collect();
let sources: BTreeMap<&str, &str> = maps
.iter()
.map(|map| (map.target.as_str(), map.source.as_str()))
.collect();
let routes: BTreeMap<&str, &'static str> = crate::xrpc::social::route_vars()
.chain(record_routes())
.map(|(nsid, var)| (var, nsid))
.collect();
for map in maps {
let Some(&nsid) = routes.get(map.target.as_str()) else {
continue;
};
if !map.default.is_empty() && !map.default.starts_with('$') {
answers
.defaults
.insert(nsid, format!("_atproto/{}", map.default));
}
let parts: Vec<Part> = map
.source
.split('|')
.map(|part| match part {
"$ks_atp_repo" => Part::Repo,
"$ks_atp_uri" => Part::Uri,
_ => match arg_names(part) {
Some(names) => Part::Args(names),
None => Part::Bucket(bucket_args(part, &sources, 0)),
},
})
.collect();
for (key, file) in &map.rows {
let values: Vec<&str> = key.split('|').collect();
if values.len() != parts.len() {
continue;
}
let mut spelled = vec![Params::new()];
let mut any = Vec::new();
let mut buckets = Vec::new();
let mut token = None;
for (part, value) in parts.iter().zip(&values) {
let name_value: Vec<(String, String)> = match part {
Part::Repo => {
let Some(did) = tokens.get(*value) else {
spelled.clear();
break;
};
token = Some(*value);
vec![("repo".into(), did.to_string())]
}
Part::Uri => vec![("uri".into(), value.to_string())],
Part::Args(names) => {
let Some(decoded) = decode(value) else {
spelled.clear();
break;
};
let decoded = canonical_did(decoded);
if decoded.is_empty() {
continue;
}
names
.iter()
.map(|name| (name.clone(), decoded.clone()))
.collect()
}
Part::Bucket(names) => {
any.extend(names.iter().cloned());
buckets.push(value.to_string());
continue;
}
};
spelled = spelled
.iter()
.flat_map(|params| {
name_value.iter().map(move |(name, value)| {
let mut params = params.clone();
params.insert(name.clone(), value.clone());
params
})
})
.collect();
}
let file = match (nsid, token) {
(RECORD, Some(token)) => format!("_atproto/{token}/{file}"),
_ => format!("_atproto/{file}"),
};
for params in spelled {
if any.is_empty() {
answers.put(nsid, params, file.clone());
} else {
let group = answers.bucketed.entry((nsid, params)).or_default();
for name in &any {
if !group.0.contains(name) {
group.0.push(name.clone());
}
}
group.1.insert((buckets.join("|"), file.clone()));
}
}
}
}
}
pub(crate) fn read_maps(generation_dir: &Path) -> anyhow::Result<Vec<MapBlock>> {
let mut maps = Vec::new();
for name in MAP_FILES {
let path = generation_dir.join("nginx/maps").join(name);
maps.extend(parse_maps(&std::fs::read_to_string(&path)?));
}
Ok(maps)
}
fn seal_key<'a>(manifest: &'a GenerationManifest, did: &str) -> Option<&'a str> {
manifest.profile["git_xrpc_entries"][did].as_str()
}
pub(crate) fn diff(
output_root: &Path,
old: (&GenerationManifest, &[MapBlock]),
new: (&GenerationManifest, &[MapBlock]),
) -> Vec<Line> {
let mut lines = Vec::new();
let old_dids: BTreeSet<&str> = old.0.repos.iter().map(|r| r.repo_did.as_str()).collect();
for entry in &new.0.repos {
let did = entry.repo_did.as_str();
if !old_dids.contains(did) {
continue;
}
let (Some(was), Some(now)) = (seal_key(old.0, did), seal_key(new.0, did)) else {
lines.push(reset(did));
continue;
};
if was == now {
continue;
}
let served = RepoStore::served(&output_root.join("repos").join(repo_key(did)));
let trace = crate::generate::Tracer::new();
let (Some(was), Some(now)) = trace.time("hooks: read seals", || {
(
crate::git_xrpc::sealed_rows(&served, was),
crate::git_xrpc::sealed_rows(&served, now),
)
}) else {
lines.push(reset(did));
continue;
};
let (mut before, mut after) = (Answers::default(), Answers::default());
trace.time("hooks: expand git rows", || {
git_answers(did, &was, Some(&now), &mut before);
git_answers(did, &now, Some(&was), &mut after);
});
trace.time("hooks: compare", || compare(&before, &after, &mut lines));
}
if old.1 != new.1 {
let dids = |manifest: &GenerationManifest| -> Vec<String> {
manifest.repos.iter().map(|r| r.repo_did.clone()).collect()
};
let (old_dids, new_dids) = (dids(old.0), dids(new.0));
let (mut before, mut after) = (Answers::default(), Answers::default());
social_answers(
old.1,
&old_dids.iter().map(String::as_str).collect::<Vec<_>>(),
&mut before,
);
social_answers(
new.1,
&new_dids.iter().map(String::as_str).collect::<Vec<_>>(),
&mut after,
);
compare(&before, &after, &mut lines);
}
lines
}
fn reset(did: &str) -> Line {
Line {
op: "reset",
nsid: None,
params: params([("repo", Some(did))]),
any: Vec::new(),
default: false,
old: None,
new: None,
}
}