use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use anyhow::Context as _;
use serde::{Deserialize, Serialize};
use super::Rows;
use crate::atproto::short_hash;
use crate::nginx::conf::{Block, NONE};
pub(crate) const WWW: &str = "www/_git_paths";
pub(crate) const POOLED: &str = "paths";
const COMMITS: &str = "c";
const SEALS: &str = "s";
/// nginx has no loops, each allowed escape gets its own rewrite line.
const ESCAPES: [(&str, char, usize); 3] = [("%2F", '/', 32), ("%40", '@', 8), ("%2B", '+', 8)];
fn either_case(escape: &str) -> String {
let (head, last) = escape.split_at(2);
match last.to_ascii_lowercase() == last {
true => escape.to_string(),
false => format!("{head}[{last}{}]", last.to_ascii_lowercase()),
}
}
const MAX_NAME_BYTES: usize = 200;
const DECODE: &str = "/_ks_git_pathd/";
const SERVE: &str = "/_ks_git_path/";
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Kind {
Tree,
Entry,
Blob,
Blame,
Log,
}
impl Kind {
pub(crate) const ALL: [Kind; 5] = [Kind::Tree, Kind::Entry, Kind::Blob, Kind::Blame, Kind::Log];
pub(crate) fn name(self) -> &'static str {
match self {
Kind::Tree => "tree",
Kind::Entry => "entry",
Kind::Blob => "blob",
Kind::Blame => "blame",
Kind::Log => "log",
}
}
fn at_root(self) -> bool {
matches!(self, Kind::Tree | Kind::Log)
}
/// decode never yields a `,`, which keeps these suffixes apart from any path.
pub(crate) fn file(self, path: &str, page: Option<(&str, &str)>) -> String {
match (self, page) {
(Kind::Log, Some((limit, cursor))) => format!("{path},log.{limit}.{cursor}.json"),
(Kind::Tree, _) => format!("{path},tree.json"),
(Kind::Entry, _) => format!("{path},entry.json"),
(Kind::Blob, _) => format!("{path},blob"),
_ => format!("{path},blame.json"),
}
}
fn rows(self, rows: &Rows) -> &BTreeMap<String, String> {
match self {
Kind::Tree => &rows.tree,
Kind::Entry => &rows.entry,
Kind::Blob => &rows.blob,
Kind::Blame => &rows.blame,
Kind::Log => &rows.log,
}
}
fn rows_mut(self, rows: &mut Rows) -> &mut BTreeMap<String, String> {
match self {
Kind::Tree => &mut rows.tree,
Kind::Entry => &mut rows.entry,
Kind::Blob => &mut rows.blob,
Kind::Blame => &mut rows.blame,
Kind::Log => &mut rows.log,
}
}
/// a key is `<commit>|<path>`, a log key adds `|<limit>|<cursor>`.
fn parse(self, key: &str) -> Option<(&str, &str, Option<(&str, &str)>)> {
let (commit, rest) = key.split_once('|')?;
if self != Kind::Log {
return Some((commit, rest, None));
}
let (rest, cursor) = rest.rsplit_once('|')?;
let (spelling, limit) = rest.rsplit_once('|')?;
let paged = |value: &str, most: usize, digit: fn(&u8) -> bool| {
value.len() <= most && value.as_bytes().iter().all(digit)
};
(paged(limit, 4, u8::is_ascii_digit)
&& paged(cursor, 64, |byte| {
byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)
}))
.then_some((commit, spelling, Some((limit, cursor))))
}
}
fn plain(byte: u8) -> bool {
byte.is_ascii_alphanumeric() || b"._*-".contains(&byte)
}
/// keep this in step with `valid` and the rewrites in `decode_location`.
pub(crate) fn decode(spelling: &str) -> Option<String> {
let raw = spelling
.bytes()
.all(|byte| plain(byte) || b"@+/".contains(&byte));
let path = match raw {
true => spelling.to_string(),
false => {
let mut out = String::with_capacity(spelling.len());
let mut counts = [0; ESCAPES.len()];
let mut rest = spelling;
while let Some(&byte) = rest.as_bytes().first() {
if plain(byte) {
out.push(byte as char);
rest = &rest[1..];
continue;
}
let at = ESCAPES.iter().position(|(escape, _, _)| {
rest.get(..3)
.is_some_and(|head| head.eq_ignore_ascii_case(escape))
})?;
counts[at] += 1;
out.push(ESCAPES[at].1);
rest = &rest[3..];
}
if counts
.iter()
.zip(ESCAPES)
.any(|(&n, (_, _, most))| n > most)
{
return None;
}
out
}
};
path.split('/')
.all(|part| !part.is_empty() && part != "." && part != "..")
.then_some(path)
}
/// `[.]` rather than `\.` keeps backslashes out of the config.
fn valid() -> String {
let raw = "(?![.][.]?(?:/|$))[A-Za-z0-9._*@+-]+";
let encoded = "(?![.][.]?(?:%2[Ff]|$))(?:[A-Za-z0-9._*-]|%40|%2[Bb])+";
format!("~^(?:{raw}(?:/{raw})*|{encoded}(?:%2[Ff]{encoded})*)$")
}
pub(crate) type Layout = BTreeMap<String, BTreeMap<String, String>>;
/// a case-folding disk can't hold names that differ only by case, those rows stay out of the layout.
pub(crate) fn split(rows: &Rows, folds: bool) -> (Layout, Rows) {
let mut groups: BTreeMap<(&str, String), Group> = BTreeMap::new();
let mut unlaid = Rows::default();
for kind in Kind::ALL {
for (key, file) in kind.rows(rows) {
let named = kind.parse(key).and_then(|(commit, spelling, page)| {
let path = match spelling {
"" if kind.at_root() => Some(String::new()),
_ => decode(spelling),
}?;
let name = kind.file(&path, page);
name.split('/')
.all(|part| part.len() <= MAX_NAME_BYTES)
.then_some((commit, name, path, spelling))
});
match named {
Some((commit, name, path, spelling)) => {
let group = groups.entry((commit, name)).or_insert_with(|| Group {
kind,
path,
rows: Vec::new(),
});
group.rows.push((key, spelling, file));
}
None => {
kind.rows_mut(&mut unlaid).insert(key.clone(), file.clone());
}
}
}
}
let mut folded: BTreeMap<(&str, String), BTreeSet<&str>> = BTreeMap::new();
for (commit, name) in groups.keys() {
for part in parts(name) {
folded
.entry((commit, part.to_ascii_lowercase()))
.or_default()
.insert(part);
}
}
let mut layout = Layout::new();
for ((commit, name), group) in &groups {
let file = group.rows[0].2;
let agreed = group.rows.iter().all(|(_, _, other)| *other == file) && group.owned();
let distinct =
parts(name).all(|part| folded[&(*commit, part.to_ascii_lowercase())].len() == 1);
match (agreed, distinct || !folds) {
(true, true) => {
layout
.entry(commit.to_string())
.or_default()
.insert(name.clone(), file.clone());
}
(true, false) => {}
(false, _) => {
for (key, _, file) in &group.rows {
group
.kind
.rows_mut(&mut unlaid)
.insert(key.to_string(), file.to_string());
}
}
}
}
(layout, unlaid)
}
struct Group<'a> {
kind: Kind,
path: String,
rows: Vec<(&'a str, &'a str, &'a String)>,
}
impl Group<'_> {
/// `%2B` and `+` both decode to `+`, a row for one spelling doesn't own the path.
fn owned(&self) -> bool {
let path = self.path.as_str();
crate::xrpc::spellings(path)
.iter()
.filter(|spelling| path.is_empty() || decode(spelling).as_deref() == Some(path))
.all(|spelling| self.rows.iter().any(|(_, held, _)| held == spelling))
}
}
pub(crate) fn folds(dir: &Path) -> std::io::Result<bool> {
std::fs::create_dir_all(dir)?;
let probe = dir.join(format!(".case-{}-A", std::process::id()));
let _ = std::fs::remove_dir(&probe);
std::fs::create_dir(&probe)?;
let folded = dir.join(format!(".case-{}-a", std::process::id())).is_dir();
std::fs::remove_dir(&probe)?;
Ok(folded)
}
fn parts(name: &str) -> impl Iterator<Item = &str> {
name.match_indices('/')
.map(|(at, _)| &name[..at])
.chain([name])
}
/// built under a staging name and renamed in, nothing ever sees a half made dir.
pub(crate) fn publish(pool: &Path, layout: &Layout) -> anyhow::Result<Laid> {
let root = pool.join(POOLED);
let commits = root.join(COMMITS);
let seals = root.join(SEALS);
std::fs::create_dir_all(&commits)?;
std::fs::create_dir_all(&seals)?;
let mut dirs = BTreeMap::new();
for (commit, names) in layout {
let hash = short_hash(&serde_json::to_vec(names)?);
let dir = commits.join(&hash);
if !dir.is_dir() {
let staging = commits.join(format!(".{hash}.{}", std::process::id()));
let _ = std::fs::remove_dir_all(&staging);
for (name, file) in names {
crate::cas::link_or_reflink(&pool.join(file), &staging.join(name))
.with_context(|| format!("link {name} of {commit}"))?;
}
settle(&staging, &dir)?;
}
dirs.insert(commit.clone(), hash);
}
let seal = short_hash(&serde_json::to_vec(&dirs)?);
let dir = seals.join(&seal);
if !dir.is_dir() {
let staging = seals.join(format!(".{seal}.{}", std::process::id()));
let _ = std::fs::remove_dir_all(&staging);
std::fs::create_dir_all(&staging)?;
for (commit, hash) in &dirs {
std::os::unix::fs::symlink(
Path::new("../..").join(COMMITS).join(hash),
staging.join(commit),
)?;
}
settle(&staging, &dir)?;
}
Ok(Laid {
seal,
commits: dirs,
})
}
#[derive(Default, Serialize, Deserialize, Clone)]
pub(crate) struct Laid {
pub(crate) seal: String,
/// the value is a dir name under `commits/`, not a path.
pub(crate) commits: BTreeMap<String, String>,
}
impl Laid {
pub(crate) fn held(&self, pool: &Path) -> bool {
seal_dir(pool, &self.seal).is_dir()
&& self
.commits
.values()
.all(|hash| pool.join(POOLED).join(COMMITS).join(hash).is_dir())
}
pub(crate) fn kept(&self, pool: &Path) -> impl Iterator<Item = std::path::PathBuf> {
let commits = pool.join(POOLED).join(COMMITS);
std::iter::once(seal_dir(pool, &self.seal))
.chain(self.commits.values().map(move |hash| commits.join(hash)))
}
}
/// dirs are named by content hash, losing the rename race to another build is fine.
fn settle(staging: &Path, dir: &Path) -> anyhow::Result<()> {
match std::fs::rename(staging, dir) {
Ok(()) => Ok(()),
Err(_) if dir.is_dir() => {
std::fs::remove_dir_all(staging)?;
Ok(())
}
Err(error) => Err(error).with_context(|| format!("publish {}", dir.display())),
}
}
pub(crate) fn seal_dir(pool: &Path, seal: &str) -> std::path::PathBuf {
pool.join(POOLED).join(SEALS).join(seal)
}
pub(crate) fn swept(pool: &Path) -> [std::path::PathBuf; 2] {
[
pool.join(POOLED).join(COMMITS),
pool.join(POOLED).join(SEALS),
]
}
pub(crate) fn maps() -> [Block; 2] {
[
crate::xrpc::map(
"$arg_path",
"$ks_git_path",
vec![(String::new(), "1".into()), (valid(), "1".into())],
),
crate::xrpc::map(
"$arg_limit|$arg_cursor",
"$ks_git_page",
vec![("~^[0-9]{0,4}[|][0-9a-f]{0,64}$".into(), "1".into())],
),
]
}
/// nginx can't decode `$arg_path`, the path rides in the uri. `valid` checked the escapes.
pub(crate) fn decode_location() -> Block {
let mut location = Block::new("location", ["^~", DECODE]).directive("internal", NONE);
for (escape, decoded, most) in ESCAPES {
for _ in 0..most {
location = location.directive(
"rewrite",
[
format!("^({DECODE}.*){}(.*)$", either_case(escape)),
format!("$1{decoded}$2"),
],
);
}
}
location.directive(
"rewrite",
[
format!("^{DECODE}(.*)$"),
format!("{SERVE}$1"),
"last".into(),
],
)
}
/// nginx refuses symlinks below the commit dir, so the names there must be real files.
pub(crate) fn serve_location(
generation_path: &str,
kind: Kind,
missing: &str,
location: impl FnOnce(Block) -> Block,
) -> Block {
let prefix = format!("{SERVE}{}/", kind.name());
let block = location(Block::new("location", ["^~", &prefix]).directive("internal", NONE));
block
.directive("alias", [format!("{generation_path}/{WWW}/")])
.directive("disable_symlinks", ["on", "from=$ks_git_from"])
.directive(
"try_files",
[
kind.file("$uri", Some(("$arg_limit", "$arg_cursor"))),
format!("/_ks_error/{missing}"),
],
)
}
pub(crate) fn target(generation_path: &str, kind: Kind) -> (String, String) {
(
format!("{generation_path}/{WWW}/$ks_git_commit"),
format!("{DECODE}{}/$ks_git_commit/$arg_path", kind.name()),
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_page_suffix_counts_toward_the_name_limit() {
let commit = "a".repeat(40);
let cursor = "b".repeat(64);
let fits = "x".repeat(MAX_NAME_BYTES - ",entry.json".len());
let mut rows = Rows::default();
rows.entry.insert(format!("{commit}|{fits}"), "e".into());
rows.log
.insert(format!("{commit}|{fits}|100|{cursor}"), "l".into());
let (layout, unlaid) = split(&rows, false);
let laid: Vec<&String> = layout.values().flat_map(|names| names.keys()).collect();
assert_eq!(laid, [&format!("{fits},entry.json")]);
assert_eq!(unlaid.log.len(), 1);
}
}