use super::locations::RETRY_AFTER_SECS;
use anyhow::Context as _;
use super::conf::{self, Block};
use crate::config::{ByteSize, ValidatedConfig};
use crate::manifest::GenerationManifest;
/// `ssh_fetch` picks from the same rows in-process, which keeps nginx and ssh on one reply.
/// a bad pattern fails the build when its row is added.
pub(crate) struct Map {
source: &'static str,
target: &'static str,
default: &'static str,
rows: Vec<(Key, String)>,
}
enum Key {
Exact(String),
Pattern(String, pcre2::bytes::Regex),
}
impl Key {
fn new(key: String) -> anyhow::Result<Self> {
let Some(pattern) = key.strip_prefix('~') else {
return Ok(Self::Exact(key));
};
let regex =
pcre2::bytes::Regex::new(pattern).with_context(|| format!("compile {pattern:?}"))?;
Ok(Self::Pattern(key, regex))
}
fn text(&self) -> &str {
match self {
Self::Exact(key) | Self::Pattern(key, _) => key,
}
}
}
impl Map {
pub(crate) fn new(source: &'static str, target: &'static str, default: &'static str) -> Self {
Self {
source,
target,
default,
rows: Vec::new(),
}
}
pub(crate) fn rows(
mut self,
rows: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
) -> anyhow::Result<Self> {
for (key, value) in rows {
self.rows.push((Key::new(key.into())?, value.into()));
}
Ok(self)
}
pub(crate) fn block(&self) -> Block {
Block::new("map", [self.source, self.target])
.directive("default", [self.default])
.entries(self.rows.iter().map(|(key, value)| (key.text(), value)))
}
pub(crate) fn pick(&self, input: &[u8]) -> anyhow::Result<Option<String>> {
for (key, value) in &self.rows {
match key {
Key::Exact(key) if key.as_bytes() == input => return Ok(Some(value.clone())),
Key::Exact(_) => {}
Key::Pattern(_, regex) => {
if let Some(captures) = regex.captures(input)? {
let first = captures.get(1).map_or(&b""[..], |group| group.as_bytes());
return Ok(Some(value.replace("$1", &String::from_utf8_lossy(first))));
}
}
}
}
Ok(None)
}
}
/// `reply` keys start with the depth-1 want, which lets depth-1 rows reject most requests early.
pub(crate) struct GitMaps {
adv: Map,
canned: Map,
depth_one: Map,
reply: Map,
file: Map,
}
pub(crate) fn git_maps(manifest: &GenerationManifest) -> anyhow::Result<GitMaps> {
let mut canned: Vec<String> = CANNED.iter().map(|(name, _)| name.to_string()).collect();
for n in 0..crate::git_export::FETCH_BASES {
canned.extend([format!("have-{n}"), format!("have-{n}-done")]);
canned.extend([format!("moved-{n}"), format!("moved-{n}-done")]);
}
canned.extend((0..crate::git_export::DEPTH_ONE_BRANCHES).map(|n| format!("shallow-{n}")));
Ok(GitMaps {
adv: Map::new("$http_git_protocol", "$ks_git_adv", "advertisement")
.rows([("~version=2", "adv-v2")])?,
canned: Map::new("$request_body", "$ks_git_canned", "none").rows([
// the static reply can't know the client's shallow commits and errors instead.
(
"~(?s)command=fetch.*00(?:15deepen 2147483647|16deepen 2147483647\n)",
"err-unshallow",
),
("~(?s)^[0-9a-f]{4}want .*[0-9a-f]{4}deepen", "err-v0-shallow"),
(
"~(?s)command=ls-refs(?=.*ref-prefix HEAD\n)(?=.*ref-prefix refs/(?:cobs/|tngl/|\n))",
"ls-refs-head-cobs",
),
("~(?s)command=ls-refs.*ref-prefix HEAD", "ls-refs-head"),
("~command=ls-refs", "ls-refs"),
("~(?s)command=fetch.*0009done\n", "fetch-done"),
("~command=fetch", "fetch-nodone"),
("~(?s)^[0-9a-f]{4}want .*0009done\n", "upload-pack"),
("~^[0-9a-f]{4}want ", "v0-nodone"),
])?,
depth_one: Map::new("$request_body", "$ks_git_depth_one", "").rows([(
"~^[0-9a-f]{4}command=fetch(?s)(?=.*?000(?:cdeepen 1|ddeepen 1\n))(?!.*?(?:deepen-|shallow )).*?want ([0-9a-f]+)\n",
"$1",
)])?,
reply: fetch_map(manifest)?,
file: Map::new("$http_x_ks_canned", "$ks_git_file", "")
.rows(canned.iter().map(|name| (name.clone(), name.clone())))?,
})
}
impl GitMaps {
pub(crate) fn blocks(&self) -> [Block; 5] {
[
&self.adv,
&self.canned,
&self.depth_one,
&self.reply,
&self.file,
]
.map(Map::block)
}
pub fn advertisement(&self, version: &str) -> anyhow::Result<String> {
Ok(self
.adv
.pick(version.as_bytes())?
.unwrap_or_else(|| self.adv.default.to_string()))
}
pub fn reply(&self, repo_did: &str, body: &[u8]) -> anyhow::Result<Option<String>> {
let canned = self
.canned
.pick(body)?
.unwrap_or_else(|| self.canned.default.to_string());
let depth_one = self.depth_one.pick(body)?.unwrap_or_default();
let key = [
depth_one.as_bytes(),
format!("/{repo_did}/git-upload-pack").as_bytes(),
body,
]
.concat();
let reply = self.reply.pick(&key)?.unwrap_or(canned);
self.file.pick(reply.as_bytes())
}
}
/// shallow clients keep the full pack because a depth-1 reply would have to start at what they have.
/// a base too big for one nginx word (~95 tips) gets no rows, a partial cut would be wrong.
pub(crate) fn fetch_map(manifest: &GenerationManifest) -> anyhow::Result<Map> {
let mut rows = Vec::new();
for repo in &manifest.repos {
let fetch = |first_want: &str| fetch_key(&repo.repo_did, first_want);
let depth_one = |tips: &str| {
format!(
"{}(?!.*?want (?!(?1)\n)).*?0009done\n",
fetch(&format!("({tips})"))
)
};
if let Some(head) = &repo.shallow_head {
rows.push((depth_one(head), "shallow-head".to_string()));
}
for (n, tip) in repo.shallow_branches.iter().enumerate() {
rows.push((depth_one(tip), format!("shallow-{n}")));
}
for (n, base) in repo.fetch_bases.iter().enumerate() {
let round = have_round(&repo.repo_did, base);
let done = have_done(&round);
if !conf::fits(&done) {
continue;
}
let moved = moved_fits_row(&repo.repo_did, base, &repo.moved_wants);
if moved {
let round = moved_round(&repo.repo_did, base, &repo.moved_wants);
rows.push((have_done(&round), format!("moved-{n}-done")));
rows.push((round, format!("moved-{n}")));
}
if crate::git_export::have_replies(n, moved) {
rows.push((done, format!("have-{n}-done")));
rows.push((round, format!("have-{n}")));
}
}
}
Map::new(
"$ks_git_depth_one$uri$request_body",
"$ks_git_reply",
"$ks_git_canned",
)
.rows(rows)
}
pub(crate) fn fetch_key(repo_did: &str, first_want: &str) -> String {
format!(
"~^{first_want}/{}/git-upload-pack[0-9a-f]{{4}}command=fetch(?s)",
repo_did.replace('.', "\\.")
)
}
pub(crate) fn have_round(repo_did: &str, base: &[String]) -> String {
let haves: String = base
.iter()
.map(|tip| format!("(?=.*?have {tip}\n)"))
.collect();
format!(
"{}(?!.*?(?:deepen|shallow |wait-for-done)){haves}",
fetch_key(repo_did, "")
)
}
pub(crate) fn have_done(round: &str) -> String {
format!("{round}.*?0009done\n")
}
pub(crate) fn base_fits_row(repo_did: &str, base: &[String]) -> bool {
conf::fits(&have_done(&have_round(repo_did, base)))
}
pub(crate) fn moved_round(repo_did: &str, base: &[String], moved: &[String]) -> String {
let moved = moved.join("|");
format!(
"{}(?!.*?want (?!(?:{moved})\n))",
have_round(repo_did, base)
)
}
pub(crate) fn moved_fits_row(repo_did: &str, base: &[String], moved: &[String]) -> bool {
!moved.is_empty() && conf::fits(&have_done(&moved_round(repo_did, base, moved)))
}
/// `$ks_client` is the client address cut to an ipv6 /64 against crawlers fanning out in a prefix.
/// empty on loopback, which the limits skip. pack replies share `$ks_pack_any`, the knot-wide cap.
pub(crate) fn limit_maps() -> anyhow::Result<[Block; 9]> {
// the text address only spots loopback, a compressed `::` makes it unreliable otherwise.
let remote = Map::new("$remote_addr", "$ks_remote", "$binary_remote_addr")
.rows([("127.0.0.1", ""), ("::1", "")])?;
let client =
Map::new("$ks_remote", "$ks_client", "$ks_remote").rows([("~(?s)^(.{8})", "$1")])?;
let counted = Map::new("$ks_client", "$ks_counted", "1").rows([("", "")])?;
let small = || {
let none = CANNED
.iter()
.filter(|(_, pack)| !pack)
.map(|(name, _)| *name);
std::iter::once("").chain(none).map(|name| (name, ""))
};
let pack_reply = Map::new("$ks_git_file", "$ks_pack_reply", "1").rows(small())?;
let handed_reply = Map::new("$ks_pack_file", "$ks_handed_reply", "1").rows(small())?;
// a pack reply counts by the proxy's `x-ks-counted` header, a handed-back one by `$ks_counted`.
let pack_any = Map::new(
"$ks_pack_reply$http_x_ks_counted:$ks_handed_reply$ks_counted",
"$ks_pack_any",
"",
)
.rows([("~^11:", "pack"), ("~:11$", "pack")])?;
let pack_rate = Map::new("$ks_pack_reply$http_x_ks_rate", "$ks_pack_rate", "0")
.rows([("~^1(.+)$", "$1")])?;
// resolve only matches on loopback, which keeps a client from asking which reply its body picks.
let resolve = Map::new("$ks_remote$http_x_ks_resolve", "$ks_resolve", "").rows([("1", "1")])?;
let retry_after = Map::new("$limit_conn_status$limit_req_status", "$ks_retry_after", "")
.rows([("~REJECTED", RETRY_AFTER_SECS.to_string())])?;
Ok([
remote,
client,
counted,
pack_reply,
handed_reply,
pack_any,
pack_rate,
resolve,
retry_after,
]
.map(|map| map.block()))
}
/// states add `have-`, `moved-` and `shallow-<n>` replies, all of which carry a pack.
pub(crate) const CANNED: [(&str, bool); 10] = [
("ls-refs-head-cobs", false),
("ls-refs-head", false),
("ls-refs", false),
("fetch-done", true),
("fetch-nodone", false),
("upload-pack", true),
("v0-nodone", false),
("shallow-head", true),
("err-unshallow", false),
("err-v0-shallow", false),
];
/// `$remote_addr` reads `X-Forwarded-For` only from the edges in `activate.real_ip_from`.
/// a client can't name a trusted address that way.
pub(crate) fn trusted_rate(
config: &ValidatedConfig,
pack_rate: &ByteSize,
) -> anyhow::Result<[Block; 2]> {
let mut geo = Block::new("geo", ["$remote_addr", "$ks_trusted_rate"])
.directive("default", [pack_rate.to_string()])
.directive("127.0.0.1", ["0".to_string()])
.directive("::1", ["0".to_string()]);
for client in &config.serve.trusted {
geo = geo.directive(&client.to_string(), [config.serve.trusted_rate()]);
}
let handed = Map::new("$ks_handed_reply$ks_trusted_rate", "$ks_handed_rate", "0")
.rows([("~^1(.+)$", "$1")])?;
Ok([geo, handed.block()])
}
pub(super) const FIREHOSE_CONNECTIONS: &str = "4";
/// rfc 6455 keys are 16 bytes, 22 base64 characters and `==`.
pub(crate) fn firehose_maps() -> anyhow::Result<[Block; 3]> {
let upgrade = Map::new("$request_method $http_upgrade", "$ks_fh_upgrade", "")
.rows([("~^GET (?i)websocket$", "1")])?;
let key = Map::new("$http_sec_websocket_key", "$ks_fh_key", "")
.rows([("~^[A-Za-z0-9+/]{22}==$", "1")])?;
let cursor = Map::new("$arg_cursor", "$ks_fh_cursor_ok", "")
.rows([("", "1"), ("~^[0-9]{1,19}$", "1")])?;
Ok([upgrade.block(), key.block(), cursor.block()])
}