use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
use std::time::{Duration, Instant};
use anyhow::Context as _;
use knot_git::{
ArchiveFormat, ArchiveLimit, ArchivePrefix, BinaryBudget, BranchTip, EntryKind, LogLimit,
LogSkip, PatchRange, PatchStatus, RefClass, Repo, SizedEntry,
};
use knot_types::{Listing, ObjectFormat, Oid, RepoPath};
use serde::{Deserialize, Serialize};
use super::wire::{
BlameBody, BlameCommitOut, BlameOut, BlameRunOut, BranchInfoOut, BranchOut, BranchesOut,
CommitOut, DefaultBranchWire, DiffOut, DiffStatOut, EntryOut, LanguageOut, LanguagesOut,
ListRefsOut, LogOut, MergeBaseOut, RefWire, SignatureOut, SubmoduleOut, TagInfoOut, TagsOut,
TreeEntryOut, TreeOut, file_diff, is_file,
};
use super::{NO_MERGE_BASE, Rows};
use crate::atproto::{Pool, short_hash};
use crate::xrpc::spellings;
pub(crate) const MAX_PROJECTED_BRANCHES: usize = 16;
pub(crate) const MAX_PROJECTED_TAGS: usize = 8;
const MAX_BLOB_BYTES: u64 = 25 * 1024 * 1024;
const MAX_BLAME_BYTES: u64 = 256 * 1024;
const MAX_ARCHIVE_BYTES: u64 = 256 * 1024 * 1024;
const MAX_TIP_PATHS: usize = 4096;
const MAX_TAG_DETAILS: usize = 200;
pub(super) const LOG_WINDOW: usize = 180;
const ATTRIBUTION_CHUNK: usize = 256;
const MAX_ATTRIBUTION_COMMITS: usize = 10_000;
const MAX_DIFF_FILES: usize = 1000;
const MAX_DIFF_BYTES: usize = 16 * 1024 * 1024;
const ATTRIBUTION_BUDGET: Duration = Duration::from_secs(10);
const BLAME_BUDGET: Duration = Duration::from_secs(2);
const DIFF_BUDGET: Duration = Duration::from_secs(20);
const LANGUAGES_BUDGET: Duration = Duration::from_secs(2);
const COUNT_BUDGET: Duration = Duration::from_secs(2);
const EMPTY_TREE_SHA1: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
const EMPTY_TREE_SHA256: &str = "6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321";
type PagePlan = &'static [(&'static [&'static str], usize, bool)];
const BRANCH_PAGES: PagePlan = &[
(&["", "50"], 50, true),
(&["100", "500"], 100, true),
(&["1"], 1, false),
(&["3"], 3, false),
(&["5"], 5, false),
(&["10"], 10, false),
(&["20"], 20, false),
];
const REF_PAGES: PagePlan = &[
(&["", "100"], 100, true),
(&["500"], 500, true),
(&["1000"], 1000, true),
];
type LogPlan = &'static [(&'static [&'static str], usize, bool, &'static [usize])];
const LOG_PAGES: LogPlan = &[
(&["1"], 1, false, &[]),
(&["3"], 3, false, &[]),
(&["", "50"], 50, true, &[0, 50, 100]),
(&["60"], 60, true, &[0, 60, 120]),
(&["100"], 100, true, &[0, 100]),
];
/// svfe only sends cursors it was handed. the first page needs the empty one.
const WALK_PAGES: LogPlan = &[(&["", "50"], 50, true, &[]), (&["100"], 100, true, &[])];
#[derive(Default, Serialize, Deserialize, Clone)]
pub(crate) struct Piece {
rows: Rows,
files: BTreeMap<String, u64>,
}
/// keys are git oids, which never change meaning. a push can keep the old entries.
#[derive(Default, Serialize, Deserialize)]
pub(crate) struct Memo {
tips: BTreeMap<String, Piece>,
diffs: BTreeMap<String, Piece>,
blame: BTreeMap<String, String>,
languages: BTreeMap<String, Vec<LanguageOut>>,
archives: BTreeMap<String, String>,
attribution: BTreeMap<String, BTreeMap<String, String>>,
rounds: BTreeMap<String, Piece>,
}
pub(crate) fn run(
repo: &Repo,
pool: &mut Pool,
shared_blobs: &std::path::Path,
name: &str,
patches: &[(Oid, Oid)],
prior: Option<&Memo>,
) -> anyhow::Result<(Rows, Memo)> {
let now = Instant::now();
let mut projector = Projector {
repo,
pool,
shared_blobs,
prior,
memo: Memo::default(),
rows: Rows::default(),
piece: None,
commits: HashMap::new(),
changed: HashMap::new(),
attribution_deadline: now + ATTRIBUTION_BUDGET,
diff_deadline: now + DIFF_BUDGET,
blame_deadline: now + BLAME_BUDGET,
count_deadline: now + COUNT_BUDGET,
};
projector.project(name, patches)?;
Ok((projector.rows, projector.memo))
}
struct Projector<'a, 'p> {
repo: &'a Repo,
pool: &'a mut Pool<'p>,
shared_blobs: &'a std::path::Path,
prior: Option<&'a Memo>,
memo: Memo,
rows: Rows,
piece: Option<Piece>,
commits: HashMap<Oid, CommitOut>,
changed: HashMap<Oid, Option<Vec<RepoPath>>>,
attribution_deadline: Instant,
diff_deadline: Instant,
blame_deadline: Instant,
count_deadline: Instant,
}
struct Tip {
oid: Oid,
label: String,
}
struct Dir {
path: String,
entries: Vec<SizedEntry>,
}
fn join(dir: &str, name: &str) -> String {
match dir.is_empty() {
true => name.to_string(),
false => format!("{dir}/{name}"),
}
}
fn prefixes(path: &str) -> impl Iterator<Item = &str> {
path.match_indices('/')
.map(|(at, _)| &path[..at])
.chain([path])
}
fn next_cursor(offset: usize, size: usize, total: usize) -> Option<String> {
Some(offset + size)
.filter(|&end| end < total)
.map(|end| end.to_string())
}
/// the first page also answers a request with no cursor.
fn offset_pages(plan: PagePlan, total: usize) -> Vec<(usize, usize, Vec<String>)> {
let mut pages = Vec::new();
for &(limits, size, chained) in plan {
let mut offset = 0;
loop {
let mut keys = Vec::new();
for limit in limits {
keys.push(format!("{limit}|{offset}"));
if offset == 0 {
keys.push(format!("{limit}|"));
}
}
pages.push((offset, size, keys));
offset += size;
if !chained || offset >= total {
break;
}
}
}
pages
}
fn archive_disposition(stem: &str, format: &str) -> String {
let safe: String = stem
.chars()
.map(|c| match c.is_ascii_alphanumeric() || "._-+".contains(c) {
true => c,
false => '-',
})
.collect();
format!("attachment; filename=\"{safe}.{format}\"")
}
impl Projector<'_, '_> {
fn project(&mut self, name: &str, patches: &[(Oid, Oid)]) -> anyhow::Result<()> {
let repo = self.repo;
let head = repo.head();
let head_commit = head
.as_ref()
.and_then(|head| repo.peel_to_commit(head.target).ok());
if let Some(commit) = head_commit {
for spelling in ["", "HEAD"] {
self.rows.commit.insert(spelling.into(), commit.to_hex());
}
}
self.list_refs(head.as_ref())?;
let branches = self.branches(head.as_ref())?;
let tags = self.tags()?;
let mut tips: Vec<Tip> = Vec::new();
let default_name = head.as_ref().map(|head| head.name.as_str().to_string());
if let Some(commit) = head_commit {
tips.push(Tip {
oid: commit,
label: default_name.clone().unwrap_or_else(|| "HEAD".into()),
});
}
for (branch, oid) in branches.iter().take(MAX_PROJECTED_BRANCHES) {
tips.push(Tip {
oid: *oid,
label: branch.clone(),
});
}
for (tag, oid) in tags.iter().take(MAX_PROJECTED_TAGS) {
tips.push(Tip {
oid: *oid,
label: tag.clone(),
});
}
let mut seen = HashSet::new();
tips.retain(|tip| seen.insert(tip.oid));
let mut logged: Vec<Oid> = Vec::new();
let mut seen = HashSet::new();
for (index, tip) in tips.iter().enumerate() {
let walked = self.tip(tip, index == 0 && head_commit.is_some())?;
logged.extend(walked.into_iter().filter(|oid| seen.insert(*oid)));
}
if let Some(commit) = head_commit {
let label = tips[0].label.clone();
self.blame(commit, name)?;
self.archives(commit, &ArchivePrefix::stem(name, &label))?;
}
let known = |oid: Oid| repo.contains(oid) && repo.peel_to_commit(oid).is_ok();
let rounds: Vec<(Oid, Oid)> = patches
.iter()
.copied()
.filter(|&(base, head)| known(base) && known(head))
.collect();
for &(base, head) in &rounds {
self.round_log(base, head)?;
}
let mut bases: HashMap<Oid, Option<Oid>> = HashMap::new();
if let Some(default) = head_commit {
for &(_, tip) in branches.iter().take(MAX_PROJECTED_BRANCHES) {
let Ok(base) = repo.merge_base(default, tip) else {
continue;
};
bases.insert(tip, base);
let file = match base {
Some(commit) => self.put_json(&MergeBaseOut { commit })?,
None => NO_MERGE_BASE.to_string(),
};
self.rows
.merge_base
.insert(format!("{}|{}", default.to_hex(), tip.to_hex()), file);
}
}
let mut ranges: Vec<(Option<Oid>, Oid)> = Vec::new();
for &commit in &logged {
let parent = self
.commit(commit)
.and_then(|out| out.parents_of().first().copied());
ranges.push((parent, commit));
}
if let Some(default) = head_commit {
for tip in tips.iter().skip(1).take(MAX_PROJECTED_BRANCHES) {
if tip.oid == default {
continue;
}
ranges.push((Some(default), tip.oid));
let base = match bases.get(&tip.oid) {
Some(&base) => base,
None => repo.merge_base(default, tip.oid).ok().flatten(),
};
if let Some(base) = base
&& base != default
{
ranges.push((Some(base), tip.oid));
}
}
}
ranges.extend(rounds.iter().map(|&(base, head)| (Some(base), head)));
for (base, head) in ranges {
self.diff(base, head)?;
}
Ok(())
}
fn empty_tree(&self) -> &'static str {
match self.repo.object_format() == ObjectFormat::SHA1 {
true => EMPTY_TREE_SHA1,
false => EMPTY_TREE_SHA256,
}
}
fn commit(&mut self, oid: Oid) -> Option<CommitOut> {
if let Some(out) = self.commits.get(&oid) {
return Some(out.clone());
}
let out = CommitOut::of(&self.repo.find_commit(oid).ok()?);
self.commits.insert(oid, out.clone());
Some(out)
}
fn rows(&mut self) -> &mut Rows {
match &mut self.piece {
Some(piece) => &mut piece.rows,
None => &mut self.rows,
}
}
fn record(&mut self, file: &str, len: u64) {
if let Some(piece) = &mut self.piece {
piece.files.insert(file.to_string(), len);
}
}
fn put(&mut self, subdir: &str, ext: &str, bytes: &[u8]) -> anyhow::Result<String> {
let file = format!("{subdir}/{}.{ext}", short_hash(bytes));
self.pool.put(&file, bytes)?;
self.record(&file, bytes.len() as u64);
Ok(file)
}
fn put_json(&mut self, value: &impl Serialize) -> anyhow::Result<String> {
self.put("json", "json", &serde_json::to_vec(value)?)
}
fn put_blob(&mut self, oid: Oid) -> anyhow::Result<Option<String>> {
let Ok(size) = self.repo.blob_size(oid) else {
return Ok(None);
};
if size > MAX_BLOB_BYTES {
return Ok(None);
}
let file = format!("blobs/{}", oid.to_hex());
let length = |path: &std::path::Path| std::fs::metadata(path).ok().map(|meta| meta.len());
let target = self.pool.dir.join(&file);
if !self.pool.files.contains_key(&file) && length(&target) != Some(size) {
let shared = self.shared_blobs.join(oid.to_hex());
if length(&shared) != Some(size) {
let bytes = self.repo.read_blob(oid)?;
crate::cas::atomic_write(&shared, &bytes)
.and_then(|()| crate::cas::seal_file(&shared))
.with_context(|| format!("write {}", shared.display()))?;
}
crate::cas::link_or_reflink(&shared, &target)
.and_then(|()| crate::cas::seal_file(&target))
.with_context(|| format!("link {}", target.display()))?;
}
self.pool.files.insert(file.clone(), size);
self.record(&file, size);
Ok(Some(file))
}
fn reuse(&mut self, piece: Option<&Piece>) -> bool {
let Some(piece) = piece else {
return false;
};
let dir = self.pool.dir;
let held = piece.files.iter().all(|(file, &len)| {
std::fs::metadata(dir.join(file)).is_ok_and(|meta| meta.len() == len)
});
if held {
self.pool.files.extend(piece.files.clone());
self.rows.absorb(&piece.rows);
}
held
}
fn begin(&mut self) {
self.piece = Some(Piece::default());
}
fn finish(&mut self) -> Piece {
let piece = self.piece.take().unwrap_or_default();
self.rows.absorb(&piece.rows);
piece
}
fn list_refs(&mut self, head: Option<&knot_git::HeadRef>) -> anyhow::Result<()> {
let mut refs: Vec<RefWire> = self
.repo
.references()?
.into_iter()
.filter(|record| RefClass::of(&record.name).is_public())
.map(|record| RefWire {
name: record.name,
sha: record.target,
})
.collect();
refs.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str()));
let default_branch = head.map(|head| DefaultBranchWire {
name: head.name.head_ref(),
head: Some(head.target),
});
for (offset, size, keys) in offset_pages(REF_PAGES, refs.len()) {
let window = &refs[offset.min(refs.len())..(offset + size).min(refs.len())];
let file = self.put_json(&ListRefsOut {
refs: window,
cursor: next_cursor(offset, size, refs.len()),
default_branch: default_branch.as_ref(),
})?;
for key in keys {
self.rows.refs.insert(key, file.clone());
}
}
Ok(())
}
fn branches(&mut self, head: Option<&knot_git::HeadRef>) -> anyhow::Result<Vec<(String, Oid)>> {
let repo = self.repo;
let mut branches = repo.branch_list()?;
branches.sort_by(|a, b| {
b.tip
.created_at()
.cmp(&a.tip.created_at())
.then_with(|| a.name.cmp(&b.name))
});
let default = head.map(|head| head.name.clone());
let listed: Vec<BranchInfoOut> = branches.iter().map(BranchInfoOut::of).collect();
for (offset, size, keys) in offset_pages(BRANCH_PAGES, listed.len()) {
let window = &listed[offset.min(listed.len())..(offset + size).min(listed.len())];
let file = self.put_json(&BranchesOut {
branches: window,
cursor: next_cursor(offset, size, listed.len()),
total: listed.len(),
default_branch: default.as_ref(),
})?;
for key in keys {
self.rows.branches.insert(key, file.clone());
}
}
let mut tips = Vec::new();
for branch in &branches {
let BranchTip::Commit(commit) = &branch.tip else {
continue;
};
let name = branch.name.as_str();
let full = format!("refs/heads/{name}");
for spelling in spellings(name).into_iter().chain(spellings(&full)) {
self.rows.commit.insert(spelling, commit.id.to_hex());
}
tips.push((name.to_string(), commit.id));
if Instant::now() >= self.count_deadline {
continue;
}
let (_, total_commits) =
repo.log_window(commit.id, LogSkip::new(0), LogLimit::new(0))?;
let file = self.put_json(&BranchOut {
name: branch.name.clone(),
hash: commit.id,
message: commit.message.clone(),
author: SignatureOut::of(&commit.author),
total_commits,
})?;
for spelling in spellings(name).into_iter().chain(spellings(&full)) {
self.rows.branch.insert(spelling, file.clone());
}
}
Ok(tips)
}
fn tags(&mut self) -> anyhow::Result<Vec<(String, Oid)>> {
let repo = self.repo;
let mut tags = repo.tag_list()?;
tags.sort_by(|a, b| {
b.created_at
.cmp(&a.created_at)
.then_with(|| a.name.cmp(&b.name))
});
let listed: Vec<TagInfoOut> = tags.iter().map(|tag| TagInfoOut::of(repo, tag)).collect();
for (offset, size, keys) in offset_pages(BRANCH_PAGES, listed.len()) {
let window = &listed[offset.min(listed.len())..(offset + size).min(listed.len())];
let file = self.put_json(&TagsOut {
tags: window,
cursor: next_cursor(offset, size, listed.len()),
total: listed.len(),
})?;
for key in keys {
self.rows.tags.insert(key, file.clone());
}
}
let mut tips = Vec::new();
for (index, (tag, out)) in tags.iter().zip(&listed).enumerate() {
let name = tag.name.as_str();
let full = format!("refs/tags/{name}");
if index < MAX_TAG_DETAILS {
let file = self.put_json(out)?;
for spelling in spellings(name).into_iter().chain(spellings(&full)) {
self.rows.tag.insert(spelling, file.clone());
}
}
// a bare name resolves to the tag over a branch of the same name.
let Ok(commit) = repo.peel_to_commit(tag.id) else {
continue;
};
for spelling in spellings(name).into_iter().chain(spellings(&full)) {
self.rows.commit.insert(spelling, commit.to_hex());
}
tips.push((name.to_string(), commit));
}
Ok(tips)
}
fn walk_tree(&self, commit: Oid) -> anyhow::Result<Vec<Dir>> {
let root = self.repo.find_commit(commit)?.tree;
let mut queue = VecDeque::from([(String::new(), root)]);
let mut dirs = Vec::new();
let mut listed = 0;
while let Some((path, tree)) = queue.pop_front() {
if listed >= MAX_TIP_PATHS {
break;
}
let entries = self.repo.tree_entries(tree)?;
listed += entries.len();
for entry in &entries {
if entry.kind == EntryKind::Tree {
queue.push_back((join(&path, &entry.name), entry.oid));
}
}
dirs.push(Dir { path, entries });
}
Ok(dirs)
}
fn changed(&mut self, commit: Oid, parent: Option<Oid>) -> Option<Vec<RepoPath>> {
if let Some(known) = self.changed.get(&commit) {
return known.clone();
}
let listed = self
.repo
.changed_paths(PatchRange {
base: parent,
head: commit,
})
.ok()
.filter(|changed| changed.listing() == Listing::Complete)
.map(|changed| changed.into_paths());
self.changed.insert(commit, listed.clone());
listed
}
fn attribute(
&mut self,
tip: Oid,
paths: &[String],
) -> anyhow::Result<(HashMap<String, Oid>, bool)> {
let mut pending: HashSet<&str> = paths.iter().map(String::as_str).collect();
let mut last = HashMap::new();
let mut walked = 0;
let mut complete = true;
'walk: loop {
let window = self.repo.log_window_bounded(
tip,
LogSkip::new(walked),
LogLimit::new(ATTRIBUTION_CHUNK),
)?;
for commit in &window {
if pending.is_empty() || walked >= MAX_ATTRIBUTION_COMMITS {
break 'walk;
}
if Instant::now() >= self.attribution_deadline {
complete = false;
break 'walk;
}
walked += 1;
self.commits
.entry(commit.id)
.or_insert_with(|| CommitOut::of(commit));
let known = self
.memo
.attribution
.get(&commit.id.to_hex())
.or_else(|| self.prior?.attribution.get(&commit.id.to_hex()));
if let Some(known) = known.filter(|_| commit.id != tip) {
for path in pending.drain() {
if let Some(oid) = known.get(path).and_then(|hex| Oid::from_hex(hex).ok()) {
last.insert(path.to_string(), oid);
}
}
break 'walk;
}
if commit.parents.len() > 1 {
continue;
}
let Some(changed) = self.changed(commit.id, commit.parents.first().copied()) else {
continue;
};
for path in &changed {
for prefix in prefixes(path.as_str()) {
if pending.remove(prefix) {
last.insert(prefix.to_string(), commit.id);
}
}
}
}
if window.len() < ATTRIBUTION_CHUNK {
break;
}
}
Ok((last, complete))
}
fn tip(&mut self, tip: &Tip, head: bool) -> anyhow::Result<Vec<Oid>> {
let key = format!("{}|{}", tip.oid.to_hex(), tip.label);
let log = self.log_window(tip.oid)?;
let logged: Vec<Oid> = log
.iter()
.take(LOG_WINDOW)
.map(|commit| commit.hash())
.collect();
let prior = self.prior.and_then(|prior| prior.tips.get(&key));
if self.reuse(prior) {
let piece = prior.expect("reused").clone();
self.memo.tips.insert(key, piece);
if let Some(known) = self
.prior
.and_then(|prior| prior.attribution.get(&tip.oid.to_hex()))
{
self.memo
.attribution
.insert(tip.oid.to_hex(), known.clone());
}
return Ok(logged);
}
self.begin();
let dirs = self.walk_tree(tip.oid)?;
let paths: Vec<String> = dirs
.iter()
.flat_map(|dir| dir.entries.iter().map(|entry| join(&dir.path, &entry.name)))
.collect();
let (last, complete) = self.attribute(tip.oid, &paths)?;
let submodules = self.repo.submodules(tip.oid).unwrap_or_default();
let tip_hex = tip.oid.to_hex();
for dir in &dirs {
let mut entries = Vec::new();
let mut newest: Option<CommitOut> = None;
for entry in &dir.entries {
let path = join(&dir.path, &entry.name);
let last_commit = last.get(&path).and_then(|oid| self.commit(*oid));
if let Some(commit) = &last_commit
&& newest
.as_ref()
.is_none_or(|newest| commit.author_time() > newest.author_time())
{
newest = Some(commit.clone());
}
let submodule = (entry.kind == EntryKind::Commit)
.then(|| {
submodules
.iter()
.find(|module| module.path.as_str() == path)
})
.flatten()
.map(|module| SubmoduleOut {
name: module.name.clone(),
url: module.url.to_string(),
branch: module.branch.clone(),
});
let file = self.put_json(&EntryOut {
name: entry.name.clone(),
mode: entry.kind.mode_octal().to_string(),
oid: entry.oid,
size: entry.size as i64,
last_commit: last_commit.clone(),
submodule,
})?;
for spelling in spellings(&path) {
self.rows()
.entry
.insert(format!("{tip_hex}|{spelling}"), file.clone());
}
if is_file(entry.kind)
&& let Some(blob) = self.put_blob(entry.oid)?
{
for spelling in spellings(&path) {
self.rows()
.blob
.insert(format!("{tip_hex}|{spelling}"), blob.clone());
}
}
entries.push(TreeEntryOut {
name: entry.name.clone(),
mode: entry.kind.mode_octal().to_string(),
size: entry.size as i64,
last_commit,
});
}
let file = self.put_json(&TreeOut {
entries,
last_commit: newest,
})?;
for spelling in spellings(&dir.path) {
self.rows()
.tree
.insert(format!("{tip_hex}|{spelling}"), file.clone());
}
}
self.languages(tip)?;
self.log_pages(tip.oid, &log)?;
if head {
self.path_logs(&tip_hex, &log)?;
}
let piece = self.finish();
if complete {
self.memo.tips.insert(key, piece);
self.memo.attribution.insert(
tip_hex,
last.into_iter()
.map(|(path, oid)| (path, oid.to_hex()))
.collect(),
);
}
Ok(logged)
}
fn languages(&mut self, tip: &Tip) -> anyhow::Result<()> {
let Some(tree) = self.commit(tip.oid).map(|commit| commit.tree_hex()) else {
return Ok(());
};
let known = self
.memo
.languages
.get(&tree)
.or_else(|| self.prior?.languages.get(&tree))
.cloned();
let languages = match known {
Some(languages) => languages,
None => {
let deadline = Instant::now() + LANGUAGES_BUDGET;
let sizes = knot_langs::analyze(self.repo, tip.oid, Some(deadline))?;
let total: u64 = sizes.values().map(|size| size.get()).sum();
let mut languages: Vec<LanguageOut> = sizes
.iter()
.filter(|(_, size)| size.get() > 0)
.map(|(name, size)| LanguageOut {
name: name.as_str().to_string(),
size: size.get(),
percentage: ((size.get() as f64) / (total as f64) * 100.0).round() as i64,
})
.collect();
languages.sort_by(|a, b| b.size.cmp(&a.size).then_with(|| a.name.cmp(&b.name)));
languages
}
};
let total: u64 = languages.iter().map(|language| language.size).sum();
let file = self.put_json(&LanguagesOut {
refspec: &tip.label,
languages: &languages,
total_size: (total > 0).then_some(total),
total_files: (total > 0).then_some(languages.len() as i64),
})?;
self.rows().languages.insert(tip.oid.to_hex(), file);
self.memo.languages.insert(tree, languages);
Ok(())
}
/// one extra commit tells a full window from a log that ends there.
fn log_window(&mut self, tip: Oid) -> anyhow::Result<Vec<CommitOut>> {
let window =
self.repo
.log_window_bounded(tip, LogSkip::new(0), LogLimit::new(LOG_WINDOW + 1))?;
Ok(window
.iter()
.map(|commit| {
self.commits
.entry(commit.id)
.or_insert_with(|| CommitOut::of(commit))
.clone()
})
.collect())
}
/// the window end gets no cursor. the last commit keeps its parents, unlike the root.
fn page(shown: &[CommitOut], complete: bool, start: usize, size: usize) -> Option<LogOut> {
let end = start + size;
if end < shown.len() {
return Some(LogOut {
commits: shown[start..end].to_vec(),
cursor: Some(shown[end - 1].hash()),
});
}
(start < shown.len() || complete && start <= shown.len()).then(|| LogOut {
commits: shown[start..].to_vec(),
cursor: None,
})
}
fn log_rows(
&mut self,
tip_hex: &str,
path: &str,
shown: &[CommitOut],
complete: bool,
plan: LogPlan,
) -> anyhow::Result<()> {
let paths = spellings(path);
for &(limits, size, chained, numeric) in plan {
let mut start = 0;
while let Some(page) = Self::page(shown, complete, start, size) {
let more = page.cursor.is_some();
let file = self.put_json(&page)?;
let mut cursors = match start {
0 => vec![String::new()],
_ => vec![shown[start - 1].hash().to_hex()],
};
if numeric.contains(&start) {
cursors.push(start.to_string());
}
for path in &paths {
for limit in limits {
for cursor in &cursors {
self.rows()
.log
.insert(format!("{tip_hex}|{path}|{limit}|{cursor}"), file.clone());
}
}
}
start += size;
if !chained || !more {
break;
}
}
}
Ok(())
}
fn log_pages(&mut self, tip: Oid, log: &[CommitOut]) -> anyhow::Result<()> {
let tip_hex = tip.to_hex();
let shown = &log[..log.len().min(LOG_WINDOW)];
self.log_rows(&tip_hex, "", shown, log.len() <= LOG_WINDOW, LOG_PAGES)?;
for (index, commit) in log.iter().take(LOG_WINDOW).enumerate() {
let hex = commit.hash().to_hex();
self.rows().commit.entry(hex.clone()).or_insert(hex.clone());
let single = LogOut {
commits: vec![commit.clone()],
cursor: (!commit.parents_of().is_empty()).then(|| commit.hash()),
};
let file = self.put_json(&single)?;
self.rows().log.insert(format!("{hex}||1|"), file);
let linear = log[index..]
.windows(2)
.take(50)
.all(|pair| pair[0].parents_of() == [pair[1].hash()]);
let tail = &log[index..];
let page = match (tail.len() > 50, log.len() <= LOG_WINDOW) {
(true, _) => Some(LogOut {
commits: tail[..50].to_vec(),
cursor: Some(tail[49].hash()),
}),
(false, true) if tail.last().is_some_and(|last| last.parents_of().is_empty()) => {
Some(LogOut {
commits: tail.to_vec(),
cursor: None,
})
}
_ => None,
};
if let Some(page) = page.filter(|_| linear && index > 0) {
let file = self.put_json(&page)?;
for limit in ["", "50"] {
self.rows()
.log
.insert(format!("{hex}||{limit}|"), file.clone());
}
}
for parent in commit.parents_of() {
let parent = parent.to_hex();
self.rows().commit.entry(parent.clone()).or_insert(parent);
}
}
Ok(())
}
fn round_log(&mut self, base: Oid, head: Oid) -> anyhow::Result<()> {
let head_hex = head.to_hex();
if self.rows.log.contains_key(&format!("{head_hex}||100|")) {
return Ok(());
}
let key = format!("{}|{head_hex}", base.to_hex());
let prior = self.prior.and_then(|prior| prior.rounds.get(&key));
if self.reuse(prior) {
self.memo.rounds.insert(key, prior.expect("reused").clone());
return Ok(());
}
self.begin();
let log = self.log_window(head)?;
let (shown, complete) = match log.iter().take(LOG_WINDOW).position(|c| c.hash() == base) {
Some(at) => (&log[..=at], true),
None => (&log[..log.len().min(LOG_WINDOW)], log.len() <= LOG_WINDOW),
};
self.log_rows(&head_hex, "", shown, complete, WALK_PAGES)?;
self.rows()
.commit
.entry(head_hex.clone())
.or_insert(head_hex);
let piece = self.finish();
self.memo.rounds.insert(key, piece);
Ok(())
}
/// matches knot2's listCommits with a path. a commit with too many changes ends the window.
fn path_logs(&mut self, tip_hex: &str, log: &[CommitOut]) -> anyhow::Result<()> {
let mut index: HashMap<String, usize> = HashMap::new();
let mut touched: Vec<Vec<CommitOut>> = Vec::new();
let mut complete = log.len() <= LOG_WINDOW;
for commit in log.iter().take(LOG_WINDOW) {
let parent = commit.parents_of().first().copied();
let Some(changed) = self.changed(commit.hash(), parent) else {
complete = false;
break;
};
for path in &changed {
for prefix in prefixes(path.as_str()) {
let at = match index.get(prefix) {
Some(&at) => at,
None if touched.len() < MAX_TIP_PATHS => {
index.insert(prefix.to_string(), touched.len());
touched.push(Vec::new());
touched.len() - 1
}
None => continue,
};
let commits = &mut touched[at];
if commits
.last()
.is_none_or(|last| last.hash() != commit.hash())
{
commits.push(commit.clone());
}
}
}
}
for (path, at) in &index {
self.log_rows(tip_hex, path, &touched[*at], complete, WALK_PAGES)?;
}
Ok(())
}
fn diff(&mut self, base: Option<Oid>, head: Oid) -> anyhow::Result<()> {
let base_hex = base
.map(Oid::to_hex)
.unwrap_or_else(|| self.empty_tree().to_string());
let key = format!("{base_hex}|{}", head.to_hex());
if self.rows.diff.contains_key(&key) {
return Ok(());
}
let prior = self.prior.and_then(|prior| prior.diffs.get(&key));
if self.reuse(prior) {
self.memo.diffs.insert(key, prior.expect("reused").clone());
return Ok(());
}
if Instant::now() >= self.diff_deadline {
return Ok(());
}
let patches = self
.repo
.commit_patches(PatchRange { base, head }, &mut BinaryBudget::Omit)?;
if patches.len() > MAX_DIFF_FILES {
return Ok(());
}
let diffs: Vec<_> = patches
.iter()
.map(|patch| file_diff(self.repo, patch))
.collect();
let body = serde_json::to_vec(&DiffOut {
diffs: &diffs,
stats: DiffStatOut::of(&patches),
})?;
if body.len() > MAX_DIFF_BYTES {
return Ok(());
}
self.begin();
let file = self.put("json", "json", &body)?;
self.rows().diff.insert(key.clone(), file);
for side in [base_hex.clone(), head.to_hex()] {
self.rows().commit.entry(side.clone()).or_insert(side);
}
for patch in &patches {
if let Some(base) = base.filter(|_| patch.status != PatchStatus::Added) {
self.side(base, &patch.path, patch.old_oid, patch.old_kind)?;
}
if patch.status != PatchStatus::Deleted {
self.side(head, &patch.path, patch.new_oid, patch.new_kind)?;
}
}
let piece = self.finish();
self.memo.diffs.insert(key, piece);
Ok(())
}
fn side(
&mut self,
commit: Oid,
path: &RepoPath,
oid: Oid,
kind: Option<EntryKind>,
) -> anyhow::Result<()> {
let Some(kind) = kind else {
return Ok(());
};
let size = match is_file(kind) {
true => self.repo.blob_size(oid).unwrap_or(0),
false => 0,
};
let file = self.put_json(&EntryOut {
name: path.file_name().to_string(),
mode: kind.mode_octal().to_string(),
oid,
size: size as i64,
last_commit: None,
submodule: None,
})?;
let commit = commit.to_hex();
let blob = match is_file(kind) {
true => self.put_blob(oid)?,
false => None,
};
for spelling in spellings(path.as_str()) {
let key = format!("{commit}|{spelling}");
self.rows().entry.insert(key.clone(), file.clone());
if let Some(blob) = &blob {
self.rows().blob.insert(key, blob.clone());
}
}
Ok(())
}
/// blame is reused while the last commit touching the file and its blob stay the same.
fn blame(&mut self, head: Oid, name: &str) -> anyhow::Result<()> {
let head_hex = head.to_hex();
let Some(last) = self.memo.attribution.get(&head_hex).cloned() else {
return Ok(());
};
for dir in self.walk_tree(head)? {
for entry in &dir.entries {
if !is_file(entry.kind) || entry.size > MAX_BLAME_BYTES {
continue;
}
let path = join(&dir.path, &entry.name);
let Some(touched) = last.get(&path) else {
continue;
};
let key = format!("{touched}|{path}|{}", entry.oid.to_hex());
let Some(body) = self.blame_body(head, &key, &path, name)? else {
continue;
};
let file = self.put_json(&BlameOut {
commit: head,
body: &body,
})?;
for spelling in spellings(&path) {
self.rows
.blame
.insert(format!("{head_hex}|{spelling}"), file.clone());
}
}
}
Ok(())
}
fn blame_body(
&mut self,
head: Oid,
key: &str,
path: &str,
name: &str,
) -> anyhow::Result<Option<BlameBody>> {
let pooled = self
.prior
.and_then(|prior| prior.blame.get(key))
.and_then(|file| {
let bytes = std::fs::read(self.pool.dir.join(file)).ok()?;
Some((
file.clone(),
serde_json::from_slice::<BlameBody>(&bytes).ok()?,
bytes.len(),
))
});
if let Some((file, body, len)) = pooled {
self.pool.files.insert(file.clone(), len as u64);
self.memo.blame.insert(key.to_string(), file);
return Ok(Some(body));
}
if Instant::now() >= self.blame_deadline {
return Ok(None);
}
let Ok(path) = RepoPath::new(path) else {
return Ok(None);
};
let (repo, deadline) = (self.repo, self.blame_deadline);
// gix-blame panics on some histories. a panic here only skips the file.
let blamed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
repo.blame_file_bounded(head, &path, MAX_BLAME_BYTES, deadline)
}));
let hunks = match blamed {
Ok(Ok(hunks)) => hunks,
Ok(Err(_)) => return Ok(None),
Err(_) => {
tracing::warn!(
repo = name,
path = path.as_str(),
"blame panicked, the file gets none"
);
return Ok(None);
}
};
let mut commits = Vec::new();
for id in hunks
.iter()
.map(|hunk| hunk.commit)
.collect::<BTreeSet<_>>()
{
commits.push(BlameCommitOut::of(&self.repo.find_commit(id)?));
}
let body = BlameBody {
commits,
runs: hunks
.iter()
.map(|hunk| BlameRunOut {
start: hunk.start,
count: hunk.lines,
commit: hunk.commit.to_hex(),
})
.collect(),
};
let file = self.put("blame", "json", &serde_json::to_vec(&body)?)?;
self.memo.blame.insert(key.to_string(), file);
Ok(Some(body))
}
fn archives(&mut self, head: Oid, prefix: &ArchivePrefix) -> anyhow::Result<()> {
let Some(tree) = self.commit(head).map(|commit| commit.tree_hex()) else {
return Ok(());
};
let head_hex = head.to_hex();
for (format, ext, spellings) in [
(ArchiveFormat::TarGz, "tar.gz", &["", "tar.gz"][..]),
(ArchiveFormat::Zip, "zip", &["zip"][..]),
] {
let key = format!("{tree}|{ext}|{}", prefix.as_str());
let dir = self.pool.dir;
let pooled = self
.prior
.and_then(|prior| prior.archives.get(&key))
.and_then(|file| {
Some((file.clone(), std::fs::metadata(dir.join(file)).ok()?.len()))
});
let file = match pooled {
Some((file, len)) => {
self.pool.files.insert(file.clone(), len);
file
}
None => {
let mut out = std::io::Cursor::new(Vec::new());
let written = self.repo.write_archive(
self.repo.peel_to_tree(head)?,
format,
Some(&prefix.clone().into_tree_prefix()),
ArchiveLimit::new(MAX_ARCHIVE_BYTES),
&mut out,
);
if written.is_err() {
continue;
}
self.put("archives", ext, out.get_ref())?
}
};
for spelling in spellings {
self.rows
.archive
.insert(format!("{head_hex}|{spelling}"), file.clone());
}
self.rows
.archive_name
.insert(file.clone(), archive_disposition(prefix.as_str(), ext));
self.memo.archives.insert(key, file);
}
Ok(())
}
}