a write reads no git reads of the repos it doesn't change
Signed-off-by: dawn <90008@klbr.net>
384 unmodified linesExpand all38538638738838938939039139239339439579 unmodified lines384 unmodified linesExpand all seals.extend(crate::git_xrpc::last_seal(&store)); let Some(kept) = crate::git_xrpc::kept_served(&store, &seals) else { return Ok(0); }; let (pool, seal_dir) = crate::git_xrpc::served_dirs(&store); let (pool, seal_dir, maps_dir) = crate::git_xrpc::served_dirs(&store); let mut removed = sweep(&seal_dir, |path| kept.contains(path))?; removed += sweep(&maps_dir, |path| kept.contains(path))?; let pooled = crate::git_xrpc::POOLED.iter().map(|dir| pool.join(dir)); for dir in pooled.chain(crate::git_xrpc::paths::swept(&pool)) { removed += sweep(&dir, |path| { kept.contains(path)79 unmodified lines27 unmodified lines28293031323334353637383916 unmodified lines5657585956576061626364656667686970717273747576777879808145 unmodified lines12712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920010 unmodified lines21121221321414614714814915015115215315415515615721521621721821922022122222316416522422522622722823 unmodified lines25225325425525625719825825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829940 unmodified lines34034134234324934434534634734834935035125735235335435535635735835936036136236336436536626036736836937037137237337437537637737837938038179 unmodified lines461462463464351352353354355356357358359360361362363364365465466467468268 unmodified linesExpand all7377387397406426437417427437447457467477487497501 unmodified line7527537547557567577586547597607617627637647657667677687697701 unmodified line27 unmodified linespub(crate) const SEAL_VERSION: u32 = 9;const POOL_KIND: &str = "git-xrpc";const SEAL_KIND: &str = "git-xrpc-seal";const LAST_SEALED: &str = "git-xrpc.last";/// per seal key: a map fragment per variable, what gc keeps, and what a carried seal needsconst MAPS_KIND: &str = "git-xrpc-maps";const CARRY_FILE: &str = "carry.json";const KEPT_FILE: &str = "kept.json";pub(crate) const POOLED: [&str; 4] = ["json", "blobs", "archives", "blame"];pub(crate) const SHARED_BLOBS: &str = "git-xrpc/blobs";
pub(crate) const NO_MERGE_BASE: &str = "none";16 unmodified linespub struct SealedRepo { pub did: RepoDid, pub key: String, pub(crate) pool: PathBuf, pub(crate) rows: Rows, pub(crate) unlaid: Rows, /// the map fragments the generation includes pub(crate) maps: PathBuf, pub(crate) paths: String, #[cfg(test)] served: RepoStore, #[cfg(test)] read_rows: std::sync::OnceLock<Rows>,}
#[cfg(test)]impl SealedRepo { pub(crate) fn rows(&self) -> &Rows { self.read_rows .get_or_init(|| sealed_rows(&self.served, &self.key).expect("the seal reads")) }}
impl SealedRepo { pub(crate) fn dir_name(&self) -> String { crate::plan::repo_token(self.did.as_str()) }}45 unmodified lines } }}
/// all a generation needs of a seal it carries unchanged#[derive(Serialize, Deserialize)]struct Carry { patches: Vec<(Oid, Oid)>, paths: String,}
/// the files a seal holds in the pool, which gc keeps#[derive(Serialize, Deserialize)]struct Kept { files: BTreeMap<String, u64>, paths: paths::Laid,}
fn read_json<T: serde::de::DeserializeOwned>(path: &Path) -> Option<T> { serde_json::from_slice(&std::fs::read(path).ok()?).ok()}
fn seal_key(input: &RepoInput, patches: &[(Oid, Oid)]) -> anyhow::Result<String> { Ok(format!( "{SEAL_VERSION}-{}", short_hash(&serde_json::to_vec(&(&input.refs_digest, patches))?) ))}
#[derive(Serialize, Deserialize)]pub(crate) struct Sealed { patches: Vec<(Oid, Oid)>, rows: Rows, files: BTreeMap<String, u64>, paths: paths::Laid, memo: project::Memo,}
/// an unchanged repo is carried by its seal key, no record of it is readpub fn seal(input: &RepoInput) -> anyhow::Result<SealedRepo> { let dir = input.served.pool(POOL_KIND); let maps_root = input.served.kind_dir(MAPS_KIND); let last = input.served.kind_dir(LAST_SEALED); let last_key = std::fs::read_to_string(&last).ok(); let sealed_repo = |key: &str, paths: String| SealedRepo { did: input.did.clone(), key: key.to_string(), pool: dir.clone(), maps: maps_root.join(key), paths, #[cfg(test)] served: input.served.clone(), #[cfg(test)] read_rows: Default::default(), }; let carried = last_key .as_deref() .filter(|_| !input.verify) .and_then(|key| { Some(( key, read_json::<Carry>(&maps_root.join(key).join(CARRY_FILE))?, )) }); if let Some((key, carry)) = carried { let patches = input.patches.as_deref().unwrap_or(&carry.patches); if seal_key(input, patches)? == key { return Ok(sealed_repo(key, carry.paths)); } } let read = |key: &str| { input .served .read_file::<Sealed>(SEAL_KIND, key)10 unmodified lines .as_ref() .map(|prior| prior.patches.clone()) .unwrap_or_default(), }; let key = format!( "{SEAL_VERSION}-{}", short_hash(&serde_json::to_vec(&(&input.refs_digest, &patches))?) ); let sealed_repo = |sealed: Sealed, unlaid: Rows| SealedRepo { did: input.did.clone(), key: key.clone(), pool: dir.clone(), unlaid, rows: sealed.rows, paths: sealed.paths.seal, }; let key = seal_key(input, &patches)?; if let Some(sealed) = read(&key).filter(|sealed| holds(&dir, &sealed.files) && sealed.paths.held(&dir)) { let unlaid = paths::split(&sealed.rows, paths::folds(&dir)?).1; write_maps(input, &maps_root.join(&key), &sealed, &unlaid)?; if last_key.as_deref() != Some(key.as_str()) { atomic_write(&last, key.as_bytes())?; } let unlaid = paths::split(&sealed.rows, paths::folds(&dir)?).1; return Ok(sealed_repo(sealed, unlaid)); return Ok(sealed_repo(&key, sealed.paths.seal)); } let repo = Repo::open(&input.export_dir) .with_context(|| format!("open export {}", input.export_dir.display()))?; let mut pool = Pool {23 unmodified lines paths: laid, memo, }; input.served.publish_file(SEAL_KIND, &key, &sealed)?; write_maps(input, &maps_root.join(&key), &sealed, &unlaid)?; atomic_write(&last, key.as_bytes())?; Ok(sealed_repo(sealed, unlaid)) Ok(sealed_repo(&key, sealed.paths.seal))}
/// the carry file goes last, a seal without one gets its fragments written againfn write_maps(input: &RepoInput, dir: &Path, sealed: &Sealed, unlaid: &Rows) -> anyhow::Result<()> { if dir.join(CARRY_FILE).is_file() { return Ok(()); } std::fs::create_dir_all(dir)?; let mut maps = Maps::default(); maps.add( &crate::plan::repo_token(input.did.as_str()), &sealed.rows, unlaid, ); for (_, variable) in sources() { let rows = Block::file().entries(maps.take(variable)); atomic_write(&dir.join(fragment(variable)), rows.to_string().as_bytes())?; } let kept = Kept { files: sealed.files.clone(), paths: sealed.paths.clone(), }; atomic_write(&dir.join(KEPT_FILE), &serde_json::to_vec(&kept)?)?; let carry = Carry { patches: sealed.patches.clone(), paths: sealed.paths.seal.clone(), }; Ok(atomic_write( &dir.join(CARRY_FILE), &serde_json::to_vec(&carry)?, )?)}
fn fragment(variable: &str) -> String { format!("{}.conf", variable.trim_start_matches('$'))}
pub(crate) fn sealed_rows(served: &RepoStore, key: &str) -> Option<Rows> { #[derive(Deserialize)] struct Only { rows: Rows,40 unmodified lines std::fs::create_dir_all(&root)?; std::fs::create_dir_all(&laid)?; let mut sorted: Vec<&SealedRepo> = repos.iter().collect(); sorted.sort_unstable_by_key(|repo| repo.did.as_str()); let mut maps = Maps::default(); let mut fragments = Vec::with_capacity(sorted.len()); for repo in sorted { std::fs::create_dir_all(&repo.pool)?; let pool = std::fs::canonicalize(&repo.pool) .with_context(|| format!("git reads of {}", repo.did))?; let name = repo.dir_name(); std::os::unix::fs::symlink(&pool, root.join(&name))?; std::os::unix::fs::symlink(paths::seal_dir(&pool, &repo.paths), laid.join(&name))?; maps.add(&name, &repo.rows, &repo.unlaid); fragments.push( std::fs::canonicalize(&repo.maps) .with_context(|| format!("git read maps of {}", repo.did))?, ); } let mut file = Block::file().blocks(paths::maps()); for (source, variable) in sources() { let block = fragments .iter() .fold(map(source, variable, Vec::new()), |block, dir| { block.directive("include", [dir.join(fragment(variable)).display()]) }); file = file.block(block); } let path = generation_dir.join("nginx/maps").join(MAPS_FILE); atomic_write(&path, maps.blocks().to_string().as_bytes()) atomic_write(&path, file.to_string().as_bytes()) .with_context(|| format!("write {}", path.display()))}
fn sources() -> impl Iterator<Item = (&'static str, &'static str)> { COMMIT_SOURCES .into_iter() .chain(FILE_MAPS) .chain([("$ks_git_archive", "$ks_git_archive_name")])}
#[derive(Default)]struct Maps { rows: BTreeMap<&'static str, Vec<(String, String)>>,}79 unmodified lines .map(|(file, name)| (format!("{dir}/{file}"), name.clone())), ); }
fn blocks(mut self) -> Block { let mut file = Block::file().blocks(paths::maps()); for (source, variable) in COMMIT_SOURCES { file = file.block(map(source, variable, self.take(variable))); } for (source, variable) in FILE_MAPS { file = file.block(map(source, variable, self.take(variable))); } file.block(map( "$ks_git_archive", "$ks_git_archive_name", self.take("$ks_git_archive_name"), )) }
fn take(&mut self, variable: &str) -> Vec<(String, String)> { without_case_twins(self.rows.remove(variable).unwrap_or_default()) }}268 unmodified linesExpand all laid_out(paths::Kind::Log, "git_not_projected"), ]}
pub fn served_dirs(served: &RepoStore) -> (PathBuf, PathBuf) { (served.pool(POOL_KIND), served.kind_dir(SEAL_KIND))pub fn served_dirs(served: &RepoStore) -> (PathBuf, PathBuf, PathBuf) { ( served.pool(POOL_KIND), served.kind_dir(SEAL_KIND), served.kind_dir(MAPS_KIND), )}
pub fn last_seal(served: &RepoStore) -> Option<String> { std::fs::read_to_string(served.kind_dir(LAST_SEALED)).ok()1 unmodified line
pub fn kept_served(served: &RepoStore, seals: &BTreeSet<String>) -> Option<BTreeSet<PathBuf>> { let pool = served.pool(POOL_KIND); let mut kept = BTreeSet::new(); let maps = served.kind_dir(MAPS_KIND); for key in seals { let sealed = read_json::<Kept>(&maps.join(key).join(KEPT_FILE)).or_else(|| { let sealed = served.read_file::<Sealed>(SEAL_KIND, key)?; let sealed = served.read_file::<Sealed>(SEAL_KIND, key)?; Some(Kept { files: sealed.files, paths: sealed.paths, }) })?; kept.insert(served.file_path(SEAL_KIND, key)); kept.insert(maps.join(key)); kept.extend(sealed.files.keys().map(|file| pool.join(file))); kept.extend(sealed.paths.kept(&pool)); } Some(kept)1 unmodified line128 unmodified linesExpand all129130131132133134135136137138139140141142143317 unmodified linesExpand all461462463464458459465466467468469470471472473474468469475476477478472479480474481482483484485351 unmodified linesExpand all128 unmodified linesExpand all
impl Served { fn load(generation: &Path) -> Self { let text = std::fs::read_to_string(generation.join("nginx/maps").join(MAPS_FILE)).unwrap(); let text: String = text .lines() .map(|line| match line.trim().strip_prefix("include ") { Some(path) => std::fs::read_to_string(path.trim_end_matches(';')).unwrap(), None => format!("{line}\n"), }) .collect(); let mut maps: HashMap<String, HashMap<String, String>> = HashMap::new(); let mut current: Option<String> = None; for line in text.lines() { let line = line.trim();317 unmodified linesExpand all let first = seal(&fixture.input(Vec::new())).unwrap(); let again = seal(&fixture.input(Vec::new())).unwrap(); assert_eq!(first.key, again.key); assert_eq!( serde_json::to_value(&first.rows).unwrap(), serde_json::to_value(&again.rows).unwrap() serde_json::to_value(first.rows()).unwrap(), serde_json::to_value(again.rows()).unwrap() );
let main = fixture.rev("main"); git(&fixture.repo, &["checkout", "-q", "feature"], 9); let moved = commit(&fixture.repo, "b.txt", "b\n", "more feature", 10); let next = seal(&fixture.input(Vec::new())).unwrap(); assert_ne!(next.key, first.key); assert_eq!( first.rows.tree.get(&format!("{main}|")), next.rows.tree.get(&format!("{main}|")), first.rows().tree.get(&format!("{main}|")), next.rows().tree.get(&format!("{main}|")), "main's tree answers carry over" ); assert!(next.rows.tree.contains_key(&format!("{moved}|"))); assert!(next.rows().tree.contains_key(&format!("{moved}|"))); let tree: serde_json::Value = serde_json::from_slice( &std::fs::read(next.pool.join(&next.rows.tree[&format!("{moved}|")])).unwrap(), &std::fs::read(next.pool.join(&next.rows().tree[&format!("{moved}|")])).unwrap(), ) .unwrap(); let names: Vec<(&str, &str)> = tree["entries"] .as_array()351 unmodified linesExpand all114 unmodified linesExpand all115116117118119119120121122123130 unmodified linesExpand all25425525625725825825926026126262 unmodified lines32532632732832932933033133233320 unmodified lines114 unmodified linesExpand all (status, body) }
fn oracle(&self, nsid: &str, path: &str) -> (u16, Vec<u8>) { let rows = &self.sealed.rows; let rows = &self.sealed.rows(); let (map, missing) = match nsid { "getEntry" => (&rows.entry, "git_entry_not_found"), "getBlob" => (&rows.blob, "git_file_not_found"), "getTree" => (&rows.tree, "git_path_not_found"),130 unmodified linesExpand all#[test]#[ignore = "requires nginx; set NGINX_BINARY"]fn merge_bases_answer_one_repo_and_refuse_two() { let knot = knot(&[("README.md", "readme\n")], &[]); let main = knot.sealed.rows.commit["main"].clone(); let main = knot.sealed.rows().commit["main"].clone(); let get = |query: &str| { let target = format!("/xrpc/org.tangled.temp.git.getMergeBase?{query}"); let (status, _, body) = http(knot.port, "GET", &target, &[], b""); let body: serde_json::Value = serde_json::from_slice(&body).unwrap();62 unmodified linesfn a_diff_limited_to_paths_is_refused() { let knot = knot(&[("README.md", "readme\n"), ("src/lib.rs", "lib\n")], &[]); let (pair, _) = knot .sealed .rows .rows() .diff .iter() .next() .expect("a projected diff");20 unmodified lines789 unmodified linesExpand all790791792793794794795796797798162 unmodified linesExpand all961962963964965965966967968969215 unmodified linesExpand all1185118611871188118911901189119011911192119311946 unmodified lines12011202120312041205120512061207120812093 unmodified lines12131214121512161217121812191220122112221217121812191220122112221223122412241225122612271228102 unmodified linesExpand all789 unmodified linesExpand all }; if matches!(nsid, "getEntry" | "getBlob") && path.is_empty() { return Want::Miss("invalid_request"); } let rows = &self.sealed.rows; let rows = &self.sealed.rows(); let (map, lower, missing) = match nsid { "getTree" => (&rows.tree, &self.lower.tree, "git_path_not_found"), "getEntry" => (&rows.entry, &self.lower.entry, "git_entry_not_found"), "getBlob" => (&rows.blob, &self.lower.blob, "git_file_not_found"),162 unmodified linesExpand all .get(&arg(query, "ref").to_ascii_lowercase()) else { return (0, 0); }; let (rows, residual) = (&self.sealed.rows, &self.residual); let (rows, residual) = (&self.sealed.rows(), &self.residual); let (all, map, key) = match nsid { "getTree" => ( &rows.tree, &residual.tree,215 unmodified linesExpand all verify: false, }; let sealed = crate::git_xrpc::seal(&input).map_err(|error| format!("seal: {error:#}"))?; let folds = paths::folds(&sealed.pool).unwrap(); let (layout, residual) = paths::split(&sealed.rows, folds); let blobs = check_blobs(&built, &sealed.rows, &sealed.pool)?; let (layout, residual) = paths::split(&sealed.rows(), folds); let blobs = check_blobs(&built, &sealed.rows(), &sealed.pool)?; crate::atproto::emit(&generation, &[]).unwrap(); crate::blobs::emit(dir, &generation, &[], &Default::default()).unwrap(); crate::git_xrpc::emit(&generation, std::slice::from_ref(&sealed)).unwrap(); let repo_map = generation.join("nginx/maps/60_atproto.conf");6 unmodified lines let outside = dir.join("outside"); std::fs::create_dir_all(&outside).unwrap(); let evil = plant(&sealed.pool, &sealed, &outside); let (port, _nginx) = start_nginx(nginx_binary, &generation); let mut refs: Vec<String> = sealed.rows.commit.keys().cloned().collect(); let mut refs: Vec<String> = sealed.rows().commit.keys().cloned().collect(); refs.retain(|spelling| !spelling.is_empty()); let mut this = Seed { rng, built: &built,3 unmodified lines insensitive: folds, generation, port, lower: Lower { commit: lowered(&sealed.rows.commit), tree: split_lowered(&sealed.rows.tree, &residual.tree), entry: split_lowered(&sealed.rows.entry, &residual.entry), blob: split_lowered(&sealed.rows.blob, &residual.blob), blame: split_lowered(&sealed.rows.blame, &residual.blame), log: split_lowered(&sealed.rows.log, &residual.log), commit: lowered(&sealed.rows().commit), tree: split_lowered(&sealed.rows().tree, &residual.tree), entry: split_lowered(&sealed.rows().entry, &residual.entry), blob: split_lowered(&sealed.rows().blob, &residual.blob), blame: split_lowered(&sealed.rows().blame, &residual.blame), log: split_lowered(&sealed.rows().log, &residual.log), }, log_keys: sealed.rows.log.keys().cloned().collect(), log_keys: sealed.rows().log.keys().cloned().collect(), residual: residual.clone(), prefixes: prefixes(&built.listed), layout, stats: BTreeMap::new(),102 unmodified linesExpand all