nginx sizes its map hashes by the git read entries the maps include
Signed-off-by: dawn <90008@klbr.net>
133 unmodified linesExpand all134135136137138139140141142143144125 unmodified linesExpand all2702712722732742752722732762772782792802812822831 unmodified line28528628728828929029129229325 unmodified lines3193203213223233163173243253263273283293303313323333343353363373381 unmodified line340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371487 unmodified linesExpand all133 unmodified linesExpand all#[derive(Serialize, Deserialize)]struct Carry { patches: Vec<(Oid, Oid)>, paths: String, /// entries per fragment, which nginx sizes its map hashes by #[serde(default)] entries: BTreeMap<String, usize>,}
/// the files a seal holds in the pool, which gc keeps#[derive(Serialize, Deserialize)]125 unmodified linesExpand all } std::fs::create_dir_all(dir)?; let mut maps = Maps::default(); maps.add(&crate::plan::repo_token(did.as_str()), &sealed.rows, unlaid); let mut entries = BTreeMap::new(); for (_, variable) in sources() { let rows = Block::file().entries(maps.take(variable)); atomic_write(&dir.join(fragment(variable)), rows.to_string().as_bytes())?; let rows = maps.take(variable); entries.insert(fragment(variable), rows.len()); let text = Block::file().entries(rows).to_string(); atomic_write(&dir.join(fragment(variable)), text.as_bytes())?; } let kept = Kept { files: sealed.files.clone(), paths: sealed.paths.clone(),1 unmodified line atomic_write(&dir.join(KEPT_FILE), &serde_json::to_vec(&kept)?)?; let carry = Carry { patches: sealed.patches.clone(), paths: sealed.paths.seal.clone(), entries, }; Ok(atomic_write( &dir.join(CARRY_FILE), &serde_json::to_vec(&carry)?,25 unmodified lines let Some(key) = last_seal(served) else { return Ok(false); }; let dir = served.kind_dir(MAPS_KIND).join(&key); if let Some(mut carry) = read_json::<Carry>(&dir.join(CARRY_FILE)) { if dir.join(CARRY_FILE).is_file() { return Ok(false); if !carry.entries.is_empty() { return Ok(false); } for (_, variable) in sources() { let name = fragment(variable); carry .entries .insert(name.clone(), counted(&dir.join(name))?); } atomic_write(&dir.join(CARRY_FILE), &serde_json::to_vec(&carry)?)?; return Ok(true); } let Some(sealed) = served.read_file::<Sealed>(SEAL_KIND, &key) else { return Ok(false); };1 unmodified line write_maps(did, &dir, &sealed, &unlaid)?; Ok(true)}
/// how many entries an included fragment adds to its map, from its seal's carry filepub(crate) fn fragment_entries( fragment: &Path, carried: &mut HashMap<PathBuf, BTreeMap<String, usize>>,) -> anyhow::Result<usize> { let dir = fragment.parent().context("fragment dir")?; let entries = carried.entry(dir.to_path_buf()).or_insert_with(|| { read_json::<Carry>(&dir.join(CARRY_FILE)) .map(|carry| carry.entries) .unwrap_or_default() }); let name = fragment.file_name().and_then(|name| name.to_str()); match name.and_then(|name| entries.get(name)) { Some(&count) => Ok(count), None => counted(fragment), }}
fn counted(fragment: &Path) -> anyhow::Result<usize> { let text = std::fs::read_to_string(fragment) .with_context(|| format!("read {}", fragment.display()))?; Ok(text.lines().filter(|line| !line.trim().is_empty()).count())}
fn fragment(variable: &str) -> String { format!("{}.conf", variable.trim_start_matches('$'))}
487 unmodified linesExpand all882 unmodified linesExpand all883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913882 unmodified linesExpand all let next = seal(&fixture.input(Vec::new())).unwrap(); assert!(collect(&next.key) >= 1); assert!(!junk.exists(), "new seals sweep the pool again");}
#[test]fn a_fragments_entries_come_from_its_carry_and_prepare_backfills_them() { let fixture = fixture(); let sealed = seal(&fixture.input(Vec::new())).unwrap(); let mut carried = HashMap::new(); for (_, variable) in sources() { let path = sealed.maps.join(fragment(variable)); assert_eq!( fragment_entries(&path, &mut carried).unwrap(), counted(&path).unwrap(), "{variable}" ); } let carry_path = sealed.maps.join(CARRY_FILE); let mut carry: serde_json::Value = serde_json::from_slice(&std::fs::read(&carry_path).unwrap()).unwrap(); let written = carry["entries"].take(); carry.as_object_mut().unwrap().remove("entries"); std::fs::write(&carry_path, serde_json::to_vec(&carry).unwrap()).unwrap(); let served = fixture.input(Vec::new()).served; let did = RepoDid::new(DID).unwrap(); assert!(super::prepare_repo(&served, &did).unwrap()); let backfilled: serde_json::Value = serde_json::from_slice(&std::fs::read(&carry_path).unwrap()).unwrap(); assert_eq!(backfilled["entries"], written);}223 unmodified linesExpand all2242252262272282292302312321 unmodified line234235236237238239240241242243244245246247286 unmodified linesExpand all223 unmodified linesExpand all
/// a ceiling far above the biggest map makes `nginx -t` slow.fn map_hash_max_size(maps_dir: &Path) -> anyhow::Result<usize> { let mut biggest: usize = 0; let mut carried = std::collections::HashMap::new(); for entry in std::fs::read_dir(maps_dir)? { let path = entry?.path(); let text = std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;1 unmodified line for line in text.lines() { match line.trim() { line if line.starts_with("map ") => keys = 0, "}" => biggest = biggest.max(keys), line if let Some(fragment) = line.strip_prefix("include ") => { keys += crate::git_xrpc::fragment_entries( Path::new(fragment.trim_end_matches(';')), &mut carried, )?; } _ => keys += 1, } } }286 unmodified linesExpand all1364 unmodified linesExpand all13651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251364 unmodified linesExpand all (501, &"MethodNotImplemented".into()) ); }}
#[test]#[ignore = "requires nginx; set NGINX_BINARY"]fn a_map_grown_through_included_fragments_gets_a_hash_that_fits() { let nginx_binary = PathBuf::from( std::env::var_os("NGINX_BINARY") .expect("NGINX_BINARY must name the real nginx binary for this ignored test"), ); let temp = tempfile::tempdir().unwrap(); let (config, manifest, generation) = test_fixture(temp.path()); let fragments = temp.path().join("fragments"); std::fs::create_dir_all(&fragments).unwrap(); let mut includes = String::new(); for part in 0..4 { let rows: String = (0..6000).map(|i| format!("k{part}x{i} v;\n")).collect(); let fragment = fragments.join(format!("{part}.conf")); std::fs::write(&fragment, rows).unwrap(); includes += &format!(" include {};\n", fragment.display()); } crate::git_xrpc::emit(&generation, &[]).unwrap(); let maps = generation .join("nginx/maps") .join(crate::git_xrpc::MAPS_FILE); let mut text = std::fs::read_to_string(&maps).unwrap(); text += &format!("map $arg_grown $ks_grown {{\n default \"\";\n{includes}}}\n"); std::fs::write(&maps, text).unwrap(); crate::atproto::emit(&generation, &[]).unwrap(); crate::blobs::emit(temp.path(), &generation, &[], &Default::default()).unwrap(); super::emit_at(&config, &manifest, &generation, &generation).unwrap();
let emitted = std::fs::read_to_string(generation.join("nginx/knot-static.conf")).unwrap(); assert!( emitted.contains("map_hash_max_size 32768;"), "24000 entries need 32768" ); let checked = generation.join("nginx/checked.conf"); let log = generation.join("nginx/error.log"); std::fs::write( &checked, format!( "pid {};\nerror_log {};\n", generation.join("nginx.pid").display(), log.display() ) + &emitted, ) .unwrap(); let check = Command::new(&nginx_binary) .arg("-e") .arg(&log) .args(["-t", "-c"]) .arg(&checked) .output() .unwrap(); let said = String::from_utf8_lossy(&check.stderr); assert!(check.status.success(), "{said}"); assert!(!said.contains("[warn]"), "{said}");}