an activation waits for nginx's reload to swap its workers before publishing the generation
289 unmodified linesExpand all29029129229329429529629729815 unmodified lines314315316317318319320321322226 unmodified linesExpand all289 unmodified linesExpand all crate::provision::chown_tree_to_dir_owner(&active_conf, &config.scan_path);
let mut reload_seconds: Option<f64> = None; if let Some(reload_cmd) = &config.activate.reload_command { let workers_before = crate::gc::active_workers(&config.output_root); let t_reload = Instant::now(); let reload_res = std::process::Command::new("sh") .arg("-c") .arg(reload_cmd)15 unmodified lines anyhow::bail!("failed to execute reload command '{reload_cmd}': {err}"); } _ => {} } crate::gc::wait_for_reload(&config.output_root, &workers_before); reload_seconds = Some(t_reload.elapsed().as_secs_f64()); }
Datastore::at(&config.output_root)226 unmodified linesExpand all63 unmodified lines646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134547 unmodified linesExpand all63 unmodified lines let _ = std::fs::remove_file(&path); false}
const WORKER: &[u8] = b"nginx: worker process";const SHUTTING_DOWN: &[u8] = b"nginx: worker process is shutting down";const RELOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);const RELOAD_POLL: std::time::Duration = std::time::Duration::from_millis(10);
/// workers of the nginx master in the pid file that still accept connections./// on SIGHUP the master starts new ones and sets the old ones to `is shutting down`.pub(crate) fn active_workers(output_root: &Path) -> BTreeSet<u32> { active_workers_in(Path::new("/proc"), output_root)}
pub(crate) fn wait_for_reload(output_root: &Path, before: &BTreeSet<u32>) { if !wait_for_reload_in(Path::new("/proc"), output_root, before, RELOAD_TIMEOUT) { tracing::warn!("nginx reload did not swap workers within {RELOAD_TIMEOUT:?}"); }}
pub(crate) fn wait_for_reload_in( proc: &Path, output_root: &Path, before: &BTreeSet<u32>, timeout: std::time::Duration,) -> bool { if before.is_empty() { return true; } let deadline = std::time::Instant::now() + timeout; loop { let now = active_workers_in(proc, output_root); if !now.is_empty() && now.is_disjoint(before) { return true; } if std::time::Instant::now() >= deadline { return false; } std::thread::sleep(RELOAD_POLL); }}
pub(crate) fn active_workers_in(proc: &Path, output_root: &Path) -> BTreeSet<u32> { let Some(master) = std::fs::read_to_string(output_root.join("runtime/nginx.pid")) .ok() .and_then(|text| text.trim().parse::<u32>().ok()) else { return BTreeSet::new(); }; let Ok(entries) = std::fs::read_dir(proc) else { return BTreeSet::new(); }; entries .flatten() .filter_map(|entry| { let pid = entry.file_name().to_str()?.parse::<u32>().ok()?; let stat = std::fs::read_to_string(entry.path().join("stat")).ok()?; let close = stat.rfind(')')?; let parent: u32 = stat.get(close + 2..)?.split(' ').nth(1)?.parse().ok()?; (parent == master).then_some(())?; let cmd = std::fs::read(entry.path().join("cmdline")).ok()?; (cmd.starts_with(WORKER) && !cmd.starts_with(SHUTTING_DOWN)).then_some(pid) }) .collect()}
/// when the oldest process that may read a retired generation started, since one started after the/// retirement can't read it. readers by role: workers of the nginx master in the pid file, cgi/// requests fcgiwrap runs, `git-ssh` sessions, and what those start. long-lived services such as/// the firehose read no generation and don't count. unknown off linux or without nginx's pid file.547 unmodified linesExpand all544 unmodified linesExpand all545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613544 unmodified linesExpand all Some(1020), "an old worker" );}
#[test]fn wait_for_reload_waits_until_new_workers_replace_every_old_one() { use std::time::{Duration, Instant};
fn write_proc(proc: &Path, pid: u32, parent: u32, cmdline: &str) { let dir = proc.join(pid.to_string()); std::fs::create_dir_all(&dir).unwrap(); let zeros = "0 ".repeat(17); std::fs::write( dir.join("stat"), format!("{pid} (nginx) S {parent} {zeros}100 0 0\n"), ) .unwrap(); std::fs::write(dir.join("cmdline"), format!("{cmdline}\0")).unwrap(); }
let root = tempfile::tempdir().unwrap(); let (proc, output) = (root.path().join("proc"), root.path().join("out")); std::fs::create_dir_all(output.join("runtime")).unwrap(); std::fs::create_dir_all(&proc).unwrap(); assert!(active_workers_in(&proc, &output).is_empty()); assert!(wait_for_reload_in( &proc, &output, &BTreeSet::new(), Duration::from_millis(1) ));
std::fs::write(output.join("runtime/nginx.pid"), "10\n").unwrap(); write_proc(&proc, 10, 1, "nginx: master process"); write_proc(&proc, 11, 10, "nginx: worker process"); write_proc(&proc, 12, 10, "nginx: worker process"); write_proc(&proc, 19, 10, "nginx: worker process is shutting down"); let before = active_workers_in(&proc, &output); assert_eq!(before, BTreeSet::from([11, 12])); assert!( !wait_for_reload_in(&proc, &output, &before, Duration::from_millis(25)), "old workers still active" );
let proc_bg = proc.clone(); let swap = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(30)); // new workers fork before old ones process SIGQUIT write_proc(&proc_bg, 13, 10, "nginx: worker process"); std::thread::sleep(Duration::from_millis(40)); std::fs::remove_dir_all(proc_bg.join("11")).unwrap(); write_proc(&proc_bg, 12, 10, "nginx: worker process is shutting down"); }); let t0 = Instant::now(); assert!(wait_for_reload_in( &proc, &output, &before, Duration::from_secs(2) )); let elapsed = t0.elapsed(); swap.join().unwrap(); assert!( elapsed >= Duration::from_millis(60), "returned while an old worker was still active: {elapsed:?}" ); assert_eq!(active_workers_in(&proc, &output), BTreeSet::from([13]));}