[config] add REPO_REFETCH_LIMIT and REPO_REFETCH_WINDOW
244 unmodified linesExpand all245246247248249250251252253254255256257258259260165 unmodified linesExpand all426427428429430431432433434435125 unmodified linesExpand all561562563564565566567568569570571572573331 unmodified linesExpand all9059069079089099109119129139149159169179189199207 unmodified lines244 unmodified linesExpand all /// `None` means unlimited. /// set via `HYDRANT_NEW_HOST_LIMIT`. pub new_host_limit: Option<u64>,
/// per `repo_refetch_window`. an operator's resync doesn't count. `None` is unlimited. /// set via `HYDRANT_REPO_REFETCH_LIMIT`. pub repo_refetch_limit: Option<u32>,
/// how long a refetch allowance lasts, counted from the first refetch in it. /// set via `HYDRANT_REPO_REFETCH_WINDOW` (humantime duration, e.g. `1h`). pub repo_refetch_window: Duration,
/// how often offline firehose sources are automatically retried. /// set via `HYDRANT_OFFLINE_HOST_RETRY_INTERVAL` (humantime duration, e.g. `30min`). /// set to `none` to disable automatic retries. pub offline_host_retry_interval: Option<Duration>,165 unmodified linesExpand all verify_cids: true, allow_private_networks: false, only_index_links: false, new_host_limit: Some(50), repo_refetch_limit: Some(6), repo_refetch_window: Duration::from_secs(60 * 60), offline_host_retry_interval: Some(Duration::from_secs(30 * 60)), tier_rules: vec![], tier_policy: { let mut tiers = HashMap::new();125 unmodified linesExpand all help = "set it to none to never retry offline hosts", "invalid HYDRANT_OFFLINE_HOST_RETRY_INTERVAL `0s`: the retry loop would spin, want a duration like 30min" ); } if self.repo_refetch_window.is_zero() { miette::bail!( "invalid HYDRANT_REPO_REFETCH_WINDOW `0s`: no refetch could be counted, want a duration like 1h" ); } if self.db_worker_threads == 0 { miette::bail!( "invalid HYDRANT_DB_WORKER_THREADS `0`: the database needs at least 1 thread to flush" );331 unmodified linesExpand all } if let Some(limit) = self.new_host_limit { config_line!(f, "max pds/day", limit)?; } match self.repo_refetch_limit { Some(limit) => config_line!( f, "repo refetches", format_args!("{limit}/{}sec", self.repo_refetch_window.as_secs()) )?, None => config_line!(f, "repo refetches", "unlimited")?, } match self.offline_host_retry_interval { Some(d) => config_line!( f, "offline retry interval",7 unmodified lines420 unmodified linesExpand all421422423424425426427428429430431432433434143 unmodified linesExpand all578579580581582583584585586587265 unmodified linesExpand all85385485585685785885986086186286315 unmodified lines8798808818828838848858868878883 unmodified lines89289389489589689789889990090190290390419 unmodified lines92492592692792892993093193293393493524 unmodified lines420 unmodified linesExpand all ); let only_index_links: bool = cfg!(lookup, "ONLY_INDEX_LINKS", defaults.only_index_links); let max_pds_added_per_day = read_or_none(&lookup, "HYDRANT_NEW_HOST_LIMIT", defaults.new_host_limit)?; let repo_refetch_limit = read_or_none( &lookup, "HYDRANT_REPO_REFETCH_LIMIT", defaults.repo_refetch_limit, )?; let repo_refetch_window = cfg!(lookup, "REPO_REFETCH_WINDOW", defaults.repo_refetch_window); let seed_poll_interval = read_or_none( &lookup, "HYDRANT_SEED_POLL_INTERVAL", defaults.seed_poll_interval,143 unmodified linesExpand all verify_cids, allow_private_networks, only_index_links, new_host_limit: max_pds_added_per_day, repo_refetch_limit, repo_refetch_window, offline_host_retry_interval: offline_retry_interval, tier_policy, tier_rules, cache_size,265 unmodified linesExpand all ("HYDRANT_OFFLINE_HOST_RETRY_INTERVAL", "None"), ("HYDRANT_OFFLINE_HOST_RETRY_INTERVAL", "30min"), ("HYDRANT_NEW_HOST_LIMIT", "NoNe"), ("HYDRANT_NEW_HOST_LIMIT", " 42 "), ("HYDRANT_REPO_REFETCH_LIMIT", "none"), ("HYDRANT_REPO_REFETCH_LIMIT", "2"), ("HYDRANT_REPO_REFETCH_WINDOW", "10s"), ("HYDRANT_DATA_COMPRESSION", "lz4"), ("HYDRANT_JOURNAL_COMPRESSION", "none"), ("HYDRANT_VERIFY_SIGNATURES", "backfill-only"), ("HYDRANT_BACKFILL_STRATEGY", "sparse-filter"),15 unmodified lines ("HYDRANT_ENABLE_FIREHOSE", "0"), ("HYDRANT_ENABLE_CRAWLER", "1"), ("HYDRANT_FIREHOSE_WORKERS", ""), ("HYDRANT_NEW_HOST_LIMIT", " 42 "), ("HYDRANT_REPO_REFETCH_LIMIT", "2"), ("HYDRANT_REPO_REFETCH_WINDOW", "10s"), ("HYDRANT_SEED_POLL_INTERVAL", "NONE"), ("HYDRANT_MAINTENANCE_TIMEOUT", "0"), ]) .unwrap();3 unmodified lines config.firehose_workers, from_pairs(&[]).unwrap().firehose_workers ); assert_eq!(config.new_host_limit, Some(42)); assert_eq!(config.repo_refetch_limit, Some(2)); assert_eq!(config.repo_refetch_window, Duration::from_secs(10)); let defaults = from_pairs(&[]).unwrap(); assert_eq!(defaults.repo_refetch_limit, Some(6)); assert_eq!(defaults.repo_refetch_window, Duration::from_secs(3600)); assert_eq!(config.seed_poll_interval, None); assert_eq!(config.maintenance_timeout, Duration::ZERO); }
19 unmodified lines ("HYDRANT_SEED_POLL_INTERVAL", "never"), ("HYDRANT_OFFLINE_HOST_RETRY_INTERVAL", "0s"), ("HYDRANT_OFFLINE_HOST_RETRY_INTERVAL", "soon"), ("HYDRANT_NEW_HOST_LIMIT", "lots"), ("HYDRANT_REPO_REFETCH_LIMIT", "lots"), ("HYDRANT_REPO_REFETCH_LIMIT", "-1"), ("HYDRANT_REPO_REFETCH_WINDOW", "0s"), ("HYDRANT_REPO_REFETCH_WINDOW", "never"), ("HYDRANT_DATA_COMPRESSION", "gzip"), ("HYDRANT_JOURNAL_COMPRESSION", "LZ4"), ("HYDRANT_VERIFY_SIGNATURES", "yes"), ("HYDRANT_BACKFILL_STRATEGY", "fast"),24 unmodified lines