use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};
use syn::visit::{self, Visit};
use syn::{
Attribute, Expr, ExprCall, ExprMethodCall, ExprPath, GenericArgument, ImplItemFn, ItemConst,
ItemEnum, ItemFn, ItemImpl, ItemMod, ItemStatic, ItemStruct, ItemUnion, PathArguments, Type,
Visibility,
};
const INVENTORY_PATH: &str = "tests/representation_inventory.tsv";
#[derive(Default)]
struct Surface {
boundaries: BTreeSet<String>,
tests: BTreeSet<String>,
}
struct Scanner<'a> {
relative_path: &'a str,
modules: Vec<String>,
impl_name: Option<String>,
function: Option<String>,
test_only: bool,
surface: &'a mut Surface,
}
impl Scanner<'_> {
fn location(&self, symbol: &str) -> String {
let mut parts = vec![self.relative_path.to_owned()];
parts.extend(self.modules.iter().cloned());
if let Some(name) = &self.impl_name {
parts.push(name.clone());
}
parts.push(symbol.to_owned());
parts.join("::")
}
fn owner(&self) -> String {
self.function
.as_deref()
.map(|function| self.location(function))
.unwrap_or_else(|| self.location("<module>"))
}
fn record(&mut self, kind: &str, symbol: &str) {
self.surface
.boundaries
.insert(format!("{kind}:{}", self.location(symbol)));
}
fn record_call(&mut self, kind: &str, callee: &str) {
self.surface
.boundaries
.insert(format!("{kind}:{}::{callee}", self.owner()));
}
fn enter_function(&mut self, name: String, block: &syn::Block) {
let previous = self.function.replace(name);
self.visit_block(block);
self.function = previous;
}
fn record_serde_attributes(&mut self, owner: &str, attrs: &[Attribute]) {
for attr in attrs.iter().filter(|attr| attr.path().is_ident("serde")) {
let value = match &attr.meta {
syn::Meta::List(list) => list.tokens.to_string(),
syn::Meta::NameValue(_) => "<name-value>".to_owned(),
syn::Meta::Path(_) => String::new(),
};
let value = value.split_whitespace().collect::<String>();
self.record("serde-attribute", &format!("{owner}::serde({value})"));
}
}
fn record_function_categories(&mut self, name: &str, visibility: &Visibility) {
let is_key_codec_module = self.relative_path.starts_with("src/db/keys/")
|| self.relative_path == "src/db/keys.rs";
let is_embedded_key_codec = matches!(
self.relative_path,
"src/backlinks/store.rs" | "src/db/filter.rs" | "src/db/pds_meta.rs"
) && looks_like_key_codec(name)
|| self.relative_path.starts_with("src/db/migration/") && looks_like_key_codec(name);
if is_key_codec_module || is_embedded_key_codec {
self.record("db-key-codec", name);
}
if looks_like_serde_helper(name) {
self.record("serde-helper", name);
}
if self.relative_path.starts_with("src/db/migration/") {
self.record("migration-transform", name);
}
if self.relative_path == "src/config/env.rs"
|| ((self.relative_path.starts_with("src/config/")
|| self.relative_path == "src/config.rs")
&& (name == "parse" || name.starts_with("parse_")))
{
self.record("config-parser", name);
}
if self.relative_path == "src/ingest/stream/codec.rs" {
self.record("firehose-codec", name);
}
if self.relative_path == "src/car.rs" {
self.record("car-codec", name);
}
if (self.relative_path.starts_with("src/api/") || self.relative_path == "src/api.rs")
&& !matches!(visibility, Visibility::Inherited)
{
self.record("api-boundary", name);
}
if self.relative_path.starts_with("src/control/stream/")
&& !matches!(visibility, Visibility::Inherited)
{
self.record("stream-projection", name);
}
}
}
impl<'ast> Visit<'ast> for Scanner<'_> {
fn visit_item_mod(&mut self, node: &'ast ItemMod) {
let previous_test_only = self.test_only;
self.test_only |= is_test_only(&node.attrs);
self.modules.push(node.ident.to_string());
if let Some((_, items)) = &node.content {
for item in items {
self.visit_item(item);
}
}
self.modules.pop();
self.test_only = previous_test_only;
}
fn visit_item_struct(&mut self, node: &'ast ItemStruct) {
if !self.test_only {
let name = node.ident.to_string();
if has_serde_derive(&node.attrs) {
self.record("serde-type", &name);
}
self.record_serde_attributes(&name, &node.attrs);
for (index, field) in node.fields.iter().enumerate() {
let field_name = field
.ident
.as_ref()
.map(ToString::to_string)
.unwrap_or_else(|| index.to_string());
self.record_serde_attributes(&format!("{name}::{field_name}"), &field.attrs);
}
}
visit::visit_item_struct(self, node);
}
fn visit_item_enum(&mut self, node: &'ast ItemEnum) {
if !self.test_only {
let name = node.ident.to_string();
if has_serde_derive(&node.attrs) {
self.record("serde-type", &name);
}
self.record_serde_attributes(&name, &node.attrs);
for variant in &node.variants {
let variant_name = format!("{name}::{}", variant.ident);
self.record_serde_attributes(&variant_name, &variant.attrs);
for (index, field) in variant.fields.iter().enumerate() {
let field_name = field
.ident
.as_ref()
.map(ToString::to_string)
.unwrap_or_else(|| index.to_string());
self.record_serde_attributes(
&format!("{variant_name}::{field_name}"),
&field.attrs,
);
}
}
}
visit::visit_item_enum(self, node);
}
fn visit_item_union(&mut self, node: &'ast ItemUnion) {
if !self.test_only {
let name = node.ident.to_string();
if has_serde_derive(&node.attrs) {
self.record("serde-type", &name);
}
self.record_serde_attributes(&name, &node.attrs);
for field in &node.fields.named {
self.record_serde_attributes(
&format!("{name}::{}", field.ident.as_ref().unwrap()),
&field.attrs,
);
}
}
visit::visit_item_union(self, node);
}
fn visit_item_const(&mut self, node: &'ast ItemConst) {
if !self.test_only && records_storage_constant(self.relative_path, &node.ident.to_string())
{
self.record("db-key-constant", &node.ident.to_string());
}
visit::visit_item_const(self, node);
}
fn visit_item_static(&mut self, node: &'ast ItemStatic) {
if !self.test_only && records_storage_constant(self.relative_path, &node.ident.to_string())
{
self.record("db-key-constant", &node.ident.to_string());
}
visit::visit_item_static(self, node);
}
fn visit_item_impl(&mut self, node: &'ast ItemImpl) {
if self.test_only || is_test_only(&node.attrs) {
return;
}
let self_name = type_name(&node.self_ty);
let conversion_owner = type_signature(&node.self_ty);
if let Some((_, path, _)) = &node.trait_
&& let Some(trait_name) = path
.segments
.last()
.map(|segment| segment.ident.to_string())
{
match trait_name.as_str() {
"Serialize" | "Deserialize" => {
self.record("manual-serde", &format!("{self_name}::{trait_name}"))
}
"FromStr" => self.record("text-parser", &format!("{self_name}::{trait_name}")),
"From" | "TryFrom" | "IntoStatic" | "Display" | "AsRef"
if records_representation_conversions(self.relative_path) =>
{
self.record(
"representation-conversion",
&format!("{conversion_owner}::{}", path_signature(path)),
)
}
"Schema" if self.relative_path == "src/db/schema.rs" => {
self.record("db-keyspace", &self_name)
}
_ => {}
}
}
let previous = self.impl_name.replace(self_name);
for item in &node.items {
self.visit_impl_item(item);
}
self.impl_name = previous;
}
fn visit_item_fn(&mut self, node: &'ast ItemFn) {
let name = node.sig.ident.to_string();
if self.test_only || is_test_only(&node.attrs) {
if is_test(&node.attrs) {
self.surface
.tests
.insert(format!("rust:{}::{name}", self.relative_path));
}
return;
}
self.record_function_categories(&name, &node.vis);
self.enter_function(name, &node.block);
}
fn visit_impl_item_fn(&mut self, node: &'ast ImplItemFn) {
let name = node.sig.ident.to_string();
if self.test_only || is_test_only(&node.attrs) {
if is_test(&node.attrs) {
self.surface
.tests
.insert(format!("rust:{}::{name}", self.relative_path));
}
return;
}
self.record_function_categories(&name, &node.vis);
self.enter_function(name, &node.block);
}
fn visit_expr_call(&mut self, node: &'ast ExprCall) {
if !self.test_only
&& let Expr::Path(path) = node.func.as_ref()
{
let callee = path_text(&path.path);
if is_codec_call(&callee) {
self.record_call("codec-call", &callee);
}
}
visit::visit_expr_call(self, node);
}
// a codec fn passed as a value, like `.map(u64::from_be_bytes)`, crosses the
// same boundary as calling it. the lowercase check keeps `Value::Null` out
fn visit_expr_path(&mut self, node: &'ast ExprPath) {
let is_function = node.path.segments.last().is_some_and(|segment| {
segment
.ident
.to_string()
.starts_with(|ch: char| ch.is_ascii_lowercase())
});
if !self.test_only && is_function {
let callee = path_text(&node.path);
if is_codec_call(&callee) {
self.record_call("codec-call", &callee);
}
}
visit::visit_expr_path(self, node);
}
fn visit_expr_method_call(&mut self, node: &'ast ExprMethodCall) {
if !self.test_only {
let method = node.method.to_string();
if matches!(
method.as_str(),
"serialize" | "deserialize" | "to_be_bytes" | "to_le_bytes"
) || (self.relative_path == "src/config/env.rs" && method == "parse")
{
self.record_call("codec-call", &format!(".{method}"));
}
}
visit::visit_expr_method_call(self, node);
}
}
fn records_representation_conversions(relative_path: &str) -> bool {
matches!(
relative_path,
"src/config.rs"
| "src/config/types.rs"
| "src/control/pds.rs"
| "src/db/types.rs"
| "src/ingest/stream/types.rs"
| "src/pds_meta.rs"
| "src/types.rs"
) || relative_path.starts_with("src/types/event/")
}
fn records_storage_constant(relative_path: &str, name: &str) -> bool {
let storage_source = relative_path.starts_with("src/db/") || relative_path == "src/db.rs";
let embedded_key_source = matches!(
relative_path,
"src/backlinks/store.rs" | "src/filter.rs" | "src/pds_meta.rs"
);
(storage_source || embedded_key_source)
&& (name == "SEP"
|| name.starts_with("TAG_")
|| name.contains("_KEY")
|| name.contains("_PREFIX")
|| name.ends_with("_SEP"))
}
fn looks_like_key_codec(name: &str) -> bool {
name.ends_with("_key")
|| name.contains("_key_")
|| name.ends_with("_prefix")
|| name.contains("_prefix_")
}
fn looks_like_serde_helper(name: &str) -> bool {
name != "serialize"
&& name != "deserialize"
&& (name.starts_with("serialize_")
|| name.starts_with("deserialize_")
|| name.starts_with("ser_")
|| name.starts_with("deser_")
|| name.contains("_serialize_")
|| name.contains("_deserialize_")
|| name.contains("_ser_")
|| name.contains("_deser_")
|| name.ends_with("_ser")
|| name.ends_with("_deser"))
}
fn is_codec_call(callee: &str) -> bool {
const ROOTS: &[&str] = &[
"rmp_serde::",
"serde_json::",
"serde_ipld_dagcbor::",
"serde_html_form::",
"iroh_car::",
"data_encoding::",
"multibase::",
"hex::",
];
ROOTS.iter().any(|root| callee.contains(root))
|| callee.contains("CarReader")
|| callee.contains("CarWriter")
|| callee.ends_with("::serialize")
|| callee.ends_with("::deserialize")
|| callee.ends_with("::from_be_bytes")
|| callee.ends_with("::from_le_bytes")
}
fn path_text(path: &syn::Path) -> String {
path.segments
.iter()
.map(|segment| segment.ident.to_string())
.collect::<Vec<_>>()
.join("::")
}
fn path_signature(path: &syn::Path) -> String {
path.segments
.iter()
.map(|segment| {
let arguments = match &segment.arguments {
PathArguments::None => String::new(),
PathArguments::AngleBracketed(arguments) => {
let types = arguments
.args
.iter()
.filter_map(|argument| match argument {
GenericArgument::Type(ty) => Some(type_signature(ty)),
GenericArgument::AssocType(assoc) => {
Some(format!("{}={}", assoc.ident, type_signature(&assoc.ty)))
}
_ => None,
})
.collect::<Vec<_>>();
format!("<{}>", types.join(","))
}
PathArguments::Parenthesized(_) => "(..)".to_owned(),
};
format!("{}{arguments}", segment.ident)
})
.collect::<Vec<_>>()
.join("::")
}
fn type_signature(ty: &Type) -> String {
match ty {
Type::Path(path) => path_signature(&path.path),
Type::Reference(reference) => format!("&{}", type_signature(&reference.elem)),
Type::Slice(slice) => format!("[{}]", type_signature(&slice.elem)),
Type::Array(array) => format!("[{};_]", type_signature(&array.elem)),
Type::Tuple(tuple) => format!(
"({})",
tuple
.elems
.iter()
.map(type_signature)
.collect::<Vec<_>>()
.join(",")
),
_ => "<type>".to_owned(),
}
}
fn type_name(ty: &Type) -> String {
match ty {
Type::Path(path) => path
.path
.segments
.last()
.map(|segment| segment.ident.to_string())
.unwrap_or_else(|| "<type>".to_owned()),
Type::Reference(reference) => type_name(&reference.elem),
_ => "<type>".to_owned(),
}
}
fn has_serde_derive(attrs: &[Attribute]) -> bool {
attrs.iter().any(|attr| {
attr.path().is_ident("derive")
&& match &attr.meta {
syn::Meta::List(list) => {
let derives = list.tokens.to_string();
derives
.split(|ch: char| !ch.is_alphanumeric() && ch != '_')
.any(|derive| matches!(derive, "Serialize" | "Deserialize"))
}
_ => false,
}
})
}
fn is_test(attrs: &[Attribute]) -> bool {
attrs.iter().any(|attr| {
attr.path()
.segments
.last()
.is_some_and(|segment| segment.ident == "test")
})
}
fn is_test_only(attrs: &[Attribute]) -> bool {
attrs.iter().any(|attr| {
is_test(std::slice::from_ref(attr))
|| (attr.path().is_ident("cfg")
&& matches!(&attr.meta, syn::Meta::List(list) if list.tokens.to_string().split(|ch: char| !ch.is_alphanumeric() && ch != '_').any(|word| word == "test")))
})
}
fn rust_files(root: &Path) -> Vec<PathBuf> {
fn collect(path: &Path, files: &mut Vec<PathBuf>) {
let mut entries = fs::read_dir(path)
.unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display()))
.collect::<Result<Vec<_>, _>>()
.unwrap_or_else(|error| panic!("failed to enumerate {}: {error}", path.display()));
entries.sort_by_key(|entry| entry.path());
for entry in entries {
let path = entry.path();
if path.is_dir() {
collect(&path, files);
} else if path.extension().is_some_and(|extension| extension == "rs") {
files.push(path);
}
}
}
let mut files = Vec::new();
collect(root, &mut files);
files
}
/// every `mod name;` in one file, as the file it loads and whether it's test-only
struct ModuleFiles<'a> {
known: &'a BTreeSet<String>,
dir: Vec<String>,
test_only: bool,
found: Vec<(String, bool)>,
}
impl<'ast> Visit<'ast> for ModuleFiles<'_> {
fn visit_item_mod(&mut self, node: &'ast ItemMod) {
let previous_test_only = self.test_only;
self.test_only |= is_test_only(&node.attrs);
self.dir.push(node.ident.to_string());
match &node.content {
Some((_, items)) => {
for item in items {
self.visit_item(item);
}
}
None => {
let base = self.dir.join("/");
let file = [format!("{base}.rs"), format!("{base}/mod.rs")]
.into_iter()
.find(|file| self.known.contains(file));
if let Some(file) = file {
self.found.push((file, self.test_only));
}
}
}
self.dir.pop();
self.test_only = previous_test_only;
}
}
/// files that only compile under `cfg(test)` because their `mod` declaration
/// says so, which the per-file scan can't see from inside the file
fn test_only_files(sources: &BTreeMap<String, syn::File>) -> BTreeSet<String> {
let known = sources.keys().cloned().collect::<BTreeSet<_>>();
let mut declared = BTreeMap::new();
for (relative, syntax) in sources {
let path = Path::new(relative);
let stem = path.file_stem().unwrap().to_string_lossy();
let parent = path.parent().unwrap().to_string_lossy().into_owned();
let dir = if matches!(stem.as_ref(), "mod" | "lib" | "main") {
vec![parent]
} else {
vec![parent, stem.into_owned()]
};
let mut modules = ModuleFiles {
known: &known,
dir,
test_only: false,
found: Vec::new(),
};
modules.visit_file(syntax);
declared.insert(relative.clone(), modules.found);
}
let mut test_only = BTreeSet::new();
let mut pending = declared
.values()
.flatten()
.filter(|(_, test_only)| *test_only)
.map(|(file, _)| file.clone())
.collect::<Vec<_>>();
while let Some(file) = pending.pop() {
if test_only.insert(file.clone()) {
pending.extend(declared[&file].iter().map(|(child, _)| child.clone()));
}
}
test_only
}
fn discover(manifest: &Path) -> Surface {
let mut sources = BTreeMap::new();
for path in rust_files(&manifest.join("src")) {
let relative = path
.strip_prefix(manifest)
.expect("source path must be below manifest")
.to_string_lossy()
.replace('\\', "/");
let source = fs::read_to_string(&path)
.unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display()));
let syntax = syn::parse_file(&source)
.unwrap_or_else(|error| panic!("failed to parse {}: {error}", path.display()));
sources.insert(relative, syntax);
}
let test_only = test_only_files(&sources);
let mut surface = Surface::default();
for (relative, syntax) in &sources {
let mut scanner = Scanner {
relative_path: relative,
modules: Vec::new(),
impl_name: None,
function: None,
test_only: test_only.contains(relative),
surface: &mut surface,
};
scanner.visit_file(syntax);
}
surface
}
fn load_inventory(manifest: &Path) -> BTreeMap<String, String> {
let path = manifest.join(INVENTORY_PATH);
let input = fs::read_to_string(&path)
.unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display()));
let mut inventory = BTreeMap::new();
for (index, line) in input.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let (boundary, coverage) = line
.split_once('\t')
.unwrap_or_else(|| panic!("{}:{} must contain one tab", path.display(), index + 1));
assert!(
inventory
.insert(boundary.to_owned(), coverage.to_owned())
.is_none(),
"{}:{} duplicates boundary {boundary}",
path.display(),
index + 1
);
}
inventory
}
fn proposed_coverage(boundary: &str, surface: &Surface) -> String {
let location = boundary
.split_once(':')
.map(|(_, location)| location)
.expect("boundary has a kind");
let source = location
.split_once("::")
.map(|(source, _)| source)
.expect("boundary has a source");
let exact = if source == "src/api/mod.rs" && boundary.contains("MultiQuery") {
Some("rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values")
} else if source == "src/api/debug.rs"
&& (boundary.contains("seed_xrpc") || boundary.contains("DebugSeedXrpc"))
{
Some(if boundary.contains("rmp_serde::to_vec") {
"nu:tests/stream_live_backfill.nu"
} else {
"nu:tests/api_xrpc.nu"
})
} else if source == "src/api/xrpc/com_atproto_describe_repo.rs" {
Some(
"rust:src/api/xrpc/com_atproto_describe_repo.rs::describe_repo_json_projection_contract",
)
} else if source == "src/api/xrpc/request_crawl.rs" {
Some("rust:src/api/xrpc/request_crawl.rs::banned_request_crawl_json_contract")
} else if source == "src/api/xrpc/subscribe_repos.rs" {
Some("nu:tests/relay_subscribe_repos_ping.nu")
} else if source.starts_with("src/api/xrpc/") {
Some("nu:tests/api_xrpc.nu")
} else if source == "src/api/db.rs" {
Some("nu:tests/api_db.nu")
} else if source == "src/api/mod.rs" {
Some("nu:tests/debug_endpoints.nu")
} else if source == "src/api/stats.rs" {
Some("nu:tests/debug_endpoints.nu")
} else if source == "src/api/ws.rs" {
Some("nu:tests/stream_ping.nu")
} else if source == "src/config/env.rs" {
Some("rust:src/config/env.rs::env_text_parser_contracts")
} else if source == "src/control/indexer.rs" {
Some("nu:tests/debug_endpoints.nu")
} else if source == "src/jetstream.rs" {
Some("nu:tests/stream_relay_jetstream_subscribe.nu")
} else if source == "src/resolver.rs" {
Some("rust:src/resolver.rs::raw_did_doc_json_contract_and_size_limit")
} else if source == "src/db/types.rs" {
Some(
if boundary.starts_with("representation-conversion:")
|| boundary.starts_with("db-key-constant:")
{
"rust:src/db/types.rs::db_representation_conversions_are_lossless"
} else if boundary.contains("TrimmedDid") {
"rust:src/control/stream/indexer.rs::legacy_string_trimmed_did_replays_after_reopen"
} else if boundary.contains("DbRkey") {
"rust:src/db/types.rs::dbrkey_string_markers_preserve_every_utf8_length"
} else if boundary.contains("DbTid") {
"rust:src/db/types.rs::dbrkey_msgpack_binary_marker_preserves_tid"
} else {
"rust:src/db/types.rs::db_action_and_did_key_representation_contracts"
},
)
} else if source == "src/patch.rs" {
Some("rust:src/patch.rs::set_update_json_markers_are_structurally_disjoint")
} else if source == "src/config/types.rs" {
Some(if boundary.contains("CrawlerMode") {
"rust:src/config/types.rs::crawler_mode_string_and_msgpack_contracts"
} else if boundary.contains("RateTier")
|| boundary.contains("CrawlerSource")
|| boundary.contains("FirehoseSource")
{
"rust:src/config/types.rs::source_and_rate_tier_parser_contracts"
} else {
"rust:src/config/types.rs::text_config_parser_contracts"
})
} else if source == "src/config.rs" && boundary.starts_with("representation-conversion:") {
Some("rust:src/config.rs::display_reports_history_ttl")
} else if source == "src/filter.rs" {
Some("rust:src/filter.rs::filter_mode_persisted_golden_fixtures")
} else if source == "src/pds_meta.rs" {
Some("rust:src/pds_meta.rs::host_status_persisted_golden_fixtures")
} else if source == "src/crawler/list_repos/checker.rs" {
Some("rust:src/crawler/list_repos/checker.rs::retry_state_persisted_golden_fixture")
} else if source == "src/crawler/relay_first.rs" {
Some("rust:src/crawler/relay_first.rs::pass_cursor_round_trips")
} else if source == "src/db/keys/indexer.rs" {
Some("rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts")
} else if source == "src/db/keys/mod.rs" || source == "src/db/keys/v1.rs" {
Some("rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts")
} else if source == "src/db/storage_mode.rs" {
Some("rust:src/db/storage_mode.rs::storage_mode_marker_persisted_golden_fixture")
} else if source == "src/db/schema.rs" && boundary.starts_with("db-keyspace:") {
Some("rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema")
} else if source == "src/db/mod.rs" && boundary.contains("FirehoseSourceMeta") {
Some("rust:src/db/mod.rs::firehose_source_meta_golden_fixture")
} else if source == "src/db/mod.rs" {
Some("rust:src/types.rs::current_persisted_state_golden_fixtures")
} else if source == "src/db/migration/mod.rs" && boundary.starts_with("db-key-") {
Some(
if boundary.contains("LEGACY_BLOCKS_KEYSPACE")
|| boundary.contains("LEGACY_RECORDS_KEYSPACE")
{
"rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage"
} else if boundary.contains("pass_cursor_key")
|| boundary.contains("MIGRATION_CURSOR_PREFIX")
{
"rust:src/db/migration/mod.rs::cursor_keys_are_distinct_per_version_and_pass"
} else {
"rust:src/db/migration/mod.rs::applied_migration_keys_track_per_migration_status"
},
)
} else if source == "src/db/migration/v10.rs" {
Some(if boundary.contains("rewrite_filter_key") {
"rust:src/db/migration/v10.rs::v10_finishes_v9_filter_keys_skipped_by_a_reduced_feature_build"
} else {
"rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage"
})
} else if matches!(
source,
"src/db/migration/v1.rs"
| "src/db/migration/v3.rs"
| "src/db/migration/v5.rs"
| "src/db/migration/v6.rs"
) {
Some(
"rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries",
)
} else if source == "src/db/migration/v2.rs" {
Some(if boundary.starts_with("serde-") {
"rust:src/db/migration/v2.rs::v1_repo_state_input_golden_fixture"
} else {
"rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries"
})
} else if source == "src/db/migration/v4.rs" {
Some(if boundary.starts_with("serde-") {
"rust:src/db/migration/v4.rs::v2_repo_state_input_golden_fixture"
} else {
"rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries"
})
} else if source == "src/types/v2.rs" {
Some("rust:src/types/v2.rs::v2_persisted_schema_golden_fixture")
} else if source == "src/types/v4.rs" {
Some("rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures")
} else if source == "src/types/v7.rs" {
Some("rust:src/types/v7.rs::v7_persisted_schema_golden_fixture")
} else if source == "src/types.rs" {
Some(if boundary.contains("DeleteBod") {
"rust:src/types.rs::operator_redaction_json_contracts"
} else if boundary.contains("RepoState") && boundary.contains("IntoStatic") {
"rust:src/types.rs::into_static_preserves_per_event_clocks"
} else if boundary.starts_with("representation-conversion:") {
"rust:src/types.rs::commit_and_repo_status_conversion_contracts"
} else {
"rust:src/types.rs::current_persisted_state_golden_fixtures"
})
} else if source == "src/ingest/indexer/shard.rs" {
Some(
"rust:src/ingest/indexer/shard.rs::resync_buffer_commit_msgpack_round_trip_preserves_path",
)
} else if source == "src/ingest/relay/sink/relay.rs" {
Some(
"rust:src/ingest/relay/sink/relay.rs::relay_ephemeral_projection_preserves_record_representation",
)
} else if source == "src/ingest/validation.rs" {
Some(
"rust:src/ingest/validation.rs::record_dag_cbor_projection_accepts_data_and_rejects_invalid_bytes",
)
} else if source == "src/ops/backlink_ops.rs" {
Some("nu:tests/backlinks.nu")
} else if source == "src/ops/record_events.rs" {
Some("nu:tests/stream_cursor_replay.nu")
} else if source == "src/ingest/stream/types.rs" {
Some(if boundary.contains("::Datetime") {
"rust:src/ingest/stream/types.rs::firehose_datetime_accepts_zoned_and_legacy_unzoned_values"
} else if boundary.contains("RepoOpAction")
|| boundary.contains("AccountStatus")
|| boundary.contains("InfoName")
{
"rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts"
} else if boundary.contains("::Sync") {
"rust:src/ingest/stream/codec.rs::firehose_frame_mock_decodes_sync_binary_blocks"
} else if boundary.contains("::Commit") || boundary.contains("::RepoOp") {
"rust:src/ingest/stream.rs::test_decode_commit_empty_since"
} else if boundary.contains("::Account") {
"rust:src/ingest/stream.rs::test_decode_account"
} else if boundary.contains("::Identity") {
"rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity"
} else if boundary.contains("::Info") {
"rust:src/ingest/stream/codec.rs::firehose_frame_maps_outdated_cursor_info"
} else {
"rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts"
})
} else if source == "src/ingest/stream/codec.rs" {
Some("rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity")
} else if source == "src/ingest/stream.rs" {
Some("rust:src/ingest/stream.rs::test_decode_commit_empty_since")
} else if source == "src/control/repos/indexer.rs" {
Some(if boundary.contains("generate_car") {
"rust:src/control/repos/indexer.rs::generate_car_reports_a_deleted_head_body"
} else if boundary.contains("get_record") || boundary.contains("list_records") {
"rust:src/control/repos/indexer.rs::get_and_list_hide_redacted_heads"
} else {
"rust:src/control/repos/indexer.rs::erase_repo_forgets_everything_stored_for_the_repo"
})
} else if source == "src/api/jetstream.rs" {
Some(
if boundary.contains("Subscriber") || boundary.contains("handle_options_message") {
"rust:src/api/jetstream.rs::jetstream_options_message_json_contract"
} else {
"rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values"
},
)
} else if source == "src/api/ingestion.rs" {
Some("rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode")
} else if source == "src/types/event/jetstream.rs" {
Some(if boundary.contains("::Relay") {
"rust:src/types/event/jetstream.rs::relay_jetstream_event_golden_fixtures"
} else if boundary.contains("::Commit") {
"rust:src/types/event/jetstream.rs::indexer_jetstream_event_golden_fixture"
} else {
"rust:src/types/event/jetstream.rs::common_jetstream_event_golden_fixtures"
})
} else if source == "src/bin/backfill_strategy_bench.rs" {
Some("rust:src/bin/backfill_strategy_bench.rs::benchmark_api_json_fixtures_decode")
} else if source == "src/backlinks/store.rs" {
Some(
if boundary.contains("forward_key") || boundary.contains("forward_did_prefix") {
"rust:src/backlinks/store.rs::forward_did_prefix_matches_key"
} else if boundary.contains("reverse_scan_prefix") {
"rust:src/backlinks/store.rs::reverse_key_scan_prefix_with_collection_and_path"
} else if boundary.contains("reverse_key")
|| boundary.contains("source_from_reverse_key")
{
"rust:src/backlinks/store.rs::source_from_reverse_key_roundtrip"
} else {
"rust:src/backlinks/store.rs::forward_value_persisted_golden_fixture"
},
)
} else if source == "src/util/mod.rs" {
Some(if boundary.contains("ser_status_code") {
"rust:src/util/mod.rs::http_status_serde_contract"
} else {
"rust:src/util/mod.rs::xrpc_output_string_serializers_are_stable"
})
} else if source == "src/control/stream/indexer.rs" {
Some(
"rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim",
)
} else if source == "src/control/repos/mod.rs" {
Some(if boundary.contains("collections") {
"rust:src/control/repos/mod.rs::collections_decodes_big_endian_counts"
} else {
"rust:src/control/repos/mod.rs::repo_info_json_contract"
})
} else if source == "src/db/indexer.rs" && boundary.contains("load_persisted_crawler_sources") {
Some("rust:src/db/indexer.rs::persisted_crawler_sources_decode_mode_and_url")
} else if source == "src/db/indexer.rs" && boundary.contains("record_count") {
Some("rust:src/db/indexer.rs::record_count_codec_updates_and_rejects_invalid_bytes")
} else if source == "src/ingest/firehose_stats/source.rs" {
Some("rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract")
} else if source == "src/ingest/firehose_stats/relay.rs" {
Some("rust:src/ingest/firehose_stats/relay.rs::relay_worker_stats_snapshot_json_contract")
} else if source == "src/control/pds.rs" && boundary.starts_with("representation-conversion:") {
Some("nu:tests/api_pds_tiers.nu")
} else if source == "src/db/filter.rs" && boundary.starts_with("db-key-") {
Some(if boundary.contains("exclude_key") {
"rust:src/db/filter.rs::test_exclude_key_trimmed"
} else {
"rust:src/db/filter.rs::test_filter_keys"
})
} else if source == "src/db/pds_meta.rs" && boundary.starts_with("db-key-") {
Some(if boundary.contains("tier") {
"nu:tests/api_pds_tiers.nu"
} else {
"nu:tests/pds_host_status_transitions.nu"
})
} else if source == "src/types/event/stream.rs" {
Some(if boundary.contains("StoredData") {
"rust:src/types/event/stream.rs::stored_data_persisted_golden_fixtures"
} else if boundary.contains("StoredEvent") {
"rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim"
} else {
"rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant"
})
} else {
None
};
if let Some(coverage) = exact {
return coverage.to_owned();
}
if let Some(test) = surface
.tests
.iter()
.find(|test| test.starts_with(&format!("rust:{source}::")))
{
return test.clone();
}
let nu = if source.starts_with("src/api/crawler.rs") {
"tests/api_crawler_sources.nu"
} else if source.starts_with("src/api/firehose.rs") {
"tests/api_firehose_sources.nu"
} else if source.starts_with("src/api/pds.rs") && boundary.to_ascii_lowercase().contains("ban")
{
"tests/api_pds_banned.nu"
} else if source.starts_with("src/api/pds.rs") {
"tests/api_pds_tiers.nu"
} else if source.starts_with("src/api/repos.rs") || source.starts_with("src/control/repos/") {
"tests/api_repos.nu"
} else if source.starts_with("src/api/debug.rs") || source.starts_with("src/db/schema.rs") {
"tests/debug_endpoints.nu"
} else if source.starts_with("src/api/filter.rs")
|| source == "src/filter.rs"
|| source == "src/control/filter.rs"
{
"tests/signal_filter.nu"
} else if source.starts_with("src/api/stream.rs")
|| source.starts_with("src/control/stream/")
|| source.starts_with("src/types/event/")
{
"tests/stream_cursor_replay.nu"
} else if source.starts_with("src/api/xrpc/subscribe_repos.rs") {
"tests/relay_subscribe_repos_ping.nu"
} else if source.starts_with("src/api/xrpc/")
|| source.starts_with("src/backfill/")
|| source.starts_with("src/ops")
{
"tests/stream_live_backfill.nu"
} else if source.starts_with("src/backlinks/") {
"tests/backlinks.nu"
} else if source.starts_with("src/crawler/") {
"tests/crawler_full_network.nu"
} else if source.starts_with("src/ingest/") {
"tests/stream_live_backfill.nu"
} else if source.starts_with("src/control/firehose.rs")
|| source.starts_with("src/ingest/firehose_stats/")
{
"tests/api_firehose_sources.nu"
} else if source.starts_with("src/control/crawler.rs") {
"tests/api_crawler_sources.nu"
} else if source.starts_with("src/control/pds.rs") || source == "src/pds_meta.rs" {
"tests/pds_host_status_transitions.nu"
} else if source.starts_with("src/control/stats.rs") {
"tests/debug_endpoints.nu"
} else if source.starts_with("src/db/") {
"tests/debug_endpoints.nu"
} else {
"tests/representation_unassigned"
};
format!("nu:{nu}")
}
fn inventory_errors(manifest: &Path) -> Vec<String> {
let surface = discover(manifest);
let inventory = load_inventory(manifest);
let inventoried = inventory.keys().cloned().collect::<BTreeSet<_>>();
let mut errors = Vec::new();
errors.extend(
surface
.boundaries
.difference(&inventoried)
.map(|boundary| format!("missing: {boundary}\t<coverage>")),
);
errors.extend(
inventoried
.difference(&surface.boundaries)
.map(|boundary| format!("stale: {boundary}")),
);
errors.extend(inventory.into_iter().filter_map(|(boundary, coverage)| {
let valid = match coverage.strip_prefix("rust:") {
Some(_) => surface.tests.contains(&coverage),
None => coverage
.strip_prefix("nu:")
.or_else(|| coverage.strip_prefix("fixture:"))
.is_some_and(|path| manifest.join(path).is_file()),
};
(!valid).then(|| format!("missing coverage target: {boundary}\t{coverage}"))
}));
errors
}
#[test]
fn representation_audit_rejects_uncovered_and_stale_boundaries() {
let tmp = tempfile::tempdir().unwrap();
fs::create_dir_all(tmp.path().join("src")).unwrap();
fs::create_dir_all(tmp.path().join("tests")).unwrap();
fs::write(
tmp.path().join("src/lib.rs"),
r#"
pub fn persist(value: &u64) -> Vec<u8> {
rmp_serde::to_vec(value).unwrap()
}
#[cfg(test)]
mod tests {
#[test]
fn round_trip() {}
}
"#,
)
.unwrap();
fs::write(
tmp.path().join(INVENTORY_PATH),
"# machine-checked representation boundary\tbehavioral coverage\n",
)
.unwrap();
let errors = inventory_errors(tmp.path());
assert_eq!(
errors,
["missing: codec-call:src/lib.rs::persist::rmp_serde::to_vec\t<coverage>"]
);
fs::write(
tmp.path().join(INVENTORY_PATH),
concat!(
"codec-call:src/lib.rs::persist::rmp_serde::to_vec",
"\trust:src/lib.rs::round_trip\n",
"serde-type:src/lib.rs::Removed",
"\trust:src/lib.rs::round_trip\n",
),
)
.unwrap();
let errors = inventory_errors(tmp.path());
assert_eq!(errors, ["stale: serde-type:src/lib.rs::Removed"]);
}
#[test]
fn representation_audit_skips_files_behind_a_test_only_mod() {
let tmp = tempfile::tempdir().unwrap();
fs::create_dir_all(tmp.path().join("src/stream/helpers")).unwrap();
fs::create_dir_all(tmp.path().join("tests")).unwrap();
fs::write(
tmp.path().join("src/lib.rs"),
"mod stream;\npub fn persist(value: &u64) -> Vec<u8> { rmp_serde::to_vec(value).unwrap() }\n",
)
.unwrap();
fs::write(
tmp.path().join("src/stream.rs"),
"#[cfg(all(test, feature = \"x\"))]\nmod helpers;\n",
)
.unwrap();
fs::write(
tmp.path().join("src/stream/helpers/mod.rs"),
concat!(
"mod nested;\n",
"fn put() -> Vec<u8> { serde_json::to_vec(&1).unwrap() }\n",
"#[test]\nfn round_trip() {}\n",
),
)
.unwrap();
fs::write(
tmp.path().join("src/stream/helpers/nested.rs"),
"fn get(bytes: &[u8]) -> u64 { rmp_serde::from_slice(bytes).unwrap() }\n",
)
.unwrap();
fs::write(
tmp.path().join(INVENTORY_PATH),
concat!(
"codec-call:src/lib.rs::persist::rmp_serde::to_vec",
"\trust:src/stream/helpers/mod.rs::round_trip\n",
),
)
.unwrap();
assert_eq!(inventory_errors(tmp.path()), Vec::<String>::new());
}
#[test]
fn representation_audit_sees_codec_functions_passed_as_values() {
let tmp = tempfile::tempdir().unwrap();
fs::create_dir_all(tmp.path().join("src")).unwrap();
fs::create_dir_all(tmp.path().join("tests")).unwrap();
fs::write(
tmp.path().join("src/lib.rs"),
concat!(
"fn seq(key: &[u8]) -> Option<u64> { key.try_into().ok().map(u64::from_be_bytes) }\n",
"fn null() -> serde_json::Value { serde_json::Value::Null }\n",
),
)
.unwrap();
fs::write(tmp.path().join(INVENTORY_PATH), "").unwrap();
assert_eq!(
inventory_errors(tmp.path()),
["missing: codec-call:src/lib.rs::seq::u64::from_be_bytes\t<coverage>"]
);
}
#[test]
#[ignore]
fn print_proposed_representation_inventory() {
let manifest = Path::new(env!("CARGO_MANIFEST_DIR"));
let surface = discover(manifest);
println!("# machine-checked representation boundary\tbehavioral coverage");
for boundary in &surface.boundaries {
println!("{boundary}\t{}", proposed_coverage(boundary, &surface));
}
}
#[test]
fn representation_audit_inventory_is_complete() {
let errors = inventory_errors(Path::new(env!("CARGO_MANIFEST_DIR")));
assert!(
errors.is_empty(),
"representation inventory is incomplete:\n{}",
errors.join("\n")
);
}