feat(tranquil-store): gc and hardening
Lewis: May this revision serve well! <lu5a@proton.me>
7403 unmodified linesExpand all740474057406740774087408740974107411741242 unmodified lines745574567457745874597459746074617462746314 unmodified lines74787479748074817482748274837484748574865 unmodified lines74927493749474957496749674977498749975009 unmodified lines75107511751275137514751475157516751775187519752075217522752275237524752575267 unmodified lines75347535753675377538753875397540754175428 unmodified lines75517552755375547555755575567557755875597 unmodified lines75677568756975707571757175727573757475752 unmodified lines75787579758075817582758275837584758575869 unmodified lines759675977598759976007600760176027603760414 unmodified lines761976207621762276237623762476257626762724 unmodified lines765276537654765576567656765776587659766082 unmodified lines77437744774577467747774777487749775077513 unmodified lines775577567757775877597759776077617762776316 unmodified lines77807781778277837784778477857786778777887 unmodified lines779677977798779978007800780178027803780412 unmodified lines78177818781978207821782178227823782478252 unmodified lines78287829783078317832783378347835783678377838783978407841784278437842784478457846784778488 unmodified lines785778587859786078597861786278637864786511 unmodified lines7877787878797880788178827883788478854 unmodified lines7890789178927893789478957896789778987899790079017902790379047905790679027907790879097910791113 unmodified lines79257926792779287924792979307931793279331331 unmodified linesExpand all7403 unmodified linesExpand all]
[[package]]name = "tranquil-api"version = "0.4.7"version = "0.5.0"dependencies = [ "anyhow", "axum", "backon",42 unmodified lines]
[[package]]name = "tranquil-auth"version = "0.4.7"version = "0.5.0"dependencies = [ "anyhow", "base32", "base64 0.22.1",14 unmodified lines]
[[package]]name = "tranquil-cache"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "base64 0.22.1", "redis",5 unmodified lines]
[[package]]name = "tranquil-comms"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "base64 0.22.1", "reqwest",9 unmodified lines]
[[package]]name = "tranquil-config"version = "0.4.7"version = "0.5.0"dependencies = [ "confique", "serde",]
[[package]]name = "tranquil-crypto"version = "0.4.7"version = "0.5.0"dependencies = [ "aes-gcm", "base64 0.22.1", "hkdf",7 unmodified lines]
[[package]]name = "tranquil-db"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "chrono", "rand 0.8.5",8 unmodified lines]
[[package]]name = "tranquil-db-traits"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "base64 0.22.1", "chrono",7 unmodified lines]
[[package]]name = "tranquil-infra"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "bytes", "futures",2 unmodified lines]
[[package]]name = "tranquil-lexicon"version = "0.4.7"version = "0.5.0"dependencies = [ "chrono", "hickory-resolver", "parking_lot",9 unmodified lines]
[[package]]name = "tranquil-oauth"version = "0.4.7"version = "0.5.0"dependencies = [ "anyhow", "axum", "base64 0.22.1",14 unmodified lines]
[[package]]name = "tranquil-oauth-server"version = "0.4.7"version = "0.5.0"dependencies = [ "axum", "base64 0.22.1", "bcrypt",24 unmodified lines]
[[package]]name = "tranquil-pds"version = "0.4.7"version = "0.5.0"dependencies = [ "aes-gcm", "anyhow", "async-trait",82 unmodified lines]
[[package]]name = "tranquil-repo"version = "0.4.7"version = "0.5.0"dependencies = [ "bytes", "cid", "jacquard-repo",3 unmodified lines]
[[package]]name = "tranquil-ripple"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "backon", "bincode 2.0.1",16 unmodified lines]
[[package]]name = "tranquil-scopes"version = "0.4.7"version = "0.5.0"dependencies = [ "axum", "futures", "hickory-resolver",7 unmodified lines]
[[package]]name = "tranquil-server"version = "0.4.7"version = "0.5.0"dependencies = [ "axum", "clap", "dotenvy",12 unmodified lines]
[[package]]name = "tranquil-signal"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "chrono", "fjall",2 unmodified lines "rand 0.9.2", "serde", "serde_json", "sqlx", "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", "tracing", "tranquil-signal", "url", "uuid",]
[[package]]name = "tranquil-storage"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "aws-config", "aws-sdk-s3",8 unmodified lines]
[[package]]name = "tranquil-store"version = "0.4.7"version = "0.5.0"dependencies = [ "async-trait", "bytes", "chrono",11 unmodified lines "parking_lot", "postcard", "proptest", "rand 0.8.5", "rayon", "serde", "serde_ipld_dagcbor", "serde_json", "sha2",4 unmodified lines "thiserror 2.0.18", "tikv-jemallocator", "tokio", "tracing", "tracing-subscriber", "tranquil-db", "tranquil-db-traits", "tranquil-oauth", "tranquil-repo", "tranquil-store", "tranquil-types", "uuid", "xxhash-rust",]
[[package]]name = "tranquil-sync"version = "0.4.7"version = "0.5.0"dependencies = [ "anyhow", "axum", "bytes",13 unmodified lines]
[[package]]name = "tranquil-types"version = "0.4.7"version = "0.5.0"dependencies = [ "chrono", "cid", "jacquard-common",1331 unmodified linesExpand all173 unmodified linesExpand all17417517617717817817918018118222 unmodified lines205206207208209210211212213214215216217218219220221222223224225226227228210229230231232233293 unmodified linesExpand all527528529530512513531532533534515535536517518537538539540541542543544545546173 unmodified linesExpand all## Can also be specified via environment variable `S3_BUCKET`.#s3_bucket =
# Custom S3 endpoint URL (for MinIO, R2, etc.).# Custom S3 endpoint URL.## Can also be specified via environment variable `S3_ENDPOINT`.#s3_endpoint =
22 unmodified lines## Can also be specified via environment variable `TRANQUIL_STORE_HANDLER_THREADS`.#handler_threads =
# Maximum total bytes of pending (unsynced) eventlog payloads. Appenders block# once this budget is exhausted until in-flight events drain via fsync. Set to# 0 to disable backpressure. Default: 1 GiB.## Can also be specified via environment variable# `TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET`.## Default value: 1073741824#eventlog_pending_bytes_budget = 1073741824
# Maximum size of an individual eventlog payload in bytes. Single events# larger than this are rejected at append time. Default: 256 MiB.## Can also be specified via environment variable# `TRANQUIL_STORE_EVENTLOG_MAX_EVENT_PAYLOAD`.## Default value: 268435456#eventlog_max_event_payload = 268435456
[cache]# Cache backend: `ripple` (default, built-in gossip) or `valkey`.# Cache backend: `ripple` by default, or `valkey`.## Can also be specified via environment variable `CACHE_BACKEND`.## Default value: "ripple"293 unmodified linesExpand all## Default value: 3600#delete_check_interval_secs = 3600
# Interval in seconds between block garbage collection cycles.# Reclaims orphaned ipld blocks that were stored but never committed.# Maximum age of events retained in the eventlog before pruning.# Per the atproto firehose spec, the relay backfill window only needs# to cover "hours or days".## Can also be specified via environment variable `BLOCK_GC_INTERVAL_SECS`.# Can also be specified via environment variable `EVENT_RETENTION_MAX_AGE_SECS`.## Default value: 21600#block_gc_interval_secs = 21600# Default value: 604800 (7 days)#event_retention_max_age_secs = 604800
# Interval in seconds between event retention prune passes.# Set to 0 to disable.## Can also be specified via environment variable `EVENT_RETENTION_INTERVAL_SECS`.## Default value: 3600#event_retention_interval_secs = 360020 unmodified lines212223242526272829303127 unmodified lines59606162636465666768697047 unmodified lines20 unmodified lines
test-store: SQLX_OFFLINE=true cargo nextest run -p tranquil-store --features tranquil-store/test-harness
test-store-sim-nightly: SQLX_OFFLINE=true TRANQUIL_SIM_SEEDS=10000 cargo nextest run -p tranquil-store --features tranquil-store/test-harness --profile sim-nightly
test-unit: SQLX_OFFLINE=true cargo test --test dpop_unit --test validation_edge_cases --test scope_edge_cases
test-auth:27 unmodified lines @just test-store @just test-unit ./scripts/run-tests.sh {{args}}
test-embedded *args: @just test-unit SQLX_OFFLINE=true TRANQUIL_TEST_BACKEND=store TRANQUIL_PDS_ALLOW_INSECURE_SECRETS=1 DISABLE_RATE_LIMITING=1 TRANQUIL_LEXICON_OFFLINE=1 SKIP_IMPORT_VERIFICATION=true cargo nextest run -E 'not binary(store_parity)' {{args}}
test-one name: ./scripts/run-tests.sh --test {{name}}
infra-start:47 unmodified lines11 unmodified lines12131415161718192021222324252627282930313233343521 unmodified lines5758596061626364656667686970717273747576777823 unmodified lines11 unmodified linesfail-fast = falsetest-threads = "num-cpus"slow-timeout = { period = "30s", terminate-after = 4 }
[profile.sim-pr]retries = 0fail-fast = truetest-threads = "num-cpus"slow-timeout = { period = "60s", terminate-after = 4 }
[profile.sim-nightly]retries = 0fail-fast = falsetest-threads = "num-cpus"slow-timeout = { period = "300s", terminate-after = 2 }
[test-groups]serial-env-tests = { max-threads = 1 }heavy-load-tests = { max-threads = 4 }io-heavy-sim = { max-threads = 2 }
[[profile.default.overrides]]filter = "test(/import_with_verification/) | test(/plc_migration/)"test-group = "serial-env-tests"21 unmodified lines[[profile.default.overrides]]filter = "binary(repo_lifecycle)"test-group = "heavy-load-tests"
[[profile.default.overrides]]filter = "binary(sim_soak)"slow-timeout = { period = "300s", terminate-after = 10 }test-group = "io-heavy-sim"
[[profile.default.overrides]]filter = "binary(metastore_crash) | binary(sim_cross_store) | binary(sim_gc_concurrent) | binary(sim_reachability) | binary(sim_eventlog) | test(/sim_/)"slow-timeout = { period = "120s", terminate-after = 8 }test-group = "io-heavy-sim"
[[profile.default.overrides]]filter = "test(/test_scale_/) | test(/full_backup_and_restore/)"slow-timeout = { period = "120s", terminate-after = 4 }
[[profile.ci.overrides]]filter = "test(/import_with_verification/) | test(/plc_migration/)"test-group = "serial-env-tests"
23 unmodified lines1234567891011121314151617181920212223242526272829303132{ "db_name": "PostgreSQL", "query": "\n SELECT seq, did, commit_cid\n FROM repo_seq\n WHERE event_type = 'commit'\n AND prev_cid IS NULL\n AND (blocks_cids IS NULL OR array_length(blocks_cids, 1) IS NULL OR array_length(blocks_cids, 1) = 0)\n ", "describe": { "columns": [ { "ordinal": 0, "name": "seq", "type_info": "Int8" }, { "ordinal": 1, "name": "did", "type_info": "Text" }, { "ordinal": 2, "name": "commit_cid", "type_info": "Text" } ], "parameters": { "Left": [] }, "nullable": [ false, false, true ] }, "hash": "033ed5638ef6e30f1648c1acdbc477c173798cb57eb0127c3a582a9633fa5012"}123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113{ "db_name": "PostgreSQL", "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2", "describe": { "columns": [ { "ordinal": 0, "name": "seq", "type_info": "Int8" }, { "ordinal": 1, "name": "did", "type_info": "Text" }, { "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" }, { "ordinal": 3, "name": "event_type: RepoEventType", "type_info": "Text" }, { "ordinal": 4, "name": "commit_cid", "type_info": "Text" }, { "ordinal": 5, "name": "prev_cid", "type_info": "Text" }, { "ordinal": 6, "name": "prev_data_cid", "type_info": "Text" }, { "ordinal": 7, "name": "ops", "type_info": "Jsonb" }, { "ordinal": 8, "name": "blobs", "type_info": "TextArray" }, { "ordinal": 9, "name": "block_cids", "type_info": "ByteaArray" }, { "ordinal": 10, "name": "block_data", "type_info": "ByteaArray" }, { "ordinal": 11, "name": "blocks_cids", "type_info": "TextArray" }, { "ordinal": 12, "name": "handle", "type_info": "Text" }, { "ordinal": 13, "name": "active", "type_info": "Bool" }, { "ordinal": 14, "name": "status", "type_info": "Text" }, { "ordinal": 15, "name": "rev", "type_info": "Text" } ], "parameters": { "Left": [ "Int8", "Int8" ] }, "nullable": [ false, false, false, false, true, true, true, true, true, true, true, true, true, true, true, true ] }, "hash": "0349b2af4d21105029ecc2bf81ad7be9987ec17ac088730596849d7a45558e57"}123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100{ "db_name": "PostgreSQL", "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq = $1", "describe": { "columns": [ { "ordinal": 0, "name": "seq", "type_info": "Int8" }, { "ordinal": 1, "name": "did", "type_info": "Text" }, { "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" }, { "ordinal": 3, "name": "event_type: RepoEventType", "type_info": "Text" }, { "ordinal": 4, "name": "commit_cid", "type_info": "Text" }, { "ordinal": 5, "name": "prev_cid", "type_info": "Text" }, { "ordinal": 6, "name": "prev_data_cid", "type_info": "Text" }, { "ordinal": 7, "name": "ops", "type_info": "Jsonb" }, { "ordinal": 8, "name": "blobs", "type_info": "TextArray" }, { "ordinal": 9, "name": "blocks_cids", "type_info": "TextArray" }, { "ordinal": 10, "name": "handle", "type_info": "Text" }, { "ordinal": 11, "name": "active", "type_info": "Bool" }, { "ordinal": 12, "name": "status", "type_info": "Text" }, { "ordinal": 13, "name": "rev", "type_info": "Text" } ], "parameters": { "Left": [ "Int8" ] }, "nullable": [ false, false, false, false, true, true, true, true, true, true, true, true, true, true ] }, "hash": "0d32a592a97ad47c65aa37cf0d45417f2966fcbd688be7434626ae5f6971fa1f"}12345678910111213141516171819202122{ "db_name": "PostgreSQL", "query": "SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1", "describe": { "columns": [ { "ordinal": 0, "name": "cid", "type_info": "Bytea" } ], "parameters": { "Left": [ "Int8" ] }, "nullable": [ false ] }, "hash": "12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b"}12345678910111213141516171819202122{ "db_name": "PostgreSQL", "query": "\n SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid)\n WHERE NOT EXISTS (\n SELECT 1 FROM user_blocks WHERE block_cid = t.cid\n )\n ", "describe": { "columns": [ { "ordinal": 0, "name": "cid", "type_info": "Bytea" } ], "parameters": { "Left": [ "ByteaArray" ] }, "nullable": [ null ] }, "hash": "18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8"}123456789101112131415{ "db_name": "PostgreSQL", "query": "UPDATE repo_seq SET blocks_cids = $1 WHERE seq = $2", "describe": { "columns": [], "parameters": { "Left": [ "TextArray", "Int8" ] }, "nullable": [] }, "hash": "3b791fdb8e29043c980963d4d18e1e492c73c39818a8648a7af70555418fb5d1"}123456789101112131415161718192021222324252627282930{ "db_name": "PostgreSQL", "query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, prev_data_cid, rev)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n RETURNING seq\n ", "describe": { "columns": [ { "ordinal": 0, "name": "seq", "type_info": "Int8" } ], "parameters": { "Left": [ "Text", "Text", "Text", "Text", "Jsonb", "TextArray", "TextArray", "Text", "Text" ] }, "nullable": [ false ] }, "hash": "40e848f48afb50f5e4820445aee874a482333e9090ef14ce44db3fd9721cfff0"}123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112{ "db_name": "PostgreSQL", "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq = $1", "describe": { "columns": [ { "ordinal": 0, "name": "seq", "type_info": "Int8" }, { "ordinal": 1, "name": "did", "type_info": "Text" }, { "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" }, { "ordinal": 3, "name": "event_type: RepoEventType", "type_info": "Text" }, { "ordinal": 4, "name": "commit_cid", "type_info": "Text" }, { "ordinal": 5, "name": "prev_cid", "type_info": "Text" }, { "ordinal": 6, "name": "prev_data_cid", "type_info": "Text" }, { "ordinal": 7, "name": "ops", "type_info": "Jsonb" }, { "ordinal": 8, "name": "blobs", "type_info": "TextArray" }, { "ordinal": 9, "name": "block_cids", "type_info": "ByteaArray" }, { "ordinal": 10, "name": "block_data", "type_info": "ByteaArray" }, { "ordinal": 11, "name": "blocks_cids", "type_info": "TextArray" }, { "ordinal": 12, "name": "handle", "type_info": "Text" }, { "ordinal": 13, "name": "active", "type_info": "Bool" }, { "ordinal": 14, "name": "status", "type_info": "Text" }, { "ordinal": 15, "name": "rev", "type_info": "Text" } ], "parameters": { "Left": [ "Int8" ] }, "nullable": [ false, false, false, false, true, true, true, true, true, true, true, true, true, true, true, true ] }, "hash": "4702a94384ce938d3f5384bc2106a48500f8749134e6bfd1c619a9151f5d3772"}