lessy noisy error logging for db auth + better testing
103 unmodified linesExpand all10410510610710810911011111211311411511611711811912042 unmodified lines103 unmodified linesExpand allBun workspaces: `packages/@wisp/*`, `apps/main-app`, `apps/hosting-service`, etc
PLEASE USE `bun check` to type check and `biome check --write` to lint.
Run tests with `bun test --isolate` (or `bun run test`, which passes it), never abare `bun test`. Several suites call `mock.module('../lib/db', ...)` with a partialset of exports, and without isolation that replacement leaks into every test fileloaded afterwards: those files fail to link with a misleading`SyntaxError: Export named '<something>' not found in module .../lib/db.ts`, andtheir tests are silently skipped rather than reported as failures. Adding any newexport to `src/lib/db.ts` is enough to trigger it. `bunfig.toml` does not support`[test] isolate`, so it has to be the CLI flag.
There are three typescript apps**`apps/main-app`** - Main backend (Bun + Elysia)
- OAuth authentication and session management42 unmodified lines1 unmodified line2345667891072 unmodified lines1 unmodified line "name": "@wispplace/main-app", "version": "1.0.50", "private": true, "scripts": { "test": "NODE_ENV=test bun test", "test": "NODE_ENV=test bun test --isolate", "dev": "NODE_ENV=development bun run build && NODE_ENV=development bun run --watch src/index.ts", "start": "bun run src/index.ts", "build:css": "tailwindcss -i public/styles/global.css -o dist/styles.css --minify", "build": "bun run build.ts && bun run build:css",72 unmodified lines120 unmodified linesExpand all121122123124125126127128129130131132133134135127136137138139140141142143144145146147148149150151152153130154155156157158478 unmodified linesExpand all120 unmodified linesExpand all
// Keeping a connection open is not an optimisation here so much as removing a// fixed cost: a cold primary connection from a remote region costs more than// every query in a sign-in put together.//// Warming is best effort. Missing a cycle only means the next request pays to// establish its own connection, which is what happened before this existed, so// a single miss is not worth an error line. Sustained failure is, because it// means the primary is unreachable.const CONNECTION_WARMING_ALARM_AFTER = 5let consecutiveWarmingFailures = 0
const connectionWarming = startPeriodicSingleFlightTask( async () => { try { await Promise.all([warmPrimaryConnections(), warmOAuthLockConnection()]) await Promise.all([warmPrimaryConnections(), warmOAuthLockConnection()]) consecutiveWarmingFailures = 0 } catch (err) { consecutiveWarmingFailures += 1 // Driver errors can carry a connection URL, so only the class is logged. const detail = { consecutiveFailures: consecutiveWarmingFailures, errorName: err instanceof Error ? err.name : 'unknown', } if (consecutiveWarmingFailures >= CONNECTION_WARMING_ALARM_AFTER) { logger.error('[Database] Connection warming has failed repeatedly', detail) } else { logger.debug('[Database] Connection warming missed a cycle', detail) } } }, connectionWarmingIntervalMs, // The task above handles its own failures, so this only catches a bug in it. () => logger.error('[Database] Connection warming failed'), () => logger.error('[Database] Connection warming task crashed'),)
const privateSiteReaper = startPrivateSiteReaper()
478 unmodified linesExpand all1234567891011121314151617181920212223242526272829303132333435363738394041424344454647import { describe, expect, test } from 'bun:test'import { probeConnectionWithRetry, resolveConnectionWarmingIntervalMs } from './connection-warming'
describe('probeConnectionWithRetry', () => { test('probes once when the connection is healthy', async () => { let calls = 0 await probeConnectionWithRetry(async () => { calls += 1 })
expect(calls).toBe(1) })
test('retries once when a connection is retired mid-query', async () => { let calls = 0 await probeConnectionWithRetry(async () => { calls += 1 // A connection reaching maxLifetime can be handed out and closed in the // same moment; the replacement connection serves the retry. if (calls === 1) throw new Error('Connection closed') })
expect(calls).toBe(2) })
test('surfaces the failure when the retry also fails', async () => { let calls = 0 const probe = probeConnectionWithRetry(async () => { calls += 1 throw new Error('primary unreachable') })
await expect(probe).rejects.toThrow('primary unreachable') expect(calls).toBe(2) })})
describe('resolveConnectionWarmingIntervalMs', () => { test('stays inside the pool idle timeout', () => { expect(resolveConnectionWarmingIntervalMs(30)).toBeLessThan(30_000) expect(resolveConnectionWarmingIntervalMs(300)).toBeLessThan(300_000) })
test('never ticks faster than ten seconds', () => { expect(resolveConnectionWarmingIntervalMs(5)).toBe(10_000) })})123456789101112131415161718192021222324252627282930313233343536/** * Keeping database pools warm. * * A node in another region pays the full Postgres handshake on a cold * connection: TCP, the SSLRequest round trip, the TLS handshake, then three * more round trips for SCRAM-SHA-256 and one for startup. Measured from a * Singapore node to a primary in Ashburn that is about 1.9 seconds — more than * every query in a sign-in put together — and with a 30 second idle timeout the * pools were empty for almost every sign-in. */
/** * Touch a pool once, retrying a single time. * * A connection that reaches `maxLifetime` can be handed out and retired in the * same moment, which fails whichever query received it. That race is inherent * to recycling connections and is transient by construction: the retry is * served by a replacement connection. */export const probeConnectionWithRetry = async (probe: () => Promise<unknown>): Promise<void> => { try { await probe() } catch { await probe() }}
/** * How often to touch an idle pool, given its idle timeout. * * Comfortably inside that timeout, and frequent enough that a connection * retired at `maxLifetime` is replaced by the warming task rather than by a * user's request. */export const resolveConnectionWarmingIntervalMs = (idleTimeoutSeconds: number): number => Math.max(10_000, Math.floor(idleTimeoutSeconds * 1_000 * 0.5))5 unmodified lines6789101112131423 unmodified lines3839404141424344454642434445464748495051565253545560565758596063646566676869707172616263646566675 unmodified linesExpand all5 unmodified lines type WebhookSecretEncryptionKeyring,} from '@wispplace/atproto-utils'import { DELETED_SITE_RECORD_CID } from '@wispplace/constants'import { SQL } from 'bun'import { probeConnectionWithRetry, resolveConnectionWarmingIntervalMs } from './connection-warming'import { resolveDatabaseConfiguration } from './database-config'import { createDatabaseReadCircuit, type DatabaseReadProbeResult } from './database-read-circuit'import { assessDatabaseReadReplication } from './database-read-replication'import { isValidHandle, toDomain } from './domain-utils'23 unmodified lines
/** * How many primary connections are held open between requests. * * A node in another region pays the full Postgres handshake on a cold * connection: TCP, the SSLRequest round trip, the TLS handshake, then three * more round trips for SCRAM-SHA-256 and one for startup. Measured from a * Singapore node to a primary in Ashburn that is about 1.9 seconds, and with a * 30 second idle timeout the pool was empty for almost every sign-in. * * Two, because the widest fan-out on the primary in one request is the pair of * lookups at the end of the OAuth callback. A third concurrent query still * connects on demand; it just is not on the common path. * See {@link probeConnectionWithRetry} for why this is worth doing at all. */const WARM_PRIMARY_CONNECTIONS = 2
/** * Hold connections open so a request never pays for establishing one. * * The queries run concurrently on purpose: one at a time would keep re-using a * The probes run concurrently on purpose: one at a time would keep re-using a * single pooled connection and leave the rest to lapse. */export const warmPrimaryConnections = async (): Promise<void> => { await Promise.all(Array.from({ length: WARM_PRIMARY_CONNECTIONS }, () => db`SELECT 1`)) await Promise.all( Array.from({ length: WARM_PRIMARY_CONNECTIONS }, () => probeConnectionWithRetry(() => db`SELECT 1`)), )}
/** * How often to touch an idle pool, in milliseconds. * * Comfortably inside the idle timeout, and frequent enough that a connection * retired at `maxLifetime` is replaced by this task rather than by a user's * request. */export const connectionWarmingIntervalMs = Math.max( 10_000, Math.floor(databaseConfiguration.primaryPool.idleTimeoutSeconds * 1_000 * 0.5),export const connectionWarmingIntervalMs = resolveConnectionWarmingIntervalMs( databaseConfiguration.primaryPool.idleTimeoutSeconds,)
// Do not create another pool unless the operator explicitly configured a// different endpoint. The raw replica client stays private so call sites must675 unmodified linesExpand all1234567836 unmodified lines45464748484950515253306 unmodified linesExpand allimport { JoseKey } from '@atproto/jwk-jose'import { NodeOAuthClient, type RuntimeLock } from '@atproto/oauth-client-node'import { SQL } from 'bun'import { probeConnectionWithRetry } from './connection-warming'import { databaseConfiguration, db } from './db'import { logger } from './logger'import { createClientMetadata } from './oauth-client-metadata'import { createOAuthFetch } from './oauth-fetch'36 unmodified lines * take. `client.callback()` always takes this lock — it revokes any prior * session before storing the new one — so the cost landed on every sign-in. */export const warmOAuthLockConnection = async (): Promise<void> => { await lockDb`SELECT 1` await probeConnectionWithRetry(() => lockDb`SELECT 1`)}
let oauthLockDatabaseClosePromise: Promise<void> | undefined
306 unmodified linesExpand all