refactor: harden multi-region data and hosting pipeline
1234567891011121314151617181920212223242526272829303132333435363731 unmodified lines# wisp services run on the host; infrastructure and the reference pds run in compose.# Keep this explicit: missing or unknown NODE_ENV values refuse the local DB default.NODE_ENV=developmentLOCAL_DEV=truePORT=8000# Default is a conservative 32 MiB. Set this only on a machine with enough# headroom for multipart parsing and per-file buffers.PUBLIC_UPLOAD_MEMORY_BUDGET_BYTES=805306368DOMAIN=http://127.0.0.1:8000CLIENT_NAME=Wisp.Place DevBASE_DOMAIN=wisp.localhostPRIVATE_HOST=priv.wisp.localhost:3001
DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/wisp# Optional: a local/replica endpoint for presentation-only eventual reads.# Use the restricted read role from the deployment runbook with# default_transaction_read_only=on, even if it points to the local primary.# Leave unset to use DATABASE_URL for every read.# DATABASE_READ_URL=postgres://wisp_read:readonly@127.0.0.1:5432/wispREDIS_URL=redis://127.0.0.1:6379
# Server-managed webhook secrets fail closed unless this is explicitly set.# Generate a fresh canonical 32-byte key and use the same key in main-app and# webhook-service. Do not commit a real production key.# WEBHOOK_SECRET_ENCRYPTION_KEY="$(openssl rand -base64 32)"# Optional comma-separated decrypt-only old keys during a local rotation.# WEBHOOK_SECRET_ENCRYPTION_PREVIOUS_KEYS=""
# `NODE_ENV=development` explicitly permits hosting's local disk source. For a# non-development single-node deployment, use HOSTING_ALLOW_DISK_SOURCE=true# only when S3_BUCKET and REDIS_URL are both intentionally absent.# HOSTING_ALLOW_DISK_SOURCE=true
S3_BUCKET=wisp-devPRIVATE_S3_BUCKET=wisp-private-devS3_REGION=us-east-1S3_ENDPOINT=http://127.0.0.1:900031 unmodified lines95 unmodified lines96979899100101102103104105106107 unmodified linesExpand all95 unmodified lines timeout: 5s retries: 30
firehose-service: # The independent leadership supervisor/watchdog must be able to SIGKILL # the worker; namespace PID 1 intentionally cannot be killed from within. init: true build: context: . dockerfile: apps/firehose-service/Dockerfile depends_on:107 unmodified linesExpand all17 unmodified lines18192021222322232425262742 unmodified lines17 unmodified lines "tailwindcss": "^4.3.2", "typescript": "^5.9.3" }, "scripts": { "test": "bun test", "dev": "bun run build && bun run --watch apps/main-app/src/index.ts", "test": "NODE_ENV=test bun test --isolate", "dev": "NODE_ENV=development bun run build && NODE_ENV=development bun run --watch apps/main-app/src/index.ts", "dev:infra": "./scripts/dev-infra.sh up", "dev:infra:down": "./scripts/dev-infra.sh down", "start": "bun run apps/main-app/src/index.ts", "build": "cd apps/main-app && bun run build",42 unmodified lines12 unmodified lines131415161717181920212223 unmodified lines12 unmodified lines "properties": { "name": { "type": "string", "format": "record-key", "description": "Unique name for this secret, scoped to the caller DID." "maxLength": 64, "description": "Unique 1–64 character secret ID scoped to the caller DID. Use ASCII letters, digits, dots, underscores, and hyphens only." } } } },23 unmodified lines9 unmodified lines1011121314141516171819202122232425262728291 unmodified line9 unmodified lines "schema": { "type": "object", "required": ["name"], "properties": { "name": { "type": "string", "format": "record-key" } "name": { "type": "string", "format": "record-key", "maxLength": 64, "description": "1–64 character server-managed secret ID. Use ASCII letters, digits, dots, underscores, and hyphens only." } } } }, "errors": [ { "name": "AuthenticationRequired" }, { "name": "InvalidRequest" }, { "name": "NotFound" } ] } }1 unmodified line9 unmodified lines10111213141415161718192021222312 unmodified lines3637383940414243441 unmodified line9 unmodified lines "schema": { "type": "object", "required": ["name"], "properties": { "name": { "type": "string", "format": "record-key" } "name": { "type": "string", "format": "record-key", "maxLength": 64, "description": "1–64 character server-managed secret ID. Use ASCII letters, digits, dots, underscores, and hyphens only." } } } }, "output": {12 unmodified lines } }, "errors": [ { "name": "AuthenticationRequired" }, { "name": "InvalidRequest" }, { "name": "NotFound" } ] } }1 unmodified line38 unmodified lines39404142434445464748494950515253545527 unmodified lines38 unmodified lines "maxLength": 3 }, "secret": { "type": "string", "minLength": 1, "maxLength": 256, "description": "Optional raw secret used to sign the webhook payload with HMAC-SHA256. Prefer secretId to avoid embedding plaintext values in PDS records." }, "secretId": { "type": "string", "format": "record-key", "description": "Name of a server-managed signing secret created via place.wisp.v2.secret.create. Takes precedence over secret if both are present." "maxLength": 64, "description": "1–64 character server-managed signing secret ID. Use ASCII letters, digits, dots, underscores, and hyphens only." }, "enabled": { "type": "boolean", "description": "Whether the webhook is active. Defaults to true if omitted."27 unmodified lines17 unmodified lines18192021222217 unmodified lines
- name: run all tests command: | export PATH="$HOME/.nix-profile/bin:$PATH" bun test bun test --isolate1234567891011121314151617181920212223242526271028293031323334353637383940414216434445464748495023515253545556575859606162631 unmodified line# Local development mode. Production rejects disk fallback and requires S3 + Redis.NODE_ENV=development
# DatabaseDATABASE_URL=postgres://postgres:postgres@localhost:5432/wisp
# FirehoseFIREHOSE_SERVICE=wss://bsky.network# Optional fallback relay used when the primary fails 3 times in a rowFIREHOSE_SERVICE_SECONDARY=FIREHOSE_MAX_CONCURRENCY=5FIREHOSE_MAX_PENDING_EVENTS=10000FIREHOSE_DRAIN_GRACE_MS=30000# Production defaults to leader election. Keep it enabled for every replica.LEADER_ELECTION=false# The production election runs in a separate process so a stalled worker is killed# before either Redis or Postgres authority is handed to a standby.LEADERSHIP_SUPERVISOR_ENABLED=false# Set true in production; the Docker image installs this compiled executable.FIREHOSE_SUPERVISOR_PATH=/usr/local/bin/firehose-supervisorFIREHOSE_WATCHDOG_PATH=/usr/local/bin/firehose-watchdogSUPERVISOR_COMMAND_TIMEOUT_MS=2000LEADER_TTL_MS=30000LEADER_RENEW_INTERVAL_MS=10000LEADER_POLL_INTERVAL_MS=5000CURSOR_SAVE_INTERVAL_MS=5000
# Redis (cache invalidation + revalidation queue)# Redis is required in production for leader election, durable revalidation,# and cache invalidation. Use rediss:// when Redis is outside a trusted network.REDIS_URL=redis://localhost:6379WISP_REVALIDATE_FAILURE_BACKOFF_SECONDS=600WISP_REVALIDATE_STREAM_MAXLEN=10000WISP_REVALIDATE_DEDUPE_TTL_SECONDS=60# Poison messages are quarantined after this many delivery attempts.WISP_REVALIDATE_MAX_ATTEMPTS=3WISP_REVALIDATE_DEADLINE_MS=300000WISP_REVALIDATE_TRANSFER_BUDGET_BYTES=1073741824WISP_REVALIDATE_RETRY_BACKOFF_BASE_MS=60000WISP_REVALIDATE_RETRY_BACKOFF_MAX_MS=900000WISP_REVALIDATE_DLQ_STREAM=wisp:revalidate:dlqWISP_REVALIDATE_DLQ_STREAM_MAXLEN=10000
# S3 Storage (leave empty for local disk fallback)# S3 Storage. S3_BUCKET is required outside development/test. Use an HTTPS# endpoint unless it is a localhost development endpoint.S3_BUCKET=S3_REGION=autoS3_ENDPOINT=S3_PREFIX=sites/S3_FORCE_PATH_STYLE=true
# AWS Credentials (required if using S3)# Static AWS credentials must be supplied as a pair. Leave both empty to use# the AWS SDK default provider chain (for example an IAM workload role).AWS_ACCESS_KEY_ID=AWS_SECRET_ACCESS_KEY=
# Disk fallback is only permitted for an explicit local development/test setup.# Set NODE_ENV=development or test when enabling this. Never enable it in production.FIREHOSE_ALLOW_DISK_STORAGE=true
# Health check serverHEALTH_PORT=3001
# For local disk fallback (when S3_BUCKET is empty)1 unmodified line19 unmodified lines202122232425262728293031323334353637383940411 unmodified line19 unmodified linesRUN bun install --frozen-lockfile
# Compile to a single binaryRUN NODE_ENV=production bun build --compile --minify apps/firehose-service/src/index.ts --outfile /app/firehoseRUN NODE_ENV=production bun build --compile --minify apps/firehose-service/src/supervisor.ts --outfile /app/firehose-supervisorRUN NODE_ENV=production bun build --compile --minify apps/firehose-service/src/watchdog.ts --outfile /app/firehose-watchdog
# Stage 2: Minimal runtimeFROM alpine:3.22
# The runtime must enable Docker's built-in init (`--init` / Compose# `init: true`) so the supervised worker is not namespace PID 1. Startup# refuses the unsafe PID-1 layout.RUN apk add --no-cache libstdc++ libgcc
COPY --from=builder /app/firehose /usr/local/bin/firehoseCOPY --from=builder /app/firehose-supervisor /usr/local/bin/firehose-supervisorCOPY --from=builder /app/firehose-watchdog /usr/local/bin/firehose-watchdog
ENV NODE_ENV=productionENV HEALTH_PORT=3001
1 unmodified line12345678910111213141516171819202122#!/usr/bin/env bunexport {}
const result = await Bun.build({ entrypoints: ['./src/index.ts', './src/supervisor.ts', './src/watchdog.ts'], outdir: './dist', target: 'bun', format: 'esm', minify: process.env.NODE_ENV === 'production', sourcemap: 'external', splitting: false,})
if (!result.success) { console.error('Build failed:') for (const log of result.logs) console.error(log) process.exit(1)}
console.log('Build successful')console.log(`Generated ${result.outputs.length} file(s)`)for (const output of result.outputs) console.log(`- ${output.path}`)1 unmodified line2345676789101171 unmodified lines1 unmodified line "name": "@wispplace/main-app", "version": "1.0.50", "private": true, "scripts": { "test": "bun test", "dev": "bun run build && bun run --watch src/index.ts", "test": "NODE_ENV=test bun test", "dev": "NODE_ENV=development bun run build && NODE_ENV=development bun run --watch src/index.ts", "start": "bun run src/index.ts", "build:css": "bunx @tailwindcss/cli -i public/styles/global.css -o dist/styles.css --minify", "build": "bun run build.ts && bun run build:css", "check": "tsc --noEmit",71 unmodified lines18 unmodified lines19202122232324252627282829303132333435363738394041424318 unmodified lines# Install dependenciesRUN bun install --frozen-lockfile
# Compile to a single binaryRUN bun build --compile --minify apps/webhook-service/src/index.ts --outfile /app/webhookRUN NODE_ENV=production bun build --compile --minify apps/webhook-service/src/index.ts --outfile /app/webhook
# Stage 2: Minimal runtimeFROM alpine:3.22
RUN apk add --no-cache libstdc++ libgccRUN apk add --no-cache libstdc++ libgcc ca-certificates
COPY --from=builder /app/webhook /usr/local/bin/webhook
ENV NODE_ENV=production# The service itself defaults to loopback. Containers opt in to an unspecified# bind address so an orchestrator can reach the single bounded health endpoint.ENV HEALTH_HOST=0.0.0.0ENV HEALTH_PORT=3003
EXPOSE 3003
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:3003/health || exit 1
CMD ["webhook"]10 unmodified lines11121314151617181920212223242526272829303110 unmodified lines
## Environment Variables
```bashNODE_ENV=productionDATABASE_URL="postgres://user:password@localhost:5432/wisp"JETSTREAM_URL="wss://jetstream2.us-east.bsky.network/subscribe"# Defaults to 127.0.0.1. The container image sets 0.0.0.0 for orchestration.HEALTH_HOST=127.0.0.1HEALTH_PORT=3003DELIVERY_TIMEOUT_MS=10000DELIVERY_MAX_RETRIES=3REDIS_URL="redis://localhost:6379"WEBHOOK_EVENTS_CHANNEL="webhook:events"# Reconnects continue indefinitely; this only caps exponential backoff.JETSTREAM_RECONNECT_MAX_EXPONENT=8```
The only HTTP listener is `GET /health`; it returns bounded service status and does not expose webhook records or endpoints.
See the [webhook documentation](../../docs/src/content/docs/lexicons/place-wisp-wh.md) for the full record schema, payload format, and signature verification.46 unmodified lines47484950515251525354555657586061596061626362 unmodified lines46 unmodified linesconst posts: Post[] = [ { slug: 'wispctl-1-3-2', title: 'wispctl 1.3.2', description: 'wispctl uses permission sets now for friendlier oauth links', description: 'wispctl uses permission sets now for friendlier oauth links', date: '2026-08-29', author: authors.nekomimi, tags: ['release', 'cli'], }, { slug: 'wispctl-1-3-1', title: 'wispctl 1.3.1', description: 'Making wispctl\'s auth more ergonomic to use', description: "Making wispctl's auth more ergonomic to use", date: '2026-08-26', author: authors.nekomimi, tags: ['release', 'cli'], },62 unmodified lines1234567891011121314151617181920212223242526import { describe, expect, it } from 'bun:test'import { normalizeServeRequestPath } from './serve'
describe('normalizeServeRequestPath', () => { it('keeps only canonical local request paths', () => { expect(normalizeServeRequestPath('/')).toBe('/') expect(normalizeServeRequestPath('/assets/app.js')).toBe('/assets/app.js') expect(normalizeServeRequestPath('/assets/')).toBe('/assets/') })
it('rejects traversal, encoded structure, Windows paths, and filesystem aliases', () => { for (const path of [ '/../secret.txt', '/assets/../secret.txt', '/assets%2f..%2fsecret.txt', '/safe%00name.txt', '/assets\\secret.txt', '/C:/Windows/system.ini', '/assets/C:/stream', '/assets/file.', '/assets/file ', ]) { expect(normalizeServeRequestPath(path)).toBeNull() } })})