update blog + fix backfill rewrite html
5 unmodified lines678910111213146 unmodified lines5 unmodified lines "dev": "astro dev", "start": "astro dev", "build": "astro build", "preview": "astro preview", "standard:publish": "bun run scripts/publish-standard.ts", "astro": "astro" }, "dependencies": { "astro": "^7.0.9",6 unmodified lines123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106import { allPosts } from '../src/posts'import { site } from '../src/site'import { standard, standardDocumentPath, standardPublicationUri, standardTimestamp } from '../src/standard'
interface PdsService { id: string type: string serviceEndpoint: string}
interface DidDocument { service?: PdsService[]}
interface Session { accessJwt: string did: string}
async function readJson<T>(response: Response, action: string): Promise<T> { if (!response.ok) { throw new Error(`${action} failed (${response.status}): ${await response.text()}`) }
return response.json() as Promise<T>}
async function resolvePds(): Promise<string> { const document = await readJson<DidDocument>( await fetch(`https://plc.directory/${standard.did}`), `Resolving ${standard.did}`, ) const pds = document.service?.find((service) => service.id === '#atproto_pds')
if (pds?.type !== 'AtprotoPersonalDataServer') { throw new Error(`No PDS service found for ${standard.did}`) }
return pds.serviceEndpoint.replace(/\/$/, '')}
async function createSession(pds: string, password: string): Promise<Session> { return readJson<Session>( await fetch(`${pds}/xrpc/com.atproto.server.createSession`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ identifier: process.env.STANDARD_HANDLE ?? 'wisp.place', password }), }), 'Creating an AT Protocol session', )}
async function putRecord( pds: string, accessJwt: string, collection: string, rkey: string, record: Record<string, unknown>,): Promise<void> { await readJson( await fetch(`${pds}/xrpc/com.atproto.repo.putRecord`, { method: 'POST', headers: { Authorization: `Bearer ${accessJwt}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ repo: standard.did, collection, rkey, record }), }), `Publishing ${collection}/${rkey}`, )}
const password = process.env.STANDARD_APP_PASSWORDif (!password) { throw new Error('Set STANDARD_APP_PASSWORD to a wisp.place app password before publishing Standard.site records.')}
const pds = await resolvePds()const session = await createSession(pds, password)if (session.did !== standard.did) { throw new Error(`STANDARD_HANDLE authenticated as ${session.did}, expected ${standard.did}`)}
await putRecord(pds, session.accessJwt, standard.publicationCollection, standard.publicationRkey, { $type: standard.publicationCollection, url: site.url, name: site.title, description: site.description, preferences: { showInDiscover: true },})
const publishedPosts = allPosts.filter((post) => !post.draft)for (const post of publishedPosts) { await putRecord(pds, session.accessJwt, standard.documentCollection, post.slug, { $type: standard.documentCollection, site: standardPublicationUri, path: standardDocumentPath(post.slug), title: post.title, description: post.description, publishedAt: standardTimestamp(post.date), ...(post.updated && { updatedAt: standardTimestamp(post.updated) }), tags: post.tags, })}
console.info(`Published ${publishedPosts.length} Standard.site documents to ${standardPublicationUri}`)66 unmodified lines676869707171727374747576777831 unmodified lines66 unmodified lines { slug: 'private-sites', title: 'New and future: private sites, permissioned data, and open web realities.', description: 'The annocunement of private sites and detailing upcoming quota systems when permissioned data lands', date: '2026-08-11', date: '2026-08-22', author: authors.nekomimi, tags: ['announcement'], } },]
/** True in `astro build`, false in `astro dev`. */const isProduction = import.meta.env.PROD31 unmodified lines123456789101112131415161718192021222324252627282930import { postHref } from './posts'
/** * Identity and stable record keys for the blog's Standard.site publication. * * These keys deliberately do not change as posts are edited, so an AT-URI * remains a durable reference to its corresponding canonical web page. */export const standard = { did: 'did:plc:7puq73yz2hkvbcpdhnsze2qw', publicationRkey: 'blog', publicationCollection: 'site.standard.publication', documentCollection: 'site.standard.document',} as const
export const standardPublicationUri = `at://${standard.did}/${standard.publicationCollection}/${standard.publicationRkey}`
export function standardDocumentUri(slug: string): string { return `at://${standard.did}/${standard.documentCollection}/${slug}`}
/** The Standard path is the blog's canonical, trailing-slash route. */export function standardDocumentPath(slug: string): string { return postHref(slug)}
/** Post dates are day-granular; publish their records at the start of that UTC day. */export function standardTimestamp(date: string): string { return `${date}T00:00:00.000Z`} blog/public/images /private-sites.png this is a binary file and will not be displayed.
10 unmodified lines111213141516171819204 unmodified lines25262728293031323314 unmodified lines48495051525354555676 unmodified lines10 unmodified lines path?: string; /** 'article' on posts, 'website' everywhere else. */ ogType?: 'website' | 'article'; publishedAt?: string; /** AT-URI of this page's Standard.site document record. */ standardDocumentUri?: string; /** Constrain the main column to reading width. */ narrow?: boolean; /** Keep the page out of search results (drafts, error pages). */ noindex?: boolean;4 unmodified lines description = site.description, path = Astro.url.pathname, ogType = 'website', publishedAt, standardDocumentUri, narrow = false, noindex = false,} = Astro.props;
14 unmodified lines {noindex && <meta name="robots" content="noindex, nofollow" />}
<link rel="icon" type="image/svg+xml" href="/favicon.svg" /> <link rel="alternate" type="application/rss+xml" title={site.title} href="/rss.xml" /> {standardDocumentUri && <link rel="site.standard.document" href={standardDocumentUri} />}
<meta property="og:type" content={ogType} /> <meta property="og:url" content={canonical} /> <meta property="og:title" content={title} />76 unmodified lines123456782 unmodified lines11121314151617181920212223242526272862 unmodified lines---import { formatDate } from '../site';import { getPost, neighbours, postHref } from '../posts';import { standardDocumentUri } from '../standard';import Base from './Base.astro';
interface Props { /** Must match an entry in src/posts.ts. */2 unmodified lines
const { slug } = Astro.props;const post = getPost(slug);const { previous, next } = neighbours(slug);const documentUri = post.draft ? undefined : standardDocumentUri(slug);---
<Base title={post.title} description={post.description} path={postHref(slug)} ogType="article" publishedAt={post.date} standardDocumentUri={documentUri} narrow noindex={post.draft}> <a class="backlink" href="/">← All posts</a>62 unmodified lines123456789101112131415161732 unmodified lines50515253545556575859606162636465666768697071727374757677787980818283848586878889909192939442 unmodified linesimport { beforeEach, describe, expect, mock, test } from 'bun:test'
const cachedKeys = new Set<string>()const promotedKeys: string[] = []/** Keys whose read throws, standing in for a concurrent invalidation removing one mid-scan. */const failingKeys = new Set<string>()
const fakeStorage = { async getWithMetadata(key: string) { promotedKeys.push(key) if (failingKeys.has(key)) { throw Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' }) } return { data: new Uint8Array([1]), metadata: { key,32 unmodified linesdescribe('HTML prewarm', () => { beforeEach(() => { cachedKeys.clear() promotedKeys.length = 0 failingKeys.clear() resetHtmlHotCacheWarmupForTests() })
// A concurrent deletePrefix() can rename a site's directory away mid-scan, so a read // that already passed listKeys can still fail. That must not cost the rest of the site. test('one failing key does not abort the rest of the site', async () => { cachedKeys.add(`${DID}/${RKEY}/index.html`) cachedKeys.add(`${DID}/${RKEY}/a.html`) cachedKeys.add(`${DID}/${RKEY}/b.html`) failingKeys.add(`${DID}/${RKEY}/a.html`)
triggerSiteHtmlHotCacheWarmup(DID, RKEY) await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY)
expect(promotedKeys.sort()).toEqual( [`${DID}/${RKEY}/a.html`, `${DID}/${RKEY}/b.html`, `${DID}/${RKEY}/index.html`].sort(), ) })
// The regression that mattered most: a thrown key skipped warmedSites.add(), so every // later request re-ran the full listing forever. test('a partially failed scan still marks the site warm', async () => { cachedKeys.add(`${DID}/${RKEY}/index.html`) cachedKeys.add(`${DID}/${RKEY}/a.html`) failingKeys.add(`${DID}/${RKEY}/a.html`)
triggerSiteHtmlHotCacheWarmup(DID, RKEY) await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY) expect(promotedKeys).toHaveLength(2)
// Second request must not rescan. triggerSiteHtmlHotCacheWarmup(DID, RKEY) await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY) expect(promotedKeys).toHaveLength(2) })
test('loads all HTML keys for a site on first hit', async () => { cachedKeys.add(`${DID}/${RKEY}/index.html`) cachedKeys.add(`${DID}/${RKEY}/nested/about.htm`) cachedKeys.add(`${DID}/${RKEY}/docs/guide.HTML`)42 unmodified lines17 unmodified lines18192021222223242526272829303132333435363132333437383940414243444546473848495051526 unmodified lines596061625363646566676869707172737475767778798032 unmodified lines17 unmodified lines
async function loadSiteHtmlKeysIntoHotTier( did: string, rkey: string,): Promise<{ scannedKeys: number; warmedHtmlKeys: number }> {): Promise<{ scannedKeys: number; warmedHtmlKeys: number; failedKeys: number }> { const prefix = `${did}/${rkey}/` let scannedKeys = 0 let warmedHtmlKeys = 0 let failedKeys = 0
for await (const key of storage.listKeys(prefix)) { scannedKeys++ if (!isHtmlStorageKey(key)) continue
// Isolate each key: one unreadable entry (e.g. a concurrent invalidation removing it // mid-scan) must not abandon the rest of the site's HTML. An escaping error also // skipped the warmedSites bookkeeping in the caller, so every later request re-ran // this entire listing — the repeated scan this cache exists to prevent. try { // getWithMetadata uses eager promotion and moves the key into hot tier. const result = await storage.getWithMetadata(key) if (result) { warmedHtmlKeys++ // getWithMetadata uses eager promotion and moves the key into hot tier. const result = await storage.getWithMetadata(key) if (result) { warmedHtmlKeys++ } } catch (err) { failedKeys++ logger.debug(`HTML prewarm skipped ${key}`, { error: err }) } }
return { scannedKeys, warmedHtmlKeys } return { scannedKeys, warmedHtmlKeys, failedKeys }}
export function triggerSiteHtmlHotCacheWarmup(did: string, rkey: string): void { const siteKey = getSiteKey(did, rkey)6 unmodified lines const entry = { generation, promise: (async () => { try { const { scannedKeys, warmedHtmlKeys } = await loadSiteHtmlKeysIntoHotTier(did, rkey) const { scannedKeys, warmedHtmlKeys, failedKeys } = await loadSiteHtmlKeysIntoHotTier(did, rkey) const latestGeneration = prewarmGeneration.get(siteKey) ?? 0 if (latestGeneration !== generation) return
// Remember successful scans even when there are no matching keys so repeated // requests for the same missing/empty prefix cannot force repeated storage scans. // Individually skipped keys still count as a completed scan: they are re-fetched // on demand by the normal serving path, which is far cheaper than re-listing. warmedSites.add(siteKey)
logger.debug(`HTML prewarm finished for ${did}/${rkey}`, { scannedKeys, warmedHtmlKeys, failedKeys, }) } catch (err) { logger.warn(`HTML prewarm failed for ${did}/${rkey}`, { error: err }) }32 unmodified lines12345678import type { APIRoute } from 'astro'import { standardPublicationUri } from '../../standard'
/** Confirms that blog.wisp.place is owned by this Standard.site publication. */export const GET: APIRoute = () => new Response(standardPublicationUri, { headers: { 'Content-Type': 'text/plain; charset=utf-8' }, })9 unmodified lines101112131414151616171819202021222324242526272828293031313233343536373738394041414243444445464748494849505152539 unmodified lines6364656667676869697071727373747576777879798081828283848586868788899091909192939 unmodified lines into commit messages, design decisions went into my head, and outages went into whatever I typed on Bluesky at the time. None of that is findable six months later. </p>
<p>So: a blog. Same domain, same design system, no analytics.</p> <p>so: a blog.</p>
<h2>What goes here</h2> <h2>what goes here</h2>
<ul> <li> <strong>Release notes.</strong> When something ships that changes how you deploy or how your <strong>release notes.</strong> when something ships that changes how you deploy or how your sites are served, it gets a post. </li> <li> <strong>Architecture write-ups.</strong> How the firehose consumer, tiered storage and hosting <strong>architecture write-ups.</strong> how the firehose consumer, tiered storage and hosting service fit together, and why they were built the way they were. </li> <li> <strong>Postmortems.</strong> When wisp.place breaks, what broke and what changed afterwards. <strong>postmortems.</strong> when wisp.place breaks, what broke and what changed afterwards. </li> <li> <strong>Protocol notes.</strong> Things learned building on AT Protocol that aren't specific <strong>protocol notes.</strong> things learned building on AT Protocol that aren't specific to wisp.place. </li> </ul>
<p> What doesn't go here: marketing. wisp.place is a passion project, not a startup — there is Wwat doesn't go here: marketing. wisp.place is a passion project, not a startup, so there is nothing to upsell you on. </p>
<h2>The short version of what wisp.place is</h2> <h2>the short version of what wisp.place is</h2>
<p> If you found this post before you found the project: wisp.place is static hosting built on the <a if you found this post before you found the project: wisp.place is static hosting built on the <a href="https://atproto.com" target="_blank" rel="noopener">AT Protocol</a >. You upload your site's files to your own PDS as blobs, and a record lists every file and the site's name. The hosting service watches the network for those records and serves what it finds. >. you upload your site's files to your own PDS as blobs, and a record lists every file and the site's name. the hosting service watches the network for those records and serves what it finds. </p>
<Terminal copy> <span class="t-prompt">$</span> npx wispctl deploy9 unmodified lines <span class="t-out"> https://sites.wisp.place/nekomimi.pet/mysite</span> </Terminal>
<p> The practical consequence is that your site isn't locked in a hosting account. The files live in the practical consequence is that your site isn't locked in a hosting account. the files live in your PDS, under your identity. If wisp.place disappeared tomorrow, someone else could serve the exact same records — the CLI already does this, and so can you. exact same records. the CLI already does this, and so can you. </p>
<Callout type="tip" title="Getting started"> The <a href="https://docs.wisp.place" target="_blank" rel="noopener">docs</a> cover the CLI, the the <a href="https://docs.wisp.place" target="_blank" rel="noopener">docs</a> cover the CLI, the lexicons, redirects and self-hosting. If you just want a site up, drag a folder into the <a href="https://wisp.place/editor">dashboard</a >. </Callout>
<h2>Subscribing</h2> <h2>subscribing</h2>
<p> There's an <a href="/rss.xml">RSS feed</a>. Announcements will also go out on <a There's an <a href="/rss.xml">RSS feed</a>. announcements will also go out on <a href="https://bsky.app/profile/wisp.place" target="_blank" rel="noopener">@wisp.place</a >. There is no mailing list and there won't be one. >. there is no mailing list and there won't be one. </p>
<p> This blog is itself a wisp.place site — a folder of HTML, uploaded to a PDS, served by the same code that serves yours. If it goes down, that's informative too. this blog is itself a wisp.place site. a folder of HTML, uploaded to a PDS, served by the same code that serves yours. if it goes down, that's informative too. </p></Post>124 unmodified linesExpand all1251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611625 unmodified lines16816917017114814915017217317417517617715615715817817918018118264 unmodified lines124 unmodified linesExpand all account. you can open it yourself, create a share link, restrict that link to a particular DID, revoke it later, or let it expire automatically. </p> <pre><code>Creating private site test from /home/regent/Developer/wisp.place/docs/│◇ ✓ Found 2 files (15.3 MB)│◇ ✓ UploadedPrivate site: https://dusky-cactus-iguana-7120.priv.wisp.place/site id: dusky-cactus-iguana-7120files: 2 size: 15.3 MBexpiry: expires 2026-08-29
Only you can open that URL while signed in.Create a shareable link with: wisp private share dusky-cactus-iguana-7120</code></pre> <p> private sites use a dedicated private origin. share tokens are generated from cryptographically random bytes, displayed once, and stored only as hashes. links can expire, be revoked, or be restricted to a specific AT Protocol account. </p> <figure> <img src="/images/private-sites.png" alt="A sign in page for a wisp private site." /> </figure> <p> you can try private sites now by going to <a href="https://wisp.place" >wisp.place</a > or by installing the cli with <code>npm install -g wispctl@1.2.1</code > </p> <h2>future: permissioned data</h2> <p> AT Protocol is developing permissioned data which allows data to remain tied to a user and their AT Protocol identity without being published5 unmodified lines <p> for private wisp sites, the intended future could look roughly like this: </p> <pre> <code>user and wisp.place create a space <pre><code>user and wisp.place create a space ↓site files live in the user's permissioned repository ↓wisp.place receives access to the space ↓wisp gathers, validates, caches, and serves the site </code> </pre>wisp gathers, validates, caches, and serves the site</code></pre> <p> this fits private sites very naturally as the files could once again be owned by the user rather than existing only inside wisp's private storage. wisp could remain the application that evaluates share links,64 unmodified lines121 unmodified linesExpand all12212312412512612612712812913012813113213313413513611 unmodified lines148149150151148149152153154155156157158159160161162163164165166167168158169170171172173854 unmodified linesExpand all121 unmodified linesExpand all if (this.isExpired(result.metadata)) { await this.delete(key) return null } // Eager promotion to hot tier (awaited - guaranteed to complete) // Eager promotion to hot tier (awaited - guaranteed to complete). // Best-effort: see the cold-tier promotion below for why a failed // promotion must not fail the read. if (this.config.tiers.hot && this.config.promotionStrategy === 'eager') { try { await this.config.tiers.hot.set(key, result.data, result.metadata) await this.config.tiers.hot.set(key, result.data, result.metadata) } catch {} } // Fire-and-forget access stats update (non-critical) void this.updateAccessStats(key, 'warm') return {11 unmodified lines await this.delete(key) return null }
// Promote to warm and hot (if configured) // Eager promotion is awaited to guarantee completion // Promote to warm and hot (if configured). // Eager promotion is awaited to guarantee completion. // // Best-effort: a promotion writes a redundant *copy* of data we already hold, // so its failure must never fail the read. Notably, deletePrefix() invalidates // a prefix by renaming its whole directory away, which can land between the // disk tier's directory check and its writeFile and surface as ENOENT. Letting // that escape would discard a successful cold-tier read and, for callers that // promote in a loop, abort the entire batch. if (this.config.promotionStrategy === 'eager') { const promotions: Promise<void>[] = [] if (this.config.tiers.warm) { promotions.push(this.config.tiers.warm.set(key, result.data, result.metadata)) } if (this.config.tiers.hot) { promotions.push(this.config.tiers.hot.set(key, result.data, result.metadata)) } await Promise.all(promotions) await Promise.allSettled(promotions) }
// Fire-and-forget access stats update (non-critical) void this.updateAccessStats(key, 'cold')854 unmodified linesExpand all634 unmodified linesExpand all635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685634 unmodified linesExpand all expect(await warm.exists('other-key')).toBe(true) expect(await cold.exists('other-key')).toBe(true) }) })
// Promotion writes a redundant copy of data the read already holds, so a failed // promotion must never fail the read. In production this surfaced as ENOENT when // deletePrefix() renamed a key's directory away between the disk tier's directory // check and its write, aborting reads that had already succeeded against S3. describe('Promotion failures', () => { const failingTier = (inner: MemoryStorageTier) => Object.assign(Object.create(Object.getPrototypeOf(inner)), inner, { async set() { throw Object.assign(new Error('ENOENT: no such file or directory, open'), { code: 'ENOENT' }) }, })
// Memory tiers throughout: these assert promotion semantics only, and staying off // disk keeps them from touching the shared testDir the other tests here reuse. const memoryTier = () => new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 })
test('cold read still succeeds when promotion to warm throws', async () => { const cold = memoryTier() const warm = failingTier(memoryTier())
await new TieredStorage({ tiers: { cold } }).set('race-key', { message: 'from cold' })
const storage = new TieredStorage({ tiers: { warm, cold }, promotionStrategy: 'eager' }) const result = await storage.getWithMetadata('race-key')
expect(result?.data).toEqual({ message: 'from cold' }) expect(result?.source).toBe('cold') })
test('warm read still succeeds when promotion to hot throws', async () => { const warm = memoryTier() const hot = failingTier(memoryTier())
await new TieredStorage({ tiers: { cold: warm } }).set('race-key', { message: 'from warm' })
const storage = new TieredStorage({ tiers: { hot, warm, cold: memoryTier() }, promotionStrategy: 'eager', }) const result = await storage.getWithMetadata('race-key')
expect(result?.data).toEqual({ message: 'from warm' }) expect(result?.source).toBe('warm') }) })})