From 189d4e7a19e6db1d95c202c9c122706e820ab30c Mon Sep 17 00:00:00 2001 From: Luke Bennett Date: Sun, 14 Jun 2026 12:16:28 +1000 Subject: [PATCH 1/3] standard.site integration --- apps/web/.gitignore | 3 + apps/web/netlify.toml | 3 + apps/web/package.json | 8 +- apps/web/scripts/sync-standard-site.test.ts | 181 ++++++ apps/web/scripts/sync-standard-site.ts | 603 ++++++++++++++++++ apps/web/src/layouts/layout.astro | 7 + apps/web/src/lib/standard-site.test.ts | 94 +++ apps/web/src/lib/standard-site.ts | 67 ++ .../.well-known/site.standard.publication.ts | 14 + apps/web/src/pages/posts/[slug].astro | 4 +- pnpm-lock.yaml | 17 +- 11 files changed, 991 insertions(+), 10 deletions(-) create mode 100644 apps/web/scripts/sync-standard-site.test.ts create mode 100644 apps/web/scripts/sync-standard-site.ts create mode 100644 apps/web/src/lib/standard-site.test.ts create mode 100644 apps/web/src/lib/standard-site.ts create mode 100644 apps/web/src/pages/.well-known/site.standard.publication.ts diff --git a/apps/web/.gitignore b/apps/web/.gitignore index 6582234..30556ba 100644 --- a/apps/web/.gitignore +++ b/apps/web/.gitignore @@ -1,2 +1,5 @@ # Local Netlify folder .netlify + +# Generated Standard.site Record Manifest +src/generated/ diff --git a/apps/web/netlify.toml b/apps/web/netlify.toml index 81d0e5d..eeac732 100644 --- a/apps/web/netlify.toml +++ b/apps/web/netlify.toml @@ -1,3 +1,6 @@ [build] command = "pnpm run build" publish = "apps/web/dist" + +[context.production.build] + command = "pnpm --filter @lukebennett/web build:production" diff --git a/apps/web/package.json b/apps/web/package.json index a822b08..3c57592 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -5,6 +5,7 @@ "scripts": { "astro": "astro", "build": "astro build", + "build:production": "pnpm run check && pnpm run standard-site:sync -- --write && pnpm run build", "check": "pnpm exec prettier 'src/**/*.astro' --cache --check && pnpm biome check && pnpm run check:types", "check:format": "pnpm exec prettier 'src/**/*.astro' --cache --check && pnpm biome format", "check:lint": "pnpm biome lint", @@ -15,7 +16,9 @@ "fix": "pnpm biome check --write", "fix:format": "pnpm exec prettier 'src/**/*.astro' --cache --write && pnpm biome format --write", "fix:lint": "pnpm biome lint --write", - "lint": "pnpm biome lint --write" + "lint": "pnpm biome lint --write", + "standard-site:sync": "node --experimental-strip-types scripts/sync-standard-site.ts", + "test:standard-site": "node --experimental-strip-types --test src/lib/standard-site.test.ts scripts/sync-standard-site.test.ts" }, "dependencies": { "@astrojs/compiler-rs": "^0.1.2", @@ -36,7 +39,8 @@ "shiki": "^4.0.2", "tailwind-merge": "^3.5.0", "tailwindcss": "^4.2.4", - "tiny-invariant": "^1.3.3" + "tiny-invariant": "^1.3.3", + "zod": "^4.4.3" }, "devDependencies": { "@astrojs/check": "^0.9.9", diff --git a/apps/web/scripts/sync-standard-site.test.ts b/apps/web/scripts/sync-standard-site.test.ts new file mode 100644 index 0000000..40e47c7 --- /dev/null +++ b/apps/web/scripts/sync-standard-site.test.ts @@ -0,0 +1,181 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + buildDocumentRecord, + compareOwnedDocumentFields, + extractPortableContent, + getDocumentUri, + isValidRecordKey, + mergeOwnedDocumentFields, + planReconciliation, + toPlainText, +} from './sync-standard-site'; + +test('extractPortableContent returns body after frontmatter', () => { + assert.equal( + extractPortableContent( + 'hello-world.mdoc', + `--- +title: Hello World +publishedAt: 2024-01-20 +isDraft: false +--- +Hello [there](https://example.com). +`, + ), + 'Hello [there](https://example.com).\n', + ); +}); + +test('extractPortableContent rejects missing frontmatter boundary', () => { + assert.throws( + () => extractPortableContent('broken.mdoc', 'Hello world'), + /broken.mdoc is missing frontmatter/, + ); +}); + +test('isValidRecordKey accepts current slug shape', () => { + assert.equal(isValidRecordKey('top-picks-2024-february'), true); + assert.equal(isValidRecordKey('uses'), true); +}); + +test('isValidRecordKey rejects invalid ATProto record keys', () => { + assert.equal(isValidRecordKey('.'), false); + assert.equal(isValidRecordKey('bad/slash'), false); + assert.equal(isValidRecordKey(''), false); +}); + +test('buildDocumentRecord creates a Standard.site document record with Portable Content', () => { + const record = buildDocumentRecord({ + portableContent: 'Hello [there](https://example.com).\n', + publishedAt: '2024-01-20', + slug: 'hello-world', + title: 'Hello World', + }); + + assert.deepEqual(record, { + $type: 'site.standard.document', + content: { + $type: 'at.markpub.markdown', + text: { + $type: 'at.markpub.text', + markdown: 'Hello [there](https://example.com).\n', + }, + }, + description: 'Hello there.', + path: '/posts/hello-world', + publishedAt: '2024-01-20T00:00:00.000Z', + site: 'at://did:plc:3z5ja7l2rhnmtr2bni5dyfe7/site.standard.publication/3mnqwgvxn372f', + textContent: 'Hello there.', + title: 'Hello World', + }); +}); + +test('getDocumentUri uses raw slug as record key', () => { + assert.equal( + getDocumentUri('did:plc:example', 'hello-world'), + 'at://did:plc:example/site.standard.document/hello-world', + ); +}); + +test('compareOwnedDocumentFields ignores unknown remote fields', () => { + const generated = buildDocumentRecord({ + portableContent: 'Hello.', + publishedAt: '2024-01-20', + slug: 'hello-world', + title: 'Hello World', + }); + + assert.equal( + compareOwnedDocumentFields(generated, { + ...generated, + unknownFutureField: true, + }), + true, + ); +}); + +test('mergeOwnedDocumentFields preserves unknown remote fields', () => { + const generated = buildDocumentRecord({ + portableContent: 'Hello.', + publishedAt: '2024-01-20', + slug: 'hello-world', + title: 'Hello World', + }); + + assert.deepEqual(mergeOwnedDocumentFields({ extra: 'keep' }, generated), { + ...generated, + extra: 'keep', + }); +}); + +test('planReconciliation creates missing records and deletes withdrawn post records', () => { + const post = { + portableContent: 'Hello.', + publishedAt: '2024-01-20', + slug: 'hello-world', + title: 'Hello World', + }; + + const plan = planReconciliation({ + did: 'did:plc:example', + existingRecords: [ + { + uri: 'at://did:plc:example/site.standard.document/old-post', + value: { + $type: 'site.standard.document', + path: '/posts/old-post', + site: 'at://did:plc:3z5ja7l2rhnmtr2bni5dyfe7/site.standard.publication/3mnqwgvxn372f', + }, + }, + ], + posts: [post], + }); + + assert.deepEqual( + plan.creates.map((item) => item.slug), + ['hello-world'], + ); + assert.deepEqual( + plan.deletes.map((item) => item.slug), + ['old-post'], + ); +}); + +test('planReconciliation fails on unexpected existing record key', () => { + assert.throws( + () => + planReconciliation({ + did: 'did:plc:example', + existingRecords: [ + { + uri: 'at://did:plc:example/site.standard.document/generated-key', + value: { + $type: 'site.standard.document', + path: '/posts/hello-world', + site: 'at://did:plc:3z5ja7l2rhnmtr2bni5dyfe7/site.standard.publication/3mnqwgvxn372f', + }, + }, + ], + posts: [ + { + portableContent: 'Hello.', + publishedAt: '2024-01-20', + slug: 'hello-world', + title: 'Hello World', + }, + ], + }), + /manual cleanup/, + ); +}); + +test('toPlainText strips Markdown links and Markdoc component blocks', () => { + assert.equal( + toPlainText(`Hello [there](https://example.com). + +{% cloudImage src="https://example.com/image" /%} +`), + 'Hello there.', + ); +}); diff --git a/apps/web/scripts/sync-standard-site.ts b/apps/web/scripts/sync-standard-site.ts new file mode 100644 index 0000000..a5418ca --- /dev/null +++ b/apps/web/scripts/sync-standard-site.ts @@ -0,0 +1,603 @@ +import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { basename } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +type PublishedPost = { + portableContent: string; + publishedAt: string; + slug: string; + title: string; +}; + +type DocumentRecord = { + $type: typeof COLLECTION; + content: { + $type: 'at.markpub.markdown'; + text: { + $type: 'at.markpub.text'; + markdown: string; + }; + }; + description: string; + path: `/posts/${string}`; + publishedAt: string; + site: typeof PUBLICATION_URI; + textContent: string; + title: string; +}; + +type DidDocument = { + service?: Array<{ + serviceEndpoint?: string; + type?: string; + }>; +}; + +type Session = { + accessJwt: string; + did: string; +}; + +type ExistingRecord = { + uri: `at://${string}`; + value: Record; +}; + +type ListRecordsResponse = { + cursor?: string; + records: Array; +}; + +type WriteRecordResponse = { + uri: `at://${string}`; +}; + +const APP_DIR = new URL('..', import.meta.url); +const POSTS_DIR = new URL('content/posts/', APP_DIR); +const GENERATED_DIR = new URL('src/generated/', APP_DIR); +const MANIFEST_URL = new URL('src/generated/standard-site.json', APP_DIR); +const COLLECTION = 'site.standard.document'; +const PUBLICATION_URI = + 'at://did:plc:3z5ja7l2rhnmtr2bni5dyfe7/site.standard.publication/3mnqwgvxn372f'; +const DEFAULT_IDENTIFIER = 'lukebennett.dev'; + +export function extractPortableContent( + filename: string, + source: string, +): string { + if (!source.startsWith('---\n')) { + throw new Error(`${filename} is missing frontmatter`); + } + + const frontmatterEnd = source.indexOf('\n---\n', 4); + + if (frontmatterEnd === -1) { + throw new Error(`${filename} has invalid frontmatter`); + } + + return source.slice(frontmatterEnd + '\n---\n'.length); +} + +const RECORD_KEY_PATTERN = /^(?!\.\.?$)[A-Za-z0-9._:~-]{1,512}$/; + +export function isValidRecordKey(value: string): boolean { + return RECORD_KEY_PATTERN.test(value); +} + +const OWNED_DOCUMENT_FIELDS = [ + '$type', + 'content', + 'description', + 'path', + 'publishedAt', + 'site', + 'textContent', + 'title', +] as const; + +export function getDocumentUri( + did: string, + slug: string, +): `at://${string}/site.standard.document/${string}` { + if (!isValidRecordKey(slug)) { + throw new Error(`${slug} is not a valid AT Protocol record key`); + } + + return `at://${did}/site.standard.document/${slug}`; +} + +export function assertExpectedDocumentUri(slug: string, uri: string): void { + if (getRkey(uri) !== slug) { + throw new Error(`${uri} has unexpected record key for ${slug}`); + } +} + +export function compareOwnedDocumentFields( + generated: DocumentRecord, + remote: Record, +): boolean { + return OWNED_DOCUMENT_FIELDS.every( + (field) => + JSON.stringify(remote[field]) === JSON.stringify(generated[field]), + ); +} + +export function mergeOwnedDocumentFields( + remote: Record, + generated: DocumentRecord, +): Record { + return { + ...remote, + ...generated, + }; +} + +type ReconciliationPlan = { + creates: Array<{ + record: DocumentRecord; + slug: string; + uri: `at://${string}`; + }>; + deletes: Array<{ record: ExistingRecord; slug: string }>; + noops: Array<{ record: ExistingRecord; slug: string; uri: `at://${string}` }>; + updates: Array<{ + record: DocumentRecord; + remote: ExistingRecord; + slug: string; + uri: `at://${string}`; + }>; +}; + +export function planReconciliation({ + did, + existingRecords, + posts, +}: { + did: string; + existingRecords: Array; + posts: Array; +}): ReconciliationPlan { + const postsByPath = new Map( + posts.map((post) => [`/posts/${post.slug}`, post]), + ); + const existingByPath = new Map(); + + for (const record of existingRecords) { + const value = record.value ?? {}; + if ( + value.site !== PUBLICATION_URI || + typeof value.path !== 'string' || + !value.path.startsWith('/posts/') + ) { + continue; + } + + const slug = value.path.slice('/posts/'.length); + + if (getRkey(record.uri) !== slug) { + throw new Error( + `${record.uri} conflicts with ${value.path}; manual cleanup required`, + ); + } + + existingByPath.set(value.path, record); + } + + const plan: ReconciliationPlan = { + creates: [], + deletes: [], + noops: [], + updates: [], + }; + + for (const post of posts) { + const record = buildDocumentRecord(post); + const uri = getDocumentUri(did, post.slug); + const remote = existingByPath.get(record.path); + + if (!remote) { + plan.creates.push({ record, slug: post.slug, uri }); + continue; + } + + assertExpectedDocumentUri(post.slug, remote.uri); + + if (compareOwnedDocumentFields(record, remote.value)) { + plan.noops.push({ record: remote, slug: post.slug, uri: remote.uri }); + continue; + } + + plan.updates.push({ record, remote, slug: post.slug, uri: remote.uri }); + } + + for (const [path, record] of existingByPath.entries()) { + if (!postsByPath.has(path)) { + plan.deletes.push({ record, slug: path.slice('/posts/'.length) }); + } + } + + return plan; +} + +export function buildDocumentRecord(post: PublishedPost): DocumentRecord { + const textContent = toPlainText(post.portableContent); + + return { + $type: COLLECTION, + content: { + $type: 'at.markpub.markdown', + text: { + $type: 'at.markpub.text', + markdown: post.portableContent, + }, + }, + description: truncateDescription(textContent), + path: `/posts/${post.slug}`, + publishedAt: new Date(`${post.publishedAt}T00:00:00.000Z`).toISOString(), + site: PUBLICATION_URI, + textContent, + title: post.title, + }; +} + +async function removeManifest(): Promise { + await rm(MANIFEST_URL, { force: true }); +} + +async function writeManifest( + documentsBySlug: Map, +): Promise { + await mkdir(GENERATED_DIR, { recursive: true }); + await writeFile( + MANIFEST_URL, + `${JSON.stringify( + { + publicationUri: PUBLICATION_URI, + documentsBySlug: Object.fromEntries( + [...documentsBySlug.entries()].sort(([a], [b]) => a.localeCompare(b)), + ), + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +export function toPlainText(markdoc: string): string { + return markdoc + .replaceAll(/```[\s\S]*?```/g, '') + .replaceAll(/!\[([^\]]*)\]\([^)]+\)/g, '$1') + .replaceAll(/\[([^\]]+)\]\([^)]+\)/g, '$1') + .replaceAll(/`([^`]+)`/g, '$1') + .replaceAll(/[*_~>#]/g, '') + .replaceAll(/\{[%#][\s\S]*?[%#]\}/g, '') + .replaceAll(/\\\n/g, '\n') + .replaceAll(/\n{3,}/g, '\n\n') + .trim(); +} + +function truncateDescription(textContent: string): string { + const maxLength = 180; + if (textContent.length <= maxLength) { + return textContent; + } + + return `${textContent.slice(0, maxLength - 1).trimEnd()}…`; +} + +export async function loadPublishedPosts(): Promise> { + const filenames = (await readdir(POSTS_DIR)) + .filter((filename) => filename.endsWith('.mdoc')) + .sort(); + const posts: Array = []; + + for (const filename of filenames) { + const source = await readFile(new URL(filename, POSTS_DIR), 'utf8'); + const slug = basename(filename, '.mdoc'); + + if (!source.startsWith('---\n')) { + continue; + } + + const frontmatterEnd = source.indexOf('\n---\n', 4); + if (frontmatterEnd === -1) { + continue; + } + + const frontmatter = source.slice(4, frontmatterEnd); + const fields = parseSimpleFrontmatter(frontmatter); + + if (fields.isDraft === 'true') { + continue; + } + + const portableContent = source.slice(frontmatterEnd + '\n---\n'.length); + + if (!fields.publishedAt || !fields.title) { + continue; + } + + posts.push({ + portableContent, + publishedAt: fields.publishedAt, + slug, + title: fields.title, + }); + } + + return posts; +} + +function parseSimpleFrontmatter( + frontmatter: string, +): Record { + const fields: Record = {}; + + for (const line of frontmatter.split('\n')) { + const separator = line.indexOf(':'); + if (separator === -1) { + continue; + } + + const key = line.slice(0, separator).trim(); + const rawValue = line.slice(separator + 1).trim(); + fields[key] = rawValue.replace(/^['"](.*)['"]$/, '$1'); + } + + return fields; +} + +async function xrpc( + pds: string, + path: string, + init: RequestInit = {}, +): Promise { + const response = await fetch(`${pds}/xrpc/${path}`, { + ...init, + headers: { + 'Content-Type': 'application/json', + ...init.headers, + }, + }); + const text = await response.text(); + const body = text ? JSON.parse(text) : undefined; + + if (!response.ok) { + throw new Error(`${response.status} ${response.statusText}: ${text}`); + } + + return body as ResponseBody; +} + +async function resolveDid(identifier: string): Promise { + if (identifier.startsWith('did:')) { + return identifier; + } + + const response = await fetch( + `https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=${encodeURIComponent(identifier)}`, + ); + const body = (await response.json()) as { did?: string }; + + if (!response.ok) { + throw new Error(`Could not resolve ${identifier}: ${JSON.stringify(body)}`); + } + + if (!body.did) { + throw new Error(`Could not resolve ${identifier}`); + } + + return body.did; +} + +async function resolvePds(did: string): Promise { + const response = await fetch(`https://plc.directory/${did}`); + const body = (await response.json()) as DidDocument; + + if (!response.ok) { + throw new Error(`Could not resolve DID document for ${did}`); + } + + const service = body.service?.find( + (item) => item.type === 'AtprotoPersonalDataServer', + ); + if (!service?.serviceEndpoint) { + throw new Error(`${did} has no AT Protocol PDS service`); + } + + return service.serviceEndpoint.replace(/\/$/, ''); +} + +async function createSession( + pds: string, + identifier: string, + password: string, +): Promise { + return xrpc(pds, 'com.atproto.server.createSession', { + body: JSON.stringify({ identifier, password }), + method: 'POST', + }); +} + +async function listRecords( + pds: string, + auth: Record, + repo: string, + collection: string, +): Promise> { + const records: Array = []; + let cursor: string | undefined; + + do { + const query = new URLSearchParams({ + collection, + limit: '100', + repo, + }); + if (cursor) { + query.set('cursor', cursor); + } + + const body = await xrpc( + pds, + `com.atproto.repo.listRecords?${query}`, + { + headers: auth, + }, + ); + records.push(...body.records); + cursor = body.cursor; + } while (cursor); + + return records; +} + +function getRkey(uri: string): string { + const index = uri.lastIndexOf('/'); + return uri.slice(index + 1); +} + +async function authenticate( + identifier: string, + password: string, +): Promise<{ auth: Record; did: string; pds: string }> { + const did = await resolveDid(identifier); + const pds = await resolvePds(did); + const session = await createSession(pds, identifier, password); + return { auth: { Authorization: `Bearer ${session.accessJwt}` }, did, pds }; +} + +function printPlan(plan: ReconciliationPlan): void { + for (const item of plan.creates) { + console.log(`create ${item.slug}`); + } + for (const item of plan.updates) { + console.log(`update ${item.slug}`); + } + for (const item of plan.deletes) { + console.log(`delete ${item.slug}`); + } + for (const item of plan.noops) { + console.log(`noop ${item.slug}`); + } + console.log( + `${plan.creates.length} creates, ${plan.updates.length} updates, ${plan.deletes.length} deletes, ${plan.noops.length} noops`, + ); +} + +async function executePlan( + pds: string, + auth: Record, + repo: string, + plan: ReconciliationPlan, +): Promise> { + const documentsBySlug = new Map(); + + for (const item of plan.creates) { + const result = await xrpc( + pds, + 'com.atproto.repo.createRecord', + { + body: JSON.stringify({ + collection: COLLECTION, + record: item.record, + repo, + validate: false, + }), + headers: auth, + method: 'POST', + }, + ); + documentsBySlug.set(item.slug, result.uri); + } + + for (const item of plan.updates) { + const result = await xrpc( + pds, + 'com.atproto.repo.putRecord', + { + body: JSON.stringify({ + collection: COLLECTION, + record: mergeOwnedDocumentFields(item.remote.value, item.record), + repo, + rkey: getRkey(item.remote.uri), + validate: false, + }), + headers: auth, + method: 'POST', + }, + ); + documentsBySlug.set(item.slug, result.uri); + } + + for (const item of plan.deletes) { + await xrpc(pds, 'com.atproto.repo.deleteRecord', { + body: JSON.stringify({ + collection: COLLECTION, + repo, + rkey: getRkey(item.record.uri), + }), + headers: auth, + method: 'POST', + }); + } + + for (const item of plan.noops) { + documentsBySlug.set(item.slug, item.uri); + } + + return documentsBySlug; +} + +async function main(): Promise { + const args = new Set(process.argv.slice(2)); + + if (args.has('--help')) { + console.log('Usage: pnpm standard-site:sync [-- --report | -- --write]'); + return; + } + + if (!args.has('--report') && !args.has('--write')) { + const posts = await loadPublishedPosts(); + for (const post of posts) { + console.log(`${post.slug} -> /posts/${post.slug}`); + } + console.log(`published posts: ${posts.length}`); + return; + } + + const password = + process.env.STANDARD_SITE_APP_PASSWORD ?? process.env.BSKY_APP_PASSWORD; + if (!password) { + throw new Error('Set STANDARD_SITE_APP_PASSWORD or BSKY_APP_PASSWORD'); + } + + const identifier = process.env.ATPROTO_IDENTIFIER ?? DEFAULT_IDENTIFIER; + const { auth, did, pds } = await authenticate(identifier, password); + const posts = await loadPublishedPosts(); + const existingRecords = await listRecords(pds, auth, did, COLLECTION); + const plan = planReconciliation({ did, existingRecords, posts }); + + if (args.has('--report')) { + printPlan(plan); + return; + } + + if (args.has('--write')) { + await removeManifest(); + const documentsBySlug = await executePlan(pds, auth, did, plan); + await writeManifest(documentsBySlug); + printPlan(plan); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main().catch((error) => { + console.error(error.message); + process.exit(1); + }); +} diff --git a/apps/web/src/layouts/layout.astro b/apps/web/src/layouts/layout.astro index fe43d69..f600ad5 100644 --- a/apps/web/src/layouts/layout.astro +++ b/apps/web/src/layouts/layout.astro @@ -11,11 +11,13 @@ import '../styles.css'; export type Props = { description?: string; + standardSiteDocumentUri?: string; title?: string; }; const { description = 'Luke Bennett’s personal website', + standardSiteDocumentUri, title = 'Luke Bennett', } = Astro.props; @@ -57,6 +59,11 @@ const ogImage = new URL('/og.jpg', Astro.site).toString(); content="index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1" /> + { + standardSiteDocumentUri ? ( + + ) : null + } diff --git a/apps/web/src/lib/standard-site.test.ts b/apps/web/src/lib/standard-site.test.ts new file mode 100644 index 0000000..560791b --- /dev/null +++ b/apps/web/src/lib/standard-site.test.ts @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { + getStandardSiteDocumentUri, + loadStandardSiteManifest, + standardSitePublicationUri, +} from './standard-site'; + +test('keeps committed publication URI available', () => { + assert.equal( + standardSitePublicationUri, + 'at://did:plc:3z5ja7l2rhnmtr2bni5dyfe7/site.standard.publication/3mnqwgvxn372f', + ); +}); + +test('returns an empty document map when manifest is missing outside production', () => { + const manifest = loadStandardSiteManifest({ + manifestPath: join(tmpdir(), 'missing-standard-site.json'), + requireManifest: false, + }); + + assert.deepEqual(manifest.documentsBySlug, {}); +}); + +test('throws when manifest is required and missing', () => { + assert.throws( + () => + loadStandardSiteManifest({ + manifestPath: join(tmpdir(), 'missing-standard-site.json'), + requireManifest: true, + }), + /Record Manifest artifact is required/, + ); +}); + +test('throws when manifest JSON is malformed', async () => { + const dir = await mkdtemp(join(tmpdir(), 'standard-site-')); + const manifestPath = join(dir, 'standard-site.json'); + + try { + await writeFile(manifestPath, '{bad json', 'utf8'); + + assert.throws( + () => + loadStandardSiteManifest({ + manifestPath, + requireManifest: false, + }), + /Record Manifest artifact is malformed/, + ); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test('reads document URIs from valid manifest', async () => { + const dir = await mkdtemp(join(tmpdir(), 'standard-site-')); + const manifestPath = join(dir, 'standard-site.json'); + const documentUri = + 'at://did:plc:3z5ja7l2rhnmtr2bni5dyfe7/site.standard.document/3testrecord'; + + try { + await writeFile( + manifestPath, + JSON.stringify({ + publicationUri: standardSitePublicationUri, + documentsBySlug: { + 'hello-world': documentUri, + }, + }), + 'utf8', + ); + + const manifest = loadStandardSiteManifest({ + manifestPath, + requireManifest: false, + }); + + assert.equal(manifest.documentsBySlug['hello-world'], documentUri); + assert.equal( + getStandardSiteDocumentUri('hello-world', { + manifestPath, + requireManifest: false, + }), + documentUri, + ); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); diff --git a/apps/web/src/lib/standard-site.ts b/apps/web/src/lib/standard-site.ts new file mode 100644 index 0000000..15d53ab --- /dev/null +++ b/apps/web/src/lib/standard-site.ts @@ -0,0 +1,67 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { z } from 'zod'; + +export const standardSitePublicationUri = + 'at://did:plc:3z5ja7l2rhnmtr2bni5dyfe7/site.standard.publication/3mnqwgvxn372f'; + +const StandardSiteManifestSchema = z + .object({ + publicationUri: z.literal(standardSitePublicationUri), + documentsBySlug: z.record(z.string(), z.string().startsWith('at://')), + }) + .readonly(); + +type StandardSiteManifest = z.infer; + +type ManifestOptions = { + manifestPath?: string; + requireManifest?: boolean; +}; + +const defaultManifestPath = fileURLToPath( + new URL('../generated/standard-site.json', import.meta.url), +); + +export function loadStandardSiteManifest({ + manifestPath = defaultManifestPath, + requireManifest = process.env.CONTEXT === 'production', +}: ManifestOptions = {}): StandardSiteManifest { + if (!existsSync(manifestPath)) { + if (requireManifest) { + throw new Error(`Record Manifest artifact is required: ${manifestPath}`); + } + + return { + documentsBySlug: {}, + publicationUri: standardSitePublicationUri, + }; + } + + let parsed: unknown; + + try { + parsed = JSON.parse(readFileSync(manifestPath, 'utf8')); + } catch (error) { + throw new Error(`Record Manifest artifact is malformed: ${manifestPath}`, { + cause: error, + }); + } + + const result = StandardSiteManifestSchema.safeParse(parsed); + + if (!result.success) { + throw new Error(`Record Manifest artifact is malformed: ${manifestPath}`, { + cause: result.error, + }); + } + + return result.data; +} + +export function getStandardSiteDocumentUri( + slug: string, + options?: ManifestOptions, +) { + return loadStandardSiteManifest(options).documentsBySlug[slug]; +} diff --git a/apps/web/src/pages/.well-known/site.standard.publication.ts b/apps/web/src/pages/.well-known/site.standard.publication.ts new file mode 100644 index 0000000..3d3312d --- /dev/null +++ b/apps/web/src/pages/.well-known/site.standard.publication.ts @@ -0,0 +1,14 @@ +import type { APIRoute } from 'astro'; +import { standardSitePublicationUri } from '../../lib/standard-site'; + +export const GET: APIRoute = () => { + if (!standardSitePublicationUri) { + return new Response(null, { status: 404 }); + } + + return new Response(standardSitePublicationUri, { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + }, + }); +}; diff --git a/apps/web/src/pages/posts/[slug].astro b/apps/web/src/pages/posts/[slug].astro index 7c694b0..e6119a3 100644 --- a/apps/web/src/pages/posts/[slug].astro +++ b/apps/web/src/pages/posts/[slug].astro @@ -3,6 +3,7 @@ import invariant from 'tiny-invariant'; import Post from '../../components/post.astro'; import Layout from '../../layouts/layout.astro'; import { reader } from '../../lib/keystatic/reader'; +import { getStandardSiteDocumentUri } from '../../lib/standard-site'; const { slug } = Astro.params; invariant(slug, 'Slug not found'); @@ -17,6 +18,7 @@ if (isDraft && process.env.NODE_ENV === 'production') { } const document = await content(); +const standardSiteDocumentUri = getStandardSiteDocumentUri(slug); // Generate static pages export async function getStaticPaths() { @@ -25,7 +27,7 @@ export async function getStaticPaths() { } --- - +

{title}

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4e61ca1..e7b060f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,6 +80,9 @@ importers: tiny-invariant: specifier: ^1.3.3 version: 1.3.3 + zod: + specifier: ^4.4.3 + version: 4.4.3 devDependencies: '@astrojs/check': specifier: ^0.9.9 @@ -5474,8 +5477,8 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - zod@4.4.2: - resolution: {integrity: sha512-IynmDyxsEsb9RKzO3J9+4SxXnl2FTFSzNBaKKaMV6tsSk0rw9gYw9gs+JFCq/qk2LCZ78KDwyj+Z289TijSkUw==} + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -5707,13 +5710,13 @@ snapshots: dependencies: fast-xml-parser: 5.7.2 piccolore: 0.1.3 - zod: 4.4.2 + zod: 4.4.3 '@astrojs/sitemap@3.7.2': dependencies: sitemap: 9.0.1 stream-replace-string: 2.0.0 - zod: 4.4.2 + zod: 4.4.3 '@astrojs/telemetry@3.3.1': dependencies: @@ -6668,7 +6671,7 @@ snapshots: validate-npm-package-name: 5.0.1 yaml: 2.8.4 yargs: 17.7.2 - zod: 4.4.2 + zod: 4.4.3 '@netlify/database-dev@0.10.1': dependencies: @@ -8748,7 +8751,7 @@ snapshots: vitefu: 1.1.3(vite@7.3.2(@types/node@24.12.2)(jiti@2.6.1)(lightningcss@1.32.0)(yaml@2.8.4)) xxhash-wasm: 1.1.0 yargs-parser: 22.0.0 - zod: 4.4.2 + zod: 4.4.3 optionalDependencies: sharp: 0.34.4 transitivePeerDependencies: @@ -11902,6 +11905,6 @@ snapshots: zod@3.25.76: {} - zod@4.4.2: {} + zod@4.4.3: {} zwitch@2.0.4: {} From 92073a32887269c64903875338fbed2af4a4290e Mon Sep 17 00:00:00 2001 From: Luke Bennett Date: Sun, 14 Jun 2026 12:21:40 +1000 Subject: [PATCH 2/3] Add docs --- docs/standard-site.md | 76 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 docs/standard-site.md diff --git a/docs/standard-site.md b/docs/standard-site.md new file mode 100644 index 0000000..5ff44a5 --- /dev/null +++ b/docs/standard-site.md @@ -0,0 +1,76 @@ +# Standard.site + +Architecture, domain language, and build orchestration. + +## Domain Language + +### Canonical Website + +The website and its Keystatic content are the source of truth for published writing. +*Avoid*: PDS source of truth, Standard.site source of truth + +### Mirrored Record + +An AT Protocol record that represents a post already published on the Canonical Website. +*Avoid*: Original post, primary content + +### Portable Content + +The full Post body stored in a Mirrored Record as authored Markdown/Markdoc, including component syntax and public asset URLs, with a plain-text fallback. +*Avoid*: Metadata-only record, stripped copy, ATProto blob copy + +### Record Manifest + +A website-owned list that maps published post slugs to their Mirrored Record AT-URIs. Written at `apps/web/src/generated/standard-site.json` during production deploys. +*Avoid*: Content source, generated content + +### Post + +A piece of writing whose public identity is its `/posts/*` path and can be mirrored as a Standard.site document. +*Avoid*: Link, bookmark + +### Withdrawn Post + +A Post that has been deleted or returned to draft and is no longer part of the Canonical Website's published set or its Mirrored Records. +*Avoid*: Deleted record, missing post + +## Architecture + +### Source of truth + +The website and Keystatic content are canonical. AT Protocol records are mirrors, deploy output not committed state. The publication AT-URI is the only committed Standard.site identifier. + +### Build modes + +| Context | Command | Manifest required | ATProto writes | +|---|---|---|---| +| Local dev / preview | `pnpm run build` | No | Never | +| Netlify production | `pnpm run build:production` | Yes (generated by sync) | Yes | + +Production build pipeline: `check`, then `standard-site:sync --write`, then `build`. Preview builds skip Standard.site entirely. + +### Record Manifest artifact + +Written to `apps/web/src/generated/standard-site.json` (gitignored). Contains `publicationUri` and `documentsBySlug`, no generation metadata. Removed before each production sync, written fresh only after successful AT-URI validation. + +Missing manifest: returns empty `documentsBySlug` outside production, fails production builds. Malformed manifest: fails all builds. + +### Sync command modes + +- No flags: offline dry run. Loads published posts, prints plan. No credentials required. +- `--report`: fetches remote records, runs reconciliation, prints plan. Credentials required. +- `--write`: removes stale manifest, executes reconciliation, writes manifest. Credentials required. + +### Record keys + +Raw post slugs are stable ATProto record keys. If a remote record exists for this publication at a path like `/posts/foo` with a record key that doesn't match `foo`, the reconciliation fails with a manual cleanup error. + +### Owned fields + +Reconciliation compares only `$type`, `content`, `description`, `path`, `publishedAt`, `site`, `textContent`, `title`. Unknown remote fields are preserved during updates. Document descriptions are generated from the first 180 characters of plain-text Portable Content. + +### Out of scope + +- `coverImage` is not included in the first implementation. +- Cleanup of old prototype or manually-created records is a separate one-off operation. +- Reconciliation only manages `site.standard.document` records for this publication with paths starting with `/posts/`. From 95903408a3a8186d4a12f929f856eb4f95e2e80e Mon Sep 17 00:00:00 2001 From: Luke Bennett Date: Sun, 14 Jun 2026 12:45:05 +1000 Subject: [PATCH 3/3] standard-site: replace type assertions with Zod inference, add runtime URI validation --- apps/web/scripts/sync-standard-site.test.ts | 2 +- apps/web/scripts/sync-standard-site.ts | 144 +++++++++++++------- apps/web/src/lib/standard-site.test.ts | 2 +- apps/web/src/lib/standard-site.ts | 2 +- apps/web/tsconfig.json | 5 +- 5 files changed, 100 insertions(+), 55 deletions(-) diff --git a/apps/web/scripts/sync-standard-site.test.ts b/apps/web/scripts/sync-standard-site.test.ts index 40e47c7..783e410 100644 --- a/apps/web/scripts/sync-standard-site.test.ts +++ b/apps/web/scripts/sync-standard-site.test.ts @@ -9,7 +9,7 @@ import { mergeOwnedDocumentFields, planReconciliation, toPlainText, -} from './sync-standard-site'; +} from './sync-standard-site.ts'; test('extractPortableContent returns body after frontmatter', () => { assert.equal( diff --git a/apps/web/scripts/sync-standard-site.ts b/apps/web/scripts/sync-standard-site.ts index a5418ca..58fb2f8 100644 --- a/apps/web/scripts/sync-standard-site.ts +++ b/apps/web/scripts/sync-standard-site.ts @@ -1,6 +1,7 @@ import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import { basename } from 'node:path'; import { pathToFileURL } from 'node:url'; +import * as z from 'zod'; type PublishedPost = { portableContent: string; @@ -26,31 +27,43 @@ type DocumentRecord = { title: string; }; -type DidDocument = { - service?: Array<{ - serviceEndpoint?: string; - type?: string; - }>; -}; +const SessionSchema = z.object({ + accessJwt: z.string(), + did: z.string(), +}); -type Session = { - accessJwt: string; - did: string; -}; +type Session = z.infer; -type ExistingRecord = { - uri: `at://${string}`; - value: Record; -}; +const ExistingRecordSchema = z.object({ + uri: z.string().startsWith('at://'), + value: z.record(z.string(), z.unknown()), +}); -type ListRecordsResponse = { - cursor?: string; - records: Array; -}; +type ExistingRecord = z.infer; -type WriteRecordResponse = { - uri: `at://${string}`; -}; +const ListRecordsResponseSchema = z.object({ + cursor: z.string().optional(), + records: z.array(ExistingRecordSchema), +}); + +const WriteRecordResponseSchema = z.object({ + uri: z.string().startsWith('at://'), +}); + +const DidDocumentSchema = z.object({ + service: z + .array( + z.object({ + serviceEndpoint: z.string().optional(), + type: z.string().optional(), + }), + ) + .optional(), +}); + +const DidResponseSchema = z.object({ + did: z.string().optional(), +}); const APP_DIR = new URL('..', import.meta.url); const POSTS_DIR = new URL('content/posts/', APP_DIR); @@ -95,10 +108,7 @@ const OWNED_DOCUMENT_FIELDS = [ 'title', ] as const; -export function getDocumentUri( - did: string, - slug: string, -): `at://${string}/site.standard.document/${string}` { +export function getDocumentUri(did: string, slug: string): string { if (!isValidRecordKey(slug)) { throw new Error(`${slug} is not a valid AT Protocol record key`); } @@ -136,15 +146,15 @@ type ReconciliationPlan = { creates: Array<{ record: DocumentRecord; slug: string; - uri: `at://${string}`; + uri: string; }>; deletes: Array<{ record: ExistingRecord; slug: string }>; - noops: Array<{ record: ExistingRecord; slug: string; uri: `at://${string}` }>; + noops: Array<{ record: ExistingRecord; slug: string; uri: string }>; updates: Array<{ record: DocumentRecord; remote: ExistingRecord; slug: string; - uri: `at://${string}`; + uri: string; }>; }; @@ -245,7 +255,7 @@ async function removeManifest(): Promise { } async function writeManifest( - documentsBySlug: Map, + documentsBySlug: Map, ): Promise { await mkdir(GENERATED_DIR, { recursive: true }); await writeFile( @@ -264,16 +274,25 @@ async function writeManifest( ); } +const FENCE_PATTERN = /```[\s\S]*?```/g; +const IMAGE_PATTERN = /!\[([^\]]*)\]\([^)]+\)/g; +const LINK_PATTERN = /\[([^\]]+)\]\([^)]+\)/g; +const INLINE_CODE_PATTERN = /`([^`]+)`/g; +const FORMATTING_CHARS_PATTERN = /[*_~>#]/g; +const TAG_PATTERN = /\{[%#][\s\S]*?[%#]\}/g; +const HARD_BREAK_PATTERN = /\\\n/g; +const EXCESS_NEWLINES_PATTERN = /\n{3,}/g; + export function toPlainText(markdoc: string): string { return markdoc - .replaceAll(/```[\s\S]*?```/g, '') - .replaceAll(/!\[([^\]]*)\]\([^)]+\)/g, '$1') - .replaceAll(/\[([^\]]+)\]\([^)]+\)/g, '$1') - .replaceAll(/`([^`]+)`/g, '$1') - .replaceAll(/[*_~>#]/g, '') - .replaceAll(/\{[%#][\s\S]*?[%#]\}/g, '') - .replaceAll(/\\\n/g, '\n') - .replaceAll(/\n{3,}/g, '\n\n') + .replaceAll(FENCE_PATTERN, '') + .replaceAll(IMAGE_PATTERN, '$1') + .replaceAll(LINK_PATTERN, '$1') + .replaceAll(INLINE_CODE_PATTERN, '$1') + .replaceAll(FORMATTING_CHARS_PATTERN, '') + .replaceAll(TAG_PATTERN, '') + .replaceAll(HARD_BREAK_PATTERN, '\n') + .replaceAll(EXCESS_NEWLINES_PATTERN, '\n\n') .trim(); } @@ -351,6 +370,7 @@ function parseSimpleFrontmatter( async function xrpc( pds: string, path: string, + schema: z.ZodType, init: RequestInit = {}, ): Promise { const response = await fetch(`${pds}/xrpc/${path}`, { @@ -361,13 +381,31 @@ async function xrpc( }, }); const text = await response.text(); - const body = text ? JSON.parse(text) : undefined; if (!response.ok) { throw new Error(`${response.status} ${response.statusText}: ${text}`); } - return body as ResponseBody; + return schema.parse(JSON.parse(text)); +} + +async function xrpcVoid( + pds: string, + path: string, + init: RequestInit = {}, +): Promise { + const response = await fetch(`${pds}/xrpc/${path}`, { + ...init, + headers: { + 'Content-Type': 'application/json', + ...init.headers, + }, + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error(`${response.status} ${response.statusText}: ${text}`); + } } async function resolveDid(identifier: string): Promise { @@ -378,7 +416,7 @@ async function resolveDid(identifier: string): Promise { const response = await fetch( `https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=${encodeURIComponent(identifier)}`, ); - const body = (await response.json()) as { did?: string }; + const body = DidResponseSchema.parse(await response.json()); if (!response.ok) { throw new Error(`Could not resolve ${identifier}: ${JSON.stringify(body)}`); @@ -393,7 +431,7 @@ async function resolveDid(identifier: string): Promise { async function resolvePds(did: string): Promise { const response = await fetch(`https://plc.directory/${did}`); - const body = (await response.json()) as DidDocument; + const body = DidDocumentSchema.parse(await response.json()); if (!response.ok) { throw new Error(`Could not resolve DID document for ${did}`); @@ -414,7 +452,7 @@ async function createSession( identifier: string, password: string, ): Promise { - return xrpc(pds, 'com.atproto.server.createSession', { + return xrpc(pds, 'com.atproto.server.createSession', SessionSchema, { body: JSON.stringify({ identifier, password }), method: 'POST', }); @@ -429,7 +467,7 @@ async function listRecords( const records: Array = []; let cursor: string | undefined; - do { + while (true) { const query = new URLSearchParams({ collection, limit: '100', @@ -439,16 +477,18 @@ async function listRecords( query.set('cursor', cursor); } - const body = await xrpc( + const body = await xrpc( pds, `com.atproto.repo.listRecords?${query}`, + ListRecordsResponseSchema, { headers: auth, }, ); records.push(...body.records); + if (!body.cursor) break; cursor = body.cursor; - } while (cursor); + } return records; } @@ -491,13 +531,14 @@ async function executePlan( auth: Record, repo: string, plan: ReconciliationPlan, -): Promise> { - const documentsBySlug = new Map(); +): Promise> { + const documentsBySlug = new Map(); for (const item of plan.creates) { - const result = await xrpc( + const result = await xrpc( pds, 'com.atproto.repo.createRecord', + WriteRecordResponseSchema, { body: JSON.stringify({ collection: COLLECTION, @@ -513,9 +554,10 @@ async function executePlan( } for (const item of plan.updates) { - const result = await xrpc( + const result = await xrpc( pds, 'com.atproto.repo.putRecord', + WriteRecordResponseSchema, { body: JSON.stringify({ collection: COLLECTION, @@ -532,7 +574,7 @@ async function executePlan( } for (const item of plan.deletes) { - await xrpc(pds, 'com.atproto.repo.deleteRecord', { + await xrpcVoid(pds, 'com.atproto.repo.deleteRecord', { body: JSON.stringify({ collection: COLLECTION, repo, @@ -561,7 +603,7 @@ async function main(): Promise { if (!args.has('--report') && !args.has('--write')) { const posts = await loadPublishedPosts(); for (const post of posts) { - console.log(`${post.slug} -> /posts/${post.slug}`); + console.log(`${post.slug} /posts/${post.slug}`); } console.log(`published posts: ${posts.length}`); return; diff --git a/apps/web/src/lib/standard-site.test.ts b/apps/web/src/lib/standard-site.test.ts index 560791b..8d45f2c 100644 --- a/apps/web/src/lib/standard-site.test.ts +++ b/apps/web/src/lib/standard-site.test.ts @@ -8,7 +8,7 @@ import { getStandardSiteDocumentUri, loadStandardSiteManifest, standardSitePublicationUri, -} from './standard-site'; +} from './standard-site.ts'; test('keeps committed publication URI available', () => { assert.equal( diff --git a/apps/web/src/lib/standard-site.ts b/apps/web/src/lib/standard-site.ts index 15d53ab..3ff50f6 100644 --- a/apps/web/src/lib/standard-site.ts +++ b/apps/web/src/lib/standard-site.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; -import { z } from 'zod'; +import * as z from 'zod'; export const standardSitePublicationUri = 'at://did:plc:3z5ja7l2rhnmtr2bni5dyfe7/site.standard.publication/3mnqwgvxn372f'; diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 8c83d4b..641edfa 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -1,3 +1,6 @@ { - "extends": "@lukebennett/tsconfig/dom" + "extends": "@lukebennett/tsconfig/dom", + "compilerOptions": { + "allowImportingTsExtensions": true + } }