feat(extensions): register custom syntax grammars

.changeset/custom-syntax-grammars.md

@@ -0,0 +1,5 @@
+---
+"hunkdiff": minor
+---
+
+Let extensions register bounded data-only TextMate grammars for custom file languages.

docs/extension-architecture.md

@@ -43,15 +43,20 @@ load issue and costs only that extension. The rules themselves are stated in
 
 ## One registry, one apply path
 
-Registrations (session behavior, themes, file languages, VCS adapters,
+Registrations (session behavior, themes, file languages, syntax grammars, VCS adapters,
 changeset transforms, panes, interactive commands, top-level CLI commands,
 lifecycle/UI events, and inter-extension bus listeners) collect into one
 `ExtensionRegistry` (`src/extensions/types.ts`) and are resolved/applied
 through `src/extensions/apply.ts` on both startup and reload. File-language registrations stay as
 declarative extension, filename, or glob selectors until `fileLanguageLookup.ts` resolves them;
 Hunk then pins that answer into Pierre's metadata so rendering cannot re-derive a conflicting
-language. A live reload replaces the compiled selector generation while preparing its changeset
-and restores the previous generation if any pre-commit step fails. Staged external-VCS bootstrap
+language. Custom syntax grammars are bounded, deeply frozen TextMate data rather than retained
+extension loaders. `core/changeset/syntaxGrammar.ts` owns their generation and digest; the UI sends
+that snapshot through the versioned highlight-worker configure handshake before matching jobs.
+Grammar changes replace the worker and participate in rendered-result cache identity. Custom regexes
+never run on the terminal thread, so worker failure or compiled-Windows worker unavailability falls
+back to plaintext without poisoning bundled languages. A live reload replaces the selector and
+grammar generations while preparing its changeset and restores both if any pre-commit step fails. Staged external-VCS bootstrap
 retains the provisional candidate/config snapshot: a final pass that
 only appends repo candidates extends the same registry, while a changed prefix
 receives bounded `shutdown` before being rebuilt. Live registry replacement uses

docs/extensions.md

@@ -280,8 +280,9 @@ new instances and run that shutdown/startup pair around the replacement.
 
 ### `hunk.apiVersion`
 
-The API generation this Hunk speaks (currently `16`). Branch on it if you want
-one file to support several Hunk versions. Version 16 adds pane-wide
+The API generation this Hunk speaks (currently `17`). Branch on it if you want
+one file to support several Hunk versions. Version 17 adds bounded data-only custom TextMate
+syntax grammars; version 16 added pane-wide
 `onActivate`; version 15 added `{ side, line }` to opted-in pane `currentLine`
 paint; version 14 added structured `rangeEndpoints`
 to two-revision VCS diff requests; version 13 added saved-note parent identities and
@@ -428,8 +429,41 @@ filenames take precedence over globs, which take precedence over extensions. The
 extension wins, and later registrations win ties within each category. Direct attempts to register
 those two reserved extensions are skipped with a notice.
 
-This API selects a language already available to Pierre/Shiki. It does not load a new syntax
-grammar; an unknown language remains plain text.
+This API selects a language already available to Pierre/Shiki or registered by the same load pass.
+An unknown language remains plain text.
+
+### `hunk.registerSyntaxGrammar(grammar)`
+
+Register a data-only TextMate grammar, then map files to its language id separately:
+
+```ts
+hunk.registerSyntaxGrammar({
+  id: "mydsl",
+  scopeName: "source.mydsl",
+  patterns: [
+    { match: "\\b(component|contract)\\b", name: "keyword.control.mydsl" },
+    { include: "#strings" },
+  ],
+  repository: {
+    strings: { begin: '"', end: '"', name: "string.quoted.double.mydsl" },
+  },
+});
+hunk.registerFileLanguage(".mydsl", "mydsl");
+```
+
+API v17 accepts a closed serializable subset: `match`, `begin`/`end`/`while`, captures, nested
+`patterns`, and local repository includes. Includes may use only `#name`, `$self`, or `$base`.
+External includes, embedded languages, injections, loaders, functions, unknown keys, and bundled
+language ids are refused. Hunk bounds serialized bytes, nesting depth, node count, and individual
+strings, then deeply copies and freezes accepted data.
+
+Custom regexes run only in the killable highlight worker. A timeout or grammar failure renders that
+file as plaintext and does not poison bundled languages. Hunk keeps custom grammars plaintext when
+worker offload is unavailable, including compiled Windows builds. Reloading atomically replaces the
+complete grammar generation and invalidates worker and rendered-result caches; the first extension
+to claim an id wins, and retired registries retain no executable loader authority.
+
+See `examples/extensions/archlang-syntax/` for a complete grammar and `.arch` mapping.
 
 ### `hunk.registerVcsAdapter(adapter)`
 

examples/extensions/archlang-syntax/README.md

@@ -0,0 +1,11 @@
+# Archlang syntax grammar example
+
+This folder extension adds a bounded, data-only TextMate grammar for `.arch` files. Run it against a
+repository containing Archlang input:
+
+```bash
+hunk --extension ./examples/extensions/archlang-syntax diff
+```
+
+The grammar deliberately uses only local repository includes. Hunk validates, copies, freezes, and
+sends the data to its killable highlight worker; it never retains an extension loader callback.

examples/extensions/archlang-syntax/index.ts

@@ -0,0 +1,40 @@
+import type { HunkExtensionAPI } from "hunkdiff/extension";
+
+/** Register a compact Archlang TextMate grammar through API v17. */
+export default function archlangSyntax(hunk: HunkExtensionAPI) {
+  hunk.registerSyntaxGrammar({
+    id: "archlang",
+    scopeName: "source.archlang",
+    patterns: [
+      { include: "#comments" },
+      { include: "#strings" },
+      { include: "#keywords" },
+      { include: "#operators" },
+      { include: "#numbers" },
+      { include: "#variables" },
+    ],
+    repository: {
+      comments: { match: "//.*$", name: "comment.line.double-slash.archlang" },
+      strings: {
+        begin: '"',
+        end: '"',
+        name: "string.quoted.double.archlang",
+        patterns: [
+          {
+            match: '\\\\(?:[\\\\"nrt]|u[0-9a-fA-F]{4})',
+            name: "constant.character.escape.archlang",
+          },
+        ],
+      },
+      keywords: {
+        match:
+          "\\b(?:arch|system|let|source|component|contract|type|record|tagged|operation|async|throws|bind|using|process|in_process|via|edges|rule|paths|components|excluding|cycles|forbidden|property|on|claim|verify|static|interface|probe|advisory|existing|version|compatibility|observed|constructed|provides|requires|roots|exclude|entrypoints)\\b",
+        name: "keyword.control.archlang",
+      },
+      operators: { match: "-/>|->|[:=$]", name: "keyword.operator.archlang" },
+      numbers: { match: "\\b\\d+(?:_\\d+)*\\b", name: "constant.numeric.archlang" },
+      variables: { match: "\\$[a-z][a-z0-9_]*", name: "variable.other.archlang" },
+    },
+  });
+  hunk.registerFileLanguage(".arch", "archlang");
+}

examples/extensions/archlang-syntax/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "hunk-archlang-syntax-example",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Example data-only Archlang syntax grammar for Hunk",
+  "hunk": {
+    "apiVersion": 17,
+    "extensions": [
+      "./index.ts"
+    ]
+  }
+}

package.json

@@ -61,6 +61,7 @@
     "install:bin": "bun run ./scripts/install-bin.ts",
     "generate:skill": "bun run ./scripts/generate-skill.ts",
     "generate:theme-colors": "bun run ./scripts/generate-theme-diff-colors.ts",
+    "generate:syntax-languages": "bun run ./scripts/generate-bundled-syntax-languages.ts",
     "generate:docs": "bun run ./scripts/generate-docs.ts",
     "generate:changelog": "bun run ./scripts/generate-changelog.ts",
     "generate:og": "bun run ./website/scripts/generate-og.ts",
@@ -151,6 +152,7 @@
     "oxfmt": "^0.41.0",
     "oxlint": "^1.56.0",
     "react": "^19.2.4",
+    "shiki": "3.23.0",
     "simple-git-hooks": "^2.13.1",
     "tuistory": "^0.11.0",
     "typescript": "^5.9.3"

scripts/check-pack.ts

@@ -39,6 +39,7 @@ import type {
   ExtensionPaneSize,
   ExtensionReviewSelection,
   ExtensionSessionOptions,
+  ExtensionSyntaxGrammar,
   ExtensionVerticalPane,
   ExtensionVcsAdapter,
   ExtensionVcsDiffInput,
@@ -86,6 +87,12 @@ export default function (hunk: HunkExtensionAPI) {
     target: "path",
   };
   hunk.registerFileLanguage(generatedTypeScript, "typescript");
+  const grammar: ExtensionSyntaxGrammar = {
+    id: "mydsl",
+    scopeName: "source.mydsl",
+    patterns: [{ match: "component", name: "keyword.control.mydsl" }],
+  };
+  hunk.registerSyntaxGrammar(grammar);
 
   const pane = (props: ExtensionPaneProps) => {
     hunk.log(\`\${props.placement}:\${props.width}x\${props.height}\`);

scripts/generate-bundled-syntax-languages.ts

@@ -0,0 +1,21 @@
+#!/usr/bin/env bun
+
+import { writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { bundledLanguages } from "shiki";
+
+/** Render the checked-in collision list in formatter-stable TypeScript. */
+export function renderBundledSyntaxLanguages(): string {
+  const ids = Object.keys(bundledLanguages).sort();
+  const entries = ids.map((id) => `  ${JSON.stringify(id)},`).join("\n");
+  return `/** Language ids bundled by the pinned Pierre release; run bun generate:syntax-languages. */
+export const BUNDLED_SYNTAX_LANGUAGE_IDS: ReadonlySet<string> = new Set([\n${entries}\n]);
+`;
+}
+
+if (import.meta.main) {
+  writeFileSync(
+    resolve(import.meta.dir, "../src/core/changeset/bundledSyntaxLanguages.generated.ts"),
+    renderBundledSyntaxLanguages(),
+  );
+}

skills/hunk-extensions/SKILL.md

@@ -98,6 +98,7 @@ bad or duplicate id is skipped with a startup notice.
 | Keep demo/training view settings temporary               | `hunk.configureSession(options)`             |
 | Add a selectable color theme                             | `hunk.registerTheme(theme)`                  |
 | Highlight an extension, exact filename, or filename glob | `hunk.registerFileLanguage(matcher, lang)`   |
+| Add a bounded data-only TextMate syntax grammar          | `hunk.registerSyntaxGrammar(grammar)`        |
 | Support another VCS (`git`/`jj`/`sl` are reserved)       | `hunk.registerVcsAdapter(adapter)`           |
 | Add a navigation/list/status pane beside the review      | `hunk.registerPane(pane)`                    |
 | Present a file as something other than a raw diff        | `hunk.registerFileView(view)` (experimental) |
@@ -110,7 +111,7 @@ bad or duplicate id is skipped with a startup notice.
 | Coordinate with another loaded extension                 | `hunk.events.emit` / `hunk.events.on`        |
 | Read user-supplied settings                              | `hunk.config` (`[extension.<id>]` table)     |
 | Snapshot stable files and every saved review note        | `ctx.review.snapshot()` in a command         |
-| Branch on the API generation (currently `15`)            | `hunk.apiVersion`                            |
+| Branch on the API generation (currently `17`)            | `hunk.apiVersion`                            |
 
 Registration is only valid while the factory runs — Hunk seals the API object
 afterwards.
@@ -190,6 +191,10 @@ through untouched.
 
 Most extension bugs are one of these:
 
+- **Custom syntax grammars are data, not loaders.** Register the grammar during the factory, map
+  files separately with `registerFileLanguage`, and use only local includes. Hunk bounds and freezes
+  the grammar, runs its regexes in the killable worker, and leaves custom languages plain when worker
+  offload is unavailable.
 - **Registering a surface does not show it.** Panes need `defaultOpen`,
   `replaces: "hunk:files"`, or a command that opens them. File views remain raw
   until selected from the **View** menu.

src/app/sessionBootstrap.test.ts

@@ -1,6 +1,10 @@
 import { describe, expect, test } from "bun:test";
 import { fileLanguageForPath } from "../core/changeset/fileLanguageLookup";
 import { replaceExtensionFileLanguages } from "../core/changeset/fileLanguage";
+import {
+  replaceExtensionSyntaxGrammars,
+  syntaxGrammarSnapshot,
+} from "../core/changeset/syntaxGrammar";
 import type { HunkConfigResolution } from "../core/run/config";
 import type { AppBootstrap } from "../core/bootstrap";
 import type { CliInput } from "../core/run/commandInputs";
@@ -58,15 +62,33 @@ describe("loadConfiguredSessionBootstrap", () => {
     expect(result.bootstrap.viewPreferencesConfigPath).toBe("/tmp/hunk-config.toml");
   });
 
-  test("restores the active file-language selectors when bootstrap loading fails", async () => {
+  test("restores file-language and syntax-grammar generations when bootstrap loading fails", async () => {
     replaceExtensionFileLanguages([
       {
         matcher: { kind: "filename", value: "CurrentHunkfile" },
         language: "python",
       },
     ]);
+    replaceExtensionSyntaxGrammars([
+      {
+        extensionId: "current",
+        grammar: Object.freeze({
+          id: "current",
+          scopeName: "source.current",
+          patterns: Object.freeze([{ match: "current" }]),
+        }),
+      },
+    ]);
     const input = createTestInput();
     const extensions = createEmptyExtensionLoadResult();
+    extensions.registry.syntaxGrammars.push({
+      extensionId: "replacement",
+      grammar: Object.freeze({
+        id: "replacement",
+        scopeName: "source.replacement",
+        patterns: Object.freeze([{ match: "replacement" }]),
+      }),
+    });
     extensions.registry.fileLanguages.push({
       extensionId: "replacement",
       matcher: { kind: "filename", value: "ReplacementHunkfile" },
@@ -80,13 +102,16 @@ describe("loadConfiguredSessionBootstrap", () => {
         extensions,
         loadAppBootstrapImpl: async () => {
           expect(fileLanguageForPath("ReplacementHunkfile")).toBe("ruby");
+          expect(syntaxGrammarSnapshot().grammars.map(({ id }) => id)).toEqual(["replacement"]);
           throw new Error("load failed");
         },
       }),
     ).rejects.toThrow("load failed");
 
     expect(fileLanguageForPath("CurrentHunkfile")).toBe("python");
     expect(fileLanguageForPath("ReplacementHunkfile")).toBe("text");
+    expect(syntaxGrammarSnapshot().grammars.map(({ id }) => id)).toEqual(["current"]);
     replaceExtensionFileLanguages([]);
+    replaceExtensionSyntaxGrammars([]);
   });
 });

src/app/sessionBootstrap.ts

@@ -3,6 +3,11 @@ import {
   restoreFileLanguageRegistrations,
   type FileLanguageRegistrationSnapshot,
 } from "../core/changeset/fileLanguage";
+import {
+  restoreSyntaxGrammars,
+  syntaxGrammarSnapshot,
+  type SyntaxGrammarSnapshot,
+} from "../core/changeset/syntaxGrammar";
 import type { HunkConfigResolution } from "../core/run/config";
 import { isVcsReviewInput } from "../core/vcs";
 import type { VcsCatalog } from "../core/vcs/types";
@@ -37,6 +42,8 @@ export interface SessionBootstrapResult {
   bootstrap: AppBootstrap;
   /** Selector set to restore if a live reload fails before its commit gate. */
   previousFileLanguages: FileLanguageRegistrationSnapshot;
+  /** Grammar set to restore if a live reload fails before its commit gate. */
+  previousSyntaxGrammars: SyntaxGrammarSnapshot;
   input: CliInput;
   sessionThemes: ReturnType<typeof collectSessionCustomThemes>;
   sessionVcs: ReturnType<typeof resolveSessionVcsId>;
@@ -59,6 +66,7 @@ export async function loadConfiguredSessionBootstrap({
   baseVcsCatalog = getBundledVcsCatalog(),
 }: SessionBootstrapOptions): Promise<SessionBootstrapResult> {
   const previousFileLanguages = fileLanguageRegistrationSnapshot();
+  const previousSyntaxGrammars = syntaxGrammarSnapshot();
 
   try {
     const sessionThemes = collectSessionCustomThemes(
@@ -91,9 +99,18 @@ export async function loadConfiguredSessionBootstrap({
     bootstrap.viewPreferencesConfigPath = configured.viewPreferencesConfigPath;
     bootstrap.keybindings = configured.keybindings;
 
-    return { applied, bootstrap, input, previousFileLanguages, sessionThemes, sessionVcs };
+    return {
+      applied,
+      bootstrap,
+      input,
+      previousFileLanguages,
+      previousSyntaxGrammars,
+      sessionThemes,
+      sessionVcs,
+    };
   } catch (error) {
     restoreFileLanguageRegistrations(previousFileLanguages);
+    restoreSyntaxGrammars(previousSyntaxGrammars);
     throw error;
   }
 }