fix(ok): drop trailing hard break that leaks a stray backslash (#2701)

* fix(open-knowledge): drop trailing hard break that leaks a stray backslash

A hard break (Shift+Enter) at the end of a paragraph serialized to a
trailing backslash in the source, and deleting it did not stick: the
WYSIWYG fragment kept the break node so the next Observer A drain
re-emitted it (a fragment/Y.Text round-trip that never settled, masked
from the invariant watchdog by parse-equivalence tolerance).

A trailing hard break has no CommonMark meaning: a backslash form decays
to a literal backslash on the next parse and a two-space form is
stripped. Drop it at serialize time via a new stripTrailingHardBreaks
mdast pre-pass in serializeMd, recursing into trailing mark wrappers and
preserving void <br> breaks (sourceRaw). Source fidelity is untouched
because a trailing backslash parsed from source is already literal text,
not a break node.

Pre-existing since the repo mirror; not introduced by the recent
serializer work.

Committed with --no-verify: raw worktree lacks the root lint-staged
install; biome, tsc, and the RED->GREEN + regression suites were run
manually and pass.

* test(open-knowledge): record bridge fuzz residual sample for the trailing hard break fix

200-seed measure:fuzz sample at rate 0.015 (3/200), inside the historical
1.0-2.6 percent band. No residual regression: the change drops a trailing
hard break so serialize produces fewer bytes toward the source, and an A/B
replay of a failing seed was identical on origin/main (pre-existing seed).

Committed with --no-verify: data-only JSONL append; the code was validated
by the full pre-push hook on the prior commit.

* chore(open-knowledge): refresh ng-anchors catalog after pipeline line shift

The trailing-hard-break strip import shifted @floor anchor line ranges in
the markdown pipeline by one; regenerate the committed catalog so the
ng-anchors freshness gate passes. Line-range refresh only (no floor added
or removed). Committed --no-verify: generated data, code already validated.

* fix(open-knowledge): strip trailing break inside mark and comment wrappers

Address review: MARK_WRAPPERS omitted OK's 'mark' (highlight) and 'comment'
phrasing wrappers, so a trailing break nested in a highlight or comment at a
block edge was not stripped. Add both; expand tests to cover the heading
container (same BLOCK_CONTAINERS path as tableCell), every mark wrapper
(emphasis/strong/strike/highlight), and consecutive trailing breaks.

Committed --no-verify: raw worktree lacks lint-staged; biome + the RED/GREEN
+ regression suites were run manually and pass.

GitOrigin-RevId: 987058fa893873bb0f5bd55964399f89a85fa206

Author: nick-inkeep

.changeset/trailing-hard-break-strip.md

@@ -0,0 +1,7 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Fix a stray backslash appearing at the end of lines after a hard break
+
+A hard break (Shift+Enter) at the end of a paragraph serialized to a trailing `\` in the markdown source, which users saw as an unexpected character at the end of a line. Worse, deleting it did not stick: the WYSIWYG fragment kept the break node, so the next sync re-emitted the `\` (a fragment/source round-trip that never settled, hidden from the invariant watchdog by parse-equivalence tolerance). A trailing hard break has no CommonMark meaning — a backslash form decays to a literal backslash on the next parse and a two-space form is stripped — so it is now dropped at serialize time. Mid-paragraph hard breaks, source-authored breaks, void `<br>` breaks, and literal backslashes in text are all unaffected and still round-trip byte-for-byte.

packages/app/tests/integration/trailing-hard-break-bridge.test.ts

@@ -0,0 +1,126 @@
+/**
+ * Trailing hardBreak through the real server bridge (Observer A/B).
+ *
+ * A WYSIWYG-created hard break at the end of a paragraph must not surface a
+ * stray `\` in Y.Text ("forward slash at end of line" report), and the
+ * fragment↔Y.Text pair must converge to the same bytes — the historical
+ * failure was a split-brain where deleting the `\` from source left the
+ * hardBreak node in the fragment and the next Observer A drain re-emitted it,
+ * masked from the watchdog by parse-equivalence tolerance.
+ */
+import { afterEach, describe, expect, test } from 'bun:test';
+import { updateYFragment } from '@tiptap/y-tiptap';
+import {
+  awaitDocQuiescence,
+  createTestClient,
+  createTestServer,
+  schema,
+  serializeFragment,
+  type TestServer,
+  wait,
+} from './test-harness';
+
+describe('trailing hardBreak bridge convergence', () => {
+  let server: TestServer | undefined;
+  afterEach(async () => {
+    await server?.cleanup();
+    server = undefined;
+  });
+
+  test('WYSIWYG trailing hardBreak: no stray backslash, fragment and Y.Text converge', async () => {
+    server = await createTestServer();
+    const docName = `trailing-hardbreak-${crypto.randomUUID()}`;
+    const client = await createTestClient(server.port, docName);
+    try {
+      await wait(300);
+
+      const pmDoc = schema.nodeFromJSON({
+        type: 'doc',
+        content: [
+          {
+            type: 'paragraph',
+            content: [{ type: 'text', text: 'hello' }, { type: 'hardBreak' }],
+          },
+        ],
+      });
+      client.doc.transact(() => {
+        updateYFragment(client.doc, client.fragment, pmDoc, {
+          mapping: new Map(),
+          isOMark: new Map(),
+        });
+      });
+      await awaitDocQuiescence(client.doc);
+      for (let i = 0; i < 60 && client.ytext.toString().length === 0; i++) await wait(100);
+      await wait(300);
+
+      const ytext = client.ytext.toString();
+      const fragMd = serializeFragment(client.fragment);
+
+      // The stray-character symptom: no backslash may reach source bytes.
+      expect(ytext).toBe('hello\n');
+      // The split-brain: fragment serialization must equal Y.Text, so a
+      // subsequent Observer A drain cannot re-emit a deleted character.
+      expect(fragMd).toBe(ytext);
+
+      // A later fragment change must not resurrect a trailing backslash.
+      const pmDoc2 = schema.nodeFromJSON({
+        type: 'doc',
+        content: [
+          {
+            type: 'paragraph',
+            content: [{ type: 'text', text: 'hello there' }, { type: 'hardBreak' }],
+          },
+        ],
+      });
+      client.doc.transact(() => {
+        updateYFragment(client.doc, client.fragment, pmDoc2, {
+          mapping: new Map(),
+          isOMark: new Map(),
+        });
+      });
+      await awaitDocQuiescence(client.doc);
+      await wait(500);
+      const ytext2 = client.ytext.toString();
+      expect(ytext2).toBe('hello there\n');
+      expect(serializeFragment(client.fragment)).toBe(ytext2);
+    } finally {
+      await client.cleanup();
+    }
+  }, 30_000);
+
+  test('mid-paragraph hard break still round-trips through the bridge', async () => {
+    server = await createTestServer();
+    const docName = `midline-hardbreak-${crypto.randomUUID()}`;
+    const client = await createTestClient(server.port, docName);
+    try {
+      await wait(300);
+      const pmDoc = schema.nodeFromJSON({
+        type: 'doc',
+        content: [
+          {
+            type: 'paragraph',
+            content: [
+              { type: 'text', text: 'a' },
+              { type: 'hardBreak', attrs: { hardBreakStyle: 'backslash', sourceRaw: null } },
+              { type: 'text', text: 'b' },
+            ],
+          },
+        ],
+      });
+      client.doc.transact(() => {
+        updateYFragment(client.doc, client.fragment, pmDoc, {
+          mapping: new Map(),
+          isOMark: new Map(),
+        });
+      });
+      await awaitDocQuiescence(client.doc);
+      for (let i = 0; i < 60 && client.ytext.toString().length === 0; i++) await wait(100);
+      await wait(300);
+
+      expect(client.ytext.toString()).toBe('a\\\nb\n');
+      expect(serializeFragment(client.fragment)).toBe('a\\\nb\n');
+    } finally {
+      await client.cleanup();
+    }
+  }, 30_000);
+});

packages/core/src/markdown/pipeline.ts

@@ -33,6 +33,7 @@ import { mermaidPromoterPlugin } from './mermaid-promoter.ts';
 import { positionAwareBlankLineJoin } from './position-aware-join.ts';
 import { remarkMdxAgnostic } from './remark-mdx-agnostic.ts';
 import { singleDollarMathPromoterPlugin } from './single-dollar-math-promoter.ts';
+import { stripTrailingHardBreaks } from './strip-trailing-hard-break.ts';
 import { remarkTags } from './tag-to-markdown.ts';
 import { voidBrPromoterPlugin } from './void-br-promoter.ts';
 import { remarkWikiLink } from './wiki-link-micromark.ts';
@@ -268,6 +269,8 @@ export function serializeMd(doc: PmNode, processor: Processor, opts: SerializeMd
     markHandlers: opts.pmMarkHandlers,
   });
 
+  stripTrailingHardBreaks(mdast);
+
   const boundary = readDocBoundary(doc.attrs?.sourceDocBoundary);
   const gaps = boundary?.gapBlankLines;
   if (gaps && gaps.length === mdast.children.length - 1) {

packages/core/src/markdown/strip-trailing-hard-break.ts

@@ -0,0 +1,46 @@
+import type { Nodes, Parent, RootContent } from 'mdast';
+
+const MARK_WRAPPERS = new Set(['strong', 'emphasis', 'delete', 'link', 'mark', 'comment']);
+
+function isBareTrailingBreak(node: RootContent | undefined): boolean {
+  if (!node || node.type !== 'break') return false;
+  const sourceRaw = (node.data as { sourceRaw?: unknown } | undefined)?.sourceRaw;
+  return typeof sourceRaw !== 'string' || sourceRaw.length === 0;
+}
+
+/** Remove trailing bare breaks from a phrasing-children array, recursing into
+ * trailing mark wrappers and pruning any wrapper the removal empties. */
+function stripTrailing(children: RootContent[]): void {
+  while (children.length > 0) {
+    const last = children[children.length - 1];
+    if (isBareTrailingBreak(last)) {
+      children.pop();
+      continue;
+    }
+    if (last && MARK_WRAPPERS.has(last.type) && 'children' in last) {
+      const inner = (last as Parent).children as RootContent[];
+      stripTrailing(inner);
+      if (inner.length === 0) {
+        children.pop();
+        continue;
+      }
+    }
+    break;
+  }
+}
+
+const BLOCK_CONTAINERS = new Set(['paragraph', 'heading', 'tableCell']);
+
+/** In-place: strip trailing editor-created hard breaks from every block whose
+ * phrasing content can end in one. */
+export function stripTrailingHardBreaks(tree: Nodes): void {
+  const visit = (node: Nodes): void => {
+    if (BLOCK_CONTAINERS.has(node.type) && 'children' in node) {
+      stripTrailing((node as Parent).children as RootContent[]);
+    }
+    if ('children' in node) {
+      for (const child of (node as Parent).children as Nodes[]) visit(child);
+    }
+  };
+  visit(tree);
+}