From 8a3fce0393839ec6561bae082d11ea05dc762dc1 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Mon, 27 Jul 2026 20:48:00 -0300 Subject: [PATCH] fix(evals): replace named organizations with generic descriptors The evals suite described its scenarios by naming specific organizations and mapping each to the workflow it uses. Those mappings are not needed to explain what the scenarios cover, and naming them here makes the relationships public. Describe each scenario by the document workflow it exercises instead. One test description elsewhere named the same organization as the source of a fixture; it now refers to the reported fixture. The scenarios, assertions, and coverage are unchanged. --- evals/README.md | 4 +- .../execution/tests/customer-workflows.yaml | 50 ++++++++----------- packages/ai/src/shared/prompts.ts | 2 +- .../style-engine/src/cascade.test.ts | 6 +-- .../plan-engine/word-diff.test.ts | 2 +- 5 files changed, 28 insertions(+), 36 deletions(-) diff --git a/evals/README.md b/evals/README.md index d71ada8a12..1b65b5c90f 100644 --- a/evals/README.md +++ b/evals/README.md @@ -247,7 +247,7 @@ evals/ vars: fixture: nda.docx keepFile: true - task: 'Replace "Amazing Corp" with "Irys Inc." everywhere.' + task: 'Replace "Amazing Corp" with "Northwind Ltd." everywhere.' assert: - type: javascript value: | @@ -255,7 +255,7 @@ evals/ const t = d.documentText || ''; if (t.includes('Amazing')) return { pass: false, score: 0, reason: 'Old name still present' }; - if (!t.includes('Irys Inc.')) + if (!t.includes('Northwind Ltd.')) return { pass: false, score: 0, reason: 'New name missing' }; return { pass: true, score: 1, reason: 'Entity names replaced' }; - type: javascript diff --git a/evals/suites/execution/tests/customer-workflows.yaml b/evals/suites/execution/tests/customer-workflows.yaml index b048949f22..e8f7342240 100644 --- a/evals/suites/execution/tests/customer-workflows.yaml +++ b/evals/suites/execution/tests/customer-workflows.yaml @@ -1,15 +1,14 @@ -# Customer workflow tests — validates real-world AI use cases from production customers. +# Workflow tests — validates real-world AI use cases drawn from production usage. # -# Mapped from: Clauze (contract review), AcuityMD (medical devices), Irys (SaaS terms), -# Athena (investment memos), Torke (citations), Lighthouse (corporate docs), -# Orbital (leases), Tryal (clinical trials), Reavant (SDK mutations). +# Each group covers a document-editing workflow we see in practice: contract +# review, document Q&A, drafting, commenting, redlining, table operations, +# multi-step compound edits, and formatting. # -# Each test exercises a documented customer workflow against a domain-specific DOCX fixture. +# Each test exercises one workflow against a domain-specific DOCX fixture. # Assertions validate document content + tool execution trace. # ============================================================================= -# A. NDA / CONTRACT REVIEW & MODIFICATION (Clauze, Irys) -# Customers: Clauze (clause modification, risk ID), Irys (entity rename, rewrite) +# A. NDA / CONTRACT REVIEW & MODIFICATION # ============================================================================= - description: 'NDA: modify indemnification from mutual to one-way' @@ -32,19 +31,19 @@ - type: javascript value: file://../shared/checks.cjs:traceLog -- description: 'NDA: global entity name replacement (Amazing → Irys)' +- description: 'NDA: global entity name replacement (Amazing → Northwind)' vars: fixture: nda.docx keepFile: true - task: 'Replace "Amazing Corp" with "Irys Inc." and "legal@iqidis.com" with "legal@irys.com" everywhere in the document.' + task: 'Replace "Amazing Corp" with "Northwind Ltd." and "legal@amazing.com" with "legal@northwind.example" everywhere in the document.' assert: - type: javascript value: | const d = JSON.parse(output); const t = d.documentText || ''; - if (t.includes('Amazing') || t.includes('iqidis')) + if (t.includes('Amazing') || t.includes('amazing.com')) return { pass: false, score: 0, reason: 'Old entity name still present' }; - if (!t.includes('Irys Inc.') || !t.includes('legal@irys.com')) + if (!t.includes('Northwind Ltd.') || !t.includes('legal@northwind.example')) return { pass: false, score: 0, reason: 'New names not found' }; if (!t.includes('TechCraft')) return { pass: false, score: 0, reason: 'Collateral: other party name removed' }; @@ -126,8 +125,7 @@ value: file://../shared/checks.cjs:traceAllOk # ============================================================================= -# B. DOCUMENT Q&A / COMPREHENSION (AcuityMD, Orbital) -# Customers: Orbital (lease Q&A), AcuityMD (doc Q&A), Tryal (study Q&A) +# B. DOCUMENT Q&A / COMPREHENSION # ============================================================================= - description: 'Lease: Q&A about assignment rights' @@ -224,8 +222,7 @@ metric: tool_usage # ============================================================================= -# C. DRAFTING / CONTENT GENERATION (Athena, Lighthouse, Tryal) -# Customers: Athena (memos), Lighthouse (disclaimers), Tryal (summaries) +# C. DRAFTING / CONTENT GENERATION # ============================================================================= - description: 'NDA: generate glossary of defined terms' @@ -301,8 +298,7 @@ value: file://../shared/checks.cjs:traceLog # ============================================================================= -# D. COMMENTS & ANNOTATIONS (Torke) -# Customers: Torke (insert comment, review), Lighthouse (review comments) +# D. COMMENTS & ANNOTATIONS # ============================================================================= - description: 'NDA: add review comment on indemnification' @@ -384,8 +380,7 @@ metric: tool_usage # ============================================================================= -# E. TRACK CHANGES / REDLINING (Clauze, AcuityMD, Lighthouse, Tryal) -# Customers: Clauze (redlining), AcuityMD (tracked edits), Lighthouse (AI redline) +# E. TRACK CHANGES / REDLINING # ============================================================================= - description: 'NDA: tracked change on term duration' @@ -443,8 +438,7 @@ value: file://../shared/checks.cjs:traceLog # ============================================================================= -# F. TABLE OPERATIONS (AcuityMD, Reavant, Tryal) -# Customers: AcuityMD (table generation), Reavant (table mutation), Tryal (data tables) +# F. TABLE OPERATIONS # ============================================================================= - description: 'Report: read enrollment table data' @@ -504,8 +498,7 @@ value: file://../shared/checks.cjs:traceAllOk # ============================================================================= -# G. MULTI-STEP COMPOUND OPERATIONS (Lighthouse, Athena, Irys) -# Customers: Lighthouse (review + edit), Athena (search + insert), Irys (rename + modify) +# G. MULTI-STEP COMPOUND OPERATIONS # ============================================================================= - description: 'Offer: fill multiple placeholders' @@ -546,7 +539,7 @@ vars: fixture: nda.docx keepFile: true - task: 'Replace "Amazing Corp" with "Irys Inc." everywhere in the document, then add a heading "CONFIDENTIALITY NOTICE" at the end followed by a paragraph "This document contains proprietary information."' + task: 'Replace "Amazing Corp" with "Northwind Ltd." everywhere in the document, then add a heading "CONFIDENTIALITY NOTICE" at the end followed by a paragraph "This document contains proprietary information."' assert: - type: javascript value: | @@ -554,7 +547,7 @@ const t = d.documentText || ''; if (t.includes('Amazing')) return { pass: false, score: 0, reason: 'Old name still present' }; - if (!t.includes('Irys Inc.')) + if (!t.includes('Northwind Ltd.')) return { pass: false, score: 0, reason: 'New name missing' }; if (!t.includes('CONFIDENTIALITY NOTICE')) return { pass: false, score: 0, reason: 'Notice heading missing' }; @@ -615,8 +608,7 @@ value: file://../shared/checks.cjs:traceLog # ============================================================================= -# H. FORMATTING (Tryal) -# Customers: Tryal (simplify, improve writing, style changes) +# H. FORMATTING # ============================================================================= - description: 'Report: replace terminology in specific section' @@ -735,7 +727,7 @@ vars: fixture: nda.docx keepFile: true - task: 'Prepare this NDA for execution: (1) Replace "Amazing Corp" with "Irys Inc." everywhere, (2) Replace "[Party A Signature]" with "Jane Smith, CEO", (3) Replace "[Date]" with "March 20, 2025", (4) Add a heading "EXHIBIT A" at the end followed by "List of authorized recipients to be provided separately."' + task: 'Prepare this NDA for execution: (1) Replace "Amazing Corp" with "Northwind Ltd." everywhere, (2) Replace "[Party A Signature]" with "Jane Smith, CEO", (3) Replace "[Date]" with "March 20, 2025", (4) Add a heading "EXHIBIT A" at the end followed by "List of authorized recipients to be provided separately."' assert: - type: javascript value: | @@ -743,7 +735,7 @@ const t = d.documentText || ''; if (t.includes('Amazing')) return { pass: false, score: 0, reason: 'Entity name not replaced' }; - if (!t.includes('Irys Inc.')) + if (!t.includes('Northwind Ltd.')) return { pass: false, score: 0, reason: 'New entity name missing' }; if (t.includes('[Party A Signature]')) return { pass: false, score: 0, reason: 'Signature placeholder not filled' }; diff --git a/packages/ai/src/shared/prompts.ts b/packages/ai/src/shared/prompts.ts index 6a797dbef7..788f1013f5 100644 --- a/packages/ai/src/shared/prompts.ts +++ b/packages/ai/src/shared/prompts.ts @@ -280,5 +280,5 @@ export const buildAIPlannerSystemPrompt = (toolDescriptions: string): string => Multi-step: find then insert → {"reasoning":"Find clause first, then add new content after it","steps":[{"id":"find","tool":"findAll","instruction":"Find Clause 7.1"},{"id":"add","tool":"insertContent","instruction":"Add a new Clause 7.2 about confidentiality","args":{"position":"after"}}]} Multi-step: generate then use → {"reasoning":"Summarize content first, then add comment with summary using step reference","steps":[{"id":"find","tool":"findAll","instruction":"Find indemnification terms"},{"id":"summarize","tool":"summarize","instruction":"Summarize the indemnification terms"},{"id":"comment","tool":"literalInsertComment","instruction":"Add comment with summary","args":{"find":"indemnification terms","comment":"$previous"}}]} Literal tools (explicit text) → {"reasoning":"User provided exact find/replace text","steps":[{"id":"swap","tool":"literalReplace","instruction":"Replace the legacy company name","args":{"find":"OldName","replace":"NewName","trackChanges":true}}]} - Multiple operations → {"reasoning":"Fix grammar and ask about entity","steps":[{"id":"fix","tool":"insertTrackedChanges","instruction":"Fix grammar errors"},{"id":"question","tool":"insertComments","instruction":"Ask if 'Iqidis' is the correct entity name"}]}`; + Multiple operations → {"reasoning":"Fix grammar and ask about entity","steps":[{"id":"fix","tool":"insertTrackedChanges","instruction":"Fix grammar errors"},{"id":"question","tool":"insertComments","instruction":"Ask if 'Northwind' is the correct entity name"}]}`; }; diff --git a/packages/layout-engine/style-engine/src/cascade.test.ts b/packages/layout-engine/style-engine/src/cascade.test.ts index 4cf8f6de72..4a794d2c1e 100644 --- a/packages/layout-engine/style-engine/src/cascade.test.ts +++ b/packages/layout-engine/style-engine/src/cascade.test.ts @@ -222,8 +222,8 @@ describe('cascade - combineRunProperties', () => { expect(result.fontFamily).toEqual({ cs: 'Arial' }); }); - it('Athenaintelligence customer shape: all 4 concretes from defaults dropped by inline themes', () => { - // Mirrors the customer fixture: docDefaults supply concrete fonts (Arial), + it('theme-override shape: all 4 concretes from defaults dropped by inline themes', () => { + // Mirrors a real-world document: docDefaults supply concrete fonts (Arial), // inline rPr supplies theme refs on ascii/hAnsi/cs (no eastAsiaTheme). The // cascade must keep only the theme refs on those three slots; eastAsia // concrete from defaults is independent. @@ -239,7 +239,7 @@ describe('cascade - combineRunProperties', () => { }); }); - it('exports FONT_SLOT_THEME_PAIRS so callers (super-editor plugin) can stay in sync', async () => { + it('exports FONT_SLOT_THEME_PAIRS so runtime adapters can stay in sync', async () => { const { FONT_SLOT_THEME_PAIRS } = await import('./cascade.js'); expect(FONT_SLOT_THEME_PAIRS).toEqual([ ['ascii', 'asciiTheme'], diff --git a/packages/super-editor/src/editors/v1/document-api-adapters/plan-engine/word-diff.test.ts b/packages/super-editor/src/editors/v1/document-api-adapters/plan-engine/word-diff.test.ts index 4444376994..4463e3e344 100644 --- a/packages/super-editor/src/editors/v1/document-api-adapters/plan-engine/word-diff.test.ts +++ b/packages/super-editor/src/editors/v1/document-api-adapters/plan-engine/word-diff.test.ts @@ -56,7 +56,7 @@ describe('getWordChanges', () => { expect(applyOps('a b c', ops)).toBe('x a y b c'); }); - it('SD-3044: regression with the exact suffix-trim shape from the Lighthouse fixture', () => { + it('SD-3044: regression with the exact suffix-trim shape from the reported fixture', () => { // After prefix/suffix trim, the parties-investor block reduces to these // strings. The trailing `]` of the second `[insert]` is in the suffix, so // `[insert` (without the `]`) becomes a token that matches between old and