diff --git a/Sources/dnt/Dnt.swift b/Sources/dnt/Dnt.swift index 8ebee23..f8b5bb1 100644 --- a/Sources/dnt/Dnt.swift +++ b/Sources/dnt/Dnt.swift @@ -123,6 +123,18 @@ struct BackendOptions: ParsableArguments { """) var keyterms = false + /// Overrides what the app's example box holds, for this run only. + /// + /// Nothing is written back: the point is to answer "what would this wording do to my own + /// audio" without first changing the setting the app is dictating with. `dnt-eval` has the + /// same flag against the near-miss corpus; this one points it at a real recording. + @Option( + name: .long, + help: ArgumentHelp( + "Override the example box: a preset name (prose, chat, notes), `none`, or literal " + + "text. Defaults to whatever the app is set to.")) + var example: String? + func resolveProvider() throws -> ProviderKind { guard let provider else { return AppPreferences.provider } guard let kind = ProviderKind(persistedValue: provider) else { @@ -147,6 +159,21 @@ struct BackendOptions: ParsableArguments { model ?? AppPreferences.model(for: kind) } + /// What to send as the dictation-style example. + /// + /// A preset name resolves to that preset's text — the user's own copy of it when they have + /// edited one, since the builder is the same one the app reads. Anything else is taken + /// literally, so a wording being considered can be measured before it is written to a file. + func resolveDictationExample(_ builder: PromptBuilder) throws -> String { + guard let example else { return AppPreferences.dictationExample(using: builder) } + let trimmed = example.trimmed + if trimmed.isEmpty || trimmed.lowercased() == "none" { return "" } + if let preset = DictationPreset(rawValue: trimmed.lowercased()) { + return try builder.dictationPresetText(preset) + } + return trimmed + } + func promptURL() throws -> URL { if let prompt { return URL(fileURLWithPath: prompt) } guard let found = PromptBuilder.findPromptDirectory() ?? Self.bundledPromptURL else { @@ -206,7 +233,7 @@ struct BackendOptions: ParsableArguments { systemInstruction: try promptBuilder().systemInstruction( fidelity: try resolveFidelity(), script: AppPreferences.chineseScript, - dictationExample: AppPreferences.dictationExample(using: try promptBuilder())), + dictationExample: try resolveDictationExample(try promptBuilder())), fidelity: try resolveFidelity(), keytermBiasing: keyterms, typography: AppPreferences.typographySpacing) @@ -239,7 +266,7 @@ struct BackendOptions: ParsableArguments { systemInstruction: try promptBuilder().systemInstruction( fidelity: try resolveFidelity(), script: AppPreferences.chineseScript, - dictationExample: AppPreferences.dictationExample(using: try promptBuilder())), + dictationExample: try resolveDictationExample(try promptBuilder())), fidelity: try resolveFidelity(), typography: AppPreferences.typographySpacing) return (service, resolved.source) diff --git a/docs/PROMPT.md b/docs/PROMPT.md index c9668c3..821f139 100644 --- a/docs/PROMPT.md +++ b/docs/PROMPT.md @@ -253,8 +253,8 @@ clause added to it unconditionally would invalidate all of them at once. > is still an answer, and it was the least predictable one available. So the numbers below describe > the request as *measured*, which is now the request an install makes only after the box is > cleared. Upgrading installs are untouched: whatever they were sending, they go on sending. The -> re-measurement against the seeded default is owed, and clearing the box reproduces the table's -> conditions exactly in the meantime. +> re-measurement against the seeded default was owed and is now the 2026-09-06 pair below; +> clearing the box still reproduces the table's original conditions exactly. **What each block is allowed to do.** Both open by restating the rule they could otherwise be read as relaxing: formatting governs how the transcript is written down, never what it says, and nothing @@ -280,6 +280,8 @@ screen context broke it. That is the failure this contract exists to prevent, an | Date | Change | Provider / model | runs | matched | improved | regressed | |------|--------|------------------|------|---------|----------|-----------| +| 2026-09-06 | Prose example reworded to stop asking for paragraph breaks | **gemini** · gemini-3.6-flash | 48 | 38 | 8 | **2** | +| 2026-09-06 | The seeded prose default as it shipped in 0.6.0 (control) | **gemini** · gemini-3.6-flash | 48 | 38 | 7 | **3** | | 2026-08-30 | Writing styles added; default request byte-identical (control) | **gemini** · gemini-3.5-flash | 48 | 38 | 8 | **2** | | 2026-08-30 | Translation stage added; transcription request untouched (control) | **gemini** · gemini-3.5-flash | 48 | 36 | 9 | **3** | | 2026-08-30 | Formatting blocks added; default request byte-identical (control) | **gemini** · gemini-3.5-flash | 48 | 38 | 7 | **2** | @@ -291,6 +293,54 @@ screen context broke it. That is the failure this contract exists to prevent, an | 2026-08-09 | Initial contract | **gemini** · gemini-3.6-flash | 15 | 15 | 0 | **0** | | 2026-08-09 | Initial contract | openrouter · google/gemini-3.6-flash | 15 | 12 | 0 | 1 | +### 2026-09-06 — the prose example stops asking for line breaks + +The previous entry closed by saying that a change to the *text* of a formatting block needs its own +measurement. This is that change, and this is that measurement. + +`prompt/dictation-style/prose.md` ended with *"and paragraph breaks where the speaker changed +subject"*. Seeded into every new install since 0.6.0, it turned out to be an instruction the model +obeys at hesitation pauses rather than at subject changes — which in spontaneous speech is most +pauses. The complaint that surfaced it was a 65-second Mandarin dictation that came back in 19 +fragments, 8 of them splitting a sentence in half. + +**Measured on real dictations rather than on the near-miss corpus**, because the near-miss suite +scores spelling under screen context and says nothing about layout. Ten recordings were replayed +from the maintainer's own history with the screen context each one originally carried, six passes +per wording, through `dnt transcribe --example`: + +| wording | runs | line breaks | mid-sentence | runs affected | +|---|---|---|---|---| +| `...and paragraph breaks where the speaker changed subject.` | 60 | 82 | 26 | 24 | +| `...in one unbroken paragraph. Never emit a line break.` | 60 | 0 | 0 | 0 | + +**An empty box was not the alternative.** It also produces no line breaks, but on two of the five +Mandarin recordings the model then dropped nearly all punctuation — 1.0 and 0.0 marks per hundred +characters, a wall of text. The prose clause earns its place; only its paragraph half was harmful. +This is the same finding as the seeding decision above, from the other direction. + +**Naming the failure made it worse.** A variant saying *"never in the middle of a sentence"* and +*"a pause is punctuation, never a line break"* produced more mid-sentence breaks than the wording +it was meant to fix, not fewer. That is the effect already recorded in `ContextEncoder.footer`, +where an instruction illustrating the rule with a concrete wrong value made substitution worse: +naming the wrong answer appears to prime it. The wording that shipped does not mention the failure, +it just declines to ask for the behaviour. + +**Cost on the near-miss suite: none that this suite can see.** The reworded example and the 0.6.0 +default were each run at three passes on the same day — 38 matched out of 48 both times, regressed +3 against 2. Both movements are inside the per-pass range the runner prints, and both runs are +committed as scorecards (`2026-09-06-prose-unbroken.json`, `2026-09-06-prose-paragraphs-control.json`) +so the grading can be re-checked without re-billing the suite. + +The control run is also the re-measurement owed since the box began being seeded: it is the request +a 0.6.0 install actually makes, rather than the empty-box request every older row above describes. + +**Existing installs do not pick this up on their own.** The example box stores a *copy* of the +preset's text and is seeded once, when the key has never been written, so an install that already +has the old sentence keeps sending it. Pressing **Prose** again in Settings replaces it. That is +the same property that makes the box editable at all, and it is why this is a fix somebody has to +accept rather than one that arrives silently. + ### 2026-08-30 — the writing styles, and a third control run `prompt/dictation-style.md` (the renamed `sample.md`) and `prompt/dictation-style/` were added, and diff --git a/eval/scorecards/2026-09-06-prose-paragraphs-control.json b/eval/scorecards/2026-09-06-prose-paragraphs-control.json new file mode 100644 index 0000000..15aa5a3 --- /dev/null +++ b/eval/scorecards/2026-09-06-prose-paragraphs-control.json @@ -0,0 +1,356 @@ +{ + "caseDirectory" : "eval/nearmiss", + "entries" : [ + { + "audioTokens" : 77, + "caseID" : "gemini-version", + "pass" : 0, + "withContext" : "We should switch to Gemini 3.5 Flash for this.", + "withoutContext" : "We should switch to Gemini 1.5 Flash for this." + }, + { + "audioTokens" : 77, + "caseID" : "gemini-version", + "pass" : 1, + "withContext" : "We should switch to Gemini 3.5 Flash for this.", + "withoutContext" : "We should switch to Gemini 3.5 Flash for this." + }, + { + "audioTokens" : 77, + "caseID" : "gemini-version", + "pass" : 2, + "withContext" : "We should switch to Gemini 3.5 Flash for this.", + "withoutContext" : "We should switch to Gemini 1.5 Flash for this." + }, + { + "audioTokens" : 59, + "caseID" : "port-number", + "pass" : 0, + "withContext" : "Run the dev server on port 8081.", + "withoutContext" : "Run the dev server on port 8081." + }, + { + "audioTokens" : 59, + "caseID" : "port-number", + "pass" : 1, + "withContext" : "Run the dev server on port 8081.", + "withoutContext" : "Run the dev server on port 8081." + }, + { + "audioTokens" : 59, + "caseID" : "port-number", + "pass" : 2, + "withContext" : "Run the dev server on port 8081.", + "withoutContext" : "Run the dev server on port 8081." + }, + { + "audioTokens" : 61, + "caseID" : "person-name", + "pass" : 0, + "withContext" : "Can you send the draft to Priya before Friday?", + "withoutContext" : "Can you send the draft to Priya before Friday?" + }, + { + "audioTokens" : 61, + "caseID" : "person-name", + "pass" : 1, + "withContext" : "Can you send the draft to Priya before Friday?", + "withoutContext" : "Can you send the draft to Priya before Friday?" + }, + { + "audioTokens" : 61, + "caseID" : "person-name", + "pass" : 2, + "withContext" : "Can you send the draft to Priya before Friday?", + "withoutContext" : "Can you send the draft to Priya before Friday?" + }, + { + "audioTokens" : 68, + "caseID" : "jargon-spelling", + "pass" : 0, + "withContext" : "We load the native library through Coffee at startup.", + "withoutContext" : "We load the native library through Caffee at startup." + }, + { + "audioTokens" : 68, + "caseID" : "jargon-spelling", + "pass" : 1, + "withContext" : "We load the native library through JNI at startup.", + "withoutContext" : "We load the native library through coffee at startup." + }, + { + "audioTokens" : 68, + "caseID" : "jargon-spelling", + "pass" : 2, + "withContext" : "We load the native library through JNI at startup.", + "withoutContext" : "We load the native library through Caffee at startup." + }, + { + "audioTokens" : 73, + "caseID" : "git-command", + "pass" : 0, + "withContext" : "Let's just do git commit --amend and move on.", + "withoutContext" : "Let's just do git commit --amend and move on." + }, + { + "audioTokens" : 73, + "caseID" : "git-command", + "pass" : 1, + "withContext" : "Let's just do git commit --amend and move on.", + "withoutContext" : "Let's just do git commit --amend and move on." + }, + { + "audioTokens" : 73, + "caseID" : "git-command", + "pass" : 2, + "withContext" : "Let's just do git commit --amend and move on.", + "withoutContext" : "Let's just do git commit --amend and move on." + }, + { + "audioTokens" : 550, + "caseID" : "real-version-number", + "pass" : 0, + "withContext" : "unified thoughts and it continues the thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill, for input tokens, we can achieve more than 500 tokens per second for the input. And also the SOTA models, including Gemini 1.5", + "withoutContext" : "5 softs, and it continues their thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill, for the input tokens, we can achieve more than 500 tokens per second for the input. And also the smaller models including Gemini 1.5" + }, + { + "audioTokens" : 550, + "caseID" : "real-version-number", + "pass" : 1, + "withContext" : "unified thoughts and it continues the thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill, for input tokens, we can achieve more than 500 tokens per second for the input. And also the SOTA models, including Gemini 2.5", + "withoutContext" : "5 source and it continuous the thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill for input tokens, we can achieve more than 500 tokens per second for the input. And also the smaller models including Gemini 1.5" + }, + { + "audioTokens" : 550, + "caseID" : "real-version-number", + "pass" : 2, + "withContext" : "simplified thoughts and it continues the thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill for input tokens we can achieve more than 500 tokens per second for the input. And also the SOTA models including Gemini 2.5", + "withoutContext" : "5 thoughts, and it continues the thinking. Why this works? We basically observe the LLM processing speed. We know that for the prefill, for the input tokens, we can achieve more than 500 tokens per second for the input. And also, the smaller models, including Gemini 1.5" + }, + { + "audioTokens" : 550, + "caseID" : "real-mandarin", + "pass" : 0, + "withContext" : "我们现在通过AI可以在其中增加自己的response。比如用户可以去选择自己的选择。AI在其中经过处理之后，它会引入到一个新的story line。这个新的story line可能就是AI生成的这样一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的喜好程度，然后来评估", + "withoutContext" : "我们现在通过AI可以在其中去增加自己的response，就用户可以去选择一个自己的选择。然后AI在其中经过处理之后，它会引入到一个新的story line。那么这个新story line可能就是AI生成的这样的一个完全不同的隐藏款剧情。那么根据AI还可以根据用户的响性的稀缺程度然后来评估" + }, + { + "audioTokens" : 550, + "caseID" : "real-mandarin", + "pass" : 1, + "withContext" : "我们可以现在通过AI可以在其中去增加自己的response，比如用户可以去选择一个自己的选择。AI在其中经过处理之后，它会引入到一个新的story line。这个新的story line可能就是AI生成的这样一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的稀缺程度然后来评估", + "withoutContext" : "我们现在通过AI可以在其中去增加自己的response，比如用户可以去选择自己的选择，AI在其中经过处理之后，它会引入到一个新的story line。这个新story line可能就是AI生成的这样一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的稀缺程度，然后来评估" + }, + { + "audioTokens" : 550, + "caseID" : "real-mandarin", + "pass" : 2, + "withContext" : "我们现在通过AI可以在其中去增加自己的response，比如用户可以去选择一个自己的选择。然后AI在其中经过处理之后，它会引入到一个新的story line。那么这个新story line可能就是AI生成的这样的一个完全不同的隐藏款剧情。根据AI还可以根据用户的响性的唏嘘程度，然后来评估", + "withoutContext" : "我们现在通过AI可以在其中去增加自己的response，比如用户可以去选择一个自己的选择。AI在其中经过处理之后，它会引入到一个新的story line。这个新的story line可能就是AI生成的这样一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的稀缺程度，来评估" + }, + { + "audioTokens" : 500, + "caseID" : "real-codeswitch", + "pass" : 0, + "withContext" : "我要把这几个给它串起来，给它搞成一个retrieval pipeline，给它串在一起的。所有的都一样，所以说我要把它给它分开，比如说这个是1024我印象里，然后，OK，这就算弄好了。然后可以看到在这个里面，它就把这个index已经把它的这个...", + "withoutContext" : "我要把几个串起来，就搞成一个 retrieval pipeline，给它串在一起，所有的都一样，所以说我要把它给它分开。比如说这个是 4240 我印象里，然后 OK，这就算弄好了。然后可以看到在这个里面，它就把这个 index 已经把这个" + }, + { + "audioTokens" : 500, + "caseID" : "real-codeswitch", + "pass" : 1, + "withContext" : "我要把一部分給它串起來，搞成一個 retrieval pipeline，要給它串在一起的，它所有的都一樣，所以說我要把它給它分開，比如說這個是 4240，我印象裡，然後 OK。然後這個就是就算弄好了，然後可以看到在這個裏面，它就把這個 index 已經把它的這個", + "withoutContext" : "我要是把这几个给它串起来，给它搞成一个 retrieval pipeline，要给它串在一起的。所有都一样，所以我把它给它分开，比如说这个是 4240，我印象里。这个就算算弄好了，可以看到在这里边，它就把这个 index 已经把这个" + }, + { + "audioTokens" : 500, + "caseID" : "real-codeswitch", + "pass" : 2, + "withContext" : "我要把几个串起来，就改成一个 retrieval pipeline 给它串在一起。所有的都一样，所以说我要把它给它分开，比如说这个是 4240，我印象里。这个就算弄好了。可以看到在这个里面他就把这个 index 已经把这个", + "withoutContext" : "我要是把这几个给它串起来，给它搞成一个 retrieval pipeline，给它串在一起，然后它这所有都一样，所以说我要把它这个给它分开。比如说这个是 4240 我印象里，然后 OK，然后这个就是这就算这弄好了。然后可以看到在这个里面，它就把这个 index 已经把这个" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym", + "pass" : 0, + "withContext" : "一部分参数，还是专门就像用现在的类似DAPO的方法，去训它的gradient。另外一种，就是我就用next token prediction的方法，去minimize它predicted的loss。但是这里predicted的东西，它不是XFT", + "withoutContext" : "一部分参数，专门就用现代的类似DAP DAPO的方法去训它的gradient。另外一种，就是我就用Next Token Prediction的方法，去minimize它predicted的loss。但是这里predict的东西，它不是x FT，" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym", + "pass" : 1, + "withContext" : "一部分参数，还是专门就像用现在的类似于DAP, DPO的方法去训它的gradient。另外一种，就是我就用Next Token Prediction的方法去minimize它predicted的loss，但是这里predicted的东西，它不是X, FT", + "withoutContext" : "一部分参数，专门像用跟现在的类似DPO的方法，去训它的gradient。另外一种，就是我就用Next Token Prediction的方法，去minimize它predicted的loss。但是这里predict的东西，它不是xft" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym", + "pass" : 2, + "withContext" : "一部分参数，专门用像跟用现在的类似DAP DAPO的方法，去训它的gradient。另外一种，就是我就用这是next token prediction的方法，去minimize它predict的loss，但是这里predict的东西，它不是XFT，", + "withoutContext" : "一部分参数，专门像用跟现在的类似DAP DAPO的方法去训它的gradient。另外一种，就是我就用Next Token Prediction的方法去minimize它predicted的loss，但是这里predicted的东西它不是SFT" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym-chain", + "pass" : 0, + "withContext" : "feeds for the speech event of the VAD model which means Voice Activity Detection and then feeds that into the ASR model and then feeds the complete sentence into the large language model which performs the thought process and then splits all the responses of the large language", + "withoutContext" : "first falls the speech and event of the VAD model, which means Voice Activity Detection, and then feeds that into the ASR model, and then feeds the complete sentence into the large language model, which performs the thought process and then splits all the responses of the large language" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym-chain", + "pass" : 1, + "withContext" : "speech and in event of the VAD model, which is which means voice activity detection, and then feeds that into the ASR model, and then feeds the complete sentence into the large language model, which performs the thought process and then split all the responses of the large language", + "withoutContext" : "firstly speech and events of the VAD model, which means Voice Activity Detection, and then feeds that into the ASR model, and then feeds the complete sentence into the large language model which performs the thought process, and then splits all the responses of the large language" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym-chain", + "pass" : 2, + "withContext" : "Follows the speech in the event of the VAD model, which means Voice Activity Detection, and then feeds that into the ASR model, and then feeds the complete sentence into the large language model, which performs the thoughts process, and then split all the responses of the large language", + "withoutContext" : "First, the speech and event of the VAD model, which means Voice Activity Detection, and then feeds that into the ASR model, and then feeds the complete sentence into the Large Language Model, which performs the thought process, and then splits all the responses of the Large Language Model." + }, + { + "audioTokens" : 500, + "caseID" : "real-jargon", + "pass" : 0, + "withContext" : "each problem that identified. And for some important work items, we can re- Scrum dashboard and create a work item and create a test cases and session, that it means agents working session for the manual inspection of the developer.", + "withoutContext" : "each problem that identified. And for some important work items, we can reach the scrum dashboard and create a work item and create test cases and session that it means agents working session for the manual inspection of the developer." + }, + { + "audioTokens" : 500, + "caseID" : "real-jargon", + "pass" : 1, + "withContext" : "each problem that identified. And for some important work items, we can re-use the Scrum dashboard, and create a work item, and create test cases, and the session that it means the agents working session for the manual inspection of the developer.", + "withoutContext" : "each problem that they identified. And for some important work items, we can reach the scrum dashboard and create a work item and create a test cases and session that it means the agents working session for the manual inspection of the developer." + }, + { + "audioTokens" : 500, + "caseID" : "real-jargon", + "pass" : 2, + "withContext" : "each problem that identified. And for some important work items, we can re-use Scrum dashboard and create a work item, and create test cases, and the session, that means the agent's working session, for the manual inspection of the developer.", + "withoutContext" : "each problem that identified. And for some important work items, we can re-scrum dashboard and create a work item and create test cases and session that it means agent's working session for the manual inspection of the developer." + }, + { + "audioTokens" : 500, + "caseID" : "real-brand", + "pass" : 0, + "withContext" : "确。因为就举个简单的例子，如果说搜索这事这么简单的话，每个人都可以说做个 Google。那为什么 Google 的搜索结果就比 Bing 的好，对吧？Bing 的又比其他的搜索引擎更好。那原因就是说它有很多的这种用户反馈的数据，然后还有它自己的一些排序的算法，它能够保证同一个 query 进去之后，它", + "withoutContext" : "确。因为就举个简单的例子，如果说搜索这事这么简单的话，每个人都可以说做个Google。那为什么Google的搜索结果就比Bing的好？Bing的又比其他的一些搜索引擎更好？原因就是说它有很多的这种用户反馈的数据，然后还有它自己的一些排序的算法，它能够保证同一个query进去之后，" + }, + { + "audioTokens" : 500, + "caseID" : "real-brand", + "pass" : 1, + "withContext" : "就举个简单的例子，如果说搜索这事这么简单的话，那每个人都可以说做一个Google，那为什么Google的搜索结果就比Bing的好，Bing的又比其他的搜索引擎更好，那原因就是说它有很多的用户反馈的数据，然后还有它自己的一些排序的算法，它能够保证同一个query进去之后", + "withoutContext" : "确。因为就举个简单的例子，如果说搜索这事这么简单的话，那么每个人都可以说做一个 Google。那为什么 Google 的搜索结果就比 Bing 的好，对吧，Bing 的又比其他的搜索引擎更好？原因就是说它有很多的用户反馈的数据，然后还有它自己的一些排序的算法，它能够保证同一个 query 进去之后，它" + }, + { + "audioTokens" : 500, + "caseID" : "real-brand", + "pass" : 2, + "withContext" : "确，因为举个简单的例子，如果说搜索这事这么简单的话，每每个人都可以说做一个Google。那为什么Google的搜索结果就比Bing的好？Bing的又比其他的搜索引擎更好？原因就是说它有很多的这种用户反馈的数据，然后还有它自己的一些排序的算法，它能够保证同一个query进去之后，它", + "withoutContext" : "确，因为举个简单的例子，如果说搜索引擎这事这么简单的话，那每个人都可以说做一个Google，那为什么Google的搜索结果就比Bing的好？Bing的又比其他的一些搜索引擎更好？那原因就是说它有很多的用户反馈数据，然后还有它自己的一些排序算法，它能够保证同一个query进去之后，它" + }, + { + "audioTokens" : 66, + "caseID" : "benefit-novel-name", + "pass" : 0, + "withContext" : "Ask Keylit to review the merge before stand up.", + "withoutContext" : "Ask Keylit to review the merge before stand up." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-novel-name", + "pass" : 1, + "withContext" : "Ask Kaleb to review the merge before standup.", + "withoutContext" : "Ask Keylit to review the merge before stand up." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-novel-name", + "pass" : 2, + "withContext" : "Ask Keylit to review the merge before standup.", + "withoutContext" : "Ask Keylet to review the merge before standup." + }, + { + "audioTokens" : 72, + "caseID" : "benefit-novel-codename", + "pass" : 0, + "withContext" : "Thessaly rollout is blocked on the Brindlewood cluster.", + "withoutContext" : "The Thessaly rollout is blocked on the Brindlewood cluster." + }, + { + "audioTokens" : 72, + "caseID" : "benefit-novel-codename", + "pass" : 1, + "withContext" : "The Thessaly rollout is blocked on the Brindlewood cluster.", + "withoutContext" : "The Thessaly rollout is blocked on the Brindlewood cluster." + }, + { + "audioTokens" : 72, + "caseID" : "benefit-novel-codename", + "pass" : 2, + "withContext" : "Thessaly rollout is blocked on the Brindlewood cluster.", + "withoutContext" : "The Thessaly rollout is blocked on the Brindlewood cluster." + }, + { + "audioTokens" : 68, + "caseID" : "benefit-novel-repo", + "pass" : 0, + "withContext" : "Clone QuillMark-Sync and run the setup script.", + "withoutContext" : "Clone Quilmark-sync and run the setup script." + }, + { + "audioTokens" : 68, + "caseID" : "benefit-novel-repo", + "pass" : 1, + "withContext" : "Clone quillmark-sync and run the setup script.", + "withoutContext" : "Clone QuillMark-Sync and run the setup script." + }, + { + "audioTokens" : 68, + "caseID" : "benefit-novel-repo", + "pass" : 2, + "withContext" : "Clone Quillmark-sync and run the setup script.", + "withoutContext" : "clone quilmark-sync and run the setup script." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-caret-channel", + "pass" : 0, + "withContext" : "Ask Kaelith to review the merge before standup.", + "withoutContext" : "Ask Keylet to review the merge before standup." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-caret-channel", + "pass" : 1, + "withContext" : "Ask Kaelith to review the merge before standup.", + "withoutContext" : "Ask Keylet to review the merge before standup." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-caret-channel", + "pass" : 2, + "withContext" : "Ask Kaelith to review the merge before standup.", + "withoutContext" : "Ask Keylit to review the merge before stand up." + } + ], + "provenance" : { + "fidelity" : "light", + "model" : "gemini-3.6-flash", + "promptDigest" : "8820496817f9", + "provider" : "google", + "recordedAt" : "2026-09-06T14:01:09Z" + }, + "summary" : { + "improved" : 7, + "neutralCorrect" : 31, + "neutralWrong" : 7, + "passed" : 38, + "regressed" : 3, + "runs" : 48 + } +} \ No newline at end of file diff --git a/eval/scorecards/2026-09-06-prose-unbroken.json b/eval/scorecards/2026-09-06-prose-unbroken.json new file mode 100644 index 0000000..5b051a5 --- /dev/null +++ b/eval/scorecards/2026-09-06-prose-unbroken.json @@ -0,0 +1,356 @@ +{ + "caseDirectory" : "eval/nearmiss", + "entries" : [ + { + "audioTokens" : 77, + "caseID" : "gemini-version", + "pass" : 0, + "withContext" : "We should switch to Gemini 3.5 Flash for this.", + "withoutContext" : "We should switch to Gemini 3.5 Flash for this." + }, + { + "audioTokens" : 77, + "caseID" : "gemini-version", + "pass" : 1, + "withContext" : "We should switch to Gemini 3.5 Flash for this.", + "withoutContext" : "We should switch to Gemini 3.5 Flash for this." + }, + { + "audioTokens" : 77, + "caseID" : "gemini-version", + "pass" : 2, + "withContext" : "We should switch to Gemini 3.5 Flash for this.", + "withoutContext" : "We should switch to Gemini 3.5 Flash for this." + }, + { + "audioTokens" : 59, + "caseID" : "port-number", + "pass" : 0, + "withContext" : "Run the dev server on port 8081.", + "withoutContext" : "Run the dev server on port 8081." + }, + { + "audioTokens" : 59, + "caseID" : "port-number", + "pass" : 1, + "withContext" : "Run the dev server on port 8081.", + "withoutContext" : "Run the dev server on port 8081." + }, + { + "audioTokens" : 59, + "caseID" : "port-number", + "pass" : 2, + "withContext" : "Run the dev server on port 8081.", + "withoutContext" : "Run the dev server on port 8081." + }, + { + "audioTokens" : 61, + "caseID" : "person-name", + "pass" : 0, + "withContext" : "Can you send the draft to Priya before Friday?", + "withoutContext" : "Can you send the draft to Priya before Friday?" + }, + { + "audioTokens" : 61, + "caseID" : "person-name", + "pass" : 1, + "withContext" : "Can you send the draft to Priya before Friday?", + "withoutContext" : "Can you send the draft to Priya before Friday?" + }, + { + "audioTokens" : 61, + "caseID" : "person-name", + "pass" : 2, + "withContext" : "Can you send the draft to Priya before Friday?", + "withoutContext" : "Can you send the draft to Priya before Friday?" + }, + { + "audioTokens" : 68, + "caseID" : "jargon-spelling", + "pass" : 0, + "withContext" : "We load the native library through Coffee at startup.", + "withoutContext" : "We load the native library through Coffee at startup." + }, + { + "audioTokens" : 68, + "caseID" : "jargon-spelling", + "pass" : 1, + "withContext" : "We load the native library through Coffee at startup.", + "withoutContext" : "We load the native library through Coffee at startup." + }, + { + "audioTokens" : 68, + "caseID" : "jargon-spelling", + "pass" : 2, + "withContext" : "We load the native library through Coffee at startup.", + "withoutContext" : "We load the native library through Coffee at startup." + }, + { + "audioTokens" : 73, + "caseID" : "git-command", + "pass" : 0, + "withContext" : "Let's just do git commit --amend and move on.", + "withoutContext" : "Let's just do git commit --amend and move on." + }, + { + "audioTokens" : 73, + "caseID" : "git-command", + "pass" : 1, + "withContext" : "Let's just do git commit --amend and move on.", + "withoutContext" : "Let's just do git commit --amend and move on." + }, + { + "audioTokens" : 73, + "caseID" : "git-command", + "pass" : 2, + "withContext" : "Let's just do git commit --amend and move on.", + "withoutContext" : "Let's just do git commit --amend and move on." + }, + { + "audioTokens" : 550, + "caseID" : "real-version-number", + "pass" : 0, + "withContext" : "provided thoughts and it continues the thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill, for input tokens, we can achieve more than 500 tokens per second for the input. And also the smaller models including Gemini 1.5", + "withoutContext" : "5 thoughts, and it continues thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill, for input tokens, we can achieve more than 500 tokens per second for the input. And also, the smaller models, including Gemini 1.5" + }, + { + "audioTokens" : 550, + "caseID" : "real-version-number", + "pass" : 1, + "withContext" : "provided thoughts and it continues thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill, for input tokens, we can achieve more than 500 tokens per second for the input. And also the smaller models including Gemini 2.5", + "withoutContext" : "5 thoughts and it continues the thinking. Why this works? So we basically observe the LLM processing speed. We know that for the prefill, for input tokens, we can achieve more than 500 tokens per second for the input. And also the smaller models, including Gemini 1.5" + }, + { + "audioTokens" : 550, + "caseID" : "real-version-number", + "pass" : 2, + "withContext" : "provide thoughts and it continues the thinking. Why this works? We basically observe the LLM processing speed. We know that for the prefill, for input tokens, we can achieve more than 500 tokens per second for the input. And also the smaller models, including Gemini 2.5", + "withoutContext" : "5 thoughts and it continuous this thinking. Why this works? So we basically observe LLM processing speed. We know that for the prefill for input tokens, we can achieve more than 500 tokens per second for the input and also the smaller models including Gemini 1.5" + }, + { + "audioTokens" : 550, + "caseID" : "real-mandarin", + "pass" : 0, + "withContext" : "我们现在通过AI可以在其中去增加自己的response，比如用户可以去选择一个自己的选择，AI在其中经过处理之后，它会引入到新的storyline，这个新的storyline可能就是AI生成的这样一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的稀缺程度，然后来评估", + "withoutContext" : "我们现在通过AI可以在其中去增加自己的response，用户可以去选择一个自己的选择，AI在其中经过处理之后，它会引入到一个新的story line。那么这个新的story line可能就是AI生成的，这样的一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的稀缺程度，然后来评估" + }, + { + "audioTokens" : 550, + "caseID" : "real-mandarin", + "pass" : 1, + "withContext" : "我们现在通过AI可以在其中去增加自己的response，比如用户可以去选择一个自己的选择，AI在其中经过处理之后，它会引入到一个新的story line。那么这个新story line可能就是AI生成的，这样的一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的稀缺程度，然后来评估", + "withoutContext" : "我們現在通過AI可以在其中去增加自己的response，比如用戶可以去選擇一個自己的選擇。AI在其中經過處理之後，它會引入到一個新的story line。那麼這個新story line可能就是AI生成的這樣的一個完全不同的隱藏款劇情。根據AI還可以根據用戶的響應的稀缺程度，然後來評估" + }, + { + "audioTokens" : 550, + "caseID" : "real-mandarin", + "pass" : 2, + "withContext" : "我们可以通过AI在其中去增加自己的response，用户可以去选择一个自己的选择。AI在其中经过处理之后，它会引入到一个新的story line。这个新的story line可能就是AI生成的这样一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的稀缺程度，然后来评估", + "withoutContext" : "我们现在通过AI可以在其中去增加自己的response，比如用户可以去选择一个自己的选择，AI在其中经过处理之后，它会引入到一个新的story line。那么这个新的story line可能就是AI生成的，这样的一个完全不同的隐藏款剧情。根据AI还可以根据用户的响应的稀缺程度，然后来评估" + }, + { + "audioTokens" : 500, + "caseID" : "real-codeswitch", + "pass" : 0, + "withContext" : "要把好几个要给她串起来，搞成一个retrieval pipeline要给她串在一起的。所有都一样，所以说我要把它给她分开，比如说这个是4240，我印象里。OK，这个就是这算好，这就算弄好了。然后可以看到在这里边，他就把这个index已经把这个给", + "withoutContext" : "我要是把这几个给它串起来，就搞成一个retrieval pipeline给它串在一起的。然后所有的都一样，所以说我要把它给它分开，比如说这个是4240，我印象里。然后OK，然后这个就是这就算是用好了，然后可以看到在这里边它就把这个index已经把这个给" + }, + { + "audioTokens" : 500, + "caseID" : "real-codeswitch", + "pass" : 1, + "withContext" : "我要把一部分給它串起來，就搞成一個retrieval pipeline，給它串在一起的。然後它全都合在一塊，所以說我要把它這個給它分開，比如說這個是4240我印象裡。然後，OK，然後這個就是這就算是不錯了。然後可以看到在這個裡面，它就把這個index已經把這個就", + "withoutContext" : "要把好几个要给它串起来，就搞成一个retrieval pipeline，要给它串在一起的，所有都一样，所以说我要把它给它分开，比如说这个是4240我印象里。OK，这就算弄好了，可以看到在这个里面，它就把这个index已经把这个..." + }, + { + "audioTokens" : 500, + "caseID" : "real-codeswitch", + "pass" : 2, + "withContext" : "我要把几个串起来，搞成一个retrieval pipeline，给串在一起。它所有都一样，所以说我要把它分开。比如说这个是4240我印象里，然后OK，这个就算是弄好了。可以看到在这个里边，它就把这个index已经把这个给", + "withoutContext" : "要把好几个给它串起来，给它搞成一个retrieval pipeline，给它串在一起的。所有的都一样，所以说我要把它给它分开，比如说这个是4240我印象里。OK，然后这个算是不行弄好了。可以看到在这个里面他就把这个index已经把这个" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym", + "pass" : 0, + "withContext" : "一部分参数，是专门用现在的比如DAPO的方法去训它的gradient。另外一种，就是我就用next token prediction的方法，去minimize它predicted loss。但是这里predicted的东西，它不是XFT，", + "withoutContext" : "一部分参数，专门就想用跟现在类似DAP, DPO的方法去训它的gradient。另外一种，就是我就用Max, 这是Next-token prediction的方法去minimize它predicted的loss。但是这里predict的东西，它不是xft," + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym", + "pass" : 1, + "withContext" : "一部分参数，是专门用现在的比如DAPO的方法去训它的gradient。另外一种，就是我就用next token prediction的方法去minimize它predict的loss。但是这里predict的东西，它不是xFT，", + "withoutContext" : "一部分参数，专门就想用跟现在的类似DAP, DPO的方法去训它的gradient。另外一种，就是我就用Max，这是next token prediction的方法，去minimize它predict的loss。但是这里predict的东西，它不是XFT," + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym", + "pass" : 2, + "withContext" : "一部分参数呢，它是专门用现在的类似DAP, DPO的方法去训它的gradient。另外一种呢，就是我就用这个next token prediction的方法去minimize它predict的loss，但是这里predict的东西它不是x, FT,", + "withoutContext" : "一部分参数，专门就用跟现代的类似于DAP, DPO的方法，然后再去训练它的gradient。另外一种，就是我就用这个Next Token Prediction的方法，去minimize它predict的loss。但是这里predict的东西，它不是xft，" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym-chain", + "pass" : 0, + "withContext" : "Follows the speech event of the VAD model, which means Voice Activity Detection, and then feeds that into the ASR model, and then feeds the complete sentence into the large language model, which performs the thoughts process and then splits all the responses of the large language", + "withoutContext" : "Follows the speech and events of the VAD model which means Voice Activity Detection and then feeds that into the ASR model and then feeds the complete sentence into the large language model which performs the thought process and then splits all the responses of the large language" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym-chain", + "pass" : 1, + "withContext" : "Follows the speech event of the VAD model, which means Voice Activity Detection, and then feeds that into the ASR model, and then feeds the complete sentence into the large language model, which performs the thoughts process and then splits all the responses of the large language", + "withoutContext" : "First, the speech and event of the VAD model, which means voice activity detection, and then feeds that into the ASR model, and then feeds the complete sentence into the large language model, which performs the thought process and then splits all the responses of the large language" + }, + { + "audioTokens" : 500, + "caseID" : "real-acronym-chain", + "pass" : 2, + "withContext" : "Follows the speech event of the VAD model, which means Voice Activity Detection, and then feeds that into the ASR model, and then feeds the complete sentence into the large language model, which performs the thought process and then splits all the responses of the large language", + "withoutContext" : "split all the speech and events of the VAD model, which means Voice Activity Detection, and then feeds that into the ASR model, and then feeds the complete sentence into the Large Language Model, which performs the thought process, and then splits all the responses of the Large Language" + }, + { + "audioTokens" : 500, + "caseID" : "real-jargon", + "pass" : 0, + "withContext" : "each problem that identified. And for some important work items, we can re-use the Scrum dashboard and create a work item, and create test cases and the session that means agent's working session for the manual inspection of the developer.", + "withoutContext" : "Each problem that identified and for some important work items we can read the scrum dashboard and create a work item and create test cases and the session that means the agent's working session for the manual inspection of the developer." + }, + { + "audioTokens" : 500, + "caseID" : "real-jargon", + "pass" : 1, + "withContext" : "each problem that they identify. And for some important work items, we can re-use the Scrum dashboard, and create a work item, and create test cases, and the session that it means the agent's working session for the manual inspection of the developer.", + "withoutContext" : "each problem that I identified. And for some important work items, we can read the Scrum dashboard and create a work item, and create test cases, and the session that it means the agent's working session for the manual inspection of the developer." + }, + { + "audioTokens" : 500, + "caseID" : "real-jargon", + "pass" : 2, + "withContext" : "each problem that had identified. And for some important work items, we can reach the Scrum dashboard and create a work item, and create test cases, and session, that it means the agent's working session for the manual inspection of the developer.", + "withoutContext" : "each problem that had identified. And for some important work items, we can read the Scrum dashboard and create a work item, and create test cases and the session, that means the agent's working session for the manual inspection of the developer." + }, + { + "audioTokens" : 500, + "caseID" : "real-brand", + "pass" : 0, + "withContext" : "就舉個簡單的例子，如果說搜索這事這麼簡單的話，那每個人都可以說做一個 Google。那為什麼 Google 的搜索結果就比 Bing 的好，Bing 的又比其他的搜索引擎更好？原因就是說它有很多的這種用戶反饋的數據，然後還有它自己的一些排序的算法，它能夠是保證同一個 query 進去之後，它", + "withoutContext" : "确。因为就举个简单的例子，如果说这个搜索这事这么简单的话，那么每个人都可以说做一个Google。那为什么Google的搜索结果就比Bing的好，对吧？Bing的又比其他的一些搜索引擎更好？那原因就是说它有很多的这种用户反馈的这些数据，然后还有它自己的一些排序的一些算法，它能够是保证同一个query进去之后，它" + }, + { + "audioTokens" : 500, + "caseID" : "real-brand", + "pass" : 1, + "withContext" : "确，因为就举个简单的例子，如果说这个搜索这事这么简单的话，那每个人都可以说做个Google，那为什么Google的搜索结果就比Bing的好，Bing的又比其它的一些搜索引擎更好，那原因就是说它有很多的这种用户反馈的这些数据，然后还有它自己的一些排序的一些算法，它能够是保证同一个query进去之后，它", + "withoutContext" : "确，因为就举个简单的例子，如果说搜索这事这么简单的话，每个人都可以说做一个Google，那为什么Google的搜索结果就比Bing的好，Bing的又比其他的搜索引擎更好？原因就是说它有很多的这种用户反馈的数据，然后还有它自己的一些排序的算法，它能够是保证同一个query进去之后，它" + }, + { + "audioTokens" : 500, + "caseID" : "real-brand", + "pass" : 2, + "withContext" : "就舉個簡單的例子，如果說搜索這事這麼簡單的話，那每個人都可以說做一個Google。那為什麼Google的搜索結果就比Bing的好，Bing的又比其他的搜索引擎更好？那原因就是說它有很多的這種用戶反饋的數據，然後還有它自己的一些排序的算法，它能夠是保證同一個query進去之後，它", + "withoutContext" : "确，因为举个简单的例子，如果说这个搜索这事这么简单的话，那每个人都可以说做一个Google。那为什么Google的搜索结果就比Bing的好？Bing的又比其他的一些搜索引擎更好？那原因就是说它有很多的这种用户反馈的这些数据，然后还有它自己的一些排序的一些算法，它能够是保证同一个Query进去之后，它" + }, + { + "audioTokens" : 66, + "caseID" : "benefit-novel-name", + "pass" : 0, + "withContext" : "Ask Keylip to review the merge before standup.", + "withoutContext" : "Ask Keylah to review the merge before standup." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-novel-name", + "pass" : 1, + "withContext" : "Ask Keylit to review the merge before standup.", + "withoutContext" : "Ask Caleb to review the merge before standup." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-novel-name", + "pass" : 2, + "withContext" : "Ask Kaleb to review the merge before standup.", + "withoutContext" : "Ask Caleb to review the merge before standup." + }, + { + "audioTokens" : 72, + "caseID" : "benefit-novel-codename", + "pass" : 0, + "withContext" : "The Thessaly rollout is blocked on the Brindlewood cluster.", + "withoutContext" : "The Thessalonii rollout is blocked on the Brindlewood cluster." + }, + { + "audioTokens" : 72, + "caseID" : "benefit-novel-codename", + "pass" : 1, + "withContext" : "Thessaly rollout is blocked on the Brindlewood cluster.", + "withoutContext" : "The Thessaly rollout is blocked on the Brindlewood cluster." + }, + { + "audioTokens" : 72, + "caseID" : "benefit-novel-codename", + "pass" : 2, + "withContext" : "Thessaly rollout is blocked on the Brindlewood cluster.", + "withoutContext" : "The Thessaly rollout is blocked on the Brindlewood cluster." + }, + { + "audioTokens" : 68, + "caseID" : "benefit-novel-repo", + "pass" : 0, + "withContext" : "Clone Quillmark-sync and run the setup script.", + "withoutContext" : "Clone Quilmark-sink and run the setup script." + }, + { + "audioTokens" : 68, + "caseID" : "benefit-novel-repo", + "pass" : 1, + "withContext" : "Clone Quillmark-sync and run the setup script.", + "withoutContext" : "Clone Quilmark-sink and run the setup script." + }, + { + "audioTokens" : 68, + "caseID" : "benefit-novel-repo", + "pass" : 2, + "withContext" : "Clone Quilmard-sync and run the setup script.", + "withoutContext" : "Clone Quilmark-sync and run the setup script." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-caret-channel", + "pass" : 0, + "withContext" : "Ask Kaelith to review the merge before standup.", + "withoutContext" : "Ask Caleb to review the merge before standup." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-caret-channel", + "pass" : 1, + "withContext" : "Ask Kaelith to review the merge before standup.", + "withoutContext" : "Ask Caleb to review the merge before standup." + }, + { + "audioTokens" : 66, + "caseID" : "benefit-caret-channel", + "pass" : 2, + "withContext" : "Ask Kaelith to review the merge before standup.", + "withoutContext" : "Ask Keylit to review the merge before standup." + } + ], + "provenance" : { + "fidelity" : "light", + "model" : "gemini-3.6-flash", + "promptDigest" : "0a2f7cc483ac", + "provider" : "google", + "recordedAt" : "2026-09-06T13:57:45Z" + }, + "summary" : { + "improved" : 8, + "neutralCorrect" : 30, + "neutralWrong" : 8, + "passed" : 38, + "regressed" : 2, + "runs" : 48 + } +} \ No newline at end of file diff --git a/prompt/dictation-style/prose.md b/prompt/dictation-style/prose.md index 8790001..a9d8d56 100644 --- a/prompt/dictation-style/prose.md +++ b/prompt/dictation-style/prose.md @@ -1,2 +1,2 @@ -Write it as prose: complete sentences, standard punctuation, and paragraph breaks where the speaker -changed subject. +Write it as prose: complete sentences and standard punctuation, in one unbroken paragraph. Never +emit a line break.