diff --git a/README.md b/README.md index a314e70dd..289b78af4 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,13 @@ npx doppler-gpu ## Evidence -Doppler has lower steady-state inference latency than Transformers.js in each +Doppler has higher steady-state inference throughput than Transformers.js in each of the comparable browser WebGPU results indexed below. -![Metal and Vulkan browser WebGPU latency distributions](https://raw.githubusercontent.com/clocksmith/doppler/main/assets/doppler-webgpu-evidence.svg) +![Metal and Vulkan browser WebGPU throughput](https://raw.githubusercontent.com/clocksmith/doppler/main/assets/doppler-webgpu-evidence.svg) -For text, faster means lower decode ms/token; for retrieval it means lower -ms/embedding or ms/rerank. Every comparison passes its declared workload +Text results use decode `tok/s`; retrieval uses `embedding/s` or `reranks/s`. +Higher is faster. Every comparison passes its declared workload correctness gate. Model loading is separate: Transformers.js loads the Vulkan embedding and reranker artifacts faster. The [competition scoreboard](https://github.com/clocksmith/doppler/blob/main/docs/model-competition-scoreboard.md) links every diff --git a/assets/doppler-webgpu-evidence.svg b/assets/doppler-webgpu-evidence.svg index cfc0f5d5a..34904b289 100644 --- a/assets/doppler-webgpu-evidence.svg +++ b/assets/doppler-webgpu-evidence.svg @@ -1,6 +1,6 @@ - Doppler and Transformers.js latency distributions - Mean latency, sample standard deviation, and observed range for nine browser WebGPU comparisons on Apple M3 Metal and Radeon 8060S Vulkan. + Doppler and Transformers.js browser WebGPU throughput + Reported throughput and receipt-reported min-to-max rate ranges for nine browser WebGPU comparisons on Apple M3 Metal and Radeon 8060S Vulkan. Higher is faster. @@ -8,73 +8,127 @@ Doppler Transformers.js - BOX ±1 SD · CENTER MEAN · WHISKER MIN–MAX + BAND MIN–MAX · CENTER REPORTED RATE - MODELMEAN ± SD - RUN SPREAD · ← LOWER IS FASTERSPEEDUP + MODELTHROUGHPUT + RECEIPT RATE RANGE · HIGHER IS FASTER →RATE RATIO Apple M3 · Metal4 MODELS - Qwen 3.5 0.8B textn=3 - 26.03 ± 0.23 ms/token36.98 ± 1.98 ms/token - - - 1.42x + Qwen 3.5 0.8B text + n=3 + 38.42 tok/s + 26.54 tok/s + + + + + + 1.45x + - Qwen 3 Embedding 0.6Bn=15 - 43.79 ± 0.42 ms/embedding51.61 ± 0.67 ms/embedding - - - 1.18x + Qwen 3 Embedding 0.6B + n=15 + 22.83 embedding/s + 19.38 embedding/s + + + + + + 1.18x + - Qwen 3 Reranker 0.6Bn=3 - 467.03 ± 17.97 ms/rerank488.87 ± 5.56 ms/rerank - - - 1.05x + Qwen 3 Reranker 0.6B + n=3 + 2.14 reranks/s + 2.05 reranks/s + + + + + + 1.05x + - Qwen 3.5 2B textn=20 paired · local model - 31.41 ± 6.00 ms/token32.62 ± 6.06 ms/token - - - 1.04x + Qwen 3.5 2B text + n=20 · local model + 33.86 tok/s + 32.75 tok/s + + + + + + 1.03x + Radeon 8060S · Vulkan5 MODELS - Qwen 3.5 0.8B textn=15 - 14.75 ± 0.15 ms/token23.88 ± 0.51 ms/token - - - 1.62x + Qwen 3.5 0.8B text + n=15 + 69.07 tok/s + 41.79 tok/s + + + + + + 1.65x + - Qwen 3 Embedding 0.6Bn=15 - 29.51 ± 0.80 ms/embedding149.44 ± 11.00 ms/embedding - - - 5.06x + Qwen 3 Embedding 0.6B + n=15 + 33.88 embedding/s + 6.69 embedding/s + + + + + + 5.06x + - Qwen 3 Reranker 0.6Bn=15 - 278.62 ± 6.01 ms/rerank470.80 ± 12.67 ms/rerank - - - 1.69x + Qwen 3 Reranker 0.6B + n=15 + 3.59 reranks/s + 2.12 reranks/s + + + + + + 1.69x + - Qwen 3.5 2B textn=15 · local model - 17.94 ± 0.11 ms/token24.50 ± 2.10 ms/token - - - 1.37x + Qwen 3.5 2B text + n=15 · local model + 56.50 tok/s + 41.50 tok/s + + + + + + 1.36x + - Gemma 4 E2B INT4-PLEn=15 · local model - 61.49 ± 0.75 ms/token107.62 ± 1.99 ms/token - - - 1.75x + Gemma 4 E2B INT4-PLE + n=15 · local model + 16.32 tok/s + 9.24 tok/s + + + + + + 1.77x + - Each row uses its own latency scale; lower is left. - Metal Qwen 3.5 2B paired difference: -1.21 ms/token · 95% CI -1.70 to -0.71. + Each row uses its own throughput scale; faster is right. + Metal Qwen 3.5 2B: 20 paired runs · 17 Doppler wins · 3 TJS wins · exact tokens. 9 LINKED RESULTS diff --git a/benchmarks/vendors/fixtures/g3-1b-p064-d064-t0-k1.compare.json b/benchmarks/vendors/fixtures/g3-1b-p064-d064-t0-k1.compare.json index 41864df87..9d4ddd10e 100644 --- a/benchmarks/vendors/fixtures/g3-1b-p064-d064-t0-k1.compare.json +++ b/benchmarks/vendors/fixtures/g3-1b-p064-d064-t0-k1.compare.json @@ -149,8 +149,8 @@ "dopplerDtype": null, "dopplerModelSource": { "source": "explicit", - "locator": "file:///Users/xyz/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", - "modelUrl": "file:///Users/xyz/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", + "locator": "file:///workspace/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", + "modelUrl": "file:///workspace/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", "modelBaseDir": null, "manifestSource": "models/local/gemma-3-1b-it-q4k-ehf16-af32/manifest.json", "manifestSha256": "fcb3136e4188eec38e65f977359957bdc5f1b3590d63fba935b21474dfed0919", @@ -358,7 +358,7 @@ "trainingBenchSteps": null, "checkpointEvery": null, "workloadType": null, - "modelUrl": "file:///Users/xyz/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", + "modelUrl": "file:///workspace/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", "cacheMode": "warm", "loadMode": "opfs", "runtimeProfile": null, @@ -1471,7 +1471,7 @@ "trainingBenchSteps": null, "checkpointEvery": null, "workloadType": null, - "modelUrl": "file:///Users/xyz/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", + "modelUrl": "file:///workspace/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", "cacheMode": "warm", "loadMode": "opfs", "runtimeProfile": null, diff --git a/benchmarks/vendors/fixtures/qwen3-5-0-8b-p064-d064-t0-k1.compare.json b/benchmarks/vendors/fixtures/qwen3-5-0-8b-p064-d064-t0-k1.compare.json index a188e8755..920b514fc 100644 --- a/benchmarks/vendors/fixtures/qwen3-5-0-8b-p064-d064-t0-k1.compare.json +++ b/benchmarks/vendors/fixtures/qwen3-5-0-8b-p064-d064-t0-k1.compare.json @@ -1210,7 +1210,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -2194,7 +2194,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -3179,7 +3179,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -4163,7 +4163,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/generate-readme-evidence-svg.js b/benchmarks/vendors/generate-readme-evidence-svg.js new file mode 100644 index 000000000..c1df70415 --- /dev/null +++ b/benchmarks/vendors/generate-readme-evidence-svg.js @@ -0,0 +1,334 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, '..', '..'); +const OUTPUT_PATH = path.join(REPO_ROOT, 'assets', 'doppler-webgpu-evidence.svg'); +const WIDTH = 1280; +const HEIGHT = 930; +const AXIS_START = 600; +const AXIS_END = 970; +const RATE_FORMATTER = new Intl.NumberFormat('en-US', { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + useGrouping: false, +}); + +const GROUPS = Object.freeze([ + Object.freeze({ + label: 'Apple M3 · Metal', + rows: Object.freeze([ + Object.freeze({ + model: 'Qwen 3.5 0.8B text', + kind: 'text', + receipt: 'benchmarks/vendors/results/compare_20260709T154633.json', + section: 'parity', + }), + Object.freeze({ + model: 'Qwen 3 Embedding 0.6B', + kind: 'embedding', + receipt: 'benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260709T180853.json', + }), + Object.freeze({ + model: 'Qwen 3 Reranker 0.6B', + kind: 'rerank', + receipt: 'benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260709T192830.json', + }), + Object.freeze({ + model: 'Qwen 3.5 2B text', + kind: 'paired-text', + receipt: 'benchmarks/vendors/results/qwen35-2b-metal-paired-p256-d512-20260710.json', + detail: 'local model', + }), + ]), + }), + Object.freeze({ + label: 'Radeon 8060S · Vulkan', + rows: Object.freeze([ + Object.freeze({ + model: 'Qwen 3.5 0.8B text', + kind: 'text', + receipt: 'benchmarks/vendors/results/compare_20260707T153509.json', + section: 'throughput', + }), + Object.freeze({ + model: 'Qwen 3 Embedding 0.6B', + kind: 'embedding', + receipt: 'benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260710T011455.json', + }), + Object.freeze({ + model: 'Qwen 3 Reranker 0.6B', + kind: 'rerank', + receipt: 'benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260710T014450.json', + }), + Object.freeze({ + model: 'Qwen 3.5 2B text', + kind: 'text', + receipt: 'benchmarks/vendors/results/compare_20260707T161623.json', + section: 'throughput', + detail: 'local model', + }), + Object.freeze({ + model: 'Gemma 4 E2B INT4-PLE', + kind: 'text', + receipt: 'benchmarks/vendors/results/compare_20260707T170557.json', + section: 'parity', + detail: 'local model', + }), + ]), + }), +]); + +function readJson(relativePath) { + return JSON.parse(fs.readFileSync(path.join(REPO_ROOT, relativePath), 'utf8')); +} + +function requiredNumber(value, label) { + const number = Number(value); + if (!Number.isFinite(number)) throw new Error(`missing finite ${label}`); + return number; +} + +function requiredArray(value, label) { + if (!Array.isArray(value) || value.length === 0) throw new Error(`missing non-empty ${label}`); + return value; +} + +function rangeFromValues(values, label) { + const finite = requiredArray(values, label).map((value, index) => requiredNumber(value, `${label}[${index}]`)); + return { + min: Math.min(...finite), + max: Math.max(...finite), + }; +} + +function rangeFromLatency(minMs, maxMs, label) { + const minimumLatency = requiredNumber(minMs, `${label}.minMs`); + const maximumLatency = requiredNumber(maxMs, `${label}.maxMs`); + if (minimumLatency <= 0 || maximumLatency <= 0) throw new Error(`${label} latency must be positive`); + return { + min: 1000 / maximumLatency, + max: 1000 / minimumLatency, + }; +} + +function loadTextRow(definition, report) { + const section = report?.sections?.compute?.[definition.section]; + if (!section) throw new Error(`${definition.receipt} has no ${definition.section} section`); + const dopplerMetrics = section?.doppler?.result?.metrics; + const dopplerStats = dopplerMetrics?.throughput?.decodeTokensPerSec; + const tjsMetrics = section?.transformersjs?.metrics; + const tjsRuns = requiredArray(section?.transformersjs?.runs, `${definition.receipt}.transformersjs.runs`); + return { + ...definition, + unit: 'tok/s', + runs: requiredNumber(report.runs, `${definition.receipt}.runs`), + doppler: { + rate: requiredNumber(dopplerMetrics?.decodeTokensPerSec, `${definition.receipt}.doppler.decodeTokensPerSec`), + min: requiredNumber(dopplerStats?.min, `${definition.receipt}.doppler.min`), + max: requiredNumber(dopplerStats?.max, `${definition.receipt}.doppler.max`), + }, + transformersjs: { + rate: requiredNumber(tjsMetrics?.decodeTokensPerSec, `${definition.receipt}.transformersjs.decodeTokensPerSec`), + ...rangeFromValues( + tjsRuns.map((run) => run?.decodeTokensPerSec), + `${definition.receipt}.transformersjs.decodeTokensPerSec`, + ), + }, + }; +} + +function loadPairedTextRow(definition, report) { + return { + ...definition, + unit: 'tok/s', + runs: requiredArray(report.pairs, `${definition.receipt}.pairs`).length, + doppler: { + rate: requiredNumber(report?.results?.doppler?.medianTokensPerSec, `${definition.receipt}.doppler.medianTokensPerSec`), + min: requiredNumber(report?.results?.doppler?.minTokensPerSec, `${definition.receipt}.doppler.minTokensPerSec`), + max: requiredNumber(report?.results?.doppler?.maxTokensPerSec, `${definition.receipt}.doppler.maxTokensPerSec`), + }, + transformersjs: { + rate: requiredNumber(report?.results?.transformersjs?.medianTokensPerSec, `${definition.receipt}.transformersjs.medianTokensPerSec`), + min: requiredNumber(report?.results?.transformersjs?.minTokensPerSec, `${definition.receipt}.transformersjs.minTokensPerSec`), + max: requiredNumber(report?.results?.transformersjs?.maxTokensPerSec, `${definition.receipt}.transformersjs.maxTokensPerSec`), + }, + pairWins: { + doppler: requiredNumber(report?.results?.comparison?.dopplerPairWins, `${definition.receipt}.dopplerPairWins`), + transformersjs: requiredNumber(report?.results?.comparison?.transformersjsPairWins, `${definition.receipt}.transformersjsPairWins`), + }, + }; +} + +function loadRetrievalRow(definition, report) { + const isEmbedding = definition.kind === 'embedding'; + const rateField = isEmbedding ? 'avgEmbeddingsPerSec' : 'avgReranksPerSec'; + const minLatencyField = isEmbedding ? 'minEmbeddingMs' : 'minRerankMs'; + const maxLatencyField = isEmbedding ? 'maxEmbeddingMs' : 'maxRerankMs'; + const timedRuns = report?.raw?.dopplerBench?.metrics?.timedRuns; + const dopplerMetrics = report?.raw?.dopplerBench?.metrics; + const tjsMetrics = report?.raw?.transformersjsBench?.metrics; + return { + ...definition, + unit: isEmbedding ? 'embedding/s' : 'reranks/s', + runs: requiredNumber(timedRuns, `${definition.receipt}.timedRuns`), + doppler: { + rate: requiredNumber(report?.summary?.doppler?.speed?.[rateField], `${definition.receipt}.doppler.${rateField}`), + ...rangeFromLatency( + dopplerMetrics?.[minLatencyField], + dopplerMetrics?.[maxLatencyField], + `${definition.receipt}.doppler`, + ), + }, + transformersjs: { + rate: requiredNumber(report?.summary?.transformersjs?.speed?.[rateField], `${definition.receipt}.transformersjs.${rateField}`), + ...rangeFromLatency( + tjsMetrics?.[minLatencyField], + tjsMetrics?.[maxLatencyField], + `${definition.receipt}.transformersjs`, + ), + }, + }; +} + +function loadRow(definition) { + const report = readJson(definition.receipt); + if (definition.kind === 'text') return loadTextRow(definition, report); + if (definition.kind === 'paired-text') return loadPairedTextRow(definition, report); + if (definition.kind === 'embedding' || definition.kind === 'rerank') { + return loadRetrievalRow(definition, report); + } + throw new Error(`unsupported evidence row kind: ${definition.kind}`); +} + +function escapeXml(value) { + return String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +function formatRate(value, unit) { + return `${RATE_FORMATTER.format(value)} ${unit}`; +} + +function scaleForRow(row) { + const values = [ + row.doppler.min, + row.doppler.max, + row.doppler.rate, + row.transformersjs.min, + row.transformersjs.max, + row.transformersjs.rate, + ]; + const minimum = Math.min(...values); + const maximum = Math.max(...values); + const span = Math.max(maximum - minimum, maximum * 0.02, 0.01); + const low = minimum - span * 0.08; + const high = maximum + span * 0.08; + return (value) => AXIS_START + ((value - low) / (high - low)) * (AXIS_END - AXIS_START); +} + +function renderRange(scale, y, values, prefix) { + const minimum = scale(values.min); + const maximum = scale(values.max); + const rate = scale(values.rate); + const width = Math.max(2, maximum - minimum); + return [ + ``, + ``, + ``, + ].join(''); +} + +function renderRow(row, top) { + const scale = scaleForRow(row); + const detail = [`n=${row.runs}`]; + if (row.detail) detail.push(row.detail); + const ratio = row.doppler.rate / row.transformersjs.rate; + return [ + `${escapeXml(row.model)}`, + `${escapeXml(detail.join(' · '))}`, + `${formatRate(row.doppler.rate, row.unit)}`, + `${formatRate(row.transformersjs.rate, row.unit)}`, + ``, + renderRange(scale, top + 19, row.doppler, 'd'), + ``, + renderRange(scale, top + 45, row.transformersjs, 't'), + ``, + `${ratio.toFixed(2)}x`, + ``, + ].join('\n '); +} + +function renderSvg(groups) { + const paired = groups.flatMap((group) => group.rows).find((row) => row.kind === 'paired-text'); + const body = [ + ``, + ' Doppler and Transformers.js browser WebGPU throughput', + ' Reported throughput and receipt-reported min-to-max rate ranges for nine browser WebGPU comparisons on Apple M3 Metal and Radeon 8060S Vulkan. Higher is faster.', + ' ', + ` `, + '', + ' Doppler', + ' Transformers.js', + ' BAND MIN–MAX · CENTER REPORTED RATE', + ' ', + ' MODELTHROUGHPUT', + ' RECEIPT RATE RANGE · HIGHER IS FASTER →RATE RATIO', + ]; + + let sectionTop = 94; + for (const [groupIndex, group] of groups.entries()) { + body.push(''); + body.push(` `); + body.push(` ${escapeXml(group.label)}${group.rows.length} MODELS`); + let rowTop = sectionTop + 46; + for (const row of group.rows) { + body.push(''); + body.push(` ${renderRow(row, rowTop)}`); + rowTop += 72; + } + sectionTop = rowTop + (groupIndex === 0 ? 12 : 0); + } + + body.push(''); + body.push(' '); + body.push(' Each row uses its own throughput scale; faster is right.'); + body.push(` Metal Qwen 3.5 2B: ${paired.runs} paired runs · ${paired.pairWins.doppler} Doppler wins · ${paired.pairWins.transformersjs} TJS wins · exact tokens.`); + body.push(' 9 LINKED RESULTS'); + body.push(''); + return `${body.join('\n')}\n`; +} + +function main() { + const unknownArgs = process.argv.slice(2).filter((arg) => arg !== '--check'); + if (unknownArgs.length > 0) throw new Error(`unknown argument: ${unknownArgs[0]}`); + const groups = GROUPS.map((group) => ({ + ...group, + rows: group.rows.map(loadRow), + })); + const expected = renderSvg(groups); + if (process.argv.includes('--check')) { + const actual = fs.readFileSync(OUTPUT_PATH, 'utf8'); + if (actual !== expected) throw new Error('assets/doppler-webgpu-evidence.svg is stale; run npm run readme:evidence:sync'); + console.log('README evidence SVG is current.'); + return; + } + fs.writeFileSync(OUTPUT_PATH, expected, 'utf8'); + console.log(`wrote ${OUTPUT_PATH}`); +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/benchmarks/vendors/local-inference-claim-matrix.json b/benchmarks/vendors/local-inference-claim-matrix.json index bac206022..a1f8bf320 100644 --- a/benchmarks/vendors/local-inference-claim-matrix.json +++ b/benchmarks/vendors/local-inference-claim-matrix.json @@ -491,7 +491,8 @@ "compression": "q4k-ehaf16", "quantization": "Q4_K_M", "manifestPath": "models/local/qwen-3-5-0-8b-q4k-ehaf16/manifest.json", - "manifestSha256": "af47de9173b29e370c7301be0212814e912f67c1e1205ce199302b701f05602e", + "manifestSha256": "f096d04990e12a9d9f43a65b37e717c38b1cc7af7e8119e43d4a2aa384895c28", + "evidenceManifestSha256": "af47de9173b29e370c7301be0212814e912f67c1e1205ce199302b701f05602e", "weightPackId": "qwen-3-5-0-8b-q4k-ehaf16-wp-catalog-v1", "weightPackHash": "sha256:fab133e49d6dc67912fc3a087222ec44ca1941d9b7bc36c60cb1379863a6dd4f", "manifestVariantId": "qwen-3-5-0-8b-q4k-ehaf16-mv-exec-v1", diff --git a/benchmarks/vendors/model-competition-scoreboard.json b/benchmarks/vendors/model-competition-scoreboard.json index 4c0facdce..78d11e0ac 100644 --- a/benchmarks/vendors/model-competition-scoreboard.json +++ b/benchmarks/vendors/model-competition-scoreboard.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "updatedAt": "2026-07-11T14:04:21.695Z", + "updatedAt": "2026-07-11T19:35:41.171Z", "sources": [ "models/catalog.json", "benchmarks/vendors/model-support-inventory.json", diff --git a/benchmarks/vendors/release-matrix.json b/benchmarks/vendors/release-matrix.json index a54205c4c..4952b485f 100644 --- a/benchmarks/vendors/release-matrix.json +++ b/benchmarks/vendors/release-matrix.json @@ -1,11 +1,11 @@ { "schemaVersion": 1, - "generatedAt": "2026-07-11T14:04:21.695Z", + "generatedAt": "2026-07-11T19:35:41.171Z", "release": { "channel": "main-snapshot", "version": "0.4.8", - "commitSha": "e0fc0604197e1fa1ba44f14f612daade69a2fea8", - "dirty": false + "commitSha": "cf1a9a38b26f25363ffc297a07877dd46fdd01cb", + "dirty": true }, "sources": { "registry": { @@ -46,8 +46,8 @@ }, "localInferenceClaimMatrix": { "path": "benchmarks/vendors/local-inference-claim-matrix.json", - "sha256": "609d3e2d0581bfac040baf97029992483db289381c47b68698e6fe59cd05f169", - "bytes": 24908, + "sha256": "9a9ae3b3800922badf2820ae3df37ab1292ab1105f781a4a0cfa98c698c73088", + "bytes": 25000, "updated": "2026-06-27" }, "modelCatalog": { diff --git a/benchmarks/vendors/results/compare_20260707T180928.json b/benchmarks/vendors/results/compare_20260707T180928.json index bf44dc859..c6058e20b 100644 --- a/benchmarks/vendors/results/compare_20260707T180928.json +++ b/benchmarks/vendors/results/compare_20260707T180928.json @@ -1,7 +1,7 @@ { "timestamp": "2026-07-07T18:04:09.865Z", "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "sourceSha256": "6e0e81bc2571a0a1eb6451f0f9a6a3165b015bff9ba4537a4f5a6f2b1a9b93c8", "updated": "2026-03-29", @@ -219,7 +219,7 @@ "dopplerModelSource": { "source": "local", "locator": "gemma-4-e2b-it-q4k-ehf16-af32-int4ple", - "modelUrl": "file:///Users/xyz/deco/doppler/models/local/gemma-4-e2b-it-q4k-ehf16-af32-int4ple", + "modelUrl": "file:///workspace/deco/doppler/models/local/gemma-4-e2b-it-q4k-ehf16-af32-int4ple", "modelBaseDir": "local", "manifestSource": "models/local/gemma-4-e2b-it-q4k-ehf16-af32-int4ple/manifest.json", "manifestSha256": "512d6e16602071cccff4cbe576bf3b2c20bd43fd32aefc271db4bb95cc9c7f17", @@ -7708,7 +7708,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -15047,7 +15047,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/compare_20260707T181516.json b/benchmarks/vendors/results/compare_20260707T181516.json index eb793cef9..ddee5fc6b 100644 --- a/benchmarks/vendors/results/compare_20260707T181516.json +++ b/benchmarks/vendors/results/compare_20260707T181516.json @@ -1,7 +1,7 @@ { "timestamp": "2026-07-07T18:09:50.414Z", "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "sourceSha256": "6e0e81bc2571a0a1eb6451f0f9a6a3165b015bff9ba4537a4f5a6f2b1a9b93c8", "updated": "2026-03-29", @@ -244,7 +244,7 @@ "dopplerModelSource": { "source": "local", "locator": "qwen-3-5-2b-q4k-ehaf16", - "modelUrl": "file:///Users/xyz/deco/doppler/models/local/qwen-3-5-2b-q4k-ehaf16", + "modelUrl": "file:///workspace/deco/doppler/models/local/qwen-3-5-2b-q4k-ehaf16", "modelBaseDir": "local", "manifestSource": "models/local/qwen-3-5-2b-q4k-ehaf16/manifest.json", "manifestSha256": "502fbd6d4c9ed6a890931665995c8ebb42a30e5cda23aa2cfd8e680bee7fa5bc", @@ -7782,7 +7782,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -15170,7 +15170,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/compare_20260707T182451.json b/benchmarks/vendors/results/compare_20260707T182451.json index 73e360a6b..695f3c361 100644 --- a/benchmarks/vendors/results/compare_20260707T182451.json +++ b/benchmarks/vendors/results/compare_20260707T182451.json @@ -1,7 +1,7 @@ { "timestamp": "2026-07-07T18:15:34.099Z", "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "sourceSha256": "6e0e81bc2571a0a1eb6451f0f9a6a3165b015bff9ba4537a4f5a6f2b1a9b93c8", "updated": "2026-03-29", @@ -244,7 +244,7 @@ "dopplerModelSource": { "source": "local", "locator": "qwen-3-5-2b-q4k-ehaf16", - "modelUrl": "file:///Users/xyz/deco/doppler/models/local/qwen-3-5-2b-q4k-ehaf16", + "modelUrl": "file:///workspace/deco/doppler/models/local/qwen-3-5-2b-q4k-ehaf16", "modelBaseDir": "local", "manifestSource": "models/local/qwen-3-5-2b-q4k-ehaf16/manifest.json", "manifestSha256": "502fbd6d4c9ed6a890931665995c8ebb42a30e5cda23aa2cfd8e680bee7fa5bc", @@ -9574,7 +9574,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -18754,7 +18754,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/compare_20260707T183552.json b/benchmarks/vendors/results/compare_20260707T183552.json index 5906d2fb9..7e8f1d46c 100644 --- a/benchmarks/vendors/results/compare_20260707T183552.json +++ b/benchmarks/vendors/results/compare_20260707T183552.json @@ -1,7 +1,7 @@ { "timestamp": "2026-07-07T18:25:00.877Z", "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "sourceSha256": "6e0e81bc2571a0a1eb6451f0f9a6a3165b015bff9ba4537a4f5a6f2b1a9b93c8", "updated": "2026-03-29", @@ -244,7 +244,7 @@ "dopplerModelSource": { "source": "local", "locator": "qwen-3-5-2b-q4k-ehaf16", - "modelUrl": "file:///Users/xyz/deco/doppler/models/local/qwen-3-5-2b-q4k-ehaf16", + "modelUrl": "file:///workspace/deco/doppler/models/local/qwen-3-5-2b-q4k-ehaf16", "modelBaseDir": "local", "manifestSource": "models/local/qwen-3-5-2b-q4k-ehaf16/manifest.json", "manifestSha256": "502fbd6d4c9ed6a890931665995c8ebb42a30e5cda23aa2cfd8e680bee7fa5bc", @@ -9574,7 +9574,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -18754,7 +18754,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/compare_20260708T152745.json b/benchmarks/vendors/results/compare_20260708T152745.json index 5ad2f8cc6..6c7807421 100644 --- a/benchmarks/vendors/results/compare_20260708T152745.json +++ b/benchmarks/vendors/results/compare_20260708T152745.json @@ -1,7 +1,7 @@ { "timestamp": "2026-07-08T15:19:54.264Z", "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "sourceSha256": "6e0e81bc2571a0a1eb6451f0f9a6a3165b015bff9ba4537a4f5a6f2b1a9b93c8", "updated": "2026-03-29", @@ -363,7 +363,7 @@ "prefillTokenTarget": 512, "prefillTokenSource": "tokenizer-accurate-synthetic", "prefillTokenSemantics": "model_input_tokens", - "promptTokenizerLocator": "/Users/xyz/.cache/doppler/tjs-models/onnx-community/Qwen3.5-0.8B-ONNX", + "promptTokenizerLocator": "/workspace/.cache/doppler/tjs-models/onnx-community/Qwen3.5-0.8B-ONNX", "promptTokenizerResolutionSource": "local-model-root", "decodeTokenTarget": 128 }, @@ -9582,7 +9582,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -18779,7 +18779,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/compare_20260709T154633.json b/benchmarks/vendors/results/compare_20260709T154633.json index 25c26729e..4e0014d06 100644 --- a/benchmarks/vendors/results/compare_20260709T154633.json +++ b/benchmarks/vendors/results/compare_20260709T154633.json @@ -1,7 +1,7 @@ { "timestamp": "2026-07-09T15:43:28.131Z", "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "sourceSha256": "6e0e81bc2571a0a1eb6451f0f9a6a3165b015bff9ba4537a4f5a6f2b1a9b93c8", "updated": "2026-03-29", @@ -8737,7 +8737,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, @@ -17077,7 +17077,7 @@ } }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260708T153537.json b/benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260708T153537.json index c41223137..3f6a508a9 100644 --- a/benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260708T153537.json +++ b/benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260708T153537.json @@ -82,24 +82,24 @@ }, "commands": { "dopplerBench": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/src/cli/doppler-cli.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/src/cli/doppler-cli.js", "bench", "--config", "{\"request\":{\"workload\":\"embedding\",\"modelId\":\"qwen-3-embedding-0-6b-q4k-ehf16-af32\",\"modelUrl\":\"https://huggingface.co/Clocksmith/rdrr/resolve/049000f49325dca7db2ed2c9de2c8881bd0f4603/models/qwen-3-embedding-0-6b-q4k-ehf16-af32\",\"cacheMode\":\"warm\",\"loadMode\":\"http\",\"runtimeConfig\":{\"shared\":{\"benchmark\":{\"run\":{\"customPrompt\":\"A small red cube rests on a wooden table beside a blue sphere.\",\"warmupRuns\":1,\"timedRuns\":15,\"loadMode\":\"http\"}}},\"inference\":{\"prompt\":\"A small red cube rests on a wooden table beside a blue sphere.\"}}},\"run\":{\"surface\":\"browser\",\"timestamp\":\"2026-07-08T15:35:37.367Z\",\"browser\":{\"opfsCache\":false}}}", "--json" ], "dopplerVerify": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/src/cli/doppler-cli.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/src/cli/doppler-cli.js", "verify", "--config", "{\"request\":{\"workload\":\"embedding\",\"modelId\":\"qwen-3-embedding-0-6b-q4k-ehf16-af32\",\"modelUrl\":\"https://huggingface.co/Clocksmith/rdrr/resolve/049000f49325dca7db2ed2c9de2c8881bd0f4603/models/qwen-3-embedding-0-6b-q4k-ehf16-af32\",\"cacheMode\":\"warm\",\"loadMode\":\"http\",\"runtimeConfig\":{\"shared\":{\"benchmark\":{\"run\":{\"customPrompt\":\"A small red cube rests on a wooden table beside a blue sphere.\",\"warmupRuns\":0,\"timedRuns\":1,\"loadMode\":\"http\"}}},\"inference\":{\"prompt\":\"A small red cube rests on a wooden table beside a blue sphere.\"}}},\"run\":{\"surface\":\"browser\",\"timestamp\":\"2026-07-08T15:35:37.367Z\",\"browser\":{\"opfsCache\":false}}}", "--json" ], "transformersjsBench": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/benchmarks/runners/transformersjs-bench.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/benchmarks/runners/transformersjs-bench.js", "--task", "embedding", "--model", @@ -127,7 +127,7 @@ "--embedding-style", "qwen3_embedding", "--semantic-fixture", - "/Users/xyz/deco/doppler/src/inference/fixtures/embedding-semantic-fixtures.json", + "/workspace/deco/doppler/src/inference/fixtures/embedding-semantic-fixtures.json", "--profile-ops", "off", "--timeout", @@ -5633,7 +5633,7 @@ "browserExecutable": null }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260709T180853.json b/benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260709T180853.json index acfe9dfd7..9db4a62ce 100644 --- a/benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260709T180853.json +++ b/benchmarks/vendors/results/embedding_compare_qwen-3-embedding-0-6b-q4k-ehf16-af32_20260709T180853.json @@ -82,24 +82,24 @@ }, "commands": { "dopplerBench": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/src/cli/doppler-cli.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/src/cli/doppler-cli.js", "bench", "--config", "{\"request\":{\"workload\":\"embedding\",\"modelId\":\"qwen-3-embedding-0-6b-q4k-ehf16-af32\",\"modelUrl\":\"https://huggingface.co/Clocksmith/rdrr/resolve/049000f49325dca7db2ed2c9de2c8881bd0f4603/models/qwen-3-embedding-0-6b-q4k-ehf16-af32\",\"cacheMode\":\"warm\",\"loadMode\":\"http\",\"runtimeConfig\":{\"shared\":{\"benchmark\":{\"run\":{\"customPrompt\":\"A small red cube rests on a wooden table beside a blue sphere.\",\"warmupRuns\":1,\"timedRuns\":15,\"loadMode\":\"http\"}}},\"inference\":{\"prompt\":\"A small red cube rests on a wooden table beside a blue sphere.\"}},\"runtimeProfile\":\"profiles/qwen-3-embedding-0-6b-metal-throughput\"},\"run\":{\"surface\":\"browser\",\"timestamp\":\"2026-07-09T18:08:53.800Z\",\"browser\":{\"timeoutMs\":1800000,\"opfsCache\":false}}}", "--json" ], "dopplerVerify": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/src/cli/doppler-cli.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/src/cli/doppler-cli.js", "verify", "--config", "{\"request\":{\"workload\":\"embedding\",\"modelId\":\"qwen-3-embedding-0-6b-q4k-ehf16-af32\",\"modelUrl\":\"https://huggingface.co/Clocksmith/rdrr/resolve/049000f49325dca7db2ed2c9de2c8881bd0f4603/models/qwen-3-embedding-0-6b-q4k-ehf16-af32\",\"cacheMode\":\"warm\",\"loadMode\":\"http\",\"runtimeConfig\":{\"shared\":{\"benchmark\":{\"run\":{\"customPrompt\":\"A small red cube rests on a wooden table beside a blue sphere.\",\"warmupRuns\":0,\"timedRuns\":1,\"loadMode\":\"http\"}}},\"inference\":{\"prompt\":\"A small red cube rests on a wooden table beside a blue sphere.\"}},\"runtimeProfile\":\"profiles/qwen-3-embedding-0-6b-metal-stability\"},\"run\":{\"surface\":\"browser\",\"timestamp\":\"2026-07-09T18:08:53.800Z\",\"browser\":{\"timeoutMs\":1800000,\"opfsCache\":false}}}", "--json" ], "transformersjsBench": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/benchmarks/runners/transformersjs-bench.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/benchmarks/runners/transformersjs-bench.js", "--task", "embedding", "--model", @@ -127,7 +127,7 @@ "--embedding-style", "qwen3_embedding", "--semantic-fixture", - "/Users/xyz/deco/doppler/src/inference/fixtures/embedding-semantic-fixtures.json", + "/workspace/deco/doppler/src/inference/fixtures/embedding-semantic-fixtures.json", "--profile-ops", "off", "--timeout", @@ -6030,7 +6030,7 @@ "browserExecutable": null }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/local-gpu-probe-mac-20260708T150919Z.json b/benchmarks/vendors/results/local-gpu-probe-mac-20260708T150919Z.json index 22f7524eb..4f5d15b1f 100644 --- a/benchmarks/vendors/results/local-gpu-probe-mac-20260708T150919Z.json +++ b/benchmarks/vendors/results/local-gpu-probe-mac-20260708T150919Z.json @@ -913,7 +913,7 @@ }, "commands": { "node": { - "command": "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node --version", + "command": "/workspace/.nvm/versions/node/v22.20.0/bin/node --version", "available": true, "status": 0, "output": "v22.20.0", diff --git a/benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260708T154713.json b/benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260708T154713.json index 5de59db96..fcc5b04f2 100644 --- a/benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260708T154713.json +++ b/benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260708T154713.json @@ -114,24 +114,24 @@ }, "commands": { "dopplerBench": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/src/cli/doppler-cli.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/src/cli/doppler-cli.js", "bench", "--config", "{\"request\":{\"workload\":\"rerank\",\"modelId\":\"qwen-3-reranker-0-6b-q4k-ehf16-af32\",\"modelUrl\":\"https://huggingface.co/Clocksmith/rdrr/resolve/f86fe245b9bbc275cd69af46b1d45d47ea685a55/models/qwen-3-reranker-0-6b-q4k-ehf16-af32\",\"cacheMode\":\"warm\",\"loadMode\":\"http\",\"runtimeConfig\":{\"shared\":{\"benchmark\":{\"run\":{\"warmupRuns\":1,\"timedRuns\":15,\"loadMode\":\"http\"}}},\"inference\":{\"rerank\":{\"query\":\"What API exposes GPU hardware for browser compute workloads?\",\"documents\":[\"WebGPU provides low-level access to graphics processors for compute workloads.\",\"WebSocket provides full-duplex communication over TCP.\",\"CSS grid arranges page layout rows and columns.\"]}}},\"runtimeProfile\":\"profiles/rerank-throughput\"},\"run\":{\"surface\":\"browser\",\"timestamp\":\"2026-07-08T15:47:13.019Z\",\"browser\":{\"timeoutMs\":900000,\"opfsCache\":false}}}", "--json" ], "dopplerVerify": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/src/cli/doppler-cli.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/src/cli/doppler-cli.js", "verify", "--config", "{\"request\":{\"workload\":\"rerank\",\"modelId\":\"qwen-3-reranker-0-6b-q4k-ehf16-af32\",\"modelUrl\":\"https://huggingface.co/Clocksmith/rdrr/resolve/f86fe245b9bbc275cd69af46b1d45d47ea685a55/models/qwen-3-reranker-0-6b-q4k-ehf16-af32\",\"cacheMode\":\"warm\",\"loadMode\":\"http\",\"runtimeConfig\":{\"shared\":{\"benchmark\":{\"run\":{\"warmupRuns\":0,\"timedRuns\":1,\"loadMode\":\"http\"}}},\"inference\":{\"rerank\":{\"query\":\"What API exposes GPU hardware for browser compute workloads?\",\"documents\":[\"WebGPU provides low-level access to graphics processors for compute workloads.\",\"WebSocket provides full-duplex communication over TCP.\",\"CSS grid arranges page layout rows and columns.\"]}}},\"runtimeProfile\":\"profiles/rerank-stability\"},\"run\":{\"surface\":\"browser\",\"timestamp\":\"2026-07-08T15:47:13.019Z\",\"browser\":{\"timeoutMs\":900000,\"opfsCache\":false}}}", "--json" ], "transformersjsBench": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/benchmarks/runners/transformersjs-bench.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/benchmarks/runners/transformersjs-bench.js", "--task", "rerank", "--model", @@ -155,7 +155,7 @@ "--format", "onnx", "--semantic-fixture", - "/Users/xyz/deco/doppler/src/inference/fixtures/rerank-semantic-fixtures.json", + "/workspace/deco/doppler/src/inference/fixtures/rerank-semantic-fixtures.json", "--profile-ops", "off", "--timeout", @@ -4424,7 +4424,7 @@ "browserExecutable": null }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260709T192830.json b/benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260709T192830.json index 9623e3b0f..3641b4099 100644 --- a/benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260709T192830.json +++ b/benchmarks/vendors/results/rerank_compare_qwen-3-reranker-0-6b-q4k-ehf16-af32_20260709T192830.json @@ -114,24 +114,24 @@ }, "commands": { "dopplerBench": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/src/cli/doppler-cli.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/src/cli/doppler-cli.js", "bench", "--config", "{\"request\":{\"workload\":\"rerank\",\"modelId\":\"qwen-3-reranker-0-6b-q4k-ehf16-af32\",\"modelUrl\":\"https://huggingface.co/Clocksmith/rdrr/resolve/f86fe245b9bbc275cd69af46b1d45d47ea685a55/models/qwen-3-reranker-0-6b-q4k-ehf16-af32\",\"cacheMode\":\"warm\",\"loadMode\":\"http\",\"runtimeConfig\":{\"shared\":{\"benchmark\":{\"run\":{\"warmupRuns\":1,\"timedRuns\":3,\"loadMode\":\"http\"}}},\"inference\":{\"rerank\":{\"query\":\"What API exposes GPU hardware for browser compute workloads?\",\"documents\":[\"WebGPU provides low-level access to graphics processors for compute workloads.\",\"WebSocket provides full-duplex communication over TCP.\",\"CSS grid arranges page layout rows and columns.\"]}}},\"runtimeProfile\":\"profiles/qwen-3-reranker-0-6b-metal-throughput\"},\"run\":{\"surface\":\"browser\",\"timestamp\":\"2026-07-09T19:28:30.640Z\",\"browser\":{\"timeoutMs\":1800000,\"opfsCache\":false}}}", "--json" ], "dopplerVerify": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/src/cli/doppler-cli.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/src/cli/doppler-cli.js", "verify", "--config", "{\"request\":{\"workload\":\"rerank\",\"modelId\":\"qwen-3-reranker-0-6b-q4k-ehf16-af32\",\"modelUrl\":\"https://huggingface.co/Clocksmith/rdrr/resolve/f86fe245b9bbc275cd69af46b1d45d47ea685a55/models/qwen-3-reranker-0-6b-q4k-ehf16-af32\",\"cacheMode\":\"warm\",\"loadMode\":\"http\",\"runtimeConfig\":{\"shared\":{\"benchmark\":{\"run\":{\"warmupRuns\":0,\"timedRuns\":1,\"loadMode\":\"http\"}}},\"inference\":{\"rerank\":{\"query\":\"What API exposes GPU hardware for browser compute workloads?\",\"documents\":[\"WebGPU provides low-level access to graphics processors for compute workloads.\",\"WebSocket provides full-duplex communication over TCP.\",\"CSS grid arranges page layout rows and columns.\"]}}},\"runtimeProfile\":\"profiles/qwen-3-reranker-0-6b-metal-stability\"},\"run\":{\"surface\":\"browser\",\"timestamp\":\"2026-07-09T19:28:30.640Z\",\"browser\":{\"timeoutMs\":1800000,\"opfsCache\":false}}}", "--json" ], "transformersjsBench": [ - "/Users/xyz/.nvm/versions/node/v22.20.0/bin/node", - "/Users/xyz/deco/doppler/benchmarks/runners/transformersjs-bench.js", + "/workspace/.nvm/versions/node/v22.20.0/bin/node", + "/workspace/deco/doppler/benchmarks/runners/transformersjs-bench.js", "--task", "rerank", "--model", @@ -155,7 +155,7 @@ "--format", "onnx", "--semantic-fixture", - "/Users/xyz/deco/doppler/src/inference/fixtures/rerank-semantic-fixtures.json", + "/workspace/deco/doppler/src/inference/fixtures/rerank-semantic-fixtures.json", "--profile-ops", "off", "--timeout", @@ -4644,7 +4644,7 @@ "browserExecutable": null }, "benchmarkPolicy": { - "source": "/Users/xyz/deco/doppler/benchmarks/vendors/benchmark-policy.json", + "source": "/workspace/deco/doppler/benchmarks/vendors/benchmark-policy.json", "schemaVersion": 1, "updated": "2026-03-29" }, diff --git a/benchmarks/vendors/schema/local-inference-claim-matrix.schema.json b/benchmarks/vendors/schema/local-inference-claim-matrix.schema.json index 3ea00264a..7c40fb8b0 100644 --- a/benchmarks/vendors/schema/local-inference-claim-matrix.schema.json +++ b/benchmarks/vendors/schema/local-inference-claim-matrix.schema.json @@ -682,6 +682,14 @@ "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "evidenceManifestSha256": { + "description": "Exact manifest hash recorded by historical compare receipts when the public manifest differs only because local metadata was sanitized.", + "type": [ + "string", + "null" + ], + "pattern": "^[a-f0-9]{64}$" + }, "weightPackId": { "type": [ "string", diff --git a/docs/model-competition-scoreboard.md b/docs/model-competition-scoreboard.md index c5ef82d71..bfde3999b 100644 --- a/docs/model-competition-scoreboard.md +++ b/docs/model-competition-scoreboard.md @@ -3,7 +3,7 @@ Generated from the catalog, support inventory, release matrix, embedding/rerank compare configs, and saved compare receipts. This file is an evidence ledger: it records what is verified, what is on Hugging Face according to catalog metadata, where Doppler has comparable performance receipts, and which gates remain. -Updated at: 2026-07-11T14:04:21.695Z +Updated at: 2026-07-11T19:35:41.171Z ## Summary diff --git a/docs/release-matrix.md b/docs/release-matrix.md index cf8ebd972..42ac2ecdc 100644 --- a/docs/release-matrix.md +++ b/docs/release-matrix.md @@ -1,7 +1,7 @@ # Release Matrix -Generated: 2026-07-11T14:04:21.695Z -Release: channel=main-snapshot, version=0.4.8, commit=e0fc0604197e1fa1ba44f14f612daade69a2fea8, dirty=no +Generated: 2026-07-11T19:35:41.171Z +Release: channel=main-snapshot, version=0.4.8, commit=cf1a9a38b26f25363ffc297a07877dd46fdd01cb, dirty=yes ## Engine Matrix diff --git a/models/local/qwen-3-5-0-8b-q4k-ehaf16/manifest.json b/models/local/qwen-3-5-0-8b-q4k-ehaf16/manifest.json index 06c54ebe1..2a80b8600 100644 --- a/models/local/qwen-3-5-0-8b-q4k-ehaf16/manifest.json +++ b/models/local/qwen-3-5-0-8b-q4k-ehaf16/manifest.json @@ -6669,7 +6669,7 @@ }, "conversion": null, "metadata": { - "source": "file:///Users/xyz/deco/doppler/downloads/qwen3.5-0.8b", + "source": "file:///workspace/deco/doppler/downloads/qwen3.5-0.8b", "convertedAt": "2026-04-24T17:34:03.192Z", "hasTokenizer": true, "manifestRefresh": { diff --git a/package.json b/package.json index f7675ec56..e8dc96016 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "goals:check": "node tools/check-goal-completion.js", "goals:report": "node tools/render-goal-completion-summary.js", "commands:surface:check": "node tools/check-command-surface-contract.js", - "check:green": "npm run agents:verify && npm run goals:check && npm run public:boundaries:check && npm run package:smoke && npm run commands:surface:check && npm run api:docs:check && npm run config:single-source:check && npm run capability-policy:check && npm run imports:check:browser && npm run pending:check && npm run exports:parity:check && npm run artifact:contract:check && npm run claims:evidence:check && npm run support:subsystems:check && npm run policy:schemas:check", + "check:green": "npm run agents:verify && npm run goals:check && npm run public:boundaries:check && npm run publication:hygiene:check && npm run package:smoke && npm run commands:surface:check && npm run api:docs:check && npm run config:single-source:check && npm run capability-policy:check && npm run imports:check:browser && npm run pending:check && npm run exports:parity:check && npm run artifact:contract:check && npm run bench:vendors:validate && npm run claims:evidence:check && npm run readme:evidence:check && npm run support:subsystems:check && npm run policy:schemas:check", "verify:model": "node src/cli/doppler-cli.js verify", "quickstart:sync": "node tools/sync-quickstart-registry.js", "quickstart:check": "node tools/sync-quickstart-registry.js --check", @@ -72,6 +72,9 @@ "onboarding:check": "node tools/onboarding-tooling.js check", "onboarding:check:strict": "node tools/onboarding-tooling.js check --strict", "bench:vendors:validate": "node tools/vendor-bench.js validate", + "publication:hygiene:check": "node tools/check-publication-hygiene.js", + "readme:evidence:sync": "node benchmarks/vendors/generate-readme-evidence-svg.js", + "readme:evidence:check": "node benchmarks/vendors/generate-readme-evidence-svg.js --check", "bench:local-gpu:challengers": "node tools/local-gpu-challengers.js", "bench:local-gpu:challengers:check": "node tools/local-gpu-challengers.js --check", "bench:embeddings:compare": "node tools/compare-embeddings.js", diff --git a/src/gpu/device.js b/src/gpu/device.js index e795488a0..d70a1db8d 100644 --- a/src/gpu/device.js +++ b/src/gpu/device.js @@ -14,10 +14,14 @@ const SHARED_DEVICE_STATE_KEY = '__dopplerGpuDeviceState'; function getSharedDeviceState() { const existing = globalThis[SHARED_DEVICE_STATE_KEY]; if (existing && typeof existing === 'object') { + if (!Object.hasOwn(existing, 'deviceInitPromise')) { + existing.deviceInitPromise = null; + } return existing; } const created = { gpuDevice: null, + deviceInitPromise: null, kernelCapabilities: null, resolvedPlatformConfig: null, lastDeviceLossInfo: null, @@ -58,14 +62,24 @@ function syncSharedDeviceState() { } function hydrateDeviceState() { + const sharedEpoch = Number.isInteger(sharedDeviceState.deviceEpoch) + ? sharedDeviceState.deviceEpoch + : 0; + if (sharedEpoch > deviceEpoch) { + gpuDevice = sharedDeviceState.gpuDevice ?? null; + kernelCapabilities = sharedDeviceState.kernelCapabilities ?? null; + resolvedPlatformConfig = sharedDeviceState.resolvedPlatformConfig ?? null; + lastDeviceLossInfo = sharedDeviceState.lastDeviceLossInfo ?? null; + platformInitialized = sharedDeviceState.platformInitialized === true; + deviceEpoch = sharedEpoch; + return; + } gpuDevice = gpuDevice ?? sharedDeviceState.gpuDevice ?? null; kernelCapabilities = kernelCapabilities ?? sharedDeviceState.kernelCapabilities ?? null; resolvedPlatformConfig = resolvedPlatformConfig ?? sharedDeviceState.resolvedPlatformConfig ?? null; lastDeviceLossInfo = lastDeviceLossInfo ?? sharedDeviceState.lastDeviceLossInfo ?? null; platformInitialized = platformInitialized || sharedDeviceState.platformInitialized === true; - if (Number.isInteger(sharedDeviceState.deviceEpoch) && sharedDeviceState.deviceEpoch > deviceEpoch) { - deviceEpoch = sharedDeviceState.deviceEpoch; - } + deviceEpoch = Math.max(deviceEpoch, sharedEpoch); } function advanceDeviceEpoch() { @@ -81,6 +95,10 @@ function clearActiveDeviceState() { syncSharedDeviceState(); } +function invalidateDeviceInitialization() { + sharedDeviceState.deviceInitPromise = null; +} + function hasUsableDeviceSlot(device) { return isUsableGPUDevice(device); } @@ -478,17 +496,7 @@ async function initializePlatformAndRegistry(adapter) { } -export async function initDevice() { - hydrateDeviceState(); - // Return cached device if available - if (gpuDevice) { - if (isUsableGPUDevice(gpuDevice)) { - return gpuDevice; - } - clearActiveDeviceState(); - advanceDeviceEpoch(); - } - +async function initializeDevice(initializationEpoch) { if (!isWebGPUAvailable()) { throw createDopplerError(ERROR_CODES.GPU_UNAVAILABLE, 'WebGPU is not available in this browser'); } @@ -510,36 +518,33 @@ export async function initDevice() { // Get adapter info (adapter.info is synchronous in modern WebGPU) const adapterInfo = adapter.info || { vendor: 'unknown', architecture: 'unknown', device: 'unknown', description: '' }; - gpuDevice = await requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits); + const device = await requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits); - if (!gpuDevice) { + if (!device) { throw createDopplerError(ERROR_CODES.GPU_DEVICE_FAILED, 'Failed to create WebGPU device'); } - lastDeviceLossInfo = null; - ensureGpuBufferConstructor(gpuDevice); - wrapDeviceCreateBindGroup(gpuDevice); - registerDeviceLostHandler(gpuDevice); - advanceDeviceEpoch(); + ensureGpuBufferConstructor(device); + wrapDeviceCreateBindGroup(device); // Wrap queue for submit tracking (when enabled) - wrapQueueForTracking(gpuDevice.queue); + wrapQueueForTracking(device.queue); // Cache kernel capabilities - let hasF16 = gpuDevice.features.has(FEATURES.SHADER_F16); + let hasF16 = device.features.has(FEATURES.SHADER_F16); if (hasF16) { - hasF16 = probeShaderF16(gpuDevice); + hasF16 = probeShaderF16(device); } - const hasSubgroups = gpuDevice.features.has(FEATURES.SUBGROUPS); + const hasSubgroups = device.features.has(FEATURES.SUBGROUPS); - kernelCapabilities = { + const capabilities = { hasSubgroups, // This is a derived compatibility bit, not a distinct WebGPU feature. hasSubgroupsF16: hasSubgroups && hasF16, hasF16, - hasTimestampQuery: gpuDevice.features.has(FEATURES.TIMESTAMP_QUERY), - maxBufferSize: gpuDevice.limits.maxStorageBufferBindingSize, - maxWorkgroupSize: gpuDevice.limits.maxComputeInvocationsPerWorkgroup, - maxWorkgroupStorageSize: gpuDevice.limits.maxComputeWorkgroupStorageSize, + hasTimestampQuery: device.features.has(FEATURES.TIMESTAMP_QUERY), + maxBufferSize: device.limits.maxStorageBufferBindingSize, + maxWorkgroupSize: device.limits.maxComputeInvocationsPerWorkgroup, + maxWorkgroupStorageSize: device.limits.maxComputeWorkgroupStorageSize, adapterInfo: { vendor: adapterInfo.vendor || 'unknown', architecture: adapterInfo.architecture || 'unknown', @@ -549,8 +554,27 @@ export async function initDevice() { submitProbeMs: null, }; - const probeMs = await probeSubmitLatency(gpuDevice); - kernelCapabilities.submitProbeMs = probeMs; + const probeMs = await probeSubmitLatency(device); + capabilities.submitProbeMs = probeMs; + + hydrateDeviceState(); + if (deviceEpoch !== initializationEpoch) { + device.destroy?.(); + if (isUsableGPUDevice(gpuDevice)) { + log.debug('GPU', 'Device initialization was superseded; using the active shared device'); + return gpuDevice; + } + throw createDopplerError( + ERROR_CODES.GPU_DEVICE_FAILED, + 'WebGPU device initialization was superseded by a device-state reset' + ); + } + + gpuDevice = device; + kernelCapabilities = capabilities; + lastDeviceLossInfo = null; + registerDeviceLostHandler(gpuDevice); + advanceDeviceEpoch(); syncSharedDeviceState(); const features = [ @@ -563,8 +587,38 @@ export async function initDevice() { return gpuDevice; } + +export async function initDevice() { + hydrateDeviceState(); + // Return cached device if available + if (gpuDevice) { + if (isUsableGPUDevice(gpuDevice)) { + return gpuDevice; + } + clearActiveDeviceState(); + advanceDeviceEpoch(); + } + + if (sharedDeviceState.deviceInitPromise) { + log.debug('GPU', 'Joining in-flight device initialization'); + return sharedDeviceState.deviceInitPromise; + } + + const initializationEpoch = deviceEpoch; + const initializationPromise = initializeDevice(initializationEpoch); + sharedDeviceState.deviceInitPromise = initializationPromise; + try { + return await initializationPromise; + } finally { + if (sharedDeviceState.deviceInitPromise === initializationPromise) { + sharedDeviceState.deviceInitPromise = null; + } + } +} + export function setDevice(device, options = {}) { hydrateDeviceState(); + invalidateDeviceInitialization(); if (!device) { clearActiveDeviceState(); advanceDeviceEpoch(); @@ -653,6 +707,8 @@ export function getLastDeviceLossInfo() { export function resetDeviceState() { + hydrateDeviceState(); + invalidateDeviceInitialization(); clearActiveDeviceState(); advanceDeviceEpoch(); } @@ -660,6 +716,7 @@ export function resetDeviceState() { export function destroyDevice() { hydrateDeviceState(); + invalidateDeviceInitialization(); if (gpuDevice) { gpuDevice.destroy(); clearActiveDeviceState(); diff --git a/src/tooling/opfs-cache.js b/src/tooling/opfs-cache.js index ea9ded344..cef3c36e7 100644 --- a/src/tooling/opfs-cache.js +++ b/src/tooling/opfs-cache.js @@ -21,9 +21,48 @@ import { const MODULE = 'OPFSCache'; let cacheOperationQueue = Promise.resolve(); +let cacheOperationSequence = 0; +let cacheQueueDepth = 0; -function runCacheOperation(run) { - const operation = cacheOperationQueue.then(run, run); +function monotonicNowMs() { + return typeof performance !== 'undefined' && typeof performance.now === 'function' + ? performance.now() + : Date.now(); +} + +function runCacheOperation(modelId, onProgress, run) { + const operationId = cacheOperationSequence + 1; + cacheOperationSequence = operationId; + const enqueuedAtMs = monotonicNowMs(); + cacheQueueDepth += 1; + const queueDepth = cacheQueueDepth; + onProgress?.({ + stage: 'cache-queued', + modelId, + operationId, + queueDepth, + queueWaitMs: 0, + message: `OPFS cache operation queued: ${modelId}`, + percent: 0, + }); + const execute = async () => { + const queueWaitMs = Math.max(0, monotonicNowMs() - enqueuedAtMs); + onProgress?.({ + stage: 'cache-start', + modelId, + operationId, + queueDepth, + queueWaitMs, + message: `OPFS cache operation started: ${modelId}`, + percent: 0, + }); + try { + return await run(); + } finally { + cacheQueueDepth = Math.max(0, cacheQueueDepth - 1); + } + }; + const operation = cacheOperationQueue.then(execute, execute); cacheOperationQueue = operation.then(() => undefined, () => undefined); return operation; } @@ -465,11 +504,15 @@ async function ensureModelCachedUnlocked(modelId, modelBaseUrl, onProgress = nul } export function ensureModelCached(modelId, modelBaseUrl, onProgress = null) { - return runCacheOperation(() => ensureModelCachedUnlocked(modelId, modelBaseUrl, onProgress)); + return runCacheOperation( + modelId, + onProgress, + () => ensureModelCachedUnlocked(modelId, modelBaseUrl, onProgress) + ); } export function ensureModelCachedSource(modelId, modelBaseUrl, onProgress = null, options = {}) { - return runCacheOperation(async () => { + return runCacheOperation(modelId, onProgress, async () => { const expectedManifestHash = normalizeExpectedManifestHash(options.expectedManifestHash); const pinnedHit = await resolvePinnedCacheHit(modelId, expectedManifestHash, onProgress); const cache = pinnedHit ?? await ensureModelCachedUnlocked(modelId, modelBaseUrl, onProgress); diff --git a/tests/fixtures/reports/gemma-3-1b-it-q4k-ehf16-af32/2026-03-12T16-35-44.418Z.json b/tests/fixtures/reports/gemma-3-1b-it-q4k-ehf16-af32/2026-03-12T16-35-44.418Z.json index d0ad2d4cf..a7fca7de8 100644 --- a/tests/fixtures/reports/gemma-3-1b-it-q4k-ehf16-af32/2026-03-12T16-35-44.418Z.json +++ b/tests/fixtures/reports/gemma-3-1b-it-q4k-ehf16-af32/2026-03-12T16-35-44.418Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "gemma3", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", + "outputDir": "/workspace/deco/doppler/models/local/gemma-3-1b-it-q4k-ehf16-af32", "shardCount": 16, "tensorCount": 340, "totalSize": 1025493248 diff --git a/tests/fixtures/reports/gemma-3-270m-it-q4k-ehf16-af32/2026-03-12T16-31-54.924Z.json b/tests/fixtures/reports/gemma-3-270m-it-q4k-ehf16-af32/2026-03-12T16-31-54.924Z.json index e39e727e3..82477eb7f 100644 --- a/tests/fixtures/reports/gemma-3-270m-it-q4k-ehf16-af32/2026-03-12T16-31-54.924Z.json +++ b/tests/fixtures/reports/gemma-3-270m-it-q4k-ehf16-af32/2026-03-12T16-31-54.924Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "gemma3", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/gemma-3-270m-it-q4k-ehf16-af32", + "outputDir": "/workspace/deco/doppler/models/local/gemma-3-270m-it-q4k-ehf16-af32", "shardCount": 6, "tensorCount": 236, "totalSize": 399357184 diff --git a/tests/fixtures/reports/lfm2-5-1-2b-instruct-q4k-ehf16-af32/2026-03-13T12-21-52.534Z.json b/tests/fixtures/reports/lfm2-5-1-2b-instruct-q4k-ehf16-af32/2026-03-13T12-21-52.534Z.json index 87b447cd8..4f0ea5c18 100644 --- a/tests/fixtures/reports/lfm2-5-1-2b-instruct-q4k-ehf16-af32/2026-03-13T12-21-52.534Z.json +++ b/tests/fixtures/reports/lfm2-5-1-2b-instruct-q4k-ehf16-af32/2026-03-13T12-21-52.534Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "lfm2", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/lfm2-5-1-2b-instruct-q4k-ehf16-af32", + "outputDir": "/workspace/deco/doppler/models/local/lfm2-5-1-2b-instruct-q4k-ehf16-af32", "shardCount": 13, "tensorCount": 148, "totalSize": 851352832 diff --git a/tests/fixtures/reports/qwen-3-5-0-8b-q4k-ehaf16/2026-03-12T16-22-26.707Z.json b/tests/fixtures/reports/qwen-3-5-0-8b-q4k-ehaf16/2026-03-12T16-22-26.707Z.json index f967aac01..1b782ddeb 100644 --- a/tests/fixtures/reports/qwen-3-5-0-8b-q4k-ehaf16/2026-03-12T16-22-26.707Z.json +++ b/tests/fixtures/reports/qwen-3-5-0-8b-q4k-ehaf16/2026-03-12T16-22-26.707Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "qwen3", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/qwen-3-5-0-8b-q4k-ehaf16", + "outputDir": "/workspace/deco/doppler/models/local/qwen-3-5-0-8b-q4k-ehaf16", "shardCount": 12, "tensorCount": 320, "totalSize": 788833984 diff --git a/tests/fixtures/reports/qwen-3-5-2b-q4k-ehaf16/2026-03-12T16-30-34.369Z.json b/tests/fixtures/reports/qwen-3-5-2b-q4k-ehaf16/2026-03-12T16-30-34.369Z.json index 161c8b9e5..a5d8f20b1 100644 --- a/tests/fixtures/reports/qwen-3-5-2b-q4k-ehaf16/2026-03-12T16-30-34.369Z.json +++ b/tests/fixtures/reports/qwen-3-5-2b-q4k-ehaf16/2026-03-12T16-30-34.369Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "qwen3", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/qwen-3-5-2b-q4k-ehaf16", + "outputDir": "/workspace/deco/doppler/models/local/qwen-3-5-2b-q4k-ehaf16", "shardCount": 27, "tensorCount": 320, "totalSize": 1789738688 diff --git a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-01-31.413Z.json b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-01-31.413Z.json index d7d5d2435..96a6b9889 100644 --- a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-01-31.413Z.json +++ b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-01-31.413Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "qwen3_vl", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", + "outputDir": "/workspace/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", "shardCount": 25, "tensorCount": 625, "totalSize": 1644748800 diff --git a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-11-09.561Z.json b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-11-09.561Z.json index c824b0867..bd0b0d411 100644 --- a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-11-09.561Z.json +++ b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-11-09.561Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "qwen3_vl", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", + "outputDir": "/workspace/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", "shardCount": 25, "tensorCount": 625, "totalSize": 1644748800 diff --git a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-42-48.162Z.json b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-42-48.162Z.json index 637b8d5ed..c6f867abe 100644 --- a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-42-48.162Z.json +++ b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-18T23-42-48.162Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "qwen3_vl", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", + "outputDir": "/workspace/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", "shardCount": 25, "tensorCount": 625, "totalSize": 1644659008 diff --git a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-08-28.694Z.json b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-08-28.694Z.json index a53bf900b..ca8d60a26 100644 --- a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-08-28.694Z.json +++ b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-08-28.694Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "qwen3_vl", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", + "outputDir": "/workspace/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", "shardCount": 25, "tensorCount": 625, "totalSize": 1644659008 diff --git a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-21-12.695Z.json b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-21-12.695Z.json index fb5aca3ed..7249518e1 100644 --- a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-21-12.695Z.json +++ b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-21-12.695Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "qwen3_vl", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", + "outputDir": "/workspace/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", "shardCount": 25, "tensorCount": 625, "totalSize": 1644659008 diff --git a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-48-21.211Z.json b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-48-21.211Z.json index 3330276c0..a0d3269d5 100644 --- a/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-48-21.211Z.json +++ b/tests/fixtures/reports/qwen3-vl-2b-q4k-ehaf16/2026-03-19T00-48-21.211Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "qwen3_vl", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", + "outputDir": "/workspace/deco/doppler/models/local/qwen3-vl-2b-q4k-ehaf16", "shardCount": 34, "tensorCount": 625, "totalSize": 2229215232 diff --git a/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-12T23-18-49.038Z.json b/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-12T23-18-49.038Z.json index 8c6f0d867..d73b386f3 100644 --- a/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-12T23-18-49.038Z.json +++ b/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-12T23-18-49.038Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "translategemma", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/translategemma-4b-it-q4k-ehf16-af32", + "outputDir": "/workspace/deco/doppler/models/local/translategemma-4b-it-q4k-ehf16-af32", "shardCount": 47, "tensorCount": 444, "totalSize": 3148102656 diff --git a/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-13T00-11-42.455Z.json b/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-13T00-11-42.455Z.json index 8a0f68ffb..1e916ed4b 100644 --- a/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-13T00-11-42.455Z.json +++ b/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-13T00-11-42.455Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "translategemma", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/translategemma-4b-it-q4k-ehf16-af32", + "outputDir": "/workspace/deco/doppler/models/local/translategemma-4b-it-q4k-ehf16-af32", "shardCount": 47, "tensorCount": 444, "totalSize": 3148102656 diff --git a/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-13T12-27-01.380Z.json b/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-13T12-27-01.380Z.json index b98a10108..c2ac7b675 100644 --- a/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-13T12-27-01.380Z.json +++ b/tests/fixtures/reports/translategemma-4b-it-q4k-ehf16-af32/2026-03-13T12-27-01.380Z.json @@ -8,7 +8,7 @@ "result": { "presetId": "translategemma", "modelType": "transformer", - "outputDir": "/Users/xyz/deco/doppler/models/local/translategemma-4b-it-q4k-ehf16-af32", + "outputDir": "/workspace/deco/doppler/models/local/translategemma-4b-it-q4k-ehf16-af32", "shardCount": 47, "tensorCount": 444, "totalSize": 3148102656 diff --git a/tests/inference/device-concurrent-init.test.js b/tests/inference/device-concurrent-init.test.js new file mode 100644 index 000000000..cd9ddb746 --- /dev/null +++ b/tests/inference/device-concurrent-init.test.js @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict'; + +globalThis.GPUBufferUsage = { + MAP_READ: 0x0001, + COPY_SRC: 0x0004, + COPY_DST: 0x0008, + STORAGE: 0x0080, +}; +globalThis.GPUMapMode = { + READ: 0x0001, +}; + +class FakeBuffer { + constructor({ size, usage }) { + this.size = size; + this.usage = usage; + } + + destroy() {} + + mapAsync() { + return Promise.resolve(); + } + + unmap() {} +} + +const limits = { + maxStorageBufferBindingSize: 1 << 20, + maxBufferSize: 1 << 20, + maxComputeWorkgroupSizeX: 256, + maxComputeWorkgroupSizeY: 1, + maxComputeWorkgroupSizeZ: 1, + maxComputeInvocationsPerWorkgroup: 256, + maxComputeWorkgroupStorageSize: 16384, + maxStorageBuffersPerShaderStage: 8, + maxUniformBufferBindingSize: 65536, +}; + +function createFakeDevice(id) { + return { + id, + features: new Set(), + limits, + queue: { + submit() {}, + onSubmittedWorkDone() { + return Promise.resolve(); + }, + }, + createBuffer(descriptor) { + return new FakeBuffer(descriptor); + }, + createBindGroup(descriptor) { + return descriptor; + }, + createCommandEncoder() { + return { + beginComputePass() { + return { + setPipeline() {}, + setBindGroup() {}, + dispatchWorkgroups() {}, + end() {}, + }; + }, + copyBufferToBuffer() {}, + finish() { + return {}; + }, + }; + }, + createComputePipeline() { + return { + getBindGroupLayout() { + return {}; + }, + }; + }, + createShaderModule() { + return {}; + }, + destroy() {}, + }; +} + +const originalNavigator = Object.getOwnPropertyDescriptor(globalThis, 'navigator'); +let adapterRequests = 0; +let deviceRequests = 0; +const adapter = { + features: new Set(), + limits, + info: { + vendor: 'unit', + architecture: 'concurrent-init', + device: 'fake', + description: 'Concurrent initialization test adapter', + }, + async requestDevice() { + deviceRequests += 1; + await Promise.resolve(); + return createFakeDevice(deviceRequests); + }, +}; + +Object.defineProperty(globalThis, 'navigator', { + configurable: true, + value: { + gpu: { + async requestAdapter() { + adapterRequests += 1; + await Promise.resolve(); + return adapter; + }, + }, + }, +}); + +const moduleA = await import('../../src/gpu/device.js?concurrent-init-a'); +const moduleB = await import('../../src/gpu/device.js?concurrent-init-b'); + +try { + moduleA.resetDeviceState(); + const [deviceA, deviceB] = await Promise.all([ + moduleA.initDevice(), + moduleB.initDevice(), + ]); + + assert.equal(adapterRequests, 1, 'concurrent callers must share one adapter request'); + assert.equal(deviceRequests, 1, 'concurrent callers must share one device request'); + assert.equal(deviceA, deviceB, 'concurrent callers must receive the same GPUDevice'); + assert.equal(moduleA.getDevice(), deviceA); + assert.equal(moduleB.getDevice(), deviceA); + assert.equal(moduleA.getDeviceEpoch(), moduleB.getDeviceEpoch()); +} finally { + moduleA.resetDeviceState(); + if (originalNavigator) { + Object.defineProperty(globalThis, 'navigator', originalNavigator); + } else { + delete globalThis.navigator; + } +} + +console.log('device-concurrent-init.test: ok'); diff --git a/tests/integration/opfs-cached-source-contract.test.js b/tests/integration/opfs-cached-source-contract.test.js index 8b718ae21..a28f52803 100644 --- a/tests/integration/opfs-cached-source-contract.test.js +++ b/tests/integration/opfs-cached-source-contract.test.js @@ -251,6 +251,10 @@ try { assert.equal(hit.manifestText, manifestText); assert.equal(hit.manifestHash, expectedManifestHash); assert.deepEqual(new Uint8Array(await hit.storageContext.loadShardRange(0, 1, 2)), shardBytes.slice(1, 3)); + assert.ok(hitEvents.some((event) => event.stage === 'cache-queued')); + assert.ok(hitEvents.some((event) => ( + event.stage === 'cache-start' && Number.isFinite(event.queueWaitMs) + ))); assert.ok(hitEvents.some((event) => event.stage === 'cache-hit')); await hit.storageContext.close(); diff --git a/tools/check-publication-hygiene.js b/tools/check-publication-hygiene.js new file mode 100644 index 000000000..d8f2fcec3 --- /dev/null +++ b/tools/check-publication-hygiene.js @@ -0,0 +1,61 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync, statSync } from 'node:fs'; + +const PRIVATE_HOME_PATTERNS = Object.freeze([ + { + label: 'macOS user home', + pattern: /\/Users\/(?!(?:|runner|Shared)(?=\/|[\s"'`,:}\]]|$))[^/\s"'`]+/g, + }, + { + label: 'Linux user home', + pattern: /\/home\/(?!(?:user|runner|x|web_user|chrome)(?=\/|[\s"'`,:}\]]|$))[^/\s"'`]+/g, + }, + { + label: 'Windows user home', + pattern: /[A-Za-z]:\\Users\\(?!(?:\\|$)|runner(?:\\|$)|Public(?:\\|$))[^\\\s"'`]+/g, + }, +]); + +function listTrackedFiles() { + return execFileSync('git', ['ls-files', '-z'], { + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }).split('\0').filter(Boolean); +} + +function scanFile(filePath) { + if (!existsSync(filePath) || !statSync(filePath).isFile()) return []; + const bytes = readFileSync(filePath); + if (bytes.includes(0)) return []; + + const findings = []; + const lines = bytes.toString('utf8').split('\n'); + for (let index = 0; index < lines.length; index++) { + for (const { label, pattern } of PRIVATE_HOME_PATTERNS) { + pattern.lastIndex = 0; + if (pattern.test(lines[index])) { + findings.push({ filePath, line: index + 1, label }); + } + } + } + return findings; +} + +function main() { + const files = listTrackedFiles(); + const findings = files.flatMap(scanFile); + if (findings.length > 0) { + for (const finding of findings) { + console.error(`${finding.filePath}:${finding.line}: ${finding.label}`); + } + console.error(`publication hygiene failed: ${findings.length} private path(s)`); + process.exitCode = 1; + return; + } + + console.log(`publication hygiene passed: ${files.length} tracked files scanned`); +} + +main(); diff --git a/tools/vendor-bench.js b/tools/vendor-bench.js index a1cd07484..5b8e71764 100644 --- a/tools/vendor-bench.js +++ b/tools/vendor-bench.js @@ -1865,9 +1865,11 @@ function assertClaimMatrixCompareEvidence(compareReport, lane, matrix, workloads const manifestSha256 = asNonEmptyStringValue(compareReport?.dopplerManifestPreflight?.manifestSha256) || asNonEmptyStringValue(compareReport?.dopplerModelSource?.manifestSha256); + const evidenceManifestSha256 = asNonEmptyStringValue(lane.artifact.evidenceManifestSha256) + || lane.artifact.manifestSha256; assertClaimCompareScalar( manifestSha256, - lane.artifact.manifestSha256, + evidenceManifestSha256, `${compareEvidenceLabel} manifestSha256` ); const manifestSource = asNonEmptyStringValue(compareReport?.dopplerModelSource?.manifestSource);