diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c08bfd..54a6e0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,20 @@ jobs: - run: npm pack --dry-run variaq-integration: - name: VariaQ 0.6.0 integration + name: VariaQ ${{ matrix.variaq_version }} integration + strategy: + fail-fast: false + matrix: + include: + - variaq_version: "0.6.0" + # Commit behind the verified legacy v0.6.0 tag. + variaq_ref: aeac83b67a9874957458d626c81ba231030f90ab + - variaq_version: "0.8.0rc1" + # Reviewed release/0.8.0rc1 candidate; never use a moving branch. + variaq_ref: 551151a2c2ccfe0301dca1b22fcc8abf5d540f56 + - variaq_version: "0.8.0" + # VariaQ 0.8.0 stable merge commit on main; never use a moving branch. + variaq_ref: 2c7ca9c488a5aea91575ec0a95e2111adeb2a0a2 runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -34,8 +47,7 @@ jobs: - uses: actions/checkout@v6 with: repository: aaronphifer/variaq - # Immutable release tag for the verified VariaQ 0.6.x integration. - ref: v0.6.0 + ref: ${{ matrix.variaq_ref }} path: .variaq-source - uses: actions/setup-node@v7 with: @@ -56,4 +68,5 @@ jobs: env: VARIAQ_TEST_PROJECT: ${{ github.workspace }}/.variaq-source VARIAQ_TEST_PYTHON: ${{ runner.temp }}/variaq-venv/bin/python + VARIAQ_TEST_VERSION: ${{ matrix.variaq_version }} run: npm run test:integration diff --git a/CHANGELOG.md b/CHANGELOG.md index 529230a..f0f7caf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,20 @@ All notable changes to this project will be documented in this file. -## 0.5.1 - Unreleased +## 0.6.0 - Unreleased -Maintenance-only release. +Unreleased compatibility update. + +- Accept stable VariaQ 0.8.x series releases and the tested 0.8.0rc1 candidate while retaining 0.6.x and schema-v1 support; paired_solvers comparisons unlock on stable VariaQ 0.8.x. +- Prepare an immutable stable VariaQ 0.8.0 integration CI leg (pin the stable tag SHA when the VariaQ stable branch is pushed). +- Pin the transitive development dependency `source-map-js` to 1.2.2 (GHSA-68fv-2mgg-jv7q) via a narrowly scoped npm override; runtime dependencies remain clean. The five remaining development advisories live inside `node_modules/npm` bundled dependencies that npm overrides cannot reach; they require an upstream npm/bb-app refresh and do not affect the shipped plugin. +- Accept the tested VariaQ 0.8.0rc1 candidate while retaining stable 0.6.x support and schema-v1 validation; preserve candidate suffixes and reject unverified versions. +- Target BB 0.46 / Plugin SDK 0.6.37, update build dependencies, and verify real local integration with both VariaQ 0.6.0 and 0.8.0rc1. +- Forward paired_solvers through existing campaign analysis/report tools and CLI, including expanded CSV paths, without plugin-side science. Reject unsupported run-comparison requests clearly. +- Validate canonical nested campaign solver parameters, including numeric candidate matrices, using non-recursive tool schemas. +- Keep GPU integration explicitly opt-in with VARIAQ_TEST_GPU=1, independently of CUDA-Q CPU tests. +- Run hosted legacy and candidate integration against immutable VariaQ commit pins. +- Document the six development-toolchain audit findings separately from runtime dependencies. - Fix duplicated plot-format requests in `variaq_report_campaign` when both `formats: ["plots"]` and `plots: true` are supplied; `formats` is now canonical and `plots: true` only ensures "plots" appears once. - Update GitHub Actions to Node 24-native action versions (`actions/checkout@v6`, `actions/setup-node@v7`, `actions/setup-python@v7`) to resolve Node 20 deprecation warnings while preserving existing workflow behavior. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 03c9bf6..5bab4a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,15 +22,20 @@ solver installation. ## Real VariaQ integration -Install VariaQ 0.2.0 separately with Python 3.12 and its `quantum` extra, then +Install VariaQ 0.6.0 or the local 0.8.0rc1 candidate separately with Python 3.12 and its `quantum` extra, then run: ```bash VARIAQ_TEST_PROJECT=/path/to/variaq \ VARIAQ_TEST_PYTHON=/path/to/variaq/.venv/bin/python \ +VARIAQ_TEST_VERSION=0.8.0rc1 \ npm run test:integration ``` +To validate the stable VariaQ 0.8.0 release instead, point +`VARIAQ_TEST_VERSION=0.8.0` at the pinned stable-tag checkout (the exact commit +SHA is pinned at release time, mirroring the CI matrix placeholder). + These tests cover the CLI contract and local exact, heuristic, and Qiskit flows. They must not reimplement VariaQ solver-correctness tests. @@ -39,10 +44,14 @@ CUDA-Q is optional. On a compatible local or self-hosted machine: ```bash VARIAQ_TEST_PROJECT=/path/to/variaq \ VARIAQ_TEST_PYTHON=/path/to/variaq/.venv/bin/python \ +VARIAQ_TEST_VERSION=0.8.0rc1 \ VARIAQ_TEST_CUDAQ=1 \ npm run test:integration ``` +GPU integration additionally requires `VARIAQ_TEST_GPU=1`. Set the expected +version to `0.6.0` when validating the legacy tagged checkout. + Use `npm run smoke` for a deterministic installed-plugin workflow. GPU work is opt-in with `VARIAQ_SMOKE_GPU=1`; neither physical-QPU nor remote-provider tests belong in the plugin's required CI. @@ -56,3 +65,8 @@ belong in the plugin's required CI. BB, or Plugin SDK versions. - Do not commit local settings, experiment databases, generated problems, credentials, dependency directories, coverage, logs, or build output. + +CI runs the legacy and candidate integration suite against the immutable VariaQ +commit pins in `.github/workflows/ci.yml`. Keep `VARIAQ_TEST_VERSION` matched to +each source revision. A candidate pin must come from reviewed release-branch +code; do not substitute main, v0.7, a moving candidate branch, or an unverified build. diff --git a/README.md b/README.md index f8f2b56..c48e94c 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,10 @@ logic — VariaQ remains authoritative for all of these. ## Prerequisites -- BB 0.43.x with Plugin SDK 0.4.104 +- BB 0.46.x with Plugin SDK 0.6.37 - Node.js 22.19 or a compatible version supported by BB - Linux-native Python 3.12 -- a separate VariaQ 0.6.x installation; 0.6.0 is the verified version +- a separate VariaQ 0.6.x installation or the local 0.8.0rc1 candidate Create a VariaQ environment separately from this repository: @@ -179,20 +179,48 @@ prose parsing or run-ID scraping. ## Version compatibility -The verified combination is: - -```text -bb-plugin-variaq 0.5.0 -VariaQ 0.6.0 -Plugin SDK 0.4.104 -BB host 0.43.x -VariaQ schema 1 +The current checkout targets BB 0.46.x / Plugin SDK 0.6.37 and schema version `1`. +It accepts stable VariaQ 0.6.x and 0.8.x series releases plus the explicitly +tested `0.8.0rc1` candidate. Real local integration was verified against the +immutable `v0.6.0` tag and the reviewed v0.8 candidate checkout. Patch +acceptance within 0.6.x and 0.8.x is a version policy; every patch was not +independently executed. VariaQ 0.5.x, 0.7.x, 0.9.x, other release candidates +(including rc2 and later) and future schemas remain unsupported. A suffix is +never silently stripped from a version. Status/version report unsupported +environments with a warning; schema parsing still rejects non-v1 output. This +does not authorize running an unverified version. + +The BB engine/SDK pins have moved from BB 0.43 / SDK 0.4.104 to the actual BB +0.46 host. Old BB 0.43 host compatibility is not claimed. Package identity is +`0.6.0` (unreleased); this work does not publish a plugin or a VariaQ release. + +### Paired-solver analysis in the 0.8 series + +On verified VariaQ 0.8.x stable releases and the tested `0.8.0rc1` candidate, +existing `variaq_analyze_campaign` and `variaq_report_campaign` tools accept +`compare: "paired_solvers"`; CLI equivalents accept `--compare paired_solvers`. +This forwards VariaQ's analysis, including instances, pairs, precision, +repeat/seed controls, null/reason values and timing scopes. Reports forward all +generated paths, including per-run, pair and repeat CSVs. The plugin computes +no objectives, gaps, pairing, timings or speedups. + +```sh +bb variaq analyze-campaign CAMPAIGN_ID --compare paired_solvers --json +bb variaq report-campaign CAMPAIGN_ID --output-dir paired --formats json,csv,markdown --compare paired_solvers --json ``` -bb-plugin-variaq 0.5.x is verified against VariaQ 0.6.x and schema version `1`. -Patch releases within the 0.5 plugin series and 0.6 VariaQ series are accepted. -Other VariaQ series or future schema versions are reported as unsupported with a -clear compatibility error. +Canonical campaign overrides use `solver_config: {qaoa: {seed: 42, +parameters: {p: 1, optimizer_trials: 2, shots: 32, warmup: false, +candidate_parameters: [[0.5, 0.25], [0.8, 0.4]]}}}`. Scalar controls, scalar +lists and numeric matrices are validated without recursive agent-tool schemas. +Legacy flat scalar override shapes remain accepted; VariaQ remains authoritative +about their interpretation. No backend math or optimizer is duplicated here. + +VariaQ's `analyze runs` CLI has no `--compare` contract. Use ordinary run analysis +for explicit run selections, and a stored campaign for comparisons. Unsupported +run-comparison requests receive a clear error. Paired campaign/report requests +on 0.6.x also fail clearly, without fallback or solver execution. Legacy campaign +comparison modes remain available on both supported versions. ## CUDA-Q and GPU behavior @@ -271,7 +299,7 @@ Starting with bb-plugin-variaq 0.5.0 and VariaQ 0.6.0: ## Limitations -- VariaQ 0.6.0 provides local execution only; there is no physical-QPU path. +- Supported VariaQ versions provide local execution only; there is no physical-QPU path. - There is no IBM Runtime/provider integration and no credential handling. - Generic CI does not require CUDA-Q, an NVIDIA GPU, a physical QPU, or remote services. @@ -312,22 +340,32 @@ npm pack --dry-run ``` Real integration tests use `VARIAQ_TEST_PROJECT` and `VARIAQ_TEST_PYTHON` when -set. They default to the local development checkout. Optional CUDA-Q CPU tests -run only with `VARIAQ_TEST_CUDAQ=1`: +set. With neither set, real tests are skipped. `VARIAQ_TEST_VERSION` is an +explicit expected version (default `0.6.0` for tagged CI). Use `0.8.0rc1` for +the candidate. Optional CUDA-Q CPU tests require `VARIAQ_TEST_CUDAQ=1`; GPU +execution separately requires `VARIAQ_TEST_GPU=1`: ```bash VARIAQ_TEST_PROJECT=/path/to/variaq \ VARIAQ_TEST_PYTHON=/path/to/variaq/.venv/bin/python \ +VARIAQ_TEST_VERSION=0.8.0rc1 \ VARIAQ_TEST_CUDAQ=1 \ npm run test:integration ``` The normal GitHub Actions plugin job uses the deterministic fake CLI. A -separate Linux integration job checks out the immutable VariaQ `v0.6.0` tag, +separate Linux integration matrix checks out immutable commits for VariaQ `v0.6.0` +and the reviewed `release/0.8.0rc1` candidate, installs `.[quantum]`, exercises local classical/Qiskit workflows, and runs a bounded campaign plan/run, analysis, and report-generation smoke. CUDA-Q, GPU, and matplotlib verification remain manual or suitable for a future self-hosted -runner. +runner. Candidate integration is pinned to VariaQ commit +`551151a2c2ccfe0301dca1b22fcc8abf5d540f56`; CI asserts the expected version. +Update this reviewed pin explicitly when the VariaQ candidate changes. +The existing public-SDK scanner test remains skipped because its string scanner +reports command strings as imports; typecheck/build and live registration remain +active checks. Linux local integration is verified; Windows plugin execution is +not claimed. The fake executable harness currently uses POSIX executable symlinks. See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution boundaries and [SECURITY.md](SECURITY.md) for the subprocess security model. diff --git a/SECURITY.md b/SECURITY.md index 49f67c1..7fa018f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -22,3 +22,43 @@ it is enabled. Otherwise contact the repository owner privately through their published profile contact. Do not include active secrets or sensitive system data in a public issue. No project-specific private security email is currently published. + +## Candidate execution and reports + +Paired analysis uses stored campaigns and VariaQ-derived evidence only; it does +not execute solvers. Reports remain beneath the configured output root with +component-by-component directory symlink rejection and VariaQ's artifact guards. +A subprocess timeout sends termination and escalates to a kill; completed v0.8 +attempts remain persisted, but an in-flight native result may be lost. The plugin +never resumes or automatically retries work. Independently running CLI campaigns +are not coordinated by VariaQ's web-worker lock. Keep one execution owner. + +Supported versions are a compatibility policy, not permission for physical-QPU +execution. No new credentials, external execution, shell, or arbitrary analytics +code is introduced by candidate compatibility. + +## Candidate dependency review + +The candidate lockfile audit reports six affected development packages (four +high, two moderate); `npm audit --omit=dev` reports zero affected production +packages. These are real upstream advisories, not a claim that development use +is harmless. The plugin runtime imports Zod and Node built-ins, invokes local +Python with argument arrays, and does not call the affected libraries directly. + +| Locked package | Development dependency path | Advisory exposure | +| --- | --- | --- | +| brace-expansion 5.0.9 | bb-app → npm → minimatch | [Stack/CPU exhaustion from hostile brace patterns](https://github.com/advisories/GHSA-qhr7-859c-m2p7) | +| http-cache-semantics 4.2.0 | bb-app → npm → make-fetch-happen | [Cross-user cached response disclosure with max-stale](https://github.com/advisories/GHSA-ch52-4w7c-c8xp) | +| ip-address 10.5.0 | bb-app → npm → proxy/SOCKS tooling | [Incorrect address classifiers/subnet checks](https://github.com/advisories/GHSA-j6r3-76f7-8jcv), plus unbounded diagnostic input | +| postcss-selector-parser 7.1.4 | bb-app → npm → dependency query tooling | [CPU exhaustion from hostile selectors](https://github.com/advisories/GHSA-rj75-hqrm-r3gf) | +| source-map-js 1.2.1 | vitest → vite → postcss | [Event-loop exhaustion from hostile indexed source maps](https://github.com/advisories/GHSA-68fv-2mgg-jv7q) | +| undici 6.28.0 | bb-app → npm → download/build tooling | [WebSocket denial of service](https://github.com/advisories/GHSA-rfgv-xxqx-mfg5), plus retry-interceptor response splitting | + +The reviewed plugin surfaces expose no user-controlled glob, HTTP cache/proxy, +CSS/source-map parsing or WebSocket path into these packages. Exploitability in +npm/build/test tooling depends on its inputs and configuration; hostile package +metadata, network peers and development assets remain relevant risks. This review +does not audit BB host internals or establish that these advisories are unreachable +in every development invocation. Use isolated CI/cache environments and review +narrow compatible dependency fixes before a broad public release. No forced audit +fix, broad upgrade or suppression was applied. diff --git a/lib/runner.ts b/lib/runner.ts index 0be67f2..57a76a0 100644 --- a/lib/runner.ts +++ b/lib/runner.ts @@ -616,14 +616,17 @@ export interface VariaqVersionCompatibility { } /** - * bb-plugin-variaq 0.5.0 is verified against VariaQ 0.6.0 / schema_version 1. - * Patch releases in the 0.6 series are accepted. Other series are reported as + * Accept stable VariaQ 0.6.x and 0.8.x series releases and the explicitly + * tested 0.8.0rc1 candidate. Other versions and candidates are reported as * unsupported so users can still inspect a mismatched environment. */ export function checkVariaqVersion(raw: string | null, schemaVersion?: string): VariaqVersionCompatibility { - const match = raw?.match(/(?:^|\s)(\d+)\.(\d+)\.(\d+)(?:\b|$)/); - const version = match ? `${match[1]}.${match[2]}.${match[3]}` : null; - const supported = match?.[1] === "0" && match?.[2] === "6"; + const match = raw?.trim().match(/^(?:variaq\s+)?(\d+)\.(\d+)\.(\d+)(rc\d+)?$/); + const version = match ? `${match[1]}.${match[2]}.${match[3]}${match[4] ?? ""}` : null; + const supported = + (match?.[1] === "0" && match?.[2] === "6" && !match?.[4]) || + (match?.[1] === "0" && match?.[2] === "8" && !match?.[4]) || + version === "0.8.0rc1"; const schemaVersionSupported = schemaVersion === undefined || schemaVersion === SUPPORTED_SCHEMA_VERSION; const parts: string[] = []; diff --git a/lib/schema.ts b/lib/schema.ts index 700e758..6eef1d9 100644 --- a/lib/schema.ts +++ b/lib/schema.ts @@ -80,7 +80,14 @@ export function isProblemFamily(value: string): value is ProblemFamily { export const campaignSolverOverrideSchema = z.record( z.string(), - z.record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.array(z.union([z.string(), z.number()]))])), + z.object({ + seed: z.number().int().min(0).optional(), + parameters: z.record(z.string(), z.union([ + z.string(), z.number(), z.boolean(), z.null(), + z.array(z.union([z.string(), z.number(), z.boolean(), z.null()])), + z.array(z.array(z.number())), + ])).optional(), + }).catchall(z.union([z.string(), z.number(), z.boolean(), z.array(z.union([z.string(), z.number()]))])), ); export const campaignDefinitionSchema = z.object({ @@ -148,6 +155,11 @@ export const campaignListItemSchema = z.object({ created_at: z.string(), }); +export const comparisonTypeSchema = z.enum([ + "classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu", "paired_solvers", +]); +export type ComparisonType = z.infer; + export const analyzeQuerySchema = z.object({ run_ids: z.array(z.string()).optional(), campaign_id: z.string().optional(), @@ -156,7 +168,7 @@ export const analyzeQuerySchema = z.object({ scaling_x: z.string().optional(), include_failed: z.boolean().optional(), include_unavailable: z.boolean().optional(), - compare: z.enum(["classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu"]).optional(), + compare: comparisonTypeSchema.optional(), }); export type AnalyzeQuery = z.infer; @@ -176,9 +188,9 @@ export interface ParsedEnvelope { export const SUPPORTED_SCHEMA_VERSION = "1"; /** Compatibility metadata, mirrored here for the schema module. */ -export const BB_PLUGIN_VARIAQ_VERSION = "0.5.1"; -export const VERIFIED_VARIAQ_VERSION = "0.6.0"; -export const SUPPORTED_VARIAQ_SERIES = "0.6.x"; +export const BB_PLUGIN_VARIAQ_VERSION = "0.6.0"; +export const VERIFIED_VARIAQ_VERSION = "0.8.0"; +export const SUPPORTED_VARIAQ_SERIES = "0.6.x, 0.8.x and 0.8.0rc1"; /** * Validate a VariaQ JSON envelope. diff --git a/package-lock.json b/package-lock.json index 2af978c..bc45f46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,21 +1,21 @@ { "name": "bb-plugin-variaq", - "version": "0.5.1", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "bb-plugin-variaq", - "version": "0.5.1", + "version": "0.6.0", "license": "Apache-2.0", "dependencies": { "zod": "^4.3.6" }, "devDependencies": { - "@get-bb/plugin-sdk": "0.4.104", + "@get-bb/plugin-sdk": "0.6.37", "@types/better-sqlite3": "^7.6.13", "@types/node": "^22.0.0", - "bb-app": "0.43.3", + "bb-app": "0.46.0", "better-sqlite3": "^12.11.1", "cron-parser": "^5.10.1", "hono": "^4.13.8", @@ -23,15 +23,19 @@ "vitest": "^5.0.1" }, "engines": { - "bb": ">=0.43.0 <0.44.0", - "bbPluginSdk": "0.4.104" + "bb": ">=0.46.0 <0.47.0", + "bbPluginSdk": "0.6.37" } }, "node_modules/@get-bb/plugin-sdk": { - "version": "0.4.104", - "resolved": "https://registry.npmjs.org/@get-bb/plugin-sdk/-/plugin-sdk-0.4.104.tgz", - "integrity": "sha512-h9+yfAa2qhg1aKaP7kZQ5oNpAk0qia4njXfslufArqifMoRWlx1WSJcQvXtfvQ8OJz/pO+JAiUn/HkySU6eTGA==", + "version": "0.6.37", + "resolved": "https://registry.npmjs.org/@get-bb/plugin-sdk/-/plugin-sdk-0.6.37.tgz", + "integrity": "sha512-fltzoFz2DTSDo35tSMaDR/nlQeTE/WFO1BLUMD0LoJ+dbKsJBCloaK+Kl1BYXP+lABPBH7dgAAfKKKSIgCBzHA==", "dev": true, + "dependencies": { + "cross-spawn": "^7.0.6", + "zod": "^4.6.5" + }, "peerDependencies": { "@testing-library/react": "^16.3.2", "@types/better-sqlite3": "^7.6.12", @@ -40,8 +44,7 @@ "cron-parser": "^5.5.0", "hono": "^4.11.9", "react": "^19.0.0", - "react-dom": "^19.0.0", - "zod": "^4.3.6" + "react-dom": "^19.0.0" }, "peerDependenciesMeta": { "@testing-library/react": { @@ -67,9 +70,6 @@ }, "react-dom": { "optional": true - }, - "zod": { - "optional": true } } }, @@ -911,27 +911,27 @@ "license": "MIT" }, "node_modules/bb-app": { - "version": "0.43.3", - "resolved": "https://registry.npmjs.org/bb-app/-/bb-app-0.43.3.tgz", - "integrity": "sha512-MzSJTvyuU316hJOxd3aierpxoWWk1AfFHXBKXpJe2UrbT3pa7rob4biTBzisFzyt44TO+/X8YIrDNGvS8t5QXw==", + "version": "0.46.0", + "resolved": "https://registry.npmjs.org/bb-app/-/bb-app-0.46.0.tgz", + "integrity": "sha512-Pr9vZAIV5FeLvTlYVwodk8Ao2qkJ5j+7WDZbAn6SYnbq8Yv3h3W8ukJ8od6Cuw0QZBHL9sCgVYbF9BclsBF1fg==", "dev": true, "os": [ "darwin", - "linux" + "linux", + "win32" ], "dependencies": { "@parcel/watcher": "2.5.6", "better-sqlite3": "13.0.3", "fs-native-extensions": "1.5.0", "hono": "^4.7.0", - "jiti": "^2.6.1", "node-pty": "1.2.0-beta.15", "npm": "11.16.0", "picocolors": "^1.1.1", "pino": "^9.6.0", "pino-pretty": "^13.0.0", "pino-roll": "^4.0.0", - "zod": "4.3.6" + "zod": "4.6.5" }, "bin": { "bb": "dist/bb.js", @@ -966,16 +966,6 @@ "node": "^18 || ^20 || >= 21" } }, - "node_modules/bb-app/node_modules/zod": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", - "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, "node_modules/better-sqlite3": { "version": "12.11.1", "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.11.1.tgz", @@ -1075,6 +1065,21 @@ "node": ">=18" } }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/date-fns": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.4.0.tgz", @@ -1334,15 +1339,12 @@ "node": ">=0.10.0" } }, - "node_modules/jiti": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", - "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true, - "license": "MIT", - "bin": { - "jiti": "lib/jiti-cli.mjs" - } + "license": "ISC" }, "node_modules/joycon": { "version": "3.1.1", @@ -3688,6 +3690,16 @@ "wrappy": "1" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -4050,6 +4062,29 @@ "node": ">=10" } }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -4115,9 +4150,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "peer": true, @@ -4449,6 +4484,22 @@ } } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/which-runtime": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/which-runtime/-/which-runtime-1.4.0.tgz", diff --git a/package.json b/package.json index 0c2fafe..cfbc654 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bb-plugin-variaq", - "version": "0.5.1", + "version": "0.6.0", "type": "module", "description": "Thin bb adapter for the standalone VariaQ hybrid-compute research platform.", "license": "Apache-2.0", @@ -17,8 +17,8 @@ "LICENSE" ], "engines": { - "bb": ">=0.43.0 <0.44.0", - "bbPluginSdk": "0.4.104" + "bb": ">=0.46.0 <0.47.0", + "bbPluginSdk": "0.6.37" }, "bb": { "name": "VariaQ", @@ -34,16 +34,17 @@ "overrides": { "bb-app": { "npm": "11.19.1" - } + }, + "source-map-js": "1.2.2" }, "dependencies": { "zod": "^4.3.6" }, "devDependencies": { - "@get-bb/plugin-sdk": "0.4.104", + "@get-bb/plugin-sdk": "0.6.37", "@types/better-sqlite3": "^7.6.13", "@types/node": "^22.0.0", - "bb-app": "0.43.3", + "bb-app": "0.46.0", "better-sqlite3": "^12.11.1", "cron-parser": "^5.10.1", "hono": "^4.13.8", diff --git a/server.ts b/server.ts index 27977ce..c704dc0 100644 --- a/server.ts +++ b/server.ts @@ -3,7 +3,7 @@ import { promises as fs } from "node:fs"; import { tmpdir } from "node:os"; import { isAbsolute, join, relative, resolve, sep } from "node:path"; import crypto from "node:crypto"; -import { problemFamilySchema, type ProblemFamily, campaignDefinitionSchema, campaignPlanDataSchema, campaignRunDataSchema, campaignListItemSchema, analyzeQuerySchema, reportFormatsSchema, type CampaignDefinition } from "./lib/schema.js"; +import { comparisonTypeSchema, type ComparisonType, problemFamilySchema, type ProblemFamily, campaignDefinitionSchema, campaignPlanDataSchema, campaignRunDataSchema, campaignListItemSchema, analyzeQuerySchema, reportFormatsSchema, type CampaignDefinition } from "./lib/schema.js"; import type { BbPluginApi } from "@get-bb/plugin-sdk"; import { PluginError, checkVariaqVersion, generateProblemArgv, parseJsonOutput, parseKeyValue, parseSolvers, probeCapabilities, quantumSupportNote, requireSuccess, resolveConfig, runVariaqJson, runVariaqStrict, solverSupportsFamily, statusFromCapabilities } from "./lib/runner.js"; import type { ResolvedConfig, VariaqSettings, CompareQuantumEnvelopeData, BenchmarkEnvelopeData, ReproduceEnvelopeData, ProblemGenerateData, SolveEnvelopeData, FamilyGenerateInput, VariaqSolver, CampaignPlanEnvelopeData, CampaignRunEnvelopeData, CampaignListEnvelopeData, CampaignShowEnvelopeData, AnalyzeEnvelopeData, ReportCampaignEnvelopeData } from "./lib/runner.js"; @@ -535,9 +535,27 @@ export default async function plugin(bb: BbPluginApi) { return argv; } + async function requireComparisonSupport(c: ResolvedConfig, comparison?: ComparisonType): Promise { + if (comparison !== "paired_solvers") return; + const result = await runVariaqStrict(c, ["--version"], { timeoutMs: 15_000 }); + const compat = checkVariaqVersion(result.stdout); + if (!(compat.supported && compat.version !== null && (compat.version === "0.8.0rc1" || /^0\.8\.\d+$/.test(compat.version)))) { + throw new PluginError( + "paired_solvers requires a verified VariaQ 0.8 stable release or the tested 0.8.0rc1 candidate.", + "Legacy comparisons remain available on VariaQ 0.6.x; no fallback or solver execution occurred.", + ); + } + } + async function opAnalyzeRuns(query: import("./lib/schema.js").AnalyzeQuery) { const c = await cfg(); const validated = analyzeQuerySchema.parse(query); + if (validated.compare !== undefined) { + throw new PluginError( + "VariaQ analyze runs does not accept comparison selectors.", + "Use variaq_analyze_campaign or bb variaq analyze-campaign for comparisons, including paired_solvers on VariaQ 0.8.x.", + ); + } const argv = buildAnalyzeArgv("runs", validated); const result = await runVariaqJson(c, argv, { timeoutMs: 60_000 }); const envelope = requireSuccess(result, ["analyze", "runs"]); @@ -548,6 +566,7 @@ export default async function plugin(bb: BbPluginApi) { async function opAnalyzeCampaign(campaignId: string, query: Omit) { const c = await cfg(); const fullQuery = analyzeQuerySchema.parse({ ...query, campaign_id: campaignId }); + await requireComparisonSupport(c, fullQuery.compare); const argv = buildAnalyzeArgv("campaign", fullQuery); const result = await runVariaqJson(c, argv, { timeoutMs: 60_000 }); const envelope = requireSuccess(result, ["analyze", "campaign"]); @@ -580,12 +599,14 @@ export default async function plugin(bb: BbPluginApi) { formats: ["json" | "csv" | "markdown" | "plots", ...("json" | "csv" | "markdown" | "plots")[]]; groupBy?: string[]; scalingX?: string; - compare?: "classical_vs_quantum" | "qiskit_vs_cudaq" | "gpu_vs_cpu"; + compare?: ComparisonType; plots?: boolean; overwrite?: boolean; }, ) { const c = await cfg(); + if (options.compare !== undefined) comparisonTypeSchema.parse(options.compare); + await requireComparisonSupport(c, options.compare); const root = resolveReportOutputDir(c); const containedDir = await resolveContainedReportDir(root, options.outputDir); @@ -851,7 +872,7 @@ export default async function plugin(bb: BbPluginApi) { formats: formats as ["json" | "csv" | "markdown" | "plots", ...("json" | "csv" | "markdown" | "plots")[]], groupBy: groupBy.length > 0 ? groupBy : undefined, scalingX: opts.get("scaling-x"), - compare: opts.get("compare") as "classical_vs_quantum" | "qiskit_vs_cudaq" | "gpu_vs_cpu" | undefined, + compare: opts.get("compare") as ComparisonType | undefined, plots: args.includes("--plots"), overwrite: args.includes("--overwrite"), }); @@ -1175,7 +1196,7 @@ export default async function plugin(bb: BbPluginApi) { bb.agents.registerTool({ name: "variaq_analyze_runs", description: "Analyze a bounded set of stored VariaQ experiment runs. Read-only; does not execute solvers or modify runs.", - instructions: "Use variaq_analyze_runs to compute VariaQ-derived quality, feasibility, timing, resource, and scaling summaries over run IDs. Analysis is read-only and returns VariaQ's AnalysisResult directly.", + instructions: "Use variaq_analyze_runs to compute VariaQ-derived quality, feasibility, timing, resource, and scaling summaries over run IDs. Analysis is read-only and returns VariaQ's AnalysisResult directly. Comparison selectors require variaq_analyze_campaign; VariaQ analyze runs has no --compare contract.", parameters: analyzeQuerySchema.omit({ campaign_id: true }), async execute(query) { try { @@ -1190,7 +1211,7 @@ export default async function plugin(bb: BbPluginApi) { bb.agents.registerTool({ name: "variaq_analyze_campaign", description: "Analyze all runs belonging to a VariaQ campaign. Read-only; does not execute solvers or modify runs.", - instructions: "Use variaq_analyze_campaign to compute VariaQ-derived quality, feasibility, timing, resource, and scaling summaries for a stored campaign. Analysis is read-only.", + instructions: "Use variaq_analyze_campaign to compute VariaQ-derived quality, feasibility, timing, resource, and scaling summaries for a stored campaign. Analysis is read-only. compare=paired_solvers requires a verified VariaQ 0.8.x stable release or the tested 0.8.0rc1 candidate and preserves its pairing, null reasons and timing scopes.", parameters: z.object({ campaignId: z.string().min(1).describe("Campaign id"), }).merge(analyzeQuerySchema.omit({ campaign_id: true, run_ids: true })), @@ -1207,14 +1228,14 @@ export default async function plugin(bb: BbPluginApi) { bb.agents.registerTool({ name: "variaq_report_campaign", description: "Generate JSON/CSV/Markdown (and optional plots) report files for a VariaQ campaign. This tool writes files under the configured report output directory.", - instructions: "Use variaq_report_campaign to produce persistent report artifacts from a stored campaign. Reports derive from stored runs, preserve source_run_ids, and are written under the configured reportOutputDir. The outputDir argument is resolved relative to that root and cannot escape it.", + instructions: "Use variaq_report_campaign to produce persistent report artifacts from a stored campaign. Reports derive from stored runs, preserve source_run_ids, and are written under the configured reportOutputDir. The outputDir argument is resolved relative to that root and cannot escape it. compare=paired_solvers requires a verified VariaQ 0.8.x stable release or the tested 0.8.0rc1 candidate and includes per-run, pair and repeat CSV artifacts.", parameters: z.object({ campaignId: z.string().min(1).describe("Campaign id"), outputDir: z.string().min(1).describe("Relative output directory beneath the configured report output root"), formats: reportFormatsSchema.describe("Report formats, e.g. ['json', 'csv', 'markdown']"), groupBy: z.array(z.string()).max(8).optional().describe("Group-by keys"), scalingX: z.string().optional().describe("Scaling x-axis metric, e.g. problem_size"), - compare: z.enum(["classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu"]).optional(), + compare: comparisonTypeSchema.optional(), plots: z.boolean().optional().describe("Request matplotlib plots if available"), overwrite: z.boolean().optional(), }), diff --git a/skills/variaq/SKILL.md b/skills/variaq/SKILL.md index 2775b76..5980c82 100644 --- a/skills/variaq/SKILL.md +++ b/skills/variaq/SKILL.md @@ -90,5 +90,8 @@ for machine-readable output. nonzero infeasible counts as a run failure. - **Graph Partition quantum support remains unadvertised** unless VariaQ capabilities change in a future release. -- bb-plugin-variaq 0.5.x is verified against VariaQ 0.6.x and schema version `1`. - Other VariaQ series or schema versions are reported as unsupported. \ No newline at end of file +- This checkout supports stable VariaQ 0.6.x and 0.8.x and the explicitly verified 0.8.0rc1 candidate, schema version `1`, on BB 0.46 / SDK 0.6.37. Other versions/candidates/schemas remain unsupported. +- On VariaQ 0.8.x (or the tested 0.8.0rc1), pass compare: "paired_solvers" to variaq_analyze_campaign or variaq_report_campaign. Forward VariaQ's pairing, precision, null reasons and timing scopes; do not compute scientific metrics or claim cold GPU state/speedup. +- VariaQ analyze runs has no --compare contract; use stored campaign analysis for comparisons. Paired analysis/reporting on 0.6.x fails clearly. +- Campaign solver_config uses nested seed/parameters overrides, including numeric candidate-parameter matrices. Preview resource limits before execution. +- A plugin subprocess timeout terminates the local command; it does not resume work or guarantee an in-flight native result was saved. Do not automatically retry. CLI campaign reruns explicitly append new work; web task deduplication is a separate VariaQ web behavior. \ No newline at end of file diff --git a/tests/fake.test.ts b/tests/fake.test.ts index 1d35371..8b21e01 100644 --- a/tests/fake.test.ts +++ b/tests/fake.test.ts @@ -82,6 +82,65 @@ describe("status via fake CLI", () => { expect(status.variaq.compatibility.warning).toBeNull(); }); + it("detects the supported 0.8.0rc1 candidate through capabilities", async () => { + vi.stubEnv("FAKE_VARIAQ_VERSION", "0.8.0rc1"); + const h = await setup(); + const status = JSON.parse(String(await tool(h, "variaq_status", {}))); + expect(status.variaq.version).toBe("0.8.0rc1"); + expect(status.variaq.schema_version).toBe("1"); + expect(status.variaq.compatibility.supported).toBe(true); + expect(status.variaq.compatibility.warning).toBeNull(); + }); + + it("detects stable 0.8.0 through capabilities without a warning", async () => { + vi.stubEnv("FAKE_VARIAQ_VERSION", "0.8.0"); + const h = await setup(); + const status = JSON.parse(String(await tool(h, "variaq_status", {}))); + expect(status.variaq.version).toBe("0.8.0"); + expect(status.variaq.compatibility.supported).toBe(true); + expect(status.variaq.compatibility.warning).toBeNull(); + }); + + it("runs paired analysis and reporting on a stable 0.8.0 stub", async () => { + vi.stubEnv("FAKE_VARIAQ_VERSION", "0.8.0"); + const h = await setup(); + const definition = { + campaign_format_version: "1", name: "stable-paired", family: "maxcut", + problem_sizes: [4], problem_seeds: [1], solvers: ["exact", "heuristic"], + repeats: 1, base_seed: 42, solver_config: {}, + }; + const run = JSON.parse(String(await tool(h, "variaq_campaign_run", { + ...definition, maxRuns: 10, overrideMaxRuns: false, + }))); + expect(run.exitCode).toBe(0); + const analysis = JSON.parse(String(await tool(h, "variaq_analyze_campaign", { + campaignId: run.campaignId, compare: "paired_solvers", + }))); + expect(analysis.analysis.warnings).toEqual([{ type: "fake_warning", message: "simulated analysis warning" }]); + expect(analysis.warnings).toEqual([]); + const report = JSON.parse(String(await tool(h, "variaq_report_campaign", { + campaignId: run.campaignId, outputDir: "stable", formats: ["json"], compare: "paired_solvers", + }))); + expect(report.reportId).toMatch(/^report-/); + }); + + it("rejects paired analysis/reporting on 0.6.x with a clear compatibility error", async () => { + const h = await setup(); + for (const [name, input] of [ + ["variaq_analyze_campaign", { campaignId: "campaign-old", compare: "paired_solvers" }], + ["variaq_report_campaign", { campaignId: "campaign-old", outputDir: "legacy", formats: ["json"], compare: "paired_solvers" }], + ] as const) { + const result = JSON.parse(String(await tool(h, name, input))); + expect(result.error).toContain("paired_solvers requires"); + expect(result.hint).toContain("Legacy comparisons remain available"); + } + const result = await cli(h, ["analyze-runs", "--run-id", "run-old", "--compare", "paired_solvers", "--json"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("analyze runs does not accept comparison selectors"); + const runs = JSON.parse(String(await tool(h, "variaq_analyze_runs", { run_ids: ["run-old"], compare: "classical_vs_quantum" }))); + expect(runs.error).toContain("analyze runs does not accept comparison selectors"); + }); + it("reports 0.5 series as unsupported", async () => { vi.stubEnv("FAKE_VARIAQ_VERSION", "0.5.2"); const h = await setup(); diff --git a/tests/runner.test.ts b/tests/runner.test.ts index ad242a3..95076c0 100644 --- a/tests/runner.test.ts +++ b/tests/runner.test.ts @@ -19,7 +19,7 @@ import { runVariaqJson, type VariaqSettings, } from "../lib/runner.js"; -import { SUPPORTED_SCHEMA_VERSION, validateEnvelope } from "../lib/schema.js"; +import { SUPPORTED_SCHEMA_VERSION, campaignDefinitionSchema, validateEnvelope } from "../lib/schema.js"; import { join } from "node:path"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -341,6 +341,47 @@ describe("VariaQ output contracts", () => { expect(checkVariaqVersion("0.6.0", "1").schemaVersionSupported).toBe(true); }); + it("accepts the explicitly verified candidate without stripping its suffix", () => { + const compatibility = checkVariaqVersion("variaq 0.8.0rc1", "1"); + expect(compatibility.version).toBe("0.8.0rc1"); + expect(compatibility.supported).toBe(true); + expect(compatibility.warning).toBeNull(); + expect(compatibility.schemaVersionSupported).toBe(true); + }); + + it.each(["0.8.0rc2", "0.7.0", "0.6.0rc1", "0.8.0rc1junk", "0.6.0.invalid", "noise 0.6.0"])( + "does not silently accept an unverified or malformed version: %s", (version) => { + expect(checkVariaqVersion(version, "1").supported).toBe(false); + }, + ); + + it("accepts stable 0.8 series releases without a warning", () => { + const compatibility = checkVariaqVersion("0.8.0", "1"); + expect(compatibility.supported).toBe(true); + expect(compatibility.warning).toBeNull(); + expect(checkVariaqVersion("variaq 0.8.3").supported).toBe(true); + expect(checkVariaqVersion("0.8.0", "1").schemaVersionSupported).toBe(true); + }); + + it("rejects unverified 0.8 release candidates such as rc2", () => { + expect(checkVariaqVersion("0.8.1rc1").supported).toBe(false); + }); + + it("accepts canonical nested campaign solver parameters without losing controls", () => { + const solver_config = { qaoa: { seed: 42, parameters: { p: 1, candidate_parameters: [[0.5, 0.25]], warmup: false } } }; + const definition = campaignDefinitionSchema.parse({ + campaign_format_version: "1", name: "controls", family: "maxcut", + problem_sizes: [4], problem_seeds: [1], solvers: ["qaoa"], solver_config, + }); + expect(definition.solver_config).toEqual(solver_config); + }); + + it("preserves additive paired data and null measurement reasons in schema-v1", () => { + const data = { instances: [{ runs: [{ precision: "fp64", timing: { warmup_seconds: null }, missing_measurements: { warmup_seconds: "not recorded" } }] }], comparisons: [{ comparison_type: "paired_solvers", pairs: [] }] }; + const parsed = validateEnvelope(JSON.stringify({ schema_version: "1", command: "analyze runs", status: "success", data })); + expect(parsed.data).toEqual(data); + }); + it("rejects the 0.5 series as unsupported", () => { const result = checkVariaqVersion("variaq 0.5.1", "1"); expect(result.supported).toBe(false); diff --git a/tests/server.test.ts b/tests/server.test.ts index cead7f0..cf2c512 100644 --- a/tests/server.test.ts +++ b/tests/server.test.ts @@ -22,6 +22,7 @@ const VARIAQ_PROJECT = process.env.VARIAQ_TEST_PROJECT ?? ""; const VARIAQ_PYTHON = process.env.VARIAQ_TEST_PYTHON ?? (VARIAQ_PROJECT === "" ? "" : join(VARIAQ_PROJECT, ".venv", "bin", "python")); const HAS_REAL_VARIAQ = VARIAQ_PROJECT !== "" && VARIAQ_PYTHON !== ""; +const VARIAQ_VERSION = process.env.VARIAQ_TEST_VERSION ?? "0.6.0"; const RUN_CUDAQ = process.env.VARIAQ_TEST_CUDAQ === "1"; const describeReal = describe.skipIf(!HAS_REAL_VARIAQ); @@ -63,7 +64,7 @@ describeReal("plugin registration", () => { const h = await setupHost(); const version = await cli(h, ["version"]); expect(version.exitCode).toBe(0); - expect(version.stdout).toContain("variaq 0.6.0"); + expect(version.stdout).toContain(`variaq ${VARIAQ_VERSION}`); }); }); @@ -73,7 +74,7 @@ describeReal("status/version", () => { const result = await tool(h, "variaq_status", {}); const s = JSON.parse(String(result)); expect(s.variaq.installed).toBe(true); - expect(s.variaq.version).toBe("0.6.0"); + expect(s.variaq.version).toBe(VARIAQ_VERSION); expect(s.variaq.schema_version).toBe("1"); expect(s.variaq.python).toBe(VARIAQ_PYTHON); expect(s.solvers.exact).toBe("available"); @@ -388,11 +389,12 @@ describeReal("campaign, analysis, and report integration", () => { scaling_x: "problem_size", include_failed: true, include_unavailable: true, + compare: "classical_vs_quantum", }))); expect(analysis.analysis.source_run_ids).toEqual(expect.arrayContaining(run.summary.run_ids)); expect(analysis.analysis.groups.length).toBeGreaterThan(0); expect(analysis.analysis.scaling_points.length).toBeGreaterThan(0); - expect(analysis.analysis.groups[0].environment_versions.variaq).toContain("0.6.0"); + expect(analysis.analysis.groups[0].environment_versions.variaq).toContain(VARIAQ_VERSION); const report = JSON.parse(String(await tool(h, "variaq_report_campaign", { campaignId: run.campaignId, @@ -400,6 +402,7 @@ describeReal("campaign, analysis, and report integration", () => { formats: ["json", "csv", "markdown"], groupBy: ["solver"], scalingX: "problem_size", + compare: "classical_vs_quantum", }))); expect(report.reportId).toMatch(/^report-/); expect(report.paths.csv.groups).toBeTruthy(); @@ -409,12 +412,73 @@ describeReal("campaign, analysis, and report integration", () => { expect(reportDocument.campaign_id).toBe(run.campaignId); expect(reportDocument.source_run_ids).toEqual(expect.arrayContaining(run.summary.run_ids)); expect(reportDocument.report_format_version).toBe("1"); - expect(reportDocument.variaq_version).toBe("0.6.0"); + expect(reportDocument.variaq_version).toBe(VARIAQ_VERSION); expect(reportDocument.generated_at).toBeTruthy(); expect(reportDocument.analysis.query.campaign_id).toBe(run.campaignId); }); }); +describe.skipIf(!HAS_REAL_VARIAQ || !["0.8.0", "0.8.0rc1"].includes(VARIAQ_VERSION))("paired-solver candidate integration", () => { + it("forwards controls, paired/null/timing evidence and expanded reports through existing tools and CLI", async () => { + const h = await setupHost(); + const parameters = { p: 1, optimizer_trials: 2, shots: 32, warmup: false, candidate_parameters: [[0.5, 0.25], [0.8, 0.4]] }; + const definition = { + campaign_format_version: "1", name: "candidate-paired", family: "maxcut", + problem_sizes: [4], problem_seeds: [11], solvers: ["exact", "heuristic", "qaoa"], + repeats: 2, base_seed: 42, generator_parameters: { edge_probability: 0.7 }, + solver_config: { qaoa: { parameters } }, + }; + const plan = JSON.parse(String(await tool(h, "variaq_campaign_plan", definition))); + expect(plan.plan.requested_runs).toBe(6); + const run = JSON.parse(String(await tool(h, "variaq_campaign_run", { ...definition, maxRuns: 6 }))); + expect(run.exitCode).toBe(0); + expect(run.summary.status_summary.success).toBe(6); + const analysis = JSON.parse(String(await tool(h, "variaq_analyze_campaign", { + campaignId: run.campaignId, compare: "paired_solvers", + }))); + const instance = analysis.analysis.instances[0]; + expect(instance.reference_optimum).toBe(4); + expect(instance.runs).toHaveLength(6); + const exact = instance.runs.find((r: { solver: string }) => r.solver === "exact"); + expect(exact.optimality_gap).toBe(0); + expect(exact.timing.first_backend_evaluation_seconds).toBeNull(); + expect(exact.missing_measurements.first_backend_evaluation_seconds).toBeTruthy(); + const qaoa = instance.runs.find((r: { solver: string }) => r.solver === "qaoa"); + expect(qaoa.parameters.candidate_parameters).toEqual(parameters.candidate_parameters); + expect(qaoa.precision).toBe("fp64"); + expect(qaoa.cold_start_verified).toBeNull(); + expect(qaoa.timing.first_backend_evaluation_seconds).toBeGreaterThanOrEqual(0); + expect(qaoa.timing_scopes.first_backend_evaluation_overlaps).toBeTruthy(); + const paired = analysis.analysis.comparisons.find((c: { comparison_type: string }) => c.comparison_type === "paired_solvers"); + expect(paired.pairs.length).toBeGreaterThanOrEqual(2); + const cliAnalysis = await cli(h, ["analyze-campaign", run.campaignId, "--compare", "paired_solvers", "--json"]); + expect(cliAnalysis.exitCode, cliAnalysis.stderr).toBe(0); + expect(JSON.parse(cliAnalysis.stdout).analysis.source_run_ids).toEqual(expect.arrayContaining(run.summary.run_ids)); + const runsAnalysis = JSON.parse(String(await tool(h, "variaq_analyze_runs", { run_ids: run.summary.run_ids }))); + expect(runsAnalysis.analysis.source_run_ids).toEqual(expect.arrayContaining(run.summary.run_ids)); + const report = JSON.parse(String(await tool(h, "variaq_report_campaign", { + campaignId: run.campaignId, outputDir: "paired", formats: ["json", "csv", "markdown"], compare: "paired_solvers", + }))); + expect(report.reportId).toMatch(/^report-/); + const document = JSON.parse(readFileSync(report.paths.json, "utf8")); + expect(document.analysis.instances).toEqual(analysis.analysis.instances); + expect(document.source_run_ids).toEqual(expect.arrayContaining(run.summary.run_ids)); + for (const name of ["runs", "pairs", "repeats"]) { + expect(readFileSync(report.paths.csv[name], "utf8").length).toBeGreaterThan(0); + } + const runCsv = readFileSync(report.paths.csv.runs, "utf8"); + expect(runCsv).toContain("timing.first_backend_evaluation_seconds"); + expect(runCsv).toContain("precision"); + for (const id of run.summary.run_ids) expect(runCsv).toContain(id); + expect(readFileSync(report.paths.markdown, "utf8")).toContain("Paired solver observations"); + const cliReport = await cli(h, ["report-campaign", run.campaignId, "--output-dir", "paired-cli", "--formats", "json,csv,markdown", "--compare", "paired_solvers", "--json"]); + expect(cliReport.exitCode, cliReport.stderr).toBe(0); + expect(JSON.parse(cliReport.stdout).paths.csv.pairs).toBeTruthy(); + const stored = JSON.parse(String(await tool(h, "variaq_runs_list", { limit: 20 }))); + expect(stored.runs).toHaveLength(6); // Read-only analysis/reporting did not execute solvers. + }); +}); + describeReal("real heuristic and Qiskit integration", () => { it("runs deterministic heuristic and local Qiskit QAOA solves on maxcut", async () => { const h = await setupHost(); @@ -524,7 +588,7 @@ describe.skipIf(!HAS_REAL_VARIAQ || !RUN_CUDAQ)("optional CUDA-Q CPU integration }); }); -describe.skipIf(!HAS_REAL_VARIAQ || !RUN_CUDAQ)("optional CUDA-Q GPU integration", () => { +describe.skipIf(!HAS_REAL_VARIAQ || process.env.VARIAQ_TEST_GPU !== "1")("optional CUDA-Q GPU integration", () => { it("runs a bounded nvidia solve when GPU is available", async () => { const h = await setupHost(); const generated = JSON.parse(String(