diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc3199e..f0dd654 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - run: npm pack --dry-run variaq-integration: - name: VariaQ 0.5.0 integration + name: VariaQ 0.6.0 integration runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -34,8 +34,8 @@ jobs: - uses: actions/checkout@v4 with: repository: aaronphifer/variaq - # Immutable release tag for the verified VariaQ 0.5.x integration. - ref: v0.5.0 + # Immutable release tag for the verified VariaQ 0.6.x integration. + ref: v0.6.0 path: .variaq-source - uses: actions/setup-node@v4 with: @@ -52,7 +52,7 @@ jobs: "$RUNNER_TEMP/variaq-venv/bin/python" -m pip install --upgrade pip "$RUNNER_TEMP/variaq-venv/bin/python" -m pip install -e './.variaq-source[quantum]' - run: npm ci - - name: Run real local integration suite + - name: Run real local integration suite with bounded campaign smoke env: VARIAQ_TEST_PROJECT: ${{ github.workspace }}/.variaq-source VARIAQ_TEST_PYTHON: ${{ runner.temp }}/variaq-venv/bin/python diff --git a/CHANGELOG.md b/CHANGELOG.md index f24bc5c..72337bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,19 @@ -## 0.4.0 (unreleased) +## 0.5.0 (unreleased) -- VariaQ 0.5.0 compatibility; `schema_version` `1` validation remains mandatory. -- Assignment quantum support via QAOA and CUDA-Q (when VariaQ's `cudaq` extra is installed). -- Subset Selection quantum support via QAOA and CUDA-Q. -- Capability-driven quantum family discovery: solver `supported_families` are - consumed directly from `variaq capabilities --json`; no plugin-side hardcoded - solver/family matrix. -- Generalized `variaq_compare_quantum`: works on any problem family supported by - the selected quantum solvers as reported by VariaQ capabilities. -- Preserve VariaQ 0.5 BQM metadata, feasibility diagnostics (`feasible_sample_count`, - `infeasible_sample_count`), penalty metadata, expectation, lowered energy, and - timing fields without recalculation or flattening. -- Graph Partition quantum attempts are rejected based on live VariaQ capabilities - (quantum support is not advertised in VariaQ 0.5.0). -- Updated human-readable CLI formatting for solve/benchmark to show family, backend, - status, objective, expectation, and feasible/total samples where present. -- Hosted CI updated to VariaQ `v0.5.0`; generic-family and Assignment/Subset quantum - coverage added. CUDA-Q remains optional in hosted CI. +- VariaQ 0.6.0 compatibility; `schema_version` `1` validation remains mandatory. +- Campaign planning (`variaq_campaign_plan`) and execution (`variaq_campaign_run`) + with VariaQ's max-run guard preserved. +- Campaign list/show (`variaq_campaign_list`, `variaq_campaign_show`). +- Analysis tools (`variaq_analyze_runs`, `variaq_analyze_campaign`) that return + VariaQ-derived groups, quality/feasibility/timing/resource summaries, and scaling + points without recalculating statistics in the plugin. +- Report generation (`variaq_report_campaign`) for JSON/CSV/Markdown (and optional + plots when matplotlib is installed), with safe output path containment under the + configured report output directory. +- Preserved source_run_ids, report IDs, null metrics, and mixed campaign status + summaries (success/failed/skipped/unavailable). +- Hosted CI updated to VariaQ `v0.6.0` with bounded campaign plan/run, + analyze-campaign, and report-generation smoke coverage. - No external domain semantics, adapter execution, Ollama Fleet, Triagewall, physical QPU, UI, or web API added. @@ -24,31 +21,31 @@ All notable changes to this project will be documented in this file. -## [0.4.0] - Unreleased +## 0.5.0 - Unreleased -- Target VariaQ 0.5.x and its stable machine-readable CLI contract. -- Recognize all four generic problem families: `maxcut`, `assignment`, - `subset-selection`, `graph-partition`. -- Consume solver `supported_families` from `variaq capabilities --json`; let - VariaQ advertise which families each quantum solver supports. -- Add quantum support for `assignment` and `subset-selection` through VariaQ 0.5. -- Generalize `compare quantum` so it works for any family supported by the selected - quantum solvers, not only MaxCut. -- Preserve VariaQ 0.5 BQM/feasibility/expectation metadata in solve, benchmark, - compare-quantum, and run records. -- Reject Graph Partition quantum runs from VariaQ capabilities rather than a - hardcoded family list. -- Update human-readable CLI formatting for multi-family results. -- Update CI to verify against VariaQ `v0.5.0` and exercise quantum workflows for - Assignment and Subset Selection. +- Target VariaQ 0.6.x and its stable machine-readable CLI contract. +- Add campaign planning/execution/list/show tools and matching `bb variaq` + subcommands. Campaign execution preserves VariaQ's default max-run guard. +- Add read-only analysis tools (`variaq_analyze_runs`, `variaq_analyze_campaign`) + that forward VariaQ-derived group summaries, quality, feasibility, timing, + resource, and scaling metrics without plugin-side recalculation. +- Add `variaq_report_campaign` with JSON/CSV/Markdown output and optional plots. + Reports are written beneath a configured/safe report output directory with + path-traversal prevention. +- Preserve provenance: `source_run_ids`, `report_id`, `campaign_id`, + `report_format_version`, `generated_at`, `variaq_version`. +- Preserve mixed campaign status summaries and null/missing metrics. +- Reject VariaQ 0.5.x and other non-0.6 series with a clear compatibility warning. +- Update CI to verify against VariaQ `v0.6.0` and exercise bounded campaign + plan/run, analysis, and report generation. Physical-QPU execution, external domain adapter execution, and downstream project integrations are not part of this release. -## [0.3.0] - Unreleased +## 0.4.0 - Unreleased -- VariaQ 0.4.x compatibility; `schema_version` `1` validation remains mandatory. -- Family-aware problem generation for all VariaQ 0.4 generic families: +- VariaQ 0.5.x compatibility; `schema_version` `1` validation remains mandatory. +- Family-aware problem generation for all VariaQ 0.5 generic families: `maxcut`, `assignment`, `subset-selection`, `graph-partition`. - Capability model consumes VariaQ's per-solver `supported_families` from `variaq capabilities --json`; no plugin-side hardcoded solver/family matrix. @@ -65,12 +62,12 @@ project integrations are not part of this release. - `runs list`, `run show`, and `run reproduce` are family-neutral. - Added `bb variaq status` human formatting for VariaQ capabilities. - Smoke script now exercises an `assignment` workflow in addition to MaxCut. -- Updated hosted CI to VariaQ `v0.4.x` and added generic-family unit and +- Updated hosted CI to VariaQ `v0.5.0` and added generic-family unit and integration coverage. - No external domain semantics, adapter execution, Ollama Fleet, Triagewall, physical QPU, UI, or web API added. -## [0.2.0] - Unreleased +## 0.2.0 - Unreleased - Target VariaQ 0.3.x and its stable machine-readable CLI contract. - Validate VariaQ `schema_version = "1"` envelopes; reject unknown future @@ -92,7 +89,7 @@ project integrations are not part of this release. Physical-QPU execution and remote quantum-provider integration are not part of this release. -## [0.1.0] - Unreleased +## 0.1.0 - Unreleased - Initial thin integration with the standalone VariaQ CLI. - Eight experiment and persisted-run tools, plus status and version tools (ten diff --git a/README.md b/README.md index 89f1059..f8f2b56 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,10 @@ VariaQ schema-v1 CLI ↓ VariaQ ├── classical - └── generic BQM-QAOA execution + ├── QAOA / CUDA-Q quantum simulators + ├── campaign planning, execution, and storage + ├── read-only analysis + └── report generation ``` Starting with bb-plugin-variaq 0.2.0, the integration boundary is VariaQ's @@ -25,21 +28,24 @@ or scrapes run IDs from prose. VariaQ remains fully usable without BB. The plugin is responsible only for BB tool/CLI registration, validated argument construction, bounded subprocess -execution, and structured result forwarding. +execution, safe temporary-file staging, report path containment, and structured +result forwarding. The plugin does not implement campaign planning, analysis, +statistics, report generation, solver logic, BQM logic, plotting, or provenance +logic — VariaQ remains authoritative for all of these. ## Prerequisites - BB 0.43.x with Plugin SDK 0.4.104 - Node.js 22.19 or a compatible version supported by BB - Linux-native Python 3.12 -- a separate VariaQ 0.5.x installation; 0.5.0 is the verified version +- a separate VariaQ 0.6.x installation; 0.6.0 is the verified version Create a VariaQ environment separately from this repository: ```bash git clone https://github.com/aaronphifer/variaq.git cd variaq -git checkout v0.5.0 # VariaQ 0.5.0 +git checkout v0.6.0 # VariaQ 0.6.0 python3.12 -m venv .venv .venv/bin/python -m pip install -e '.[quantum]' ``` @@ -50,7 +56,7 @@ The `quantum` extra supplies the local Qiskit workflow. CUDA-Q is optional: .venv/bin/python -m pip install -e '.[quantum,cudaq]' ``` -VariaQ 0.5.0 constrains CUDA-Q to `>=0.16,<0.17`; do not independently +VariaQ 0.6.0 constrains CUDA-Q to `>=0.16,<0.17`; do not independently upgrade it beyond VariaQ's supported range. ## Installation @@ -81,6 +87,7 @@ Source defaults contain no machine-specific paths. | `projectDir` | empty | standalone VariaQ checkout; otherwise auto-discovered | | `dbPath` | empty | use VariaQ's `data/variaq.sqlite3` default | | `problemsDir` | empty | use VariaQ's `data/problems` default | +| `reportOutputDir` | empty | safe default under the project workspace | | `timeoutMs` | `120000` | per-command timeout, allowed range 1,000–3,600,000 ms | Configured paths must be absolute. `pythonPath` must be an executable file and @@ -92,21 +99,33 @@ bb plugin config variaq set pythonPath /path/to/variaq/.venv/bin/python bb plugin config variaq set projectDir /path/to/variaq bb plugin config variaq set dbPath /path/to/variaq/data/variaq.sqlite3 bb plugin config variaq set problemsDir /path/to/variaq/data/problems +bb plugin config variaq set reportOutputDir /path/to/variaq/reports bb plugin config variaq set timeoutMs 120000 bb plugin reload variaq ``` ## Agent tools -The plugin registers ten BB tools: +The plugin registers seventeen BB tools: - `variaq_status` and `variaq_version` -- `variaq_problem_generate` and `variaq_problem_show` +- `variaq_problem_generate`, `variaq_problem_show`, and `variaq_problem_import` - `variaq_solve`, `variaq_benchmark`, and `variaq_compare_quantum` +- `variaq_campaign_plan`, `variaq_campaign_run`, `variaq_campaign_list`, and `variaq_campaign_show` +- `variaq_analyze_runs` and `variaq_analyze_campaign` +- `variaq_report_campaign` - `variaq_runs_list`, `variaq_run_show`, and `variaq_run_reproduce` -The eight experiment/store tools are accompanied by the two status/version -tools so agents can validate the local environment before running work. +Tools are divided into read-only and execution/write categories: + +**Read-only:** `variaq_status`, `variaq_version`, `variaq_problem_show`, +`variaq_runs_list`, `variaq_run_show`, `variaq_campaign_plan`, +`variaq_campaign_list`, `variaq_campaign_show`, `variaq_analyze_runs`, +`variaq_analyze_campaign`. + +**Execution / write:** `variaq_problem_generate`, `variaq_problem_import`, +`variaq_solve`, `variaq_benchmark`, `variaq_compare_quantum`, +`variaq_campaign_run`, `variaq_run_reproduce`, `variaq_report_campaign`. ## BB CLI and example workflow @@ -125,29 +144,53 @@ bb variaq solve "$problem_id" --solver qaoa --seed 42 --json bb variaq benchmark "$problem_id" \ --solvers exact,heuristic,qaoa --repeats 1 --seed 42 --json bb variaq compare-quantum "$problem_id" --p 1 --repeats 1 --json + +# Campaign workflow: plan first, then run, then analyze and report. +campaign_json=$(cat <<'EOF' +{ + "campaign_format_version": "1", + "name": "maxcut-scaling", + "family": "maxcut", + "problem_sizes": [4, 6, 8], + "problem_seeds": [1, 2], + "solvers": ["exact", "heuristic"], + "repeats": 1, + "base_seed": 42, + "generator_parameters": {"edge_probability": 0.4}, + "solver_config": {} +} +EOF +) +bb variaq campaign-plan "$campaign_json" --json +bb variaq campaign-run "$campaign_json" --max-runs 100 --json +bb variaq campaigns --limit 10 --json +bb variaq campaign --json +bb variaq analyze-campaign --group-by problem_id --group-by solver --scaling-x problem_size --json +bb variaq report-campaign --output-dir ./reports --formats json,csv,markdown --json + bb variaq runs --limit 10 --json bb variaq run run- --json bb variaq reproduce run- --json ``` -All `solve`, `benchmark`, `compare-quantum`, and `reproduce` commands now -return VariaQ's structured schema-v1 envelope directly. There is no prose -parsing or run-ID scraping. +All `solve`, `benchmark`, `compare-quantum`, `campaign-run`, and `reproduce` +commands return VariaQ's structured schema-v1 envelope directly. There is no +prose parsing or run-ID scraping. ## Version compatibility The verified combination is: ```text -bb-plugin-variaq 0.4.0 -VariaQ 0.5.0 +bb-plugin-variaq 0.5.0 +VariaQ 0.6.0 Plugin SDK 0.4.104 BB host 0.43.x VariaQ schema 1 ``` -bb-plugin-variaq 0.4.x is verified against VariaQ 0.5.x and schema version `1`. -Patch releases within the 0.4 plugin series and 0.5 VariaQ series are accepted. +bb-plugin-variaq 0.5.x is verified against VariaQ 0.6.x and schema version `1`. +Patch releases within the 0.5 plugin series and 0.6 VariaQ series are accepted. Other VariaQ series or future schema versions are reported as unsupported with a clear compatibility error. @@ -208,18 +251,39 @@ bb-plugin-variaq Project-specific adapters live outside this plugin. +## Campaign, analysis, and reporting + +Starting with bb-plugin-variaq 0.5.0 and VariaQ 0.6.0: + +- **Campaign plan** (`variaq_campaign_plan` / `bb variaq campaign-plan`) is + non-executing and safe. It previews requested run counts, solver breakdown, + unavailable combinations, and max-run warnings. +- **Campaign run** (`variaq_campaign_run` / `bb variaq campaign-run`) executes + solvers. It passes an explicit `maxRuns` value to VariaQ and only adds + `--override-max-runs` when the caller sets `overrideMaxRuns: true`. The plugin + does not bypass VariaQ's safety guard. +- **Analysis** is read-only and returns VariaQ's `AnalysisResult` directly. The + plugin does not compute statistics, gaps, or scaling points. +- **Reports** derive from stored runs and preserve `source_run_ids`, + `report_id`, `report_format_version`, `generated_at`, and `variaq_version`. + Report outputs are written beneath the configured `reportOutputDir` and cannot + escape that root (path traversal and absolute-outside-root paths are rejected). + ## Limitations -- VariaQ 0.5.0 provides local execution only; there is no physical-QPU path. +- VariaQ 0.6.0 provides local execution only; there is no physical-QPU path. - There is no IBM Runtime/provider integration and no credential handling. - Generic CI does not require CUDA-Q, an NVIDIA GPU, a physical QPU, or remote services. +- Campaign execution is local and single-threaded. - This plugin does not schedule work across Fleet and has no Triagewall integration. - Solver correctness belongs to VariaQ's own test suite; plugin tests cover the integration boundary. - A quantum run may return some infeasible samples; the plugin preserves VariaQ's feasibility diagnostics and does not treat them as a plugin error. +- Optional matplotlib plots are only generated when VariaQ's reporting layer + detects matplotlib; they are not required for core functionality. ## Deterministic smoke test @@ -259,9 +323,10 @@ npm run test:integration ``` The normal GitHub Actions plugin job uses the deterministic fake CLI. A -separate Linux integration job checks out the immutable VariaQ `v0.5.0` tag, -installs `.[quantum]`, and exercises local classical/Qiskit workflows. -CUDA-Q and GPU verification remain manual or suitable for a future self-hosted +separate Linux integration job checks out the immutable VariaQ `v0.6.0` tag, +installs `.[quantum]`, exercises local classical/Qiskit workflows, and runs a +bounded campaign plan/run, analysis, and report-generation smoke. CUDA-Q, GPU, +and matplotlib verification remain manual or suitable for a future self-hosted runner. See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution boundaries and diff --git a/lib/runner.ts b/lib/runner.ts index f961a41..0be67f2 100644 --- a/lib/runner.ts +++ b/lib/runner.ts @@ -24,6 +24,7 @@ export interface VariaqSettings { projectDir: string; dbPath: string; problemsDir: string; + reportOutputDir: string; timeoutMs: number; } @@ -102,6 +103,7 @@ export interface ResolvedConfig { projectDir: string; dbPath: string | null; // null → let VariaQ use its built-in default problemsDir: string | null; // null → let VariaQ use its built-in default + reportOutputDir: string | null; // null → plugin resolves a safe default timeoutMs: number; source: ResolvedSource; } @@ -213,6 +215,7 @@ export function resolveConfig(raw: VariaqSettings): Omit const projectDir = configuredPath(raw.projectDir, "projectDir"); const dbPath = configuredPath(raw.dbPath, "dbPath"); const problemsDir = configuredPath(raw.problemsDir, "problemsDir"); + const reportOutputDir = configuredPath(raw.reportOutputDir, "reportOutputDir"); let resolvedPython = ""; let resolvedProjectDir = ""; @@ -283,6 +286,7 @@ export function resolveConfig(raw: VariaqSettings): Omit projectDir: resolvedProjectDir, dbPath: dbPath === "" ? null : dbPath, problemsDir: problemsDir === "" ? null : problemsDir, + reportOutputDir: reportOutputDir === "" ? null : reportOutputDir, timeoutMs, source, }; @@ -536,7 +540,7 @@ export async function runPython( } /** - * VariaQ 0.5.x capabilities --json data shape. We only model the pieces the + * VariaQ 0.6.x capabilities --json data shape. We only model the pieces the * plugin reads; everything else is forwarded as unknown. Quantum solver * supported families are reported dynamically by VariaQ and must not be * hardcoded in the plugin. @@ -612,14 +616,14 @@ export interface VariaqVersionCompatibility { } /** - * bb-plugin-variaq 0.4.0 is verified against VariaQ 0.5.0 / schema_version 1. - * Patch releases in the 0.5 series are accepted. Other series are reported as + * bb-plugin-variaq 0.5.0 is verified against VariaQ 0.6.0 / schema_version 1. + * Patch releases in the 0.6 series are accepted. Other series are reported as * unsupported so users can still inspect a mismatched environment. */ export function checkVariaqVersion(raw: string | null, schemaVersion?: string): VariaqVersionCompatibility { const match = raw?.match(/(?:^|\s)(\d+)\.(\d+)\.(\d+)(?:\b|$)/); const version = match ? `${match[1]}.${match[2]}.${match[3]}` : null; - const supported = match?.[1] === "0" && match?.[2] === "5"; + const supported = match?.[1] === "0" && match?.[2] === "6"; const schemaVersionSupported = schemaVersion === undefined || schemaVersion === SUPPORTED_SCHEMA_VERSION; const parts: string[] = []; @@ -957,5 +961,87 @@ export function exitCodeFromEnvelope(envelope: ParsedEnvelope, processCode: numb return 0; } +export interface CampaignPlanEnvelopeData { + campaign_id: string; + default_max_runs: number; + estimated_quantum_runs: number; + exceeds_default_max: boolean; + family: string; + max_binary_variables: number; + name: string; + problem_instance_count: number; + repeats: number; + requested_runs: number; + solver_breakdown: Array<{ + available: boolean; + installed: boolean; + requested_runs: number; + solver: string; + supported: boolean; + }>; + unavailable: Array<{ solver: string; reason: string }>; + warnings: string[]; +} + +export interface CampaignRunEnvelopeData { + campaign_id: string; + completed_runs: number; + family: string; + name: string; + problem_ids: string[]; + requested_runs: number; + run_ids: string[]; + status_summary: { + success: number; + failed: number; + skipped: number; + unavailable: number; + }; +} + +export interface CampaignListEnvelopeData { + campaigns: Array<{ + campaign_id: string; + created_at: string; + family: string; + name: string; + }>; +} + +export interface CampaignShowEnvelopeData { + base_seed: number; + campaign_format_version: string; + created_at: string; + family: string; + generator_parameters: Record; + name: string; + notes: string; + problem_seeds: number[]; + problem_sizes: number[]; + repeats: number; + solver_config: Record>; + solvers: string[]; + tags: string[]; +} + +export interface AnalyzeEnvelopeData { + groups: unknown[]; + comparisons: unknown[]; + query: Record; + scaling_points: unknown[]; + source_run_ids: string[]; + warnings: unknown[]; +} + +export interface ReportCampaignEnvelopeData { + report_id: string; + paths: { + json?: string; + csv?: { groups?: string; scaling?: string }; + markdown?: string; + plots?: Record; + }; +} + /** Re-export schema constants for convenience. */ export { SUPPORTED_SCHEMA_VERSION, SUPPORTED_VARIAQ_SERIES, VERIFIED_VARIAQ_VERSION }; diff --git a/lib/schema.ts b/lib/schema.ts index 8afc1e2..5bbdae2 100644 --- a/lib/schema.ts +++ b/lib/schema.ts @@ -67,11 +67,102 @@ export type ProblemFamily = (typeof PROBLEM_FAMILIES)[number]; export const problemFamilySchema = z.enum(PROBLEM_FAMILIES); -/** Validate a problem family name against VariaQ 0.5 generic families. */ +/** Validate a problem family name against VariaQ 0.6 generic families. */ export function isProblemFamily(value: string): value is ProblemFamily { return PROBLEM_FAMILIES.includes(value as ProblemFamily); } +// --------------------------------------------------------------------------- +// Campaign / analysis / report schemas (VariaQ 0.6.0) +// The plugin validates only the envelope and a few shared fields; inner data +// is forwarded as-is so the plugin does not become coupled to VariaQ analytics. +// --------------------------------------------------------------------------- + +export const campaignSolverOverrideSchema = z.record( + z.string(), + z.record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.array(z.union([z.string(), z.number()]))])), +); + +export const campaignDefinitionSchema = z.object({ + campaign_format_version: z.literal("1"), + name: z.string().min(1).max(256), + family: problemFamilySchema, + problem_sizes: z.array(z.number().int().min(1)).min(1).max(64), + problem_seeds: z.array(z.number().int().min(0)).min(1).max(64), + solvers: z.array(z.enum(["exact", "heuristic", "qaoa", "cudaq-cpu", "cudaq-gpu"])).min(1), + repeats: z.number().int().min(1).max(100).default(1), + base_seed: z.number().int().min(0).default(0), + solver_config: campaignSolverOverrideSchema.default({}), + generator_parameters: z.record(z.string(), z.unknown()).default({}), + tags: z.array(z.string().max(64)).max(32).default([]), + notes: z.string().max(4096).default(""), +}); + +export type CampaignDefinition = z.infer; + +export const campaignPlanDataSchema = z.object({ + campaign_id: z.string(), + name: z.string(), + family: problemFamilySchema, + requested_runs: z.number().int(), + problem_instance_count: z.number().int(), + repeats: z.number().int(), + default_max_runs: z.number().int(), + exceeds_default_max: z.boolean(), + estimated_quantum_runs: z.number().int(), + max_binary_variables: z.number().int(), + solver_breakdown: z.array(z.object({ + solver: z.string(), + supported: z.boolean(), + installed: z.boolean(), + available: z.boolean(), + requested_runs: z.number().int(), + })), + unavailable: z.array(z.object({ + solver: z.string(), + reason: z.string(), + })), + warnings: z.array(z.string()), +}); + +export const campaignRunDataSchema = z.object({ + campaign_id: z.string(), + name: z.string(), + family: problemFamilySchema, + requested_runs: z.number().int(), + completed_runs: z.number().int(), + problem_ids: z.array(z.string()), + run_ids: z.array(z.string()), + status_summary: z.object({ + success: z.number().int(), + failed: z.number().int(), + skipped: z.number().int(), + unavailable: z.number().int(), + }), +}); + +export const campaignListItemSchema = z.object({ + campaign_id: z.string(), + name: z.string(), + family: problemFamilySchema, + created_at: z.string(), +}); + +export const analyzeQuerySchema = z.object({ + run_ids: z.array(z.string()).optional(), + campaign_id: z.string().optional(), + group_by: z.array(z.string()).min(1).max(8).optional(), + filters: z.record(z.string(), z.string()).optional(), + scaling_x: z.string().optional(), + include_failed: z.boolean().optional(), + include_unavailable: z.boolean().optional(), + compare: z.enum(["classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu"]).optional(), +}); + +export type AnalyzeQuery = z.infer; + +export const reportFormatsSchema = z.array(z.enum(["json", "csv", "markdown", "plots"])).min(1); + export interface ParsedEnvelope { command: string; status: EnvelopeStatus; @@ -85,9 +176,9 @@ export interface ParsedEnvelope { export const SUPPORTED_SCHEMA_VERSION = "1"; /** Compatibility metadata, mirrored here for the schema module. */ -export const BB_PLUGIN_VARIAQ_VERSION = "0.4.0"; -export const VERIFIED_VARIAQ_VERSION = "0.5.0"; -export const SUPPORTED_VARIAQ_SERIES = "0.5.x"; +export const BB_PLUGIN_VARIAQ_VERSION = "0.5.0"; +export const VERIFIED_VARIAQ_VERSION = "0.6.0"; +export const SUPPORTED_VARIAQ_SERIES = "0.6.x"; /** * Validate a VariaQ JSON envelope. diff --git a/package-lock.json b/package-lock.json index a464bfc..879595e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "bb-plugin-variaq", - "version": "0.4.0", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "bb-plugin-variaq", - "version": "0.4.0", + "version": "0.5.0", "license": "Apache-2.0", "dependencies": { "zod": "^4.3.6" diff --git a/package.json b/package.json index 9c30ca6..f2d6b64 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bb-plugin-variaq", - "version": "0.4.0", + "version": "0.5.0", "type": "module", "description": "Thin bb adapter for the standalone VariaQ hybrid-compute research platform.", "license": "Apache-2.0", @@ -59,7 +59,9 @@ "variaq", "quantum-computing", "qaoa", - "maxcut" + "maxcut", + "campaign", + "benchmark" ], "repository": { "type": "git", diff --git a/server.ts b/server.ts index 6889ff1..bd9bdfe 100644 --- a/server.ts +++ b/server.ts @@ -1,12 +1,12 @@ import { z } from "zod"; import { promises as fs } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { isAbsolute, join, relative, resolve, sep } from "node:path"; import crypto from "node:crypto"; -import { problemFamilySchema, type ProblemFamily } from "./lib/schema.js"; +import { problemFamilySchema, type ProblemFamily, campaignDefinitionSchema, campaignPlanDataSchema, campaignRunDataSchema, campaignListItemSchema, analyzeQuerySchema, reportFormatsSchema, type CampaignDefinition } from "./lib/schema.js"; import type { BbPluginApi } from "@get-bb/plugin-sdk"; import { PluginError, checkVariaqVersion, generateProblemArgv, parseJsonOutput, parseKeyValue, parseSolvers, probeCapabilities, quantumSupportNote, requireSuccess, resolveConfig, runVariaqJson, runVariaqStrict, solverSupportsFamily, statusFromCapabilities } from "./lib/runner.js"; -import type { ResolvedConfig, VariaqSettings, CompareQuantumEnvelopeData, BenchmarkEnvelopeData, ReproduceEnvelopeData, ProblemGenerateData, SolveEnvelopeData, FamilyGenerateInput, VariaqSolver } from "./lib/runner.js"; +import type { ResolvedConfig, VariaqSettings, CompareQuantumEnvelopeData, BenchmarkEnvelopeData, ReproduceEnvelopeData, ProblemGenerateData, SolveEnvelopeData, FamilyGenerateInput, VariaqSolver, CampaignPlanEnvelopeData, CampaignRunEnvelopeData, CampaignListEnvelopeData, CampaignShowEnvelopeData, AnalyzeEnvelopeData, ReportCampaignEnvelopeData } from "./lib/runner.js"; const SOLVERS = ["exact", "heuristic", "qaoa", "cudaq-cpu", "cudaq-gpu"] as const; const VARIAQ_SOLVERS = SOLVERS; @@ -22,11 +22,19 @@ const USAGE = `bb variaq — Run VariaQ classical/quantum experiments from bb bb variaq solve --solver <${SOLVERS.join("|")}> [--seed S] [--param KEY=VALUE ...] [--json] bb variaq benchmark [--solvers a,b] [--repeats N] [--seed S] [--json] bb variaq compare-quantum [--p N] [--repeats N] [--json] + bb variaq campaign-plan [--json] + bb variaq campaign-run [--max-runs N] [--override-max-runs] [--json] + bb variaq campaigns [--limit N] [--json] + bb variaq campaign [--json] + bb variaq analyze-runs [--run-id ID ...] [--group-by KEY ...] [--filter KEY=VALUE ...] [--scaling-x METRIC] [--include-failed] [--include-unavailable] [--compare COMPARISON] [--json] + bb variaq analyze-campaign [--group-by KEY ...] [--scaling-x METRIC] [--include-failed] [--include-unavailable] [--compare COMPARISON] [--json] + bb variaq report-campaign --output-dir DIR --formats f1,f2 [--group-by KEY ...] [--scaling-x METRIC] [--plots] [--overwrite] [--json] bb variaq runs [--limit N] [--json] bb variaq run [--json] bb variaq reproduce [--json] -Problem families: maxcut, assignment, subset-selection, graph-partition. +Campaign tools are read-only (plan/analyze) or execute solver runs (run). +Reports create files under the configured report output directory. This plugin shells out to the standalone VariaQ CLI. Solver/quantum logic lives in VariaQ — never here. Quantum solver availability and supported @@ -62,6 +70,12 @@ export default async function plugin(bb: BbPluginApi) { description: "Directory for saved problem JSON. Leave empty to use VariaQ's built-in default (/data/problems).", default: "", }, + reportOutputDir: { + type: "string", + label: "Report output directory", + description: "Directory for generated campaign reports. Leave empty to use a safe default under the project workspace.", + default: "", + }, timeoutMs: { type: "number", label: "Command timeout (ms)", @@ -78,6 +92,7 @@ export default async function plugin(bb: BbPluginApi) { projectDir: values.projectDir, dbPath: values.dbPath, problemsDir: values.problemsDir, + reportOutputDir: values.reportOutputDir, timeoutMs: values.timeoutMs, }; return resolveConfig(raw); @@ -359,6 +374,225 @@ export default async function plugin(bb: BbPluginApi) { }; } + // Helpers for safe temporary file handling and report path containment. + + async function writeCampaignTemp(c: ResolvedConfig, definition: CampaignDefinition): Promise { + const workDir = c.problemsDir ?? tmpdir(); + await fs.mkdir(workDir, { recursive: true }); + const tempFile = join(workDir, `campaign-${crypto.randomUUID()}.json`); + await fs.writeFile(tempFile, JSON.stringify(definition, null, 2), "utf8"); + return tempFile; + } + + function resolveReportOutputDir(c: ResolvedConfig): string { + if (c.reportOutputDir !== null) return c.reportOutputDir; + // Keep the default stable so reports remain discoverable and VariaQ's + // overwrite protection applies across separate plugin calls. + return join(c.projectDir, "reports"); + } + + /** + * Ensure `outputDir` is contained within `root`. Rejects traversal attempts, + * absolute paths outside root, and symlinks that escape root. + */ + async function resolveContainedReportDir(root: string, outputDir: string): Promise { + await fs.mkdir(root, { recursive: true }); + const absRoot = await fs.realpath(resolve(root)); + const candidate = resolve(absRoot, outputDir); + const relativeCandidate = relative(absRoot, candidate); + if ( + relativeCandidate === ".." || + relativeCandidate.startsWith(`..${sep}`) || + isAbsolute(relativeCandidate) + ) { + throw new PluginError( + `Report output directory '${outputDir}' is outside the allowed report root.`, + "Use a relative path beneath the configured report output directory, or ask the user to change the reportOutputDir setting.", + ); + } + + // Walk one component at a time and reject symlinks. Resolving only the + // complete candidate is unsafe when its final component does not yet + // exist but an existing parent symlink escapes the root. + let current = absRoot; + for (const part of relativeCandidate.split(sep).filter(Boolean)) { + current = join(current, part); + try { + const stat = await fs.lstat(current); + if (stat.isSymbolicLink()) { + throw new PluginError( + `Report output directory '${outputDir}' contains a symbolic link.`, + "Use a real directory beneath the configured report output directory.", + ); + } + if (!stat.isDirectory()) { + throw new PluginError(`Report output directory '${outputDir}' is not a directory.`); + } + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; + await fs.mkdir(current); + } + } + return current; + } + + // ------------------------------------------------------------------ campaigns -- + + async function opCampaignPlan(definition: CampaignDefinition) { + const c = await cfg(); + const validated = campaignDefinitionSchema.parse(definition); + const tempFile = await writeCampaignTemp(c, validated); + try { + const result = await runVariaqJson(c, ["campaign", "plan", tempFile], { timeoutMs: 30_000 }); + const envelope = requireSuccess(result, ["campaign", "plan"]); + const { data, warnings } = successData(envelope, "campaign plan"); + return { plan: campaignPlanDataSchema.parse(data), warnings }; + } finally { + try { + await fs.unlink(tempFile); + } catch { + /* ignore */ + } + } + } + + async function opCampaignRun(definition: CampaignDefinition, maxRuns: number, overrideMaxRuns: boolean) { + const c = await cfg(); + const validated = campaignDefinitionSchema.parse(definition); + const tempFile = await writeCampaignTemp(c, validated); + try { + const argv = ["campaign", "run", tempFile, "--max-runs", String(maxRuns)]; + if (overrideMaxRuns) argv.push("--override-max-runs"); + const result = await runVariaqJson(c, argv); + if (result.code !== 0) { + return { + exitCode: result.code, + timedOut: result.timedOut, + campaignId: undefined, + summary: undefined, + error: result.envelope.error, + warnings: result.envelope.warnings, + stderr: null, + }; + } + const envelope = result.envelope; + const { data, warnings } = successData(envelope, "campaign run"); + return { + exitCode: result.code, + timedOut: result.timedOut, + campaignId: data.campaign_id, + summary: campaignRunDataSchema.parse(data), + error: undefined, + warnings, + stderr: null, + }; + } finally { + try { + await fs.unlink(tempFile); + } catch { + /* ignore */ + } + } + } + + async function opCampaignList(limit: number) { + const c = await cfg(); + const result = await runVariaqJson(c, ["campaign", "list", "--limit", String(limit)], { timeoutMs: 15_000 }); + const envelope = requireSuccess(result, ["campaign", "list"]); + const { data, warnings } = successData(envelope, "campaign list"); + const campaigns = (data as Record[]).map((item) => campaignListItemSchema.parse(item)); + return { campaigns, warnings }; + } + + async function opCampaignShow(campaignId: string) { + const c = await cfg(); + const result = await runVariaqJson(c, ["campaign", "show", campaignId], { timeoutMs: 15_000 }); + const envelope = requireSuccess(result, ["campaign", "show", campaignId]); + const { data, warnings } = successData(envelope, "campaign show"); + return { campaignId, campaign: data, warnings }; + } + + // ------------------------------------------------------------------ analysis -- + + function buildAnalyzeArgv(kind: "runs" | "campaign", query: import("./lib/schema.js").AnalyzeQuery): string[] { + const argv: string[] = ["analyze", kind]; + if (kind === "runs" && query.run_ids !== undefined && query.run_ids.length > 0) { + for (const id of query.run_ids) argv.push("--run-id", id); + } + if (kind === "campaign" && query.campaign_id !== undefined) { + argv.push(query.campaign_id); + } + if (query.group_by !== undefined) { + for (const key of query.group_by) argv.push("--group-by", key); + } + if (query.filters !== undefined) { + for (const [k, v] of Object.entries(query.filters)) argv.push("--filter", `${k}=${v}`); + } + if (query.scaling_x !== undefined) argv.push("--scaling-x", query.scaling_x); + if (query.include_failed) argv.push("--include-failed"); + if (query.include_unavailable) argv.push("--include-unavailable"); + if (query.compare !== undefined) argv.push("--compare", query.compare); + return argv; + } + + async function opAnalyzeRuns(query: import("./lib/schema.js").AnalyzeQuery) { + const c = await cfg(); + const validated = analyzeQuerySchema.parse(query); + const argv = buildAnalyzeArgv("runs", validated); + const result = await runVariaqJson(c, argv, { timeoutMs: 60_000 }); + const envelope = requireSuccess(result, ["analyze", "runs"]); + const { data, warnings } = successData(envelope, "analyze runs"); + return { analysis: data, warnings }; + } + + async function opAnalyzeCampaign(campaignId: string, query: Omit) { + const c = await cfg(); + const fullQuery = analyzeQuerySchema.parse({ ...query, campaign_id: campaignId }); + const argv = buildAnalyzeArgv("campaign", fullQuery); + const result = await runVariaqJson(c, argv, { timeoutMs: 60_000 }); + const envelope = requireSuccess(result, ["analyze", "campaign"]); + const { data, warnings } = successData(envelope, "analyze campaign"); + return { analysis: data, warnings }; + } + + // ------------------------------------------------------------------ reports -- + + async function opReportCampaign( + campaignId: string, + options: { + outputDir: string; + formats: ["json" | "csv" | "markdown" | "plots", ...("json" | "csv" | "markdown" | "plots")[]]; + groupBy?: string[]; + scalingX?: string; + compare?: "classical_vs_quantum" | "qiskit_vs_cudaq" | "gpu_vs_cpu"; + plots?: boolean; + overwrite?: boolean; + }, + ) { + const c = await cfg(); + const root = resolveReportOutputDir(c); + const containedDir = await resolveContainedReportDir(root, options.outputDir); + + const argv = ["report", "campaign", campaignId, "--output-dir", containedDir, "--formats", options.formats.join(",")]; + if (options.groupBy !== undefined) { + for (const key of options.groupBy) argv.push("--group-by", key); + } + if (options.scalingX !== undefined) argv.push("--scaling-x", options.scalingX); + if (options.compare !== undefined) argv.push("--compare", options.compare); + if (options.plots) argv.push("--plots"); + if (options.overwrite) argv.push("--overwrite"); + + const result = await runVariaqJson(c, argv, { timeoutMs: 60_000 }); + const envelope = requireSuccess(result, ["report", "campaign"]); + const { data, warnings } = successData(envelope, "report campaign"); + return { + reportId: data.report_id, + paths: data.paths, + outputDir: containedDir, + warnings, + }; + } + const errorResult = (err: unknown): string => { if (err instanceof PluginError) { return JSON.stringify( @@ -404,6 +638,33 @@ export default async function plugin(bb: BbPluginApi) { summary: "Run matched Qiskit/CUDA-Q QAOA comparison on a family supported by the selected quantum solvers", usage: "bb variaq compare-quantum [--p N] [--repeats N] [--json]", }, + { + name: "campaign-plan", + summary: "Preview a campaign without executing any solver runs", + usage: "bb variaq campaign-plan [--json]", + }, + { + name: "campaign-run", + summary: "Execute a campaign: generates problems and runs all configured solvers", + usage: "bb variaq campaign-run [--max-runs N] [--override-max-runs] [--json]", + }, + { name: "campaigns", summary: "List stored campaigns", usage: "bb variaq campaigns [--limit N] [--json]" }, + { name: "campaign", summary: "Show a stored campaign definition", usage: "bb variaq campaign [--json]" }, + { + name: "analyze-runs", + summary: "Analyze a bounded set of stored experiment runs", + usage: "bb variaq analyze-runs [--run-id ID ...] [--group-by KEY ...] [--filter KEY=VALUE ...] [--scaling-x METRIC] [--include-failed] [--include-unavailable] [--compare COMPARISON] [--json]", + }, + { + name: "analyze-campaign", + summary: "Analyze all runs belonging to a campaign", + usage: "bb variaq analyze-campaign [--group-by KEY ...] [--scaling-x METRIC] [--include-failed] [--include-unavailable] [--compare COMPARISON] [--json]", + }, + { + name: "report-campaign", + summary: "Generate JSON/CSV/Markdown (optionally plots) report files for a campaign", + usage: "bb variaq report-campaign --output-dir DIR --formats f1,f2 [--group-by KEY ...] [--scaling-x METRIC] [--plots] [--overwrite] [--json]", + }, { name: "runs", summary: "List recent experiment runs", usage: "bb variaq runs [--limit N] [--json]" }, { name: "run", summary: "Show one experiment run as JSON", usage: "bb variaq run [--json]" }, { name: "reproduce", summary: "Reproduce a durable experiment run", usage: "bb variaq reproduce [--json]" }, @@ -515,6 +776,68 @@ export default async function plugin(bb: BbPluginApi) { repeats: optionalInt(opts, "repeats", 1), }); } + case "campaign-plan": { + const content = requiredPositional(args, "campaign-json-content"); + const definition = parseCampaignJson(content); + return opCampaignPlan(definition); + } + case "campaign-run": { + const content = requiredPositional(args, "campaign-json-content"); + const definition = parseCampaignJson(content); + const opts = readOptions(args); + const maxRuns = optionalInt(opts, "max-runs", 500); + const overrideMaxRuns = opts.has("override-max-runs"); + return opCampaignRun(definition, maxRuns, overrideMaxRuns); + } + case "campaigns": { + const opts = readOptions(args); + return opCampaignList(optionalInt(opts, "limit", 20)); + } + case "campaign": + return opCampaignShow(requiredPositional(args, "campaign-id")); + case "analyze-runs": { + const runIds = collectRepeated(args, "--run-id"); + const groupBy = collectRepeated(args, "--group-by"); + const filters = collectRepeated(args, "--filter").map((kv) => parseKeyValue(kv, "--filter")); + const opts = readOptions(args); + return opAnalyzeRuns({ + run_ids: runIds.length > 0 ? runIds : undefined, + group_by: groupBy.length > 0 ? groupBy : undefined, + filters: filters.length > 0 ? Object.fromEntries(filters) : undefined, + scaling_x: opts.get("scaling-x"), + include_failed: args.includes("--include-failed"), + include_unavailable: args.includes("--include-unavailable"), + compare: opts.get("compare") as import("./lib/schema.js").AnalyzeQuery["compare"], + }); + } + case "analyze-campaign": { + const campaignId = requiredPositional(args, "campaign-id"); + const groupBy = collectRepeated(args, "--group-by"); + const opts = readOptions(args); + return opAnalyzeCampaign(campaignId, { + group_by: groupBy.length > 0 ? groupBy : undefined, + scaling_x: opts.get("scaling-x"), + include_failed: args.includes("--include-failed"), + include_unavailable: args.includes("--include-unavailable"), + compare: opts.get("compare") as import("./lib/schema.js").AnalyzeQuery["compare"], + }); + } + case "report-campaign": { + const campaignId = requiredPositional(args, "campaign-id"); + const opts = readOptions(args); + const outputDir = requiredOption(opts, "output-dir"); + const formats = requiredOption(opts, "formats").split(",").map((s) => s.trim()).filter(Boolean); + const groupBy = collectRepeated(args, "--group-by"); + return opReportCampaign(campaignId, { + outputDir, + formats: formats as ["json" | "csv" | "markdown" | "plots", ...("json" | "csv" | "markdown" | "plots")[]], + groupBy: groupBy.length > 0 ? groupBy : undefined, + scalingX: opts.get("scaling-x"), + compare: opts.get("compare") as "classical_vs_quantum" | "qiskit_vs_cudaq" | "gpu_vs_cpu" | undefined, + plots: args.includes("--plots"), + overwrite: args.includes("--overwrite"), + }); + } case "runs": { const opts = readOptions(args); return opRuns(optionalInt(opts, "limit", 20)); @@ -763,6 +1086,138 @@ export default async function plugin(bb: BbPluginApi) { }, }); + bb.agents.registerTool({ + name: "variaq_campaign_plan", + description: "Plan a VariaQ campaign without executing any solver runs. Returns requested run counts, solver breakdown, and max-run warnings.", + instructions: "Use variaq_campaign_plan to preview a campaign before running it. Campaign plan is non-executing and safe. If the plan warns about exceeding the default run maximum, decide whether to reduce the campaign or explicitly pass maxRuns + overrideMaxRuns to variaq_campaign_run.", + parameters: campaignDefinitionSchema, + async execute(definition) { + try { + const result = await opCampaignPlan(definition); + return JSON.stringify(result, null, 2); + } catch (err) { + return errorResult(err); + } + }, + }); + + bb.agents.registerTool({ + name: "variaq_campaign_run", + description: "Execute a VariaQ campaign: generate problem instances and run the configured solvers. This can run many solver invocations.", + instructions: "Use variaq_campaign_run after planning. Always prefer to call variaq_campaign_plan first. The tool respects VariaQ's max-run guard; campaigns that exceed the default maximum require an explicit maxRuns value and overrideMaxRuns=true. The plugin does not orchestrate solvers or bypass VariaQ's safety guard.", + parameters: campaignDefinitionSchema.extend({ + maxRuns: z.number().int().min(1).max(10_000).optional().describe("Maximum runs allowed (default 500, matching VariaQ's safe default)"), + overrideMaxRuns: z.boolean().optional().describe("Set true to allow running a campaign whose requested_runs exceed maxRuns"), + }), + async execute(definition) { + try { + const { maxRuns, overrideMaxRuns, ...campaign } = definition as CampaignDefinition & { maxRuns?: number; overrideMaxRuns?: boolean }; + const result = await opCampaignRun(campaign, maxRuns ?? 500, overrideMaxRuns ?? false); + return JSON.stringify(result, null, 2); + } catch (err) { + return errorResult(err); + } + }, + }); + + bb.agents.registerTool({ + name: "variaq_campaign_list", + description: "List stored VariaQ campaigns (campaign_id, name, family, created_at).", + instructions: "Use variaq_campaign_list to discover campaigns before analyzing or reporting on one.", + parameters: z.object({ + limit: z.number().int().min(1).max(500).optional().describe("Max campaigns to return (default 20)"), + }), + async execute({ limit }) { + try { + const result = await opCampaignList(limit ?? 20); + return JSON.stringify(result, null, 2); + } catch (err) { + return errorResult(err); + } + }, + }); + + bb.agents.registerTool({ + name: "variaq_campaign_show", + description: "Show a stored VariaQ campaign definition by id.", + instructions: "Use variaq_campaign_show to inspect the full definition of a stored campaign before analyzing or reproducing it.", + parameters: z.object({ + campaignId: z.string().min(1).describe("Campaign id"), + }), + async execute({ campaignId }) { + try { + const result = await opCampaignShow(campaignId); + return JSON.stringify(result, null, 2); + } catch (err) { + return errorResult(err); + } + }, + }); + + bb.agents.registerTool({ + name: "variaq_analyze_runs", + description: "Analyze a bounded set of stored VariaQ experiment runs. Read-only; does not execute solvers or modify runs.", + instructions: "Use variaq_analyze_runs to compute VariaQ-derived quality, feasibility, timing, resource, and scaling summaries over run IDs. Analysis is read-only and returns VariaQ's AnalysisResult directly.", + parameters: analyzeQuerySchema.omit({ campaign_id: true }), + async execute(query) { + try { + const result = await opAnalyzeRuns(query); + return JSON.stringify(result, null, 2); + } catch (err) { + return errorResult(err); + } + }, + }); + + bb.agents.registerTool({ + name: "variaq_analyze_campaign", + description: "Analyze all runs belonging to a VariaQ campaign. Read-only; does not execute solvers or modify runs.", + instructions: "Use variaq_analyze_campaign to compute VariaQ-derived quality, feasibility, timing, resource, and scaling summaries for a stored campaign. Analysis is read-only.", + parameters: z.object({ + campaignId: z.string().min(1).describe("Campaign id"), + }).merge(analyzeQuerySchema.omit({ campaign_id: true, run_ids: true })), + async execute({ campaignId, ...query }) { + try { + const result = await opAnalyzeCampaign(campaignId, query); + return JSON.stringify(result, null, 2); + } catch (err) { + return errorResult(err); + } + }, + }); + + bb.agents.registerTool({ + name: "variaq_report_campaign", + description: "Generate JSON/CSV/Markdown (and optional plots) report files for a VariaQ campaign. This tool writes files under the configured report output directory.", + instructions: "Use variaq_report_campaign to produce persistent report artifacts from a stored campaign. Reports derive from stored runs, preserve source_run_ids, and are written under the configured reportOutputDir. The outputDir argument is resolved relative to that root and cannot escape it.", + parameters: z.object({ + campaignId: z.string().min(1).describe("Campaign id"), + outputDir: z.string().min(1).describe("Relative output directory beneath the configured report output root"), + formats: reportFormatsSchema.describe("Report formats, e.g. ['json', 'csv', 'markdown']"), + groupBy: z.array(z.string()).max(8).optional().describe("Group-by keys"), + scalingX: z.string().optional().describe("Scaling x-axis metric, e.g. problem_size"), + compare: z.enum(["classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu"]).optional(), + plots: z.boolean().optional().describe("Request matplotlib plots if available"), + overwrite: z.boolean().optional(), + }), + async execute({ campaignId, outputDir, formats, groupBy, scalingX, compare, plots, overwrite }) { + try { + const result = await opReportCampaign(campaignId, { + outputDir, + formats: formats as ["json" | "csv" | "markdown" | "plots", ...("json" | "csv" | "markdown" | "plots")[]], + groupBy, + scalingX, + compare, + plots, + overwrite, + }); + return JSON.stringify(result, null, 2); + } catch (err) { + return errorResult(err); + } + }, + }); + bb.agents.registerTool({ name: "variaq_runs_list", description: "List recent VariaQ experiment runs (run_id, problem, solver, status, objective, timing).", @@ -826,10 +1281,13 @@ function readOptions(args: string[]): Map { if (a !== undefined && a.startsWith("--") && a !== "--param") { const next = args[i + 1]; if (next === undefined || next.startsWith("--")) { - throw new PluginError(`Missing value for ${a}`); + // Boolean flag: store it with the empty string sentinel so it is still + // discoverable via has(). + map.set(a.slice(2), ""); + } else { + map.set(a.slice(2), next); + i++; } - map.set(a.slice(2), next); - i++; } } return map; @@ -887,6 +1345,26 @@ function parseInteger(raw: string, name: string): number { return Number.parseInt(raw, 10); } +function parseCampaignJson(content: string): CampaignDefinition { + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch (err) { + throw new PluginError( + "Campaign content is not valid JSON.", + err instanceof Error ? err.message : String(err), + ); + } + const result = campaignDefinitionSchema.safeParse(parsed); + if (!result.success) { + throw new PluginError( + `Invalid campaign definition: ${result.error.message}`, + "Campaign must declare campaign_format_version '1', a valid family, problem_sizes, problem_seeds, and solvers.", + ); + } + return result.data; +} + function successData( envelope: import("./lib/schema.js").ParsedEnvelope, command: string, @@ -915,13 +1393,118 @@ function unwrapEnvelope( }; } -/** Human-readable default for the CLI; full JSON under --json. */ /** Human-readable default for the CLI; full JSON under --json. */ function formatCliOutput(value: unknown, json: boolean): string { if (json) return JSON.stringify(value, null, 2); if (typeof value !== "object" || value === null) return String(value); const record = value as Record; + const warningLines = (...sources: unknown[]): string[] => { + const messages = sources + .flatMap((source) => Array.isArray(source) ? source : []) + .map((warning) => { + if (typeof warning === "string") return warning; + if (typeof warning === "object" && warning !== null && "message" in warning) { + return String((warning as { message: unknown }).message); + } + return JSON.stringify(warning); + }); + return [...new Set(messages)].map((message) => ` warning: ${message}`); + }; + + if ("plan" in record && typeof record.plan === "object" && record.plan !== null) { + const plan = record.plan as Record; + const solvers = (plan.solver_breakdown as Array> | undefined) ?? []; + return [ + `Campaign plan: ${String(plan.name ?? "unnamed")}`, + ` family: ${String(plan.family ?? "unknown")}`, + ` requested runs: ${String(plan.requested_runs ?? "?")}`, + ` problem instances: ${String(plan.problem_instance_count ?? "?")}`, + ` exceeds default maximum: ${plan.exceeds_default_max === true ? "yes" : "no"}`, + ` solvers: ${solvers.map((solver) => `${String(solver.solver)}=${String(solver.requested_runs)}`).join(", ") || "none"}`, + ...warningLines(plan.warnings, record.warnings), + ].join("\n"); + } + + if ("campaignId" in record && "summary" in record && typeof record.summary === "object" && record.summary !== null) { + const summary = record.summary as Record; + const statuses = (summary.status_summary as Record | undefined) ?? {}; + return [ + `Campaign run: ${String(record.campaignId)}`, + ` name: ${String(summary.name ?? "unnamed")}`, + ` family: ${String(summary.family ?? "unknown")}`, + ` runs: ${String(summary.completed_runs ?? "?")}/${String(summary.requested_runs ?? "?")} completed`, + ` success: ${String(statuses.success ?? 0)}, failed: ${String(statuses.failed ?? 0)}, skipped: ${String(statuses.skipped ?? 0)}, unavailable: ${String(statuses.unavailable ?? 0)}`, + ...warningLines(record.warnings), + ].join("\n"); + } + + if ("campaigns" in record && Array.isArray(record.campaigns)) { + const campaigns = record.campaigns as Array>; + return [ + `Campaigns: ${campaigns.length}`, + ...campaigns.map((campaign) => + ` ${String(campaign.campaign_id)} ${String(campaign.name)} ${String(campaign.family)} ${String(campaign.created_at)}`), + ...warningLines(record.warnings), + ].join("\n"); + } + + if ("campaign" in record && typeof record.campaign === "object" && record.campaign !== null) { + const campaign = record.campaign as Record; + return [ + `Campaign: ${String(record.campaignId ?? "unknown")}`, + ` name: ${String(campaign.name ?? "unnamed")}`, + ` family: ${String(campaign.family ?? "unknown")}`, + ` sizes: ${((campaign.problem_sizes as unknown[] | undefined) ?? []).join(", ")}`, + ` seeds: ${((campaign.problem_seeds as unknown[] | undefined) ?? []).join(", ")}`, + ` solvers: ${((campaign.solvers as unknown[] | undefined) ?? []).join(", ")}`, + ` repeats: ${String(campaign.repeats ?? "?")}`, + ` created: ${String(campaign.created_at ?? "unknown")}`, + ...warningLines(record.warnings), + ].join("\n"); + } + + if ("analysis" in record && typeof record.analysis === "object" && record.analysis !== null) { + const analysis = record.analysis as Record; + const groups = (analysis.groups as Array> | undefined) ?? []; + const sourceRunIds = (analysis.source_run_ids as unknown[] | undefined) ?? []; + const scalingPoints = (analysis.scaling_points as unknown[] | undefined) ?? []; + const lines = [ + "Analysis", + ` source runs: ${sourceRunIds.length}`, + ` groups: ${groups.length}`, + ` scaling points: ${scalingPoints.length}`, + ]; + for (const group of groups.slice(0, 8)) { + const key = JSON.stringify(group.group_key ?? {}); + const quality = (group.quality as Record | undefined) ?? {}; + const feasibility = (group.feasibility as Record | undefined) ?? {}; + lines.push( + ` ${key}: count=${String(group.count ?? "?")}, mean_objective=${String(quality.mean_objective ?? "null")}, feasible_runs=${String(feasibility.feasible_runs ?? "?")}`, + ); + } + if (groups.length > 8) lines.push(` … ${groups.length - 8} more groups`); + lines.push(...warningLines(analysis.warnings, record.warnings)); + return lines.join("\n"); + } + + if ("reportId" in record && "paths" in record) { + const paths = record.paths as Record; + const files: string[] = []; + for (const value of Object.values(paths)) { + if (typeof value === "string") files.push(value); + if (typeof value === "object" && value !== null) { + files.push(...Object.values(value as Record).filter((entry): entry is string => typeof entry === "string")); + } + } + return [ + `Report: ${String(record.reportId)}`, + ` generated files: ${files.length}`, + ...files.map((file) => ` ${file}`), + ...warningLines(record.warnings), + ].join("\n"); + } + if ("variaq" in record && "solvers" in record) { const status = value as { variaq: { @@ -1050,4 +1633,4 @@ function formatCliOutput(value: unknown, json: boolean): string { if ("runs" in record) return JSON.stringify(record.runs, null, 2); return JSON.stringify(value, null, 2); -} \ No newline at end of file +} diff --git a/skills/variaq/SKILL.md b/skills/variaq/SKILL.md index f3f8e14..2775b76 100644 --- a/skills/variaq/SKILL.md +++ b/skills/variaq/SKILL.md @@ -1,14 +1,15 @@ --- name: variaq -description: "Use when running VariaQ hybrid-compute experiments (classical/Qiskit/CUDA-Q) from bb — generating problems, solving, benchmarking, or inspecting/reproducing experiment runs." +description: "Use when running VariaQ hybrid-compute experiments (classical/Qiskit/CUDA-Q) from bb — generating problems, solving, benchmarking, planning campaigns, executing campaigns, analyzing results, generating reports, or inspecting/reproducing experiment runs." --- # VariaQ from bb The VariaQ plugin is a thin adapter: every capability below shells out to the **standalone** VariaQ CLI (`python -m variaq`) from its installed checkout. -No solver/quantum logic lives in this plugin — read VariaQ's own -ARCHITECTURE.md for solver internals. +No solver/quantum logic, campaign planning, analysis, statistics, report +generation, BQM logic, plotting, or provenance logic lives in this plugin — +read VariaQ's own ARCHITECTURE.md for solver internals. ## Tools @@ -21,7 +22,7 @@ ARCHITECTURE.md for solver internals. family (`maxcut`, `assignment`, `subset-selection`, `graph-partition`) → `problemId`. - `variaq_problem_show` — full problem document (family-specific data, - generation seed, sense). + generation seed, sense). **Read-only.** - `variaq_solve` — one solver run on a problem; returns `runId` + full run record (objective, optimality gap, solution bitstring, backend metrics, feasibility diagnostics, BQM metadata). Solver/family compatibility is checked @@ -34,12 +35,33 @@ ARCHITECTURE.md for solver internals. Returns persisted run ids and records. - `variaq_runs_list` / `variaq_run_show` / `variaq_run_reproduce` — browse, inspect, and re-execute durable runs from the SQLite experiment store. +- `variaq_campaign_plan` — non-executing preview of a campaign: requested run + count, solver breakdown, unavailable combinations, and max-run warnings. + **Read-only.** +- `variaq_campaign_run` — execute all planned solver runs bounded by an explicit + `maxRuns` value; only sets `--override-max-runs` when the caller explicitly + opts in. Campaigns can contain mixed success/failed/skipped/unavailable results; + the plugin preserves VariaQ's exact status summary. +- `variaq_campaign_list` / `variaq_campaign_show` — browse and inspect stored + campaigns. **Read-only.** +- `variaq_analyze_runs` / `variaq_analyze_campaign` — read-only aggregate + analysis over selected runs or a single campaign. Returns VariaQ's + `AnalysisResult` directly (groups, quality, feasibility, timing, resources, + repeat statistics, scaling points, comparisons, warnings, source run IDs). + The plugin does not compute statistics itself. +- `variaq_report_campaign` — generate JSON/CSV/Markdown reports (and optional + plots if matplotlib is installed) from a campaign's stored runs. Writes files + beneath the configured `reportOutputDir` and rejects path traversal. The + report preserves `report_id`, `source_run_ids`, `report_format_version`, + `generated_at`, and `variaq_version`. ## CLI `bb variaq --help` lists the same operations (`bb variaq status`, `solve …`, -`bb variaq benchmark …`, `bb variaq runs …`, …); add `--json` for -machine-readable output. +`bb variaq benchmark …`, `bb variaq runs …`, `bb variaq campaign-plan …`, +`bb variaq campaign-run …`, `bb variaq campaigns …`, `bb variaq campaign …`, +`bb variaq analyze-campaign …`, `bb variaq report-campaign …`, …); add `--json` +for machine-readable output. ## Constraints @@ -51,11 +73,22 @@ machine-readable output. - A discoverable CUDA-Q `nvidia` target is not enough for GPU availability; status also requires a usable NVIDIA driver and a positive CUDA-Q GPU count. - **No physical QPU execution exists anywhere in this VariaQ release.** +- **Campaign execution is local and single-threaded.** Plan before large runs; + `variaq_campaign_plan` is safe and non-executing. +- **Analysis is read-only.** The plugin does not calculate statistics, gaps, + or scaling points; it forwards VariaQ's `AnalysisResult` unchanged. +- **Reports derive from stored runs.** Report generation may create files, but + it never mutates run records, problems, or campaign definitions. - Problems default to `/data/problems/` and the experiment DB to `/data/variaq.sqlite3` — change via `bb plugin config variaq set ` - (`dbPath`, `problemsDir`, `pythonPath`, `projectDir`, `timeoutMs`). + (`dbPath`, `problemsDir`, `pythonPath`, `projectDir`, `reportOutputDir`, + `timeoutMs`). Report outputs are constrained to `reportOutputDir`. - All runs are deterministic given their seed; VariaQ appends runs, it never mutates them. - Quantum runs may return some feasible and some infeasible samples. The plugin reports VariaQ's own feasibility counts and objective; it does not treat nonzero infeasible counts as a run failure. +- **Graph Partition quantum support remains unadvertised** unless VariaQ + capabilities change in a future release. +- bb-plugin-variaq 0.5.x is verified against VariaQ 0.6.x and schema version `1`. + Other VariaQ series or schema versions are reported as unsupported. \ No newline at end of file diff --git a/tests/fake.test.ts b/tests/fake.test.ts index 39c83a0..c03ac0a 100644 --- a/tests/fake.test.ts +++ b/tests/fake.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createFakePluginHost } from "@get-bb/plugin-sdk/testing"; -import { mkdtempSync, rmSync, symlinkSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -25,7 +25,7 @@ afterEach(async () => { vi.unstubAllEnvs(); }); -async function setup(tools: { timeoutMs?: number } = {}) { +async function setup(options: { timeoutMs?: number; reportOutputDir?: string } = {}) { const dir = mkdtempSync(join(tmpdir(), "variaq-fake-")); temps.push(dir); const fakePython = join(dir, "python"); @@ -34,12 +34,15 @@ async function setup(tools: { timeoutMs?: number } = {}) { const h = createFakePluginHost({ pluginId: "variaq" }); hosts.push(h); await plugin(h.bb); + const campaignState = join(dir, "campaigns.json"); + vi.stubEnv("FAKE_CAMPAIGN_STATE", campaignState); await h.harness.behavior.setSettings({ pythonPath: fakePython, projectDir: dir, dbPath: null, problemsDir: null, - timeoutMs: tools.timeoutMs ?? 120_000, + reportOutputDir: options.reportOutputDir ?? join(dir, "reports"), + timeoutMs: options.timeoutMs ?? 120_000, }); return h; } @@ -71,20 +74,20 @@ describe("status via fake CLI", () => { expect(status.solvers["cudaq-cpu"]).toBe("unavailable"); }); - it("accepts supported VariaQ 0.5 patch versions without a warning", async () => { - vi.stubEnv("FAKE_VARIAQ_VERSION", "0.5.2"); + it("accepts supported VariaQ 0.6 patch versions without a warning", async () => { + vi.stubEnv("FAKE_VARIAQ_VERSION", "0.6.2"); const h = await setup(); const status = JSON.parse(String(await tool(h, "variaq_status", {}))); expect(status.variaq.compatibility.supported).toBe(true); expect(status.variaq.compatibility.warning).toBeNull(); }); - it("reports 0.4 series as unsupported", async () => { - vi.stubEnv("FAKE_VARIAQ_VERSION", "0.4.2"); + it("reports 0.5 series as unsupported", async () => { + vi.stubEnv("FAKE_VARIAQ_VERSION", "0.5.2"); const h = await setup(); const status = JSON.parse(String(await tool(h, "variaq_status", {}))); expect(status.variaq.compatibility.supported).toBe(false); - expect(status.variaq.compatibility.warning).toMatch(/Unsupported VariaQ version 0\.4\.2/); + expect(status.variaq.compatibility.warning).toMatch(/Unsupported VariaQ version 0\.5\.2/); }); }); @@ -310,6 +313,206 @@ describe("solve via fake CLI", () => { }, 20_000); }); +describe("campaign tools", () => { + const tinyCampaign = { + campaign_format_version: "1", + name: "maxcut-tiny", + family: "maxcut", + problem_sizes: [4], + problem_seeds: [1], + solvers: ["exact", "heuristic"], + repeats: 1, + base_seed: 42, + generator_parameters: { edge_probability: 0.4 }, + solver_config: {}, + }; + + it("plans a campaign without executing solvers", async () => { + const h = await setup(); + const result = JSON.parse(String(await tool(h, "variaq_campaign_plan", tinyCampaign))); + expect(result.plan.requested_runs).toBe(2); + expect(result.plan.exceeds_default_max).toBe(false); + expect(result.plan.solver_breakdown).toHaveLength(2); + expect(result.warnings).toEqual([]); + expect(result).not.toHaveProperty("tempFile"); + }); + + it("preserves run-count warning in plan", async () => { + const h = await setup(); + const big = { ...tinyCampaign, problem_sizes: [4, 6, 8, 10, 12, 14, 16, 18, 20], problem_seeds: [1, 2, 3, 4, 5, 6], solvers: ["exact", "heuristic", "qaoa"], repeats: 4 }; + const result = JSON.parse(String(await tool(h, "variaq_campaign_plan", big))); + expect(result.plan.requested_runs).toBeGreaterThan(500); + expect(result.plan.exceeds_default_max).toBe(true); + expect(result.plan.warnings.some((w: string) => w.includes("default maximum"))).toBe(true); + }); + + it("runs a campaign and preserves mixed status summary", async () => { + const h = await setup(); + const result = JSON.parse(String(await tool(h, "variaq_campaign_run", { ...tinyCampaign, maxRuns: 10, overrideMaxRuns: false }))); + expect(result.exitCode).toBe(0); + expect(result.campaignId).toMatch(/^campaign-/); + expect(result.summary.status_summary).toMatchObject({ success: expect.any(Number), failed: 1, skipped: 1, unavailable: 0 }); + }); + + it("refuses to run a campaign exceeding max-runs without override", async () => { + const h = await setup(); + const big = { ...tinyCampaign, problem_sizes: [4, 6, 8, 10, 12, 14, 16, 18], problem_seeds: [1, 2, 3, 4, 5, 6], solvers: ["exact", "heuristic", "qaoa"], repeats: 4 }; + const result = JSON.parse(String(await tool(h, "variaq_campaign_run", { ...big, maxRuns: 500, overrideMaxRuns: false }))); + expect(result.exitCode).toBe(2); + expect(result.error?.type).toBe("ValidationError"); + expect(result.error?.message).toMatch(/exceeding maximum/); + }); + + it("lists and shows campaigns", async () => { + const h = await setup(); + const run = JSON.parse(String(await tool(h, "variaq_campaign_run", { ...tinyCampaign, name: "listable", maxRuns: 10, overrideMaxRuns: false }))); + const list = JSON.parse(String(await tool(h, "variaq_campaign_list", { limit: 10 }))); + expect(list.campaigns.some((c: { campaign_id: string }) => c.campaign_id === run.campaignId)).toBe(true); + const show = JSON.parse(String(await tool(h, "variaq_campaign_show", { campaignId: run.campaignId }))); + expect(show.campaign.name).toBe("listable"); + }); + + it("rejects malformed campaign JSON via CLI", async () => { + const h = await setup(); + const cliResult = await cli(h, ["campaign-plan", JSON.stringify({ campaign_format_version: "1", name: "bad" })]); + expect(cliResult.exitCode).toBe(1); + expect(String(cliResult.stderr)).toMatch(/Invalid campaign definition/); + }); + + it("formats campaign plan and run output concisely for humans", async () => { + const h = await setup(); + const plan = await cli(h, ["campaign-plan", JSON.stringify(tinyCampaign)]); + expect(plan.stdout).toContain("Campaign plan: maxcut-tiny"); + expect(plan.stdout).toContain("requested runs: 2"); + expect(plan.stdout).not.toContain('"solver_breakdown"'); + const run = await cli(h, ["campaign-run", JSON.stringify(tinyCampaign), "--max-runs", "10"]); + expect(run.stdout).toContain("Campaign run: campaign-"); + expect(run.stdout).toContain("failed: 1, skipped: 1"); + expect(run.stdout).not.toContain('"status_summary"'); + }); +}); + +describe("analyze tools", () => { + it("analyzes runs and preserves source_run_ids", async () => { + const h = await setup(); + const result = JSON.parse(String(await tool(h, "variaq_analyze_runs", { + run_ids: ["run-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"], + group_by: ["solver"], + scaling_x: "problem_size", + }))); + expect(result.analysis.source_run_ids).toContain("run-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"); + expect(result.analysis.groups).toHaveLength(1); + expect(result.analysis.scaling_points).toHaveLength(1); + expect(result.analysis.scaling_points[0].x_value).toBe(4); + }); + + it("null metrics remain null", async () => { + const h = await setup(); + const result = JSON.parse(String(await tool(h, "variaq_analyze_runs", { run_ids: ["run-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"] }))); + expect(result.analysis.groups[0].feasibility.feasible_sample_count).toBeNull(); + expect(result.analysis.groups[0].resource.qubits).toBeNull(); + }); + + it("formats analysis output as a compact summary", async () => { + const h = await setup(); + const result = await cli(h, [ + "analyze-runs", + "--run-id", + "run-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "--group-by", + "solver", + ]); + expect(result.stdout).toContain("Analysis"); + expect(result.stdout).toContain("source runs: 1"); + expect(result.stdout).toContain("groups: 1"); + expect(result.stdout).not.toContain('"analysis"'); + }); +}); + +describe("report campaign", () => { + it("generates JSON/CSV/Markdown files and preserves report_id", async () => { + const h = await setup(); + const run = JSON.parse(String(await tool(h, "variaq_campaign_run", { + campaign_format_version: "1", + name: "reportable", + family: "maxcut", + problem_sizes: [4], + problem_seeds: [1], + solvers: ["exact"], + repeats: 1, + base_seed: 42, + generator_parameters: {}, + solver_config: {}, + maxRuns: 10, + }))); + const result = JSON.parse(String(await tool(h, "variaq_report_campaign", { + campaignId: run.campaignId, + outputDir: "test-reports", + formats: ["json", "csv", "markdown"], + }))); + expect(result.reportId).toMatch(/^report-/); + expect(result.paths.json).toContain("report-"); + expect(result.outputDir).toContain("test-reports"); + }); + + it("rejects traversal outputDir", async () => { + const h = await setup(); + const result = JSON.parse(String(await tool(h, "variaq_report_campaign", { + campaignId: "campaign-doesnotmatter", + outputDir: "../../outside", + formats: ["json"], + }))); + expect(result.error).toMatch(/outside the allowed report root/); + }); + + it("uses a stable default root and requires explicit overwrite", async () => { + const h = await setup({ reportOutputDir: "" }); + const input = { + campaignId: "campaign-stable-report", + outputDir: "repeat", + formats: ["json"], + }; + const first = JSON.parse(String(await tool(h, "variaq_report_campaign", input))); + expect(first.reportId).toMatch(/^report-/); + const second = JSON.parse(String(await tool(h, "variaq_report_campaign", input))); + expect(second.error).toMatch(/already exists/); + const replaced = JSON.parse(String(await tool(h, "variaq_report_campaign", { ...input, overwrite: true }))); + expect(replaced.reportId).toBe(first.reportId); + }); + + it("rejects a symlink component beneath the report root", async () => { + const base = mkdtempSync(join(tmpdir(), "variaq-report-link-")); + temps.push(base); + const reportRoot = join(base, "reports"); + const outside = join(base, "outside"); + mkdirSync(reportRoot); + mkdirSync(outside); + symlinkSync(outside, join(reportRoot, "escape")); + const h = await setup({ reportOutputDir: reportRoot }); + const result = JSON.parse(String(await tool(h, "variaq_report_campaign", { + campaignId: "campaign-doesnotmatter", + outputDir: "escape/nested", + formats: ["json"], + }))); + expect(result.error).toMatch(/symbolic link/); + }); + + it("formats report output as a compact file summary", async () => { + const h = await setup(); + const result = await cli(h, [ + "report-campaign", + "campaign-human-output", + "--output-dir", + "human-output", + "--formats", + "json,markdown", + ]); + expect(result.stdout).toContain("Report: report-"); + expect(result.stdout).toContain("generated files: 2"); + expect(result.stdout).not.toContain('"paths"'); + }); +}); + describe("benchmark", () => { it("returns partial when a solver/family combination is unsupported", async () => { const h = await setup(); diff --git a/tests/fake_variaq.py b/tests/fake_variaq.py index 1ff0712..4272c84 100755 --- a/tests/fake_variaq.py +++ b/tests/fake_variaq.py @@ -2,12 +2,13 @@ """ Deterministic stand-in for the VariaQ CLI used by bb-plugin-variaq tests. -Emulates VariaQ 0.5.0 schema-v1 envelopes so tests can verify the plugin's +Emulates VariaQ 0.6.0 schema-v1 envelopes so tests can verify the plugin's JSON contract without touching a real VariaQ install or solver. Recognized argv shape: python fake_variaq.py [--db P] [--problems-dir P] """ +import hashlib import json import os import sys @@ -15,12 +16,31 @@ import uuid OUTPUT_SCHEMA_VERSION = "1" +DEFAULT_MAX_RUNS = 500 +FAKE_VARIAQ_VERSION = "0.6.0" FAMILIES = ["maxcut", "assignment", "subset-selection", "graph-partition"] CLASSICAL_FAMILIES = FAMILIES -# VariaQ 0.5 advertises these quantum families dynamically; the fake mirrors that. +# VariaQ 0.6 advertises these quantum families dynamically; the fake mirrors that. QUANTUM_FAMILIES = ["maxcut", "assignment", "subset-selection"] +CAMPAIGNS = {} +CAMPAIGN_STATE_FILE = os.environ.get("FAKE_CAMPAIGN_STATE", ".fake_campaigns.json") + + +def _load_campaigns(): + global CAMPAIGNS + try: + with open(CAMPAIGN_STATE_FILE) as f: + CAMPAIGNS = json.load(f) + except Exception: + CAMPAIGNS = {} + + +def _save_campaigns(): + with open(CAMPAIGN_STATE_FILE, "w") as f: + json.dump(CAMPAIGNS, f) + def envelope(command, status, data, warnings=None, error=None): out = { @@ -40,6 +60,50 @@ def fake_run_id(): return f"run-{uuid.uuid4()}" +def make_analysis_group(group_by, run_id, problem_size=4): + group_key = {k: (run_id if k == "run_id" else ("exact" if k == "solver" else f"{k}-value")) for k in group_by} + return { + "count": 1, + "environment_versions": {"variaq": [FAKE_VARIAQ_VERSION], "python": ["3.12.3"]}, + "feasibility": { + "best_feasible_objective": 2.0, + "best_infeasible_energy": None, + "count": 1, + "feasible_runs": 1, + "feasible_sample_count": None, + "infeasible_runs": 0, + "infeasible_sample_count": None, + "max_feasible_rate": None, + "mean_feasible_rate": None, + "median_feasible_rate": None, + "min_feasible_rate": None, + "zero_feasible_runs": 0, + }, + "group_key": group_key, + "problem_ids": ["maxcut-fake000000001"], + "quality": { + "approximation_ratio": 1.0, + "best_objective": 2.0, + "count": 1, + "mean_gap_percent": 0.0, + "mean_objective": 2.0, + "median_objective": 2.0, + "success_at_optimum_rate": 1.0, + "worst_objective": 2.0, + }, + "resource": {"auxiliary_variables": None, "binary_variables": None, "circuit_depth": None, "count": 1, "estimated_statevector_bytes": None, "gate_count": None, "logical_variables": problem_size, "qubits": None}, + "run_ids": [run_id], + "timing": {"count": 1, "expectation_evaluation_seconds": None, "initialization_seconds": None, "parameter_search_seconds": None, "sampling_seconds": None, "solver_time_seconds": None, "total_wall_time_seconds": None, "warmup_seconds": None}, + } + + +def make_scaling_point(group_by, run_id, x_metric, x_value): + point = make_analysis_group(group_by, run_id, problem_size=int(x_value)) + point["x_metric"] = x_metric + point["x_value"] = float(x_value) + return point + + def supported_family(family): return family in FAMILIES @@ -79,7 +143,7 @@ def main(argv): cmd = args[0] if cmd == "--version": - version = os.environ.get("FAKE_VARIAQ_VERSION", "9.9.9") + version = os.environ.get("FAKE_VARIAQ_VERSION", FAKE_VARIAQ_VERSION) print(f"variaq {version}") return 0 @@ -87,6 +151,275 @@ def main(argv): time.sleep(30) return 0 + if cmd == "campaign" and len(args) > 1 and args[1] == "plan": + path = args[2] if len(args) > 2 else "missing" + try: + with open(path) as f: + definition = json.load(f) + except Exception as e: + print(json.dumps(envelope("campaign plan", "error", None, error={ + "type": "ValidationError", + "message": f"Cannot read campaign file: {e}", + }))) + return 2 + sizes = definition.get("problem_sizes", []) + seeds = definition.get("problem_seeds", []) + solvers = definition.get("solvers", []) + repeats = definition.get("repeats", 1) + requested = len(sizes) * len(seeds) * len(solvers) * repeats + instance_count = len(sizes) * len(seeds) + max_binary = max(sizes) if sizes else 0 + quantum_runs = requested if any(s in ("qaoa", "cudaq-cpu", "cudaq-gpu") for s in solvers) else 0 + breakdown = [] + for s in solvers: + sup = s in ("exact", "heuristic") or (s in ("qaoa", "cudaq-cpu", "cudaq-gpu") and quantum_supported_family(definition.get("family", "maxcut"))) + breakdown.append({ + "solver": s, + "supported": sup, + "installed": sup, + "available": sup, + "requested_runs": instance_count * repeats, + }) + warnings = [] + if requested > DEFAULT_MAX_RUNS: + warnings.append(f"Campaign requests {requested} runs; default maximum is {DEFAULT_MAX_RUNS}. Use --override-max-runs to execute.") + print(json.dumps(envelope("campaign plan", "success", { + "campaign_id": "not-yet-executed", + "default_max_runs": DEFAULT_MAX_RUNS, + "estimated_quantum_runs": quantum_runs, + "exceeds_default_max": requested > DEFAULT_MAX_RUNS, + "family": definition.get("family", "maxcut"), + "max_binary_variables": max_binary, + "name": definition.get("name", "fake-campaign"), + "problem_instance_count": instance_count, + "repeats": repeats, + "requested_runs": requested, + "solver_breakdown": breakdown, + "unavailable": [], + "warnings": warnings, + }))) + return 0 + + if cmd == "campaign" and len(args) > 1 and args[1] == "run": + path = args[2] if len(args) > 2 else "missing" + max_runs = DEFAULT_MAX_RUNS + override = False + for i, a in enumerate(args): + if a == "--max-runs" and i + 1 < len(args): + max_runs = int(args[i + 1]) + if a == "--override-max-runs": + override = True + try: + with open(path) as f: + definition = json.load(f) + except Exception as e: + print(json.dumps(envelope("campaign run", "error", None, error={ + "type": "ValidationError", + "message": f"Cannot read campaign file: {e}", + }))) + return 2 + sizes = definition.get("problem_sizes", []) + seeds = definition.get("problem_seeds", []) + solvers = definition.get("solvers", []) + repeats = definition.get("repeats", 1) + requested = len(sizes) * len(seeds) * len(solvers) * repeats + if requested > max_runs and not override: + print(json.dumps(envelope("campaign run", "error", None, error={ + "type": "ValidationError", + "message": f"Campaign requests {requested} runs, exceeding maximum {max_runs}. Use --override-max-runs to execute.", + }))) + return 2 + # Deterministic campaign id based on canonical content. + campaign_id = f"campaign-{abs(hash(json.dumps(definition, sort_keys=True))) & 0xffffffffffffffff:016x}" + run_ids = [fake_run_id() for _ in range(min(requested, max_runs))] + problem_ids = [f"{definition.get('family', 'maxcut')}-fake{campaign_id[-8:]}" for _ in sizes] + # Simulate mixed results: first run fails, second skipped, rest success. + summary = {"success": 0, "failed": 0, "skipped": 0, "unavailable": 0} + for i, _ in enumerate(run_ids): + if i == 0: + summary["failed"] += 1 + elif i == 1: + summary["skipped"] += 1 + else: + summary["success"] += 1 + CAMPAIGNS[campaign_id] = definition + _save_campaigns() + print(json.dumps(envelope("campaign run", "success", { + "campaign_id": campaign_id, + "completed_runs": len(run_ids), + "family": definition.get("family", "maxcut"), + "name": definition.get("name", "fake-campaign"), + "problem_ids": problem_ids[:len(sizes)], + "requested_runs": requested, + "run_ids": run_ids, + "status_summary": summary, + }))) + return 0 + + if cmd == "campaign" and len(args) > 1 and args[1] == "list": + _load_campaigns() + limit = 20 + for i, a in enumerate(args): + if a == "--limit" and i + 1 < len(args): + limit = int(args[i + 1]) + campaigns = [ + {"campaign_id": cid, "name": defn.get("name", "fake"), "family": defn.get("family", "maxcut"), "created_at": "2026-01-01T00:00:00+00:00"} + for cid, defn in CAMPAIGNS.items() + ] + print(json.dumps(envelope("campaign list", "success", campaigns[:limit]))) + return 0 + + if cmd == "campaign" and len(args) > 1 and args[1] == "show": + _load_campaigns() + cid = args[2] if len(args) > 2 else "missing" + definition = CAMPAIGNS.get(cid) + if definition is None: + print(json.dumps(envelope("campaign show", "error", None, error={ + "type": "ValidationError", + "message": f"no such campaign {cid}", + }))) + return 2 + show = dict(definition) + show["campaign_format_version"] = definition.get("campaign_format_version", "1") + show["created_at"] = "2026-01-01T00:00:00+00:00" + show.setdefault("notes", "") + show.setdefault("tags", []) + print(json.dumps(envelope("campaign show", "success", show))) + return 0 + + if cmd == "analyze" and len(args) > 1 and args[1] == "runs": + run_ids = [] + group_by = [] + scaling_x = None + for i, a in enumerate(args): + if a == "--run-id" and i + 1 < len(args): + run_ids.append(args[i + 1]) + if a == "--group-by" and i + 1 < len(args): + group_by.append(args[i + 1]) + if a == "--scaling-x" and i + 1 < len(args): + scaling_x = args[i + 1] + groups = [] + scaling_points = [] + for rid in run_ids: + groups.append(make_analysis_group(group_by or ["solver"], rid, problem_size=4)) + if scaling_x: + scaling_points.append(make_scaling_point(group_by or ["solver"], rid, scaling_x, 4)) + print(json.dumps(envelope("analyze runs", "success", { + "groups": groups, + "comparisons": [], + "query": {"run_ids": run_ids, "group_by": group_by or ["solver"], "filters": {}, "scaling_x": scaling_x, "include_failed": False, "include_unavailable": False, "campaign_id": None}, + "scaling_points": scaling_points, + "source_run_ids": run_ids, + "warnings": [], + }))) + return 0 + + if cmd == "analyze" and len(args) > 1 and args[1] == "campaign": + _load_campaigns() + cid = args[2] if len(args) > 2 else "missing" + definition = CAMPAIGNS.get(cid) + if definition is None: + print(json.dumps(envelope("analyze campaign", "error", None, error={ + "type": "ValidationError", + "message": f"no such campaign {cid}", + }))) + return 2 + group_by = [] + scaling_x = None + for i, a in enumerate(args): + if a == "--group-by" and i + 1 < len(args): + group_by.append(args[i + 1]) + if a == "--scaling-x" and i + 1 < len(args): + scaling_x = args[i + 1] + run_ids = [fake_run_id() for _ in range(3)] + groups = [make_analysis_group(group_by or ["solver"], rid, problem_size=4) for rid in run_ids] + scaling_points = [make_scaling_point(group_by or ["solver"], rid, scaling_x, 4) for rid in run_ids] if scaling_x else [] + print(json.dumps(envelope("analyze campaign", "success", { + "groups": groups, + "comparisons": [], + "query": {"campaign_id": cid, "group_by": group_by or ["solver"], "filters": {}, "scaling_x": scaling_x, "include_failed": False, "include_unavailable": False}, + "scaling_points": scaling_points, + "source_run_ids": run_ids, + "warnings": [{"type": "fake_warning", "message": "simulated analysis warning"}], + }))) + return 0 + + if cmd == "report" and len(args) > 1 and args[1] == "campaign": + cid = args[2] if len(args) > 2 else "missing" + output_dir = "." + formats = ["json"] + group_by = [] + scaling_x = None + plots = False + overwrite = False + for i, a in enumerate(args): + if a == "--output-dir" and i + 1 < len(args): + output_dir = args[i + 1] + if a == "--formats" and i + 1 < len(args): + formats = [s.strip() for s in args[i + 1].split(",") if s.strip()] + if a == "--group-by" and i + 1 < len(args): + group_by.append(args[i + 1]) + if a == "--scaling-x" and i + 1 < len(args): + scaling_x = args[i + 1] + if a == "--plots": + plots = True + if a == "--overwrite": + overwrite = True + os.makedirs(output_dir, exist_ok=True) + report_key = cid + json.dumps(formats, sort_keys=True) + report_id = f"report-{hashlib.sha256(report_key.encode()).hexdigest()[:16]}" + prospective_paths = [] + if "json" in formats: + prospective_paths.append(os.path.join(output_dir, f"{report_id}.json")) + if "csv" in formats: + prospective_paths.extend([ + os.path.join(output_dir, f"{report_id}_groups.csv"), + os.path.join(output_dir, f"{report_id}_scaling.csv"), + ]) + if "markdown" in formats: + prospective_paths.append(os.path.join(output_dir, f"{report_id}.md")) + if plots and "plots" in formats: + prospective_paths.append(os.path.join(output_dir, f"{report_id}_plot.png")) + if not overwrite and any(os.path.exists(p) for p in prospective_paths): + print(json.dumps(envelope("report campaign", "error", None, error={ + "type": "ValidationError", + "message": "Report output already exists; pass --overwrite to replace it.", + }))) + return 2 + paths = {} + if "json" in formats: + json_path = os.path.join(output_dir, f"{report_id}.json") + with open(json_path, "w") as f: + json.dump({ + "report_format_version": "1", + "report_id": report_id, + "campaign_id": cid, + "generated_at": "2026-01-01T00:00:00+00:00", + "variaq_version": os.environ.get("FAKE_VARIAQ_VERSION", FAKE_VARIAQ_VERSION), + "source_run_ids": [fake_run_id()], + "analysis": {"groups": []}, + }, f) + paths["json"] = json_path + if "csv" in formats: + groups_path = os.path.join(output_dir, f"{report_id}_groups.csv") + scaling_path = os.path.join(output_dir, f"{report_id}_scaling.csv") + for p in (groups_path, scaling_path): + with open(p, "w") as f: + f.write("group,count\n") + paths["csv"] = {"groups": groups_path, "scaling": scaling_path} + if "markdown" in formats: + md_path = os.path.join(output_dir, f"{report_id}.md") + with open(md_path, "w") as f: + f.write(f"# Report {report_id}\n") + paths["markdown"] = md_path + if plots and "plots" in formats: + plot_path = os.path.join(output_dir, f"{report_id}_plot.png") + with open(plot_path, "wb") as f: + f.write(b"PNG") + paths["plots"] = {"summary": plot_path} + print(json.dumps(envelope("report campaign", "success", {"report_id": report_id, "paths": paths}))) + return 0 + if cmd == "capabilities": mode = os.environ.get("FAKE_CUDAQ", "unavailable") available = mode in ("available", "driver-down") @@ -102,7 +435,7 @@ def main(argv): "type": "backend_availability", "message": "CUDA-Q NVIDIA target present but reports no compatible GPU", }) - version = os.environ.get("FAKE_VARIAQ_VERSION", "9.9.9") + version = os.environ.get("FAKE_VARIAQ_VERSION", FAKE_VARIAQ_VERSION) solvers = [ {"name": "exact", "supported": True, "installed": True, "available": True, "supported_families": list(CLASSICAL_FAMILIES)}, @@ -655,7 +988,7 @@ def make_run(problem, solver, seed, family=None): }))) return 2 new_id = fake_run_id() - version = os.environ.get("FAKE_VARIAQ_VERSION", "0.5.0") + version = os.environ.get("FAKE_VARIAQ_VERSION", FAKE_VARIAQ_VERSION) print(json.dumps(envelope("runs reproduce", "success", { "original_run_id": rid, "new_run_id": new_id, diff --git a/tests/runner.test.ts b/tests/runner.test.ts index 95487b0..ad242a3 100644 --- a/tests/runner.test.ts +++ b/tests/runner.test.ts @@ -29,6 +29,7 @@ const base: VariaqSettings = { projectDir: "", dbPath: "", problemsDir: "", + reportOutputDir: "", timeoutMs: 1000, }; @@ -334,16 +335,16 @@ describe("runVariaqJson", () => { }); describe("VariaQ output contracts", () => { - it("accepts patch releases in the verified 0.5 series", () => { - expect(checkVariaqVersion("variaq 0.5.0").supported).toBe(true); - expect(checkVariaqVersion("0.5.7").supported).toBe(true); - expect(checkVariaqVersion("0.5.0", "1").schemaVersionSupported).toBe(true); + it("accepts patch releases in the verified 0.6 series", () => { + expect(checkVariaqVersion("variaq 0.6.0").supported).toBe(true); + expect(checkVariaqVersion("0.6.7").supported).toBe(true); + expect(checkVariaqVersion("0.6.0", "1").schemaVersionSupported).toBe(true); }); - it("rejects the 0.4 series as unsupported", () => { - const result = checkVariaqVersion("variaq 0.4.1", "1"); + it("rejects the 0.5 series as unsupported", () => { + const result = checkVariaqVersion("variaq 0.5.1", "1"); expect(result.supported).toBe(false); - expect(result.warning).toMatch(/Unsupported VariaQ version 0\.4\.1/); + expect(result.warning).toMatch(/Unsupported VariaQ version 0\.5\.1/); }); it("warns without hard-failing for unsupported versions", () => { diff --git a/tests/server.test.ts b/tests/server.test.ts index 5e78ef5..cead7f0 100644 --- a/tests/server.test.ts +++ b/tests/server.test.ts @@ -44,6 +44,7 @@ async function setupHost() { projectDir: VARIAQ_PROJECT, dbPath: join(tmp, "exp.db"), problemsDir: join(tmp, "problems"), + reportOutputDir: join(tmp, "reports"), timeoutMs: 180_000, }); return h; @@ -62,7 +63,7 @@ describeReal("plugin registration", () => { const h = await setupHost(); const version = await cli(h, ["version"]); expect(version.exitCode).toBe(0); - expect(version.stdout).toContain("variaq 0.5.0"); + expect(version.stdout).toContain("variaq 0.6.0"); }); }); @@ -72,7 +73,7 @@ describeReal("status/version", () => { const result = await tool(h, "variaq_status", {}); const s = JSON.parse(String(result)); expect(s.variaq.installed).toBe(true); - expect(s.variaq.version).toBe("0.5.0"); + expect(s.variaq.version).toBe("0.6.0"); expect(s.variaq.schema_version).toBe("1"); expect(s.variaq.python).toBe(VARIAQ_PYTHON); expect(s.solvers.exact).toBe("available"); @@ -348,6 +349,72 @@ describeReal("problem import", () => { }); }); +describeReal("campaign, analysis, and report integration", () => { + it("plans and runs a bounded campaign, analyzes it, and writes provenance-preserving reports", async () => { + const h = await setupHost(); + const campaign = { + campaign_format_version: "1", + name: "ci-maxcut-scaling", + family: "maxcut", + problem_sizes: [4, 5], + problem_seeds: [1], + solvers: ["exact", "heuristic"], + repeats: 1, + base_seed: 42, + generator_parameters: { edge_probability: 0.4 }, + solver_config: {}, + }; + + const plan = JSON.parse(String(await tool(h, "variaq_campaign_plan", campaign))); + expect(plan.plan.requested_runs).toBe(4); + expect(plan.plan.exceeds_default_max).toBe(false); + expect(plan.plan.solver_breakdown).toHaveLength(2); + expect(plan).not.toHaveProperty("tempFile"); + + const run = JSON.parse(String(await tool(h, "variaq_campaign_run", { + ...campaign, + maxRuns: 10, + overrideMaxRuns: false, + }))); + expect(run.exitCode).toBe(0); + expect(run.campaignId).toMatch(/^campaign-/); + expect(run.summary.requested_runs).toBe(4); + expect(run.summary.completed_runs).toBe(4); + expect(run.summary.status_summary).toEqual({ success: 4, failed: 0, skipped: 0, unavailable: 0 }); + + const analysis = JSON.parse(String(await tool(h, "variaq_analyze_campaign", { + campaignId: run.campaignId, + group_by: ["solver"], + scaling_x: "problem_size", + include_failed: true, + include_unavailable: true, + }))); + expect(analysis.analysis.source_run_ids).toEqual(expect.arrayContaining(run.summary.run_ids)); + expect(analysis.analysis.groups.length).toBeGreaterThan(0); + expect(analysis.analysis.scaling_points.length).toBeGreaterThan(0); + expect(analysis.analysis.groups[0].environment_versions.variaq).toContain("0.6.0"); + + const report = JSON.parse(String(await tool(h, "variaq_report_campaign", { + campaignId: run.campaignId, + outputDir: "ci-smoke", + formats: ["json", "csv", "markdown"], + groupBy: ["solver"], + scalingX: "problem_size", + }))); + expect(report.reportId).toMatch(/^report-/); + expect(report.paths.csv.groups).toBeTruthy(); + expect(report.paths.markdown).toBeTruthy(); + const reportDocument = JSON.parse(readFileSync(report.paths.json as string, "utf8")); + expect(reportDocument.report_id).toBe(report.reportId); + expect(reportDocument.campaign_id).toBe(run.campaignId); + expect(reportDocument.source_run_ids).toEqual(expect.arrayContaining(run.summary.run_ids)); + expect(reportDocument.report_format_version).toBe("1"); + expect(reportDocument.variaq_version).toBe("0.6.0"); + expect(reportDocument.generated_at).toBeTruthy(); + expect(reportDocument.analysis.query.campaign_id).toBe(run.campaignId); + }); +}); + describeReal("real heuristic and Qiskit integration", () => { it("runs deterministic heuristic and local Qiskit QAOA solves on maxcut", async () => { const h = await setupHost();